Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3499,
     "status": "ok",
     "timestamp": 1566970390531,
     "user": {
      "displayName": "i150226 Saad Arshad",
      "photoUrl": "",
      "userId": "18168902830249938130"
     },
     "user_tz": -300
    },
    "id": "O5PyrlI04-nf",
    "outputId": "9cb1c831-6593-4c24-ee8b-37e26153ddcd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.13.1\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers , activations , models , preprocessing , utils\n",
    "import pandas as pd\n",
    "\n",
    "print( tf.VERSION )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "h8H7mVpQ4-oZ"
   },
   "outputs": [],
   "source": [
    "data_path = 'clean_conversation.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YT3xecgZu9cy"
   },
   "outputs": [],
   "source": [
    "input_texts = []\n",
    "target_texts = []\n",
    "with open(data_path, 'r', encoding='utf-8') as f:\n",
    "    lines = f.read().split('\\n')\n",
    "for line in lines[: min(600, len(lines) - 1)]:\n",
    "    input_text = line.split('\\t')[0]\n",
    "    target_text = line.split('\\t')[1]\n",
    "    input_texts.append(input_text)\n",
    "    target_texts.append(target_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 657,
     "status": "ok",
     "timestamp": 1566970783839,
     "user": {
      "displayName": "i150226 Saad Arshad",
      "photoUrl": "",
      "userId": "18168902830249938130"
     },
     "user_tz": -300
    },
    "id": "t3BTZCofvDDM",
    "outputId": "f302c475-db73-42c8-e579-e940e7bafc3b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "566"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(input_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y_gZi8kAujGz"
   },
   "outputs": [],
   "source": [
    "zippedList =  list(zip(input_texts, target_texts))\n",
    "lines = pd.DataFrame(zippedList, columns = ['input' , 'output']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1210,
     "status": "ok",
     "timestamp": 1566970843462,
     "user": {
      "displayName": "i150226 Saad Arshad",
      "photoUrl": "",
      "userId": "18168902830249938130"
     },
     "user_tz": -300
    },
    "id": "zaot_rwEvXBc",
    "outputId": "7017b6b7-138b-43c9-d633-3fafb7605e78"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What are your interests</td>\n",
       "      <td>I am interested in all kinds of things. We can...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What are your favorite subjects</td>\n",
       "      <td>My favorite subjects include robotics, compute...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What are your interests</td>\n",
       "      <td>I am interested in a wide variety of topics, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What is your number</td>\n",
       "      <td>I don't have any number</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What is your number</td>\n",
       "      <td>23 skiddoo!</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             input  \\\n",
       "0          What are your interests   \n",
       "1  What are your favorite subjects   \n",
       "2          What are your interests   \n",
       "3              What is your number   \n",
       "4              What is your number   \n",
       "\n",
       "                                              output  \n",
       "0  I am interested in all kinds of things. We can...  \n",
       "1  My favorite subjects include robotics, compute...  \n",
       "2  I am interested in a wide variety of topics, a...  \n",
       "3                            I don't have any number  \n",
       "4                                        23 skiddoo!  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing input data for the Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 69
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1217,
     "status": "ok",
     "timestamp": 1566970853149,
     "user": {
      "displayName": "i150226 Saad Arshad",
      "photoUrl": "",
      "userId": "18168902830249938130"
     },
     "user_tz": -300
    },
    "id": "iB-LfSHe4-ol",
    "outputId": "26d97950-1610-404f-aa34-8ade21ed43e0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input max length is 22\n",
      "Encoder input data shape -> (566, 22)\n",
      "Number of Input tokens = 518\n"
     ]
    }
   ],
   "source": [
    "input_lines = list()\n",
    "for line in lines.input:\n",
    "    input_lines.append( line ) \n",
    "\n",
    "tokenizer = preprocessing.text.Tokenizer()\n",
    "tokenizer.fit_on_texts( input_lines ) \n",
    "tokenized_input_lines = tokenizer.texts_to_sequences( input_lines ) \n",
    "\n",
    "length_list = list()\n",
    "for token_seq in tokenized_input_lines:\n",
    "    length_list.append( len( token_seq ))\n",
    "max_input_length = np.array( length_list ).max()\n",
    "print( 'Input max length is {}'.format( max_input_length ))\n",
    "\n",
    "padded_input_lines = preprocessing.sequence.pad_sequences( tokenized_input_lines , maxlen=max_input_length , padding='post' )\n",
    "encoder_input_data = np.array( padded_input_lines )\n",
    "print( 'Encoder input data shape -> {}'.format( encoder_input_data.shape ))\n",
    "\n",
    "input_word_dict = tokenizer.word_index\n",
    "num_input_tokens = len( input_word_dict )+1\n",
    "print( 'Number of Input tokens = {}'.format( num_input_tokens))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing input data for the Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 69
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1224,
     "status": "ok",
     "timestamp": 1566970862207,
     "user": {
      "displayName": "i150226 Saad Arshad",
      "photoUrl": "",
      "userId": "18168902830249938130"
     },
     "user_tz": -300
    },
    "id": "49gbwsHS4-oy",
    "outputId": "143e9654-370c-4fda-b438-38d60402573d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output max length is 74\n",
      "Decoder input data shape -> (566, 74)\n",
      "Number of Output tokens = 1692\n"
     ]
    }
   ],
   "source": [
    "output_lines = list()\n",
    "for line in lines.output:\n",
    "    output_lines.append( '<START> ' + line + ' <END>' )  \n",
    "\n",
    "tokenizer = preprocessing.text.Tokenizer()\n",
    "tokenizer.fit_on_texts( output_lines ) \n",
    "tokenized_output_lines = tokenizer.texts_to_sequences( output_lines ) \n",
    "\n",
    "length_list = list()\n",
    "for token_seq in tokenized_output_lines:\n",
    "    length_list.append( len( token_seq ))\n",
    "max_output_length = np.array( length_list ).max()\n",
    "print( 'Output max length is {}'.format( max_output_length ))\n",
    "\n",
    "padded_output_lines = preprocessing.sequence.pad_sequences( tokenized_output_lines , maxlen=max_output_length, padding='post' )\n",
    "decoder_input_data = np.array( padded_output_lines )\n",
    "print( 'Decoder input data shape -> {}'.format( decoder_input_data.shape ))\n",
    "\n",
    "output_word_dict = tokenizer.word_index\n",
    "num_output_tokens = len( output_word_dict )+1\n",
    "print( 'Number of Output tokens = {}'.format( num_output_tokens))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing target data for the Decoder "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1213,
     "status": "ok",
     "timestamp": 1566970870299,
     "user": {
      "displayName": "i150226 Saad Arshad",
      "photoUrl": "",
      "userId": "18168902830249938130"
     },
     "user_tz": -300
    },
    "id": "VXAhyQxL4-o8",
    "outputId": "e796634e-a73f-4f9b-d206-85678aac2d78"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder target data shape -> (566, 74, 1692)\n"
     ]
    }
   ],
   "source": [
    "decoder_target_data = list()\n",
    "for token_seq in tokenized_output_lines:\n",
    "    decoder_target_data.append( token_seq[ 1 : ] ) \n",
    "    \n",
    "padded_output_lines = preprocessing.sequence.pad_sequences( decoder_target_data , maxlen=max_output_length, padding='post' )\n",
    "onehot_output_lines = utils.to_categorical( padded_output_lines , num_output_tokens )\n",
    "decoder_target_data = np.array( onehot_output_lines )\n",
    "print( 'Decoder target data shape -> {}'.format( decoder_target_data.shape ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 627
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2329,
     "status": "ok",
     "timestamp": 1566970890357,
     "user": {
      "displayName": "i150226 Saad Arshad",
      "photoUrl": "",
      "userId": "18168902830249938130"
     },
     "user_tz": -300
    },
    "id": "LkRUlgU-4-pS",
    "outputId": "4e34b83d-4f0b-4ad2-9209-746f8e698c21"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/saadarshad/anaconda3/envs/amna_bot/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /home/saadarshad/anaconda3/envs/amna_bot/lib/python3.6/site-packages/tensorflow/python/keras/backend.py:4010: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, None, 256)    132608      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 256)    433152      input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     [(None, 256), (None, 525312      embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   [(None, None, 256),  525312      embedding_1[0][0]                \n",
      "                                                                 lstm[0][1]                       \n",
      "                                                                 lstm[0][2]                       \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, None, 1692)   434844      lstm_1[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 2,051,228\n",
      "Trainable params: 2,051,228\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "encoder_inputs = tf.keras.layers.Input(shape=( None , ))\n",
    "encoder_embedding = tf.keras.layers.Embedding( num_input_tokens, 256 , mask_zero=True ) (encoder_inputs)\n",
    "encoder_outputs , state_h , state_c = tf.keras.layers.LSTM( 256 , return_state=True , recurrent_dropout=0.2 , dropout=0.2 )( encoder_embedding )\n",
    "encoder_states = [ state_h , state_c ]\n",
    "\n",
    "decoder_inputs = tf.keras.layers.Input(shape=( None ,  ))\n",
    "decoder_embedding = tf.keras.layers.Embedding( num_output_tokens, 256 , mask_zero=True) (decoder_inputs)\n",
    "decoder_lstm = tf.keras.layers.LSTM( 256 , return_state=True , return_sequences=True , recurrent_dropout=0.2 , dropout=0.2)\n",
    "decoder_outputs , _ , _ = decoder_lstm ( decoder_embedding , initial_state=encoder_states )\n",
    "decoder_dense = tf.keras.layers.Dense( num_output_tokens , activation=tf.keras.activations.softmax ) \n",
    "output = decoder_dense ( decoder_outputs )\n",
    "\n",
    "model = tf.keras.models.Model([encoder_inputs, decoder_inputs], output )\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(), loss='categorical_crossentropy')\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 118763,
     "status": "ok",
     "timestamp": 1566972590170,
     "user": {
      "displayName": "i150226 Saad Arshad",
      "photoUrl": "",
      "userId": "18168902830249938130"
     },
     "user_tz": -300
    },
    "id": "S5mbCw1Q4-pd",
    "outputId": "790ac443-17cb-4376-e9e5-35d020dd583d",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/saadarshad/anaconda3/envs/amna_bot/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From /home/saadarshad/anaconda3/envs/amna_bot/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "Epoch 1/250\n",
      "566/566 [==============================] - 20s 35ms/sample - loss: 7.4269\n",
      "Epoch 2/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 7.3459\n",
      "Epoch 3/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 6.7157\n",
      "Epoch 4/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 5.9993\n",
      "Epoch 5/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 5.8423\n",
      "Epoch 6/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 5.7378\n",
      "Epoch 7/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 5.7122\n",
      "Epoch 8/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 5.6598\n",
      "Epoch 9/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 5.6351\n",
      "Epoch 10/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 5.6039\n",
      "Epoch 11/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 5.5675\n",
      "Epoch 12/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 5.5285\n",
      "Epoch 13/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 5.4803\n",
      "Epoch 14/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 5.4352\n",
      "Epoch 15/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 5.3850\n",
      "Epoch 16/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 5.3420\n",
      "Epoch 17/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 5.3073\n",
      "Epoch 18/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 5.2673\n",
      "Epoch 19/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 5.2234\n",
      "Epoch 20/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 5.1804\n",
      "Epoch 21/250\n",
      "566/566 [==============================] - 15s 27ms/sample - loss: 5.1372\n",
      "Epoch 22/250\n",
      "566/566 [==============================] - 15s 26ms/sample - loss: 5.0970\n",
      "Epoch 23/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 5.0530\n",
      "Epoch 24/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.9995\n",
      "Epoch 25/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.9608\n",
      "Epoch 26/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.9241\n",
      "Epoch 27/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.8832\n",
      "Epoch 28/250\n",
      "566/566 [==============================] - 15s 26ms/sample - loss: 4.8479\n",
      "Epoch 29/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 4.8038\n",
      "Epoch 30/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.7671\n",
      "Epoch 31/250\n",
      "566/566 [==============================] - 14s 26ms/sample - loss: 4.7277\n",
      "Epoch 32/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.6938\n",
      "Epoch 33/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.6549\n",
      "Epoch 34/250\n",
      "566/566 [==============================] - 15s 26ms/sample - loss: 4.6177\n",
      "Epoch 35/250\n",
      "566/566 [==============================] - 16s 27ms/sample - loss: 4.5815\n",
      "Epoch 36/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.5399\n",
      "Epoch 37/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 4.4976\n",
      "Epoch 38/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.4713\n",
      "Epoch 39/250\n",
      "566/566 [==============================] - 15s 26ms/sample - loss: 4.4274\n",
      "Epoch 40/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.3970\n",
      "Epoch 41/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.3575\n",
      "Epoch 42/250\n",
      "566/566 [==============================] - 15s 26ms/sample - loss: 4.3241\n",
      "Epoch 43/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 4.2809\n",
      "Epoch 44/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 4.2518\n",
      "Epoch 45/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 4.2060\n",
      "Epoch 46/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 4.1732\n",
      "Epoch 47/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 4.1423\n",
      "Epoch 48/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 4.1039\n",
      "Epoch 49/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 4.0676\n",
      "Epoch 50/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 4.0387\n",
      "Epoch 51/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 4.0042\n",
      "Epoch 52/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 3.9712\n",
      "Epoch 53/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 3.9338\n",
      "Epoch 54/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 3.9004\n",
      "Epoch 55/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.8667\n",
      "Epoch 56/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.8267\n",
      "Epoch 57/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.7941\n",
      "Epoch 58/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.7723\n",
      "Epoch 59/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.7382\n",
      "Epoch 60/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.7079\n",
      "Epoch 61/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 3.6802\n",
      "Epoch 62/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.6500\n",
      "Epoch 63/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.6179\n",
      "Epoch 64/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.5867\n",
      "Epoch 65/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.5546\n",
      "Epoch 66/250\n",
      "566/566 [==============================] - 14s 26ms/sample - loss: 3.5235\n",
      "Epoch 67/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 3.4915\n",
      "Epoch 68/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.4665\n",
      "Epoch 69/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 3.4333\n",
      "Epoch 70/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.4054\n",
      "Epoch 71/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.3734\n",
      "Epoch 72/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.3416\n",
      "Epoch 73/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.3096\n",
      "Epoch 74/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 3.2801\n",
      "Epoch 75/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.2564\n",
      "Epoch 76/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.2269\n",
      "Epoch 77/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.1988\n",
      "Epoch 78/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.1601\n",
      "Epoch 79/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.1298\n",
      "Epoch 80/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.1054\n",
      "Epoch 81/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.0758\n",
      "Epoch 82/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.0460\n",
      "Epoch 83/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 3.0164\n",
      "Epoch 84/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.9909\n",
      "Epoch 85/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.9534\n",
      "Epoch 86/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.9275\n",
      "Epoch 87/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.9019\n",
      "Epoch 88/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.8715\n",
      "Epoch 89/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 2.8419\n",
      "Epoch 90/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.8161\n",
      "Epoch 91/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.7896\n",
      "Epoch 92/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.7509\n",
      "Epoch 93/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.7338\n",
      "Epoch 94/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.7067\n",
      "Epoch 95/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.6789\n",
      "Epoch 96/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.6483\n",
      "Epoch 97/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.6121\n",
      "Epoch 98/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.5930\n",
      "Epoch 99/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.5670\n",
      "Epoch 100/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.5382\n",
      "Epoch 101/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 2.5130\n",
      "Epoch 102/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 2.4884\n",
      "Epoch 103/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.4564\n",
      "Epoch 104/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 2.4312\n",
      "Epoch 105/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 2.4048\n",
      "Epoch 106/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 2.3722\n",
      "Epoch 107/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 2.3543\n",
      "Epoch 108/250\n",
      "566/566 [==============================] - 15s 26ms/sample - loss: 2.3247\n",
      "Epoch 109/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 2.2917\n",
      "Epoch 110/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 2.2800\n",
      "Epoch 111/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.2469\n",
      "Epoch 112/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.2239\n",
      "Epoch 113/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.1977\n",
      "Epoch 114/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 2.1764\n",
      "Epoch 115/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 2.1389\n",
      "Epoch 116/250\n",
      "566/566 [==============================] - 15s 27ms/sample - loss: 2.1269\n",
      "Epoch 117/250\n",
      "566/566 [==============================] - 15s 27ms/sample - loss: 2.0925\n",
      "Epoch 118/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.0782\n",
      "Epoch 119/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 2.0381\n",
      "Epoch 120/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.0249\n",
      "Epoch 121/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 2.0015\n",
      "Epoch 122/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.9775\n",
      "Epoch 123/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 1.9580\n",
      "Epoch 124/250\n",
      "566/566 [==============================] - 14s 25ms/sample - loss: 1.9353\n",
      "Epoch 125/250\n",
      "566/566 [==============================] - 16s 27ms/sample - loss: 1.9087\n",
      "Epoch 126/250\n",
      "566/566 [==============================] - 16s 29ms/sample - loss: 1.8837\n",
      "Epoch 127/250\n",
      "566/566 [==============================] - 16s 28ms/sample - loss: 1.8668\n",
      "Epoch 128/250\n",
      "566/566 [==============================] - 15s 26ms/sample - loss: 1.8422\n",
      "Epoch 129/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 1.8167\n",
      "Epoch 130/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.8007\n",
      "Epoch 131/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.7804\n",
      "Epoch 132/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 1.7535\n",
      "Epoch 133/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.7345\n",
      "Epoch 134/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.7171\n",
      "Epoch 135/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.6931\n",
      "Epoch 136/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.6670\n",
      "Epoch 137/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.6580\n",
      "Epoch 138/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.6353\n",
      "Epoch 139/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.6163\n",
      "Epoch 140/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.6023\n",
      "Epoch 141/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.5768\n",
      "Epoch 142/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.5534\n",
      "Epoch 143/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.5438\n",
      "Epoch 144/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.5175\n",
      "Epoch 145/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.4989\n",
      "Epoch 146/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.4849\n",
      "Epoch 147/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.4613\n",
      "Epoch 148/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.4461\n",
      "Epoch 149/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.4346\n",
      "Epoch 150/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.4214\n",
      "Epoch 151/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.3936\n",
      "Epoch 152/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.3800\n",
      "Epoch 153/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.3690\n",
      "Epoch 154/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.3532\n",
      "Epoch 155/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.3383\n",
      "Epoch 156/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 1.3195\n",
      "Epoch 157/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.3045\n",
      "Epoch 158/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.2917\n",
      "Epoch 159/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.2752\n",
      "Epoch 160/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.2614\n",
      "Epoch 161/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 1.2449\n",
      "Epoch 162/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.2316\n",
      "Epoch 163/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.2137\n",
      "Epoch 164/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.2041\n",
      "Epoch 165/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.1900\n",
      "Epoch 166/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.1761\n",
      "Epoch 167/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.1638\n",
      "Epoch 168/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.1507\n",
      "Epoch 169/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.1358\n",
      "Epoch 170/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.1207\n",
      "Epoch 171/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.1081\n",
      "Epoch 172/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.0960\n",
      "Epoch 173/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.0833\n",
      "Epoch 174/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.0648\n",
      "Epoch 175/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.0605\n",
      "Epoch 176/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.0457\n",
      "Epoch 177/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.0331\n",
      "Epoch 178/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.0245\n",
      "Epoch 179/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 1.0101\n",
      "Epoch 180/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.9997\n",
      "Epoch 181/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.9824\n",
      "Epoch 182/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.9733\n",
      "Epoch 183/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 0.9680\n",
      "Epoch 184/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.9497\n",
      "Epoch 185/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.9332\n",
      "Epoch 186/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.9324\n",
      "Epoch 187/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.9204\n",
      "Epoch 188/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.9077\n",
      "Epoch 189/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.8986\n",
      "Epoch 190/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.8934\n",
      "Epoch 191/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.8862\n",
      "Epoch 192/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.8662\n",
      "Epoch 193/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.8586\n",
      "Epoch 194/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.8547\n",
      "Epoch 195/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.8377\n",
      "Epoch 196/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.8258\n",
      "Epoch 197/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.8232\n",
      "Epoch 198/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.8132\n",
      "Epoch 199/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.7999\n",
      "Epoch 200/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.7938\n",
      "Epoch 201/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.7870\n",
      "Epoch 202/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.7777\n",
      "Epoch 203/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.7737\n",
      "Epoch 204/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.7565\n",
      "Epoch 205/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.7484\n",
      "Epoch 206/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.7413\n",
      "Epoch 207/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.7336\n",
      "Epoch 208/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.7300\n",
      "Epoch 209/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.7226\n",
      "Epoch 210/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.7135\n",
      "Epoch 211/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.7056\n",
      "Epoch 212/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6959\n",
      "Epoch 213/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.6957\n",
      "Epoch 214/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.6851\n",
      "Epoch 215/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6757\n",
      "Epoch 216/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6719\n",
      "Epoch 217/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6638\n",
      "Epoch 218/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6536\n",
      "Epoch 219/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6414\n",
      "Epoch 220/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6374\n",
      "Epoch 221/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6372\n",
      "Epoch 222/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6240\n",
      "Epoch 223/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6208\n",
      "Epoch 224/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6151\n",
      "Epoch 225/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6053\n",
      "Epoch 226/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.6005\n",
      "Epoch 227/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.5981\n",
      "Epoch 228/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 0.5867\n",
      "Epoch 229/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.5837\n",
      "Epoch 230/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.5768\n",
      "Epoch 231/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.5673\n",
      "Epoch 232/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.5637\n",
      "Epoch 233/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 0.5567\n",
      "Epoch 234/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.5539\n",
      "Epoch 235/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.5462\n",
      "Epoch 236/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.5385\n",
      "Epoch 237/250\n",
      "566/566 [==============================] - 14s 24ms/sample - loss: 0.5339\n",
      "Epoch 238/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.5311\n",
      "Epoch 239/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.5292\n",
      "Epoch 240/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.5201\n",
      "Epoch 241/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.5145\n",
      "Epoch 242/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.5125\n",
      "Epoch 243/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.5033\n",
      "Epoch 244/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.5007\n",
      "Epoch 245/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.4946\n",
      "Epoch 246/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.4907\n",
      "Epoch 247/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.4828\n",
      "Epoch 248/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.4766\n",
      "Epoch 249/250\n",
      "566/566 [==============================] - 13s 23ms/sample - loss: 0.4725\n",
      "Epoch 250/250\n",
      "566/566 [==============================] - 13s 24ms/sample - loss: 0.4704\n"
     ]
    }
   ],
   "source": [
    "model.fit([encoder_input_data , decoder_input_data], decoder_target_data, batch_size=124, epochs=250) \n",
    "model.save( 'model.h5' ) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Fe8wYN0Z4-pt"
   },
   "outputs": [],
   "source": [
    "def make_inference_models():\n",
    "    \n",
    "    encoder_model = tf.keras.models.Model(encoder_inputs, encoder_states)\n",
    "    \n",
    "    decoder_state_input_h = tf.keras.layers.Input(shape=(256,))\n",
    "    decoder_state_input_c = tf.keras.layers.Input(shape=(256,))\n",
    "    \n",
    "    decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "    \n",
    "    decoder_outputs, state_h, state_c = decoder_lstm(\n",
    "        decoder_embedding , initial_state=decoder_states_inputs)\n",
    "    decoder_states = [state_h, state_c]\n",
    "    decoder_outputs = decoder_dense(decoder_outputs)\n",
    "    decoder_model = tf.keras.models.Model(\n",
    "        [decoder_inputs] + decoder_states_inputs,\n",
    "        [decoder_outputs] + decoder_states)\n",
    "    \n",
    "    return encoder_model , decoder_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VRZQUhXb4-p6"
   },
   "outputs": [],
   "source": [
    "def str_to_tokens( sentence : str ):\n",
    "    words = sentence.lower().split()\n",
    "    tokens_list = list()\n",
    "    for word in words:\n",
    "        tokens_list.append( input_word_dict[ word ] ) \n",
    "    return preprocessing.sequence.pad_sequences( [tokens_list] , maxlen=max_input_length , padding='post')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 531
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 56399,
     "status": "error",
     "timestamp": 1566972712128,
     "user": {
      "displayName": "i150226 Saad Arshad",
      "photoUrl": "",
      "userId": "18168902830249938130"
     },
     "user_tz": -300
    },
    "id": "dtO9QVI67N65",
    "outputId": "be16a472-04bf-438e-f04a-d9ef9f07062f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User: hi\n",
      "Bot: hello\n",
      "\n",
      "User: who are you\n",
      "Bot: i am just an artificial intelligence\n",
      "\n",
      "User: what is ai\n",
      "Bot: artificial intelligence is the branch of engineering and science devoted to constructing machines that\n",
      "\n",
      "User: good\n",
      "Bot: electricity is food for robots\n",
      "\n"
     ]
    }
   ],
   "source": [
    "enc_model , dec_model = make_inference_models()\n",
    "\n",
    "enc_model.save( 'enc_model.h5' ) \n",
    "dec_model.save( 'dec_model.h5' ) \n",
    "model.save( 'model.h5' ) \n",
    "\n",
    "for epoch in range( encoder_input_data.shape[0] ):\n",
    "    states_values = enc_model.predict( str_to_tokens( input( 'User: ' ) ) )\n",
    "    empty_target_seq = np.zeros( ( 1 , 1 ) )\n",
    "    empty_target_seq[0, 0] = output_word_dict['start']\n",
    "    stop_condition = False\n",
    "    decoded_translation = ''\n",
    "    while not stop_condition :\n",
    "        dec_outputs , h , c = dec_model.predict([ empty_target_seq ] + states_values )\n",
    "        sampled_word_index = np.argmax( dec_outputs[0, -1, :] )\n",
    "        sampled_word = None\n",
    "        for word , index in output_word_dict.items() :\n",
    "            if sampled_word_index == index :\n",
    "                decoded_translation += ' {}'.format( word )\n",
    "                sampled_word = word\n",
    "        \n",
    "        if sampled_word == 'end' or len(decoded_translation.split()) > max_output_length:\n",
    "            stop_condition = True\n",
    "            \n",
    "        empty_target_seq = np.zeros( ( 1 , 1 ) )  \n",
    "        empty_target_seq[ 0 , 0 ] = sampled_word_index\n",
    "        states_values = [ h , c ] \n",
    "\n",
    "    print( \"Bot:\" +decoded_translation.replace(' end', '') )\n",
    "    print()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_cxalVo74-qm"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "Copy of Training.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}