neural-machine-translation/44.conv-seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensor2tensor.utils import beam_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('train-test.json') as fopen:\n",
    "    dataset = json.load(fopen)\n",
    "    \n",
    "with open('dictionary.json') as fopen:\n",
    "    dictionary = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = dataset['train_X']\n",
    "train_Y = dataset['train_Y']\n",
    "test_X = dataset['test_X']\n",
    "test_Y = dataset['test_Y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['from', 'to'])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dictionary.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary_from = dictionary['from']['dictionary']\n",
    "rev_dictionary_from = dictionary['from']['rev_dictionary']\n",
    "\n",
    "dictionary_to = dictionary['to']['dictionary']\n",
    "rev_dictionary_to = dictionary['to']['rev_dictionary']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Rachel Pike : The science behind a climate headline EOS'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(len(train_X)):\n",
    "    train_X[i] += ' EOS'\n",
    "    \n",
    "train_X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'How can I speak in <NUM> minutes about the bonds of women over three generations , about how the astonishing strength of those bonds took hold in the life of a four - year - old girl huddled with her young sister , her mother and her grandmother for five days and nights in a small boat in the China Sea more than <NUM> years ago , bonds that took hold in the life of that small girl and never let go - - that small girl now living in San Francisco and speaking to you today ? EOS'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(len(test_X)):\n",
    "    test_X[i] += ' EOS'\n",
    "    \n",
    "test_X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_second_dim(x, desired_size):\n",
    "    padding = tf.tile([[[0.0]]], tf.stack([tf.shape(x)[0], desired_size - tf.shape(x)[1], tf.shape(x)[2]], 0))\n",
    "    return tf.concat([x, padding], 1)\n",
    "\n",
    "def encoder_block(inp, n_hidden, filter_size):\n",
    "    inp = tf.pad(inp, [[0, 0], [(filter_size[0]-1)//2, (filter_size[0]-1)//2], [0, 0]])\n",
    "    conv = tf.layers.conv1d(inp, n_hidden, filter_size, padding=\"VALID\", activation=None)\n",
    "    return conv\n",
    "\n",
    "def decoder_block(inp, n_hidden, filter_size):\n",
    "    inp = tf.pad(inp, [[0, 0], [filter_size[0]-1, 0], [0, 0]])\n",
    "    conv = tf.layers.conv1d(inp, n_hidden, filter_size, padding=\"VALID\", activation=None)\n",
    "    return conv\n",
    "\n",
    "def glu(x):\n",
    "    return tf.multiply(x[:, :, :tf.shape(x)[2]//2], tf.sigmoid(x[:, :, tf.shape(x)[2]//2:]))\n",
    "\n",
    "def layer(inp, conv_block, kernel_width, n_hidden, residual=None):\n",
    "    z = conv_block(inp, n_hidden, (kernel_width,))\n",
    "    return glu(z) + (residual if residual is not None else 0)\n",
    "\n",
    "def sinusoidal_position_encoding(inputs, mask, repr_dim):\n",
    "    T = tf.shape(inputs)[1]\n",
    "    pos = tf.reshape(tf.range(0.0, tf.to_float(T), dtype=tf.float32), [-1, 1])\n",
    "    i = np.arange(0, repr_dim, 2, np.float32)\n",
    "    denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1])\n",
    "    enc = tf.expand_dims(tf.concat([tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0)\n",
    "    return tf.tile(enc, [tf.shape(inputs)[0], 1, 1]) * tf.expand_dims(tf.to_float(mask), -1)\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, from_dict_size, to_dict_size, size_layer, num_layers,\n",
    "                 learning_rate, n_attn_heads = 16, beam_width = 5):\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        encoder_embedding = tf.Variable(tf.random_uniform([from_dict_size, size_layer], -1, 1))\n",
    "        decoder_embedding = tf.Variable(tf.random_uniform([to_dict_size, size_layer], -1, 1))\n",
    "        \n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        def forward(x, y, reuse = False):\n",
    "            encoder_embedded = tf.nn.embedding_lookup(encoder_embedding, x)\n",
    "            decoder_embedded = tf.nn.embedding_lookup(decoder_embedding, y)\n",
    "            \n",
    "            en_masks = tf.sign(x)\n",
    "            encoder_embedded += sinusoidal_position_encoding(x, en_masks, size_layer)\n",
    "            \n",
    "            de_masks = tf.sign(y)\n",
    "            decoder_embedded += sinusoidal_position_encoding(y, de_masks, size_layer)\n",
    "            \n",
    "            e = tf.identity(encoder_embedded)\n",
    "            \n",
    "            for i in range(num_layers):\n",
    "                z = layer(encoder_embedded, encoder_block, 3, size_layer * 2, encoder_embedded)\n",
    "                encoder_embedded = z\n",
    "                \n",
    "            encoder_output, output_memory = z, z + e\n",
    "            g = tf.identity(decoder_embedded)\n",
    "            \n",
    "            for i in range(num_layers):\n",
    "                attn_res = h = layer(decoder_embedded, decoder_block, 3, size_layer * 2, \n",
    "                                         residual=tf.zeros_like(decoder_embedded))\n",
    "                C = []\n",
    "                for j in range(n_attn_heads):\n",
    "                    h_ = tf.layers.dense(h, size_layer//n_attn_heads)\n",
    "                    g_ = tf.layers.dense(g, size_layer//n_attn_heads)\n",
    "                    zu_ = tf.layers.dense(encoder_output, size_layer//n_attn_heads)\n",
    "                    ze_ = tf.layers.dense(output_memory, size_layer//n_attn_heads)\n",
    "\n",
    "                    d = tf.layers.dense(h_, size_layer//n_attn_heads) + g_\n",
    "                    dz = tf.matmul(d, tf.transpose(zu_, [0, 2, 1]))\n",
    "                    a = tf.nn.softmax(dz)\n",
    "                    c_ = tf.matmul(a, ze_)\n",
    "                    C.append(c_)\n",
    "\n",
    "                c = tf.concat(C, 2)\n",
    "                h = tf.layers.dense(attn_res + c, size_layer)\n",
    "                decoder_embedded = h\n",
    "            \n",
    "            dec = decoder_embedded\n",
    "            weights = tf.transpose(decoder_embedding)\n",
    "            logits = tf.einsum('ntd,dk->ntk', dec, weights)\n",
    "            return logits\n",
    "        \n",
    "        self.training_logits = forward(self.X, decoder_input)\n",
    "\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "        initial_ids = tf.fill([batch_size], GO)\n",
    "        \n",
    "        def symbols_to_logits(ids):\n",
    "            x = tf.contrib.seq2seq.tile_batch(self.X, beam_width)\n",
    "            logits = forward(x, ids, reuse = True)\n",
    "            return logits[:, tf.shape(ids)[1]-1, :]\n",
    "        \n",
    "        final_ids, final_probs, _ = beam_search.beam_search(\n",
    "            symbols_to_logits,\n",
    "            initial_ids,\n",
    "            beam_width,\n",
    "            tf.reduce_max(self.X_seq_len),\n",
    "            to_dict_size,\n",
    "            0.0,\n",
    "            eos_id = EOS)\n",
    "        \n",
    "        self.predicting_ids = final_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 512\n",
    "num_layers = 4\n",
    "learning_rate = 1e-4\n",
    "batch_size = 96\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-25-56f334ed76da>:70: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py:1354: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Translator(len(dictionary_from), len(dictionary_to), size_layer, num_layers, learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i.split():\n",
    "            ints.append(dic.get(k,UNK))\n",
    "        X.append(ints)\n",
    "    return X\n",
    "\n",
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = str_idx(train_X, dictionary_from)\n",
    "test_X = str_idx(test_X, dictionary_from)\n",
    "train_Y = str_idx(train_Y, dictionary_to)\n",
    "test_Y = str_idx(test_Y, dictionary_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 5, 11)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(model.predicting_ids, feed_dict = {model.X: [train_X[0]]}).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [08:21<00:00,  2.77it/s, accuracy=0.0634, cost=9.18]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:07<00:00,  3.79it/s, accuracy=0.0594, cost=9.18]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 13.009109, training avg acc 0.049766\n",
      "epoch 1, testing avg loss 8.495492, testing avg acc 0.065890\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:58<00:00,  2.90it/s, accuracy=0.0983, cost=7.48]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.97it/s, accuracy=0.112, cost=7.36]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, training avg loss 7.716085, training avg acc 0.097395\n",
      "epoch 2, testing avg loss 6.930654, testing avg acc 0.113499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:58<00:00,  2.90it/s, accuracy=0.119, cost=6.89] \n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.93it/s, accuracy=0.133, cost=6.76]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3, training avg loss 6.553644, training avg acc 0.139216\n",
      "epoch 3, testing avg loss 6.168635, testing avg acc 0.151609\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:58<00:00,  2.90it/s, accuracy=0.138, cost=6.37]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.99it/s, accuracy=0.151, cost=6.35]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4, training avg loss 5.948684, training avg acc 0.174122\n",
      "epoch 4, testing avg loss 5.739917, testing avg acc 0.179151\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:55<00:00,  2.92it/s, accuracy=0.164, cost=6.05]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.90it/s, accuracy=0.177, cost=6.04]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 5, training avg loss 5.557820, training avg acc 0.203419\n",
      "epoch 5, testing avg loss 5.417099, testing avg acc 0.206283\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:53<00:00,  2.93it/s, accuracy=0.172, cost=5.78]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.85it/s, accuracy=0.202, cost=5.85]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 6, training avg loss 5.238049, training avg acc 0.232062\n",
      "epoch 6, testing avg loss 5.140901, testing avg acc 0.233100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:54<00:00,  2.93it/s, accuracy=0.197, cost=5.52]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.00it/s, accuracy=0.226, cost=5.66]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 7, training avg loss 4.947861, training avg acc 0.259573\n",
      "epoch 7, testing avg loss 4.912915, testing avg acc 0.260262\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:53<00:00,  2.93it/s, accuracy=0.211, cost=5.32]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.96it/s, accuracy=0.229, cost=5.57]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 8, training avg loss 4.714117, training avg acc 0.283986\n",
      "epoch 8, testing avg loss 4.754980, testing avg acc 0.279387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:52<00:00,  2.94it/s, accuracy=0.22, cost=5.1]  \n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.41it/s, accuracy=0.242, cost=5.36]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 9, training avg loss 4.494548, training avg acc 0.305655\n",
      "epoch 9, testing avg loss 4.606326, testing avg acc 0.295167\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:54<00:00,  2.93it/s, accuracy=0.255, cost=4.77]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.78it/s, accuracy=0.247, cost=5.27]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, training avg loss 4.306141, training avg acc 0.324865\n",
      "epoch 10, testing avg loss 4.491202, testing avg acc 0.306867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:54<00:00,  2.93it/s, accuracy=0.279, cost=4.47]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.79it/s, accuracy=0.257, cost=5.17]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 11, training avg loss 4.137705, training avg acc 0.342264\n",
      "epoch 11, testing avg loss 4.411122, testing avg acc 0.314787\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:53<00:00,  2.93it/s, accuracy=0.31, cost=4.19] \n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.81it/s, accuracy=0.266, cost=5.1] \n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 12, training avg loss 3.977038, training avg acc 0.358672\n",
      "epoch 12, testing avg loss 4.345131, testing avg acc 0.322070\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:54<00:00,  2.93it/s, accuracy=0.353, cost=3.89]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.77it/s, accuracy=0.268, cost=5.05]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 13, training avg loss 3.834134, training avg acc 0.373652\n",
      "epoch 13, testing avg loss 4.308117, testing avg acc 0.327052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:52<00:00,  2.94it/s, accuracy=0.382, cost=3.64]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.05it/s, accuracy=0.269, cost=5.04]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 14, training avg loss 3.703263, training avg acc 0.387587\n",
      "epoch 14, testing avg loss 4.288002, testing avg acc 0.329598\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:54<00:00,  2.93it/s, accuracy=0.414, cost=3.37]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.85it/s, accuracy=0.269, cost=5.01]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 15, training avg loss 3.580431, training avg acc 0.400632\n",
      "epoch 15, testing avg loss 4.258347, testing avg acc 0.334915\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:53<00:00,  2.93it/s, accuracy=0.441, cost=3.07]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.84it/s, accuracy=0.274, cost=5.02]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 16, training avg loss 3.467622, training avg acc 0.412436\n",
      "epoch 16, testing avg loss 4.284437, testing avg acc 0.332509\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:53<00:00,  2.93it/s, accuracy=0.476, cost=2.81]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.91it/s, accuracy=0.269, cost=5]   \n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 17, training avg loss 3.368611, training avg acc 0.423133\n",
      "epoch 17, testing avg loss 4.288836, testing avg acc 0.332730\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:53<00:00,  2.93it/s, accuracy=0.516, cost=2.59]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:05<00:00,  5.95it/s, accuracy=0.271, cost=5.04]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 18, training avg loss 3.278569, training avg acc 0.433086\n",
      "epoch 18, testing avg loss 4.284799, testing avg acc 0.337076\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:52<00:00,  2.94it/s, accuracy=0.533, cost=2.4] \n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.44it/s, accuracy=0.266, cost=5.04]\n",
      "minibatch loop:   0%|          | 0/1389 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 19, training avg loss 3.197485, training avg acc 0.441848\n",
      "epoch 19, testing avg loss 4.293097, testing avg acc 0.339903\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1389/1389 [07:47<00:00,  2.97it/s, accuracy=0.567, cost=2.22]\n",
      "minibatch loop: 100%|██████████| 30/30 [00:04<00:00,  6.42it/s, accuracy=0.272, cost=5.07]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 20, training avg loss 3.118063, training avg acc 0.450676\n",
      "epoch 20, testing avg loss 4.315866, testing avg acc 0.337291\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "\n",
    "for e in range(epoch):\n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc, test_loss, test_acc = [], [], [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        maxlen = max([len(s) for s in train_X[i : index] + train_Y[i : index]])\n",
    "        batch_x, seq_x = pad_sentence_batch(train_X[i : index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(train_Y[i : index], PAD)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    \n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x, seq_x = pad_sentence_batch(test_X[i : index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(test_Y[i : index], PAD)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y,}\n",
    "        accuracy, loss = sess.run([model.accuracy,model.cost],\n",
    "                                    feed_dict = feed)\n",
    "\n",
    "        test_loss.append(loss)\n",
    "        test_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))\n",
    "    print('epoch %d, testing avg loss %f, testing avg acc %f'%(e+1,\n",
    "                                                              np.mean(test_loss),np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "rev_dictionary_to = {int(k): v for k, v in rev_dictionary_to.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 100)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_size = 20\n",
    "\n",
    "batch_x, seq_x = pad_sentence_batch(test_X[: test_size], PAD)\n",
    "batch_y, seq_y = pad_sentence_batch(test_Y[: test_size], PAD)\n",
    "feed = {model.X: batch_x}\n",
    "logits = sess.run(model.predicting_ids, feed_dict = feed)[:,0,:]\n",
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 predict: chuyện Walkman Rucell leo ADD Leni Hoả Authur xôi Lawmen esta Đi Baxter Now phích XML Studio Elah Bowles Earle đén Triceratops Chinook giáo Lawmen Russel gới Lawmen Lawmen khưr ngất lú Maude Helium Authur Blade Lakewood Fiennes ADD Maude gàu ADD Gleason esta Kowan Lawmen Kit aurochs gới rôbốt gới ADD Lawmen break chạc esta mệng ập chuyện chạc Lawmen chuyện câ Kendall Firewire đươc Brother Deutsch gàu x4 dò Crafty Nang Rôi How báu Medicare tđiều myxin Authur Cruz Excel Lawmen calorie ảo Bangalore Lawmen U.N. Lim Sussman aurochs Luskin FGM Satchidananda Forbes Ronson Gettysburg Jatra Elah\n",
      "0 actual: Làm sao tôi có thể trình bày trong <NUM> phút về sợi dây liên kết những người phụ nữ qua ba thế hệ , về việc làm thế nào những sợi dây mạnh mẽ đáng kinh ngạc ấy đã níu chặt lấy cuộc sống của một cô bé bốn tuổi co quắp với đứa em gái nhỏ của cô bé , với mẹ và bà trong suốt năm ngày đêm trên con thuyền nhỏ lênh đênh trên Biển Đông hơn <NUM> năm trước , những sợi dây liên kết đã níu lấy cuộc đời cô bé ấy và không bao giờ rời đi - - cô bé ấy giờ sống ở San Francisco và đang nói chuyện với các bạn hôm nay ?\n",
      "\n",
      "1 predict: Meyerowitz Harris vấp Centipede Malay Lewes Kardinsky Ghosh duyên tím vấp Lewes Beethoven break Mev lú luống nhoẻn Julián 1,667 hoa Elah Naran ่ Rutan hơi Koons Koons calorie giàm hơi Koons duyên Black 1,667 R.A. sư Koons Koons duyên Koons Tamburini Koons Koons calorie giàm lú Koons Koons 1,667 Để hơi Eden Koons Koons Koons Hal Mooc Maude hơi Koons Daratu break Tajikistan giàm clavus CyArt Maude Koons nhẻm Maude Koons chứa Mobius Koons lú Harris Manning Asia Mobius Downing Jesus Maude Koons Koons Koons Daratu Đoán Anita carboxylic ổi Koons Asia ổi Rutan RSI lú Manning Pagel\n",
      "1 actual: Câu chuyện này chưa kết thúc .\n",
      "\n",
      "2 predict: polyester Misha vĩnh Pagel Vết Koons Healey thiét nhậm break cú androstenone USC Birhan Pin break break Koons break androstenone Mileece Geeks pen huân Pagel play Edelman Diller Koons Eden Koons tuợng duyên Koons nhậm Maude Koons Koons break Koons Koons Centauri Maude Rôi Pernod Koons Pleistocene terroir Koons Pagel Koons Trồng Koons Pagel Pagel Lawmen Healey Columbus U lú Hosni Erik Pagel Maude Low lú chén break Koons Koons terroir Koons Koons Production Koons break Trồng Koons Kardinsky nhậm Koons Koons Koons Healey Eden câ CyArt Pilot Mister Low break Koons Koons Koons Koons Koons Koons Koons Koons\n",
      "2 actual: Nó là một trò chơi ghép hình vẫn đang được xếp .\n",
      "\n",
      "3 predict: Thứ ่ Heifer Edelman androstenone Maude Sony lú Edelman Phấn Edelman 1,667 Thiên Jennifer calorie Narcisa ngủ Yoruba break Sony calorie Edelman Ayatollah thừng Magie calorie Edelman Edelman ADD Rôi Vanier Edelman Edelman Low Edelman duyên Koons Malay Edelman Beauvoir 1,667 break Koons Đoán break Edelman while Koons Edelman CyArt Koons Waldo Nile sư Koons Mileece Cathy Meyerowitz Edelman Edelman 12,500 Aladdin androstenone Meyer Mama Bodnar Koons ngủ Edelman CyArt Mileece Waldo Singing ghép Nile vấp B.J. Koons Misery Bề lú kenya Ayatollah Koons Sony Milinda Yutaka Dodson Koons Ajit break break Vanier Cancun Waldo kenya break oái break\n",
      "3 actual: Hãy để tôi kể cho các bạn về vài mảnh ghép nhé .\n",
      "\n",
      "4 predict: bẽo re Rocky cú Carr ti Chancellor U Pagel Maude break stones viền Maude Mileece Pari Scott Manning Cederquist Maude Koons Electronic Mileece truớc Maude Maude ngã Manning Mobius Koons oái Chancellor Maude Manning Chauvet viền lú megabit ่ nứt gàu Pagel Maysoon calorie Yutaka Treo Mileece nứt A Manning A break Koons Kardinsky bun statin ่ statin Koons Mileece Mileece terroir Edelman August lú lú 16,17 ti Mileece lú Lục DreamWorks Conran Manning 370,000 Treo ่ lú lú lú carboxylic SG Leadbeater tongkonan Floria Mileece Rutan Truyền Treo Mileece Centauri Linda Fiennes break lú hơi Manning ngoằn Koons\n",
      "4 actual: Hãy tưởng tượng mảnh đầu tiên : một người đàn ông đốt cháy sự nghiệp cả đời mình .\n",
      "\n",
      "5 predict: bẽo Areca 1,667 Q Koons ADD Lawmen rhinovirus 370,000 Lawmen CRP argh Jehand . Yutaka Koons Scoop Birhan Birhan ่ Narcisa trạnh Research Yutaka ngủ break 1,667 Wikipedia isoprene Chauvet bun Hosni Chướng Budev bun đựơc ่ opiate Koons Siera break Cá FGM ADD Gagarin TBN Lawmen bun Koons carboxylic Waldo Beauvoir GeV Boyle Dunlap Kardinsky măt Lawmen TBN gới Narcisa Lawmen dàn vấp bẹp ngủ Mugsy Kenneth Béatrice Elah Misery Treo Wikipedia quà Ockelford Koons Webb 370,000 Porfirio ่ rain bẽo mỉ rỗ Waldo Yu Lawmen Khẩu Qualin Lovelock Amen Pari Kéré ่ lú Night Queens Garage Forte\n",
      "5 actual: Ông là nhà thơ , nhà viết kịch , một người mà cả cuộc đời chênh vênh trên tia hi vọng duy nhất rằng đất nước ông sẽ độc lập tự do .\n",
      "\n",
      "6 predict: Gagarin Nang lú Lawmen lú Lawmen 1,667 câ Koons Chancellor tư NaUy Cường Intercative Lower nủa Pilot nghiẹm Montmartre ACE lú 1,95 câ tĩnh break duyên Sebastien Kit nhậm câ Sebastien terroir Pari dẻ Koons shaman break calorie oái trò Julián 1,667 Koons Chancellor Zach chênh Pilot Pari headset shaman FGM lú chênh Kiểm WhipCar Gafic Saddiqui 16,17 Đoán trường Chướng rèn calorie Hollande Conran Thiên rìa Elah Kiểm Đen rắn Karel Pagel Koons myxin Dirk lú Carr Garage Lawmen 1,667 calorie 1,667 Areca carboxylic Koons nhậm duckface Kardinsky Manning break Rộng nhậm Lawmen Chancellor Lawmen Đoán carboxylic nhẽ\n",
      "6 actual: Hãy tưởng tượng ông , một người cộng sản tiến vào , đối diện sự thật rằng cả cuộc đời ông đã phí hoài .\n",
      "\n",
      "7 predict: Waldo TEDxMidAtlantic kìa Diller Bạch lệch Gleason resveratrol Ộng 1,667 đôla Là hơi 1,667 hùng Kwong Koons TEDxKibera Koons Narcisa Floria Manning lú Sato Kwong dây Misery Koons Station Adenauer Phấn Koons Koons Pagel dông trái nản Koons feel đập Koons Koons Koons Mileece Koons Daytona Koons Koons Apatzingán Koons Koons Maude oái Maude hơi Koons Koons Dickinson calorie Electronic 16,17 Floria huyêt Centauri hơi Teddy mỏm Koons nghiẹm Markoff pen lúng Truyền Koons Hotmail ổi Truyền McWhorter Raghupati Xenluloza IUD Albert Boo rock Manning Hah Rò feel oái Hah sảng Adenauer Manning Krill sen Production nứt CyArt sầu\n",
      "7 actual: Ngôn từ , qua bao năm tháng là bạn đồng hành với ông , giờ quay ra chế giễu ông .\n",
      "\n",
      "8 predict: sen br Cleveland amip Milton ngủ Celtel break Intercative stream tecno Intercative Maude Hanks vồ Lasley Pagel nầy nhậm Maude Wald Koons 3,8 Maude break Markoff Siera Stony ngủ Pablos Hanks Maude Pagel Tán Koons Electronic kenya toạ Maude Siera nhiềm 12,500 Koons Koons ngủ Electronic Gafic Maude Edelman Edelman Maude Xenluloza tĩnh tĩnh 12,500 bomb giàm Koons ớt Ắn lắng Pagel Maude Julián Bilbao Pablos Crafty câ Wald Koons kenya Maude Maude Intercative Chảng 12,500 Rags Maude Cobley lú Venice ổi Wald Wald Bragg Siera giang tĩnh Cathy Kanai Wald Crete thạch Maude ớt Mengatoue Minnesota Intercative Maude\n",
      "8 actual: Ông rút lui vào yên lặng .\n",
      "\n",
      "9 predict: 12,500 Mang ngủ Narcisa 12,500 giàm xét Chekhov SVU Yuyu 12,500 trủng Trạch Maude break etude Tour alanine break huyêt Narcisa Gleason Mobius ghép Sony break Lặc Floria lú Koons Floria Floria Jesse Axel Conran lú lú Koons Anita break Jesus Axel phích Maude nghêu Koons Garcia Odin CyArt Maude Jesus Patagonia break RTTT Floria 12,500 Pagel chầm tags Maude Maude Maude Maude tags Jesse 12,500 Pagel Albert Victoria tags Gafic 1,667 Viz Fitbit Mister 12,500 Phấn Lotte Jahnani Electronic nua Gleason Pagel Mileece Jahnani Ấn giàm Jahnani Trở Koons lú Chicago break break lú phích xà Mobius MacGregor\n",
      "9 actual: Ông qua đời , bị lịch sử quật ngã .\n",
      "\n",
      "10 predict: kenya Friedman Anh Pagel Graves 12,500 Maude Asia Maude Algore Graves break Maude Maude Maude Maude Maude Maude Maude Maude Maude Maude Maude Maude ốm Maude Maude Gafic Khều Gafic Maude Koons Koons Koons Koons Maysoon Pagel Maude Maude Maude Pagel cú Maude Beethoven ngủ cú Reverend Floria West Koons Koons Koons Maude huyêt rock Kỳ Maude 12,500 26,000 log dại Maude hơi Maude Maude Maude Maude Koons Koons Cascino Koons Moss Maude đaã Koons NBA break Koons Koons nguowif Koons 26,000 Kardinsky place CyArt Apple Maude Kardinsky câ Devil break ngủ Edelman Maude Maude Maude Maude Maude Maude\n",
      "10 actual: Ông là ông của tôi .\n",
      "\n",
      "11 predict: Albert 370,000 duyên Mần nhậm Maude ảo Hạ Mần ảo ảo Bilbao Rwando Maude dại hơi Truyền Mần Rôi Coelho Rò Kodak place sư Pagel câ lú re Coelho Maude Kodak choắt Federation Huxley Maude Coelho Coelho Kodak choắt Coelho nhậm Maude Maude Atlantic Koons sư re cú sư Eglash Samhita Maude Koons Sing Maude uyên duyên Friendster Pritsker sư Rôi Edelman Edelman rạp Maude câ ót Koons pH Retriever Rò A.E. lú Cascino Coelho sư Kardinsky ala Recluse gàu Centauri Edelman rối Maude sư lú uyên uyên uyên Coelho Maude rối Khều Welles Maude Bragg UNIVAC re uyên\n",
      "11 actual: Tôi chưa bao giờ gặp ông ngoài đời .\n",
      "\n",
      "12 predict: Jamil Dashboard đôla nhẽ tĩnh Shanghai Koons pastel Barefoot XX 1,667 Eisner 1,667 ่ Joes mươi ่ myxin Chauvet Hosni Huyết break Wald 16,17 Hermes Sim ngủ Rôi phảng Jesus calorie Mileece cụp Narcisa thứ Omenetoo Apatzingán Recluse Lipkis 16,17 lú tags Lotte đựơc Koons Dodi Sunni projections hại pbạch Koons Koons projections Avaz Treo Treak RTTT Recluse nàu Barefoot lú pajamas Kiểm break Chauvet ngủ Rôi lú Lawmen Koons Lời Studio Koons Gonzalez ่ Kodak Milinda sống lú Fi 16,17 Koons ó đoan chẵn Rôi DVR calorie pen break Pagel break lú calorie Koons Wald Adenauer Corp DVR\n",
      "12 actual: Nhưng cuộc đời ta nhiều hơn những gì ta lưu trong kí ức nhiều .\n",
      "\n",
      "13 predict: Chandrasekhar ala ngủ Now stream Edelman Manning xóc ngủ bomb nhậm Ngao Edelman break Edelman Edelman Edelman Koons Pilot Hal nhấtt ngủ Maude Hosni Linda Koons Maude tags ngủ Lipkis nhậm Pritsker câ Koons Maude câ break ngủ Ye break Koons xóc ngủ Mạo Kardinsky Koons ngủ đag Koons Cat ngủ Mạo Koons Fiore Koons ngủ Mileece dại 16,17 lú Hosni Pritsker ngủ Stapleton Rutan Mạo choắt March R.A. 16,17 Koons Stieglitz 1,667 1,667 1,667 lú 1,667 Hermes Cường 1,667 bun Britney Koons uyên Dailypath Bề Kardinsky bác bẽo khõi Kardinsky Nikolaus Kardinsky Kardinsky Treo DVR Huang Centauri Kardinsky\n",
      "13 actual: Bà tôi chưa bao giờ cho phép tôi quên cuộc đời của ông .\n",
      "\n",
      "14 predict: live giài Misery Misery Elah bụt đảng bởi chồng Elah Theresa Misery trợ Đi Đi Mileece Conran Ngươi Garage Elah Skittles Elah silo Skittles xuồng Kidd Welles Koons clavus Monbiot break calorie carboxylic NÚI ่ Ráp tuyết Arduino Keefe carboxylic Pin lường chênh u như Yosemite nắn Chướng Magie Chinook Elah nắn iO break nô Pilot lường duyên carboxylic esta Ace Amr IUD Misery Koons Conran nắn skhacs vườn Srebrenica Elah tọng Speedway nành Misery Manning Elah bĩu oxide lậnh Misery trợ Rutan esta Conran _ vẫ Mister beatjazz Weingartin Elah Dhaka SolarImpulse Normandy statin biophony 370,000 Yutaka Khều\n",
      "14 actual: Nhiệm vụ của tôi là không để cuộc đời ấy qua trong vô vọng , và bài học của tôi là nhận ra rằng , vâng , lịch sử đã cố quật ngã chúng tôi , nhưng chúng tôi đã chịu đựng được .\n",
      "\n",
      "15 predict: sen Pagel Là break tongkonan Hứng break Nội _ Koons 1,667 Cán carboxylic Edelman phá calorie To Birhan Gyatso Maude Rutan Gleason Areca Diller Adenauer Hứng Mechnical break Koons break Dailypath Đi KB Birhan carboxylic Waldo ADD Electronic sulfates Edelman GlobalFamilyReunion Theresa Theresa Manning Prrkinson bun break Nephites nhẽ trái Hosni break xét esta Hứng calorie Siera Healey tư silo hại break Edelman Anatotitan sành break Elysees Đen 1,667 Maude break oái Syndrome Siera break oái _ 1,25 esta break Manning Healey ExxonMobil Chancellor Rutan break Malay oái opiate Skittles Birhan Edelman break Gleason Trồng ADD Gleason ADD Magie\n",
      "15 actual: Mảnh ghép tiếp theo của tấm hình là một con thuyền trong sớm hoàng hôn lặng lẽ trườn ra biển .\n",
      "\n",
      "16 predict: nhồng myxin Manning Audis Elah Elah Garage Đi Elah Meyerowitz lện phảng Jehand vìi terroir break Jacqueline Audis Mileece Manning Ara Soloveitchik Ngươi break Mugsy 1,667 nhắt xỉ Maude đươc Truyền Gevinson nhồng Heuser styrene Ráp break Maude baby tongkonan Maude quà chạng Maude Manning psychiatrists break trạnh Conran trạnh Chancellor NHÂN Britney tags Skittles Popper đời Elah Elah Edelman Ballet Elah Manning Sẻ Manning Yahweh Cowley Minority terroir Kubat Areca polyester Pinatubo Simone Jarreth Đen break 1,667 Addis Cowley break Maude hoản lú styrene Elah Gamow nghiẹm Bọng boomer tày Ndebele remarkable Mugsy Pyramids Pagel Manning Diệt Alisa\n",
      "16 actual: Mẹ tôi , Mai , mới <NUM> tuổi khi ba của mẹ mất - - đã lập gia đình , một cuộc hôn nhân sắp đặt trước , đã có hai cô con gái nhỏ .\n",
      "\n",
      "17 predict: Routemaster ché Helium back Edelman soundscapes Lawmen calorie Magie duyên quà carboxylic oái Oedipus Kit carboxylic nghiẹm ADD bomb break break rỗ Ngưởi đảng pen quà Chandrasekhar 16,17 Maude quà calorie ่ U Lawmen Pagel Mileece cụp pen androstenone androstenone Mileece quà Chancellor Mileece ่ progerin carboxylic Addis U 57,000 Milanovic Blicket Edelman Mileece Pagel Rutan đáo Corvisart Chandrasekhar rhinovirus Chancellor calorie Birhan Dân Beauvoir carboxylic Huyết Lawmen terroir boomer Rutan Columbia thớ rối thước Jahnani Refuge Lời nghiẹm Maude Pezitron BiblioTech chầm opiate calorie băng Mileece calorie carboxylic ngủ Edelman Dunlap Elgin chạng Dass Lenny Mileece ่ terroir\n",
      "17 actual: Với mẹ , cuộc đời cô đọng vào nhiệm vụ duy nhất : để gia đình mẹ trốn thoát và bắt đầu cuộc sống mới ở Úc .\n",
      "\n",
      "18 predict: khinh giống bun Sato Rutan Kéré dùì Eetwidomayloh nhin break ti độnt break break 1,600 lú rệu Anti nhát Koons 1,667 1,667 tags câ câ 1,667 Koons Koons Adenauer Koons calorie Manning câ Vanier iO Koons Koons Koons Koons Annunziata Hal Mileece Edelman Koons chén 1,667 thứ Edelman Maude Koons androstenone calorie Birhan calorie Nuremberg Coelho Dian rôbốt Barefoot DVR lú Edelman Edelman Walkman Tăm 1,95 Koons ti dùì Edelman break Barefoot Lasley 1,95 1,667 break Koons Hah terroir ngủ DVR dại break bun break ngủ 1,667 calorie Khều rệu Edelman break Hah thạch thạch TEDxKibera cut Healey lú\n",
      "18 actual: Mẹ không bao giờ chấp nhận được là mẹ sẽ không thành công .\n",
      "\n",
      "19 predict: Waldo Jawad Hoả boomer ngác Phấn Chướng opiate tuợng Tacoma Beatle Petra hoa trường Edelman Khều Hah Koons Cường điệu tags Misery ó Siera Areca Britney Khều Elah loài Deutsch Pors Lionel Elah ่ Koons Elah nhậm 125,000 yet Pilot RelayRides Phản Mileece Jehand ่ tuồn RSI Tar Koons tọng tags ่ Sing Pari Britney Elah Chauvet Graham Mileece GeV Nhiên Kafkaesque Addis Họp còm Elah Sing tọng Marra da Họp chả IMDB rock whiteout Manning xê carboxylic Cường hơi Abernathy break Pagel Lawmen U calorie Pari NHÂN Addis Wald hangry Chướng Misery Britney Misery ó Rôi Gevinson Britney\n",
      "19 actual: Thế là sau bốn năm , một trường thiên đằng đẵng hơn cả trong truyện , một chiếc thuyền trườn ra biển nguỵ trang là thuyền đánh cá .\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rejected = ['PAD', 'EOS', 'UNK', 'GO']\n",
    "\n",
    "for i in range(test_size):\n",
    "    predict = [rev_dictionary_to[i] for i in logits[i] if rev_dictionary_to[i] not in rejected]\n",
    "    actual = [rev_dictionary_to[i] for i in batch_y[i] if rev_dictionary_to[i] not in rejected]\n",
    "    print(i, 'predict:', ' '.join(predict))\n",
    "    print(i, 'actual:', ' '.join(actual))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}