topic-generator/2.tav.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "123"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = os.listdir('news')\n",
    "news = ['news/' + i for i in labels if '.json' in i]\n",
    "labels = [i.replace('.json','') for i in labels]\n",
    "len(news)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(news[0]) as fopen:\n",
    "    first_news = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A+ A-\\n\\nKetua Umum PKR Datuk Seri Anwar Ibrahim membidas kenyataan anggota Majlis Pimpinan Tertinggi BERSATU Datuk Abdul Kadir Jasin berhubung peruntukan kerajaan kepada Yang di-Pertuan Agong (YDPA).\\n\\nBeliau berkata walaupun hak untuk menyuarakan pandangan perlu dihormati, wartawan veteran itu perlu menunjukkan penghormatan dan ketertiban (decorum) dalam hal ini.\\n\\nKatanya, kritikan itu juga dibuat tanpa memberi peluang kepada Institusi Raja-raja Melayu untuk memberi respons.\\n\\n\"Sekiranya buat kritikan sebegini tanpa memberi peluang kepada Raja-Raja Melayu untuk jelaskan jawapan, ia tidak sihat terutamanya menggunakan kedudukan yang ada.\\n\\n\"Saya juga lihat sejumlah penekanan, dan ada fakta (yang) dipertikai, saya rasa kita perlu lebih berwaspada,\" katanya pada sidang media selepas pertemuan dengan wakil rakyat PKR di sebuah hotel di Selangor, hari ini.\\n\\nBeliau diminta mengulas artikel Abdul Kadir yang juga ketua media dan komunikasi Majlis Penasihat Kerajaan di blognya hari ini, yang mendakwa kerajaan memperuntukkan RM256.9 juta antara Januari hingga April tahun ini untuk YDPA.\\n\\nNamun beliau menyertakan penjelasan bahawa artikel itu ditulis atas kapasiti peribadi dan tidak mempunyai kaitan dengan sebarang jawatan yang dipegangnya.\\n\\nMengulas kontroversi sekitar pelantikan pakar perlembagaan Tommy Thomas sebagai peguam negara, Kadir dalam artikel itu berkata Raja-raja Melayu yakin dan tidak merasakan kedudukan mereka tergugat.\\n\\n\"Selama 16 bulan sehingga April tahun ini, sejumlah RM256.9 juta diperuntukkan kepada Yang di-Pertuan Agong untuk tujuan tempat tinggal, istana, barangan peribadi, pengangkutan dan kapal terbang, peralatan dan latihan pengiring, pakaian dan barangan upacara, hadiah dan cenderamata, lawatan luar negara dan gaji pekerja istana,\" katanya.\\n\\nBeliau bagaimanapun tidak menyatakan sumber maklumat itu.\\n\\nMenurut Perbendaharaan, kerajaan hanya memperuntukkan RM13.5 juta setiap tahun kepada YDPA.\\n\\nBina hubungan dengan raja\\n\\nMengulas lanjut, Anwar berkata beliau akan terus mengadakan pertemuan dengan Raja-raja Melayu bagi membuang persepsi Pakatan Harapan (HARAPAN) tidak mesra golongan itu.\\n\\n\"Kami menghabiskan banyak masa dan usaha untuk mendapatkan raja-raja Melayu untuk melindungi pentadbiran ini dan memastikan peralihan kuasa yang lancar,\" katanya.\\n\\nDalam pertemuan itu, katanya, beliau juga memberi jaminan HARAPAN kekal komited mempertahan peruntukan Perlembagaan Persekutuan dalam soal hak Melayu, serta kedudukan Bahasa Melayu dan Islam.\\n\\nKatanya lagi, memang sukar untuk meyakinkan Raja-raja Melayu terhadap pentadbiran Tun Dr Mahathir Mohamad, justeru semua pihak perlu berhati-hati dalam membuat kenyataan seumpama itu.\\n\\nAnwar juga memaklumkan beliau kemungkinan besar tidak akan kembali ke Parlimen dalam sesi pertama yang akan bermula 16 Julai ini perlu memenuhi komitmen tempatan dan antarabangsa.\\n\\nBeliau juga mengesahkan hanya akan kembali sebagai anggota parlimen, tanpa memegang mana-mana kementerian.\\n\\nPKR sedia untuk pemilihan parti\\n\\nSementara itu, Setiausaha Agung PKR Datuk Saifuddin Nasution Ismail berkata PKR akan memulakan persediaan mengadakan pemilihan parti bermula Ogos ini.\\n\\nPada masa ini, Saifuddin berkata PKR telah diberikan tempoh penangguhan 18 bulan - sehingga November 2018 - untuk mengadakan kongres, manakala pemilihan perlu diadakan sebelum Februari tahun depan.\\n\\nBeliau juga berkata Anwar telah menyeru agar semua yang hadir pada pertemuan ini memberi sokongan penuh kepada kerajaan pimpinan Mahathir.\\n\\n\"Anwar berkata sama seperti komitmennya terhadap PKR dan HARAPAN, beliau mahu semua wakil rakyat juga demikian,\" katanya lagi.\\n\\nBeliau memaklumkan pertemuan hari ini dihadiri 51 daripada 67 wakil rakyat PKR.\\n\\nMenteri Hal Ehwal Ekonomi Datuk Seri Mohamed Azmin Ali dan Menteri Perumahan dan Kerajaan Tempatan Zuraida Kamaruddin tidak hadir kerana berada di luar negara.'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_news[0]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import malaya"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = malaya.preprocessing._SocialTokenizer().tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "accept_tokens = ',-.()\"\\''\n",
    "\n",
    "def is_number_regex(s):\n",
    "    if re.match(\"^\\d+?\\.\\d+?$\", s) is None:\n",
    "        return s.isdigit()\n",
    "    return True\n",
    "\n",
    "def detect_money(word):\n",
    "    if word[:2] == 'rm' and is_number_regex(word[2:]):\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "def preprocessing(string):\n",
    "    tokenized = tokenizer(string)\n",
    "    tokenized = [w.lower() for w in tokenized if len(w) > 1 or w in accept_tokens]\n",
    "    tokenized = ['<NUM>' if is_number_regex(w) else w for w in tokenized]\n",
    "    tokenized = ['<MONEY>' if detect_money(w) else w for w in tokenized]\n",
    "    return tokenized\n",
    "\n",
    "def clean_label(label):\n",
    "    string = re.sub('[^A-Za-z\\- ]+', ' ', label)\n",
    "    return re.sub(r'[ ]+', ' ', string.lower()).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [clean_label(label) for label in labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "\n",
    "maxlen = 150\n",
    "min_len = 20\n",
    "\n",
    "x, y = [], []\n",
    "for no, n in enumerate(news):\n",
    "    with open(n) as fopen: \n",
    "        news_ = json.load(fopen)\n",
    "    for row in news_:\n",
    "        if len(row['text'].split()) > min_len:\n",
    "            p = preprocessing(row['text'])\n",
    "            p = p[:maxlen]\n",
    "            x.append(p)\n",
    "            y.append(labels[no])\n",
    "            \n",
    "x, y = shuffle(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14471, 14471)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x), len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "def build_dataset(words, n_words, atleast=2):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 63032\n",
      "Most common words [(',', 90498), ('.', 80674), ('yang', 32102), ('-', 29965), ('the', 29732), ('dan', 28171)]\n",
      "Sample data [1659, 5, 237, 7, 7, 756, 716, 594, 1239, 12] ['tribunnews', '.', 'com', '-', '-', 'pt', 'xl', 'axiata', 'tbk', '(']\n",
      "filtered vocab size: 38621\n",
      "% of vocab used: 61.27%\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "\n",
    "concat = list(itertools.chain(*x)) + ' '.join(labels).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])\n",
    "print('filtered vocab size:',len(dictionary))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary)/vocabulary_size,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(x)):\n",
    "    x.append('EOS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Generator:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, \n",
    "                 dict_size, learning_rate, beam_width = 5):\n",
    "        \n",
    "        def lstm_cell(size, reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size, initializer=tf.orthogonal_initializer(),\n",
    "                                           reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        topic_embedded = tf.nn.embedding_lookup(embeddings, self.X)\n",
    "        topic_average = tf.reduce_mean(topic_embedded, axis=1)\n",
    "        topic_state = tf.layers.dense(topic_average, size_layer)\n",
    "        \n",
    "        lstm_state = tf.nn.rnn_cell.LSTMStateTuple(c=topic_state, h=topic_state)\n",
    "        self.encoder_state = tuple([lstm_state] * num_layers)\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        decoder_cells = tf.nn.rnn_cell.MultiRNNCell([lstm_cell(size_layer) for _ in range(num_layers)])\n",
    "        dense_layer = tf.layers.Dense(dict_size)\n",
    "        \n",
    "        training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(embeddings, decoder_input),\n",
    "                sequence_length = self.Y_seq_len,\n",
    "                time_major = False)\n",
    "        training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = decoder_cells,\n",
    "                helper = training_helper,\n",
    "                initial_state = self.encoder_state,\n",
    "                output_layer = dense_layer)\n",
    "        training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = training_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        \n",
    "        predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = decoder_cells,\n",
    "                embedding = embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = tf.contrib.seq2seq.tile_batch(self.encoder_state, beam_width),\n",
    "                beam_width = beam_width,\n",
    "                output_layer = dense_layer,\n",
    "                length_penalty_weight = 0.0)\n",
    "        \n",
    "        predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = maxlen)\n",
    "        \n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        self.predicting_ids = predicting_decoder_output.predicted_ids[:, :, 0]\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 256\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "learning_rate = 0.001\n",
    "batch_size = 8\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Generator(size_layer, num_layers, embedded_size, len(dictionary), \n",
    "                learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i:\n",
    "            ints.append(dic.get(k,UNK))\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = [i.split() for i in y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(y, dictionary)\n",
    "Y = str_idx(x, dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:10<00:00,  1.15s/it, accuracy=0.131, cost=6.35] \n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 7.127246, avg accuracy: 0.074340\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:12<00:00,  1.16s/it, accuracy=0.175, cost=5.52]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, avg loss: 5.861953, avg accuracy: 0.156625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:13<00:00,  1.16s/it, accuracy=0.192, cost=5.09]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, avg loss: 5.299645, avg accuracy: 0.193236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:15<00:00,  1.16s/it, accuracy=0.218, cost=4.77]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, avg loss: 4.936064, avg accuracy: 0.215528\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:15<00:00,  1.16s/it, accuracy=0.233, cost=4.51]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, avg loss: 4.655775, avg accuracy: 0.233165\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:15<00:00,  1.16s/it, accuracy=0.254, cost=4.29]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, avg loss: 4.428398, avg accuracy: 0.249557\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:14<00:00,  1.16s/it, accuracy=0.26, cost=4.1]  \n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, avg loss: 4.241566, avg accuracy: 0.264621\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:14<00:00,  1.16s/it, accuracy=0.272, cost=3.94]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, avg loss: 4.083615, avg accuracy: 0.279595\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:15<00:00,  1.16s/it, accuracy=0.297, cost=3.81]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9, avg loss: 3.948315, avg accuracy: 0.294211\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:15<00:00,  1.16s/it, accuracy=0.307, cost=3.69]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 10, avg loss: 3.830216, avg accuracy: 0.307300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:15<00:00,  1.16s/it, accuracy=0.328, cost=3.58]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 11, avg loss: 3.725735, avg accuracy: 0.319824\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:14<00:00,  1.16s/it, accuracy=0.337, cost=3.48]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 12, avg loss: 3.631060, avg accuracy: 0.331617\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:14<00:00,  1.16s/it, accuracy=0.347, cost=3.38]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 13, avg loss: 3.543331, avg accuracy: 0.342568\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:14<00:00,  1.16s/it, accuracy=0.366, cost=3.28]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 14, avg loss: 3.462204, avg accuracy: 0.353233\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:14<00:00,  1.16s/it, accuracy=0.371, cost=3.2] \n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 15, avg loss: 3.387544, avg accuracy: 0.363161\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:15<00:00,  1.16s/it, accuracy=0.38, cost=3.15] \n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 16, avg loss: 3.317969, avg accuracy: 0.372744\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:14<00:00,  1.16s/it, accuracy=0.401, cost=3.09]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 17, avg loss: 3.251911, avg accuracy: 0.381816\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:14<00:00,  1.15s/it, accuracy=0.404, cost=3.02]\n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 18, avg loss: 3.189790, avg accuracy: 0.390488\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:14<00:00,  1.16s/it, accuracy=0.42, cost=2.96] \n",
      "minibatch loop:   0%|          | 0/1809 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 19, avg loss: 3.130544, avg accuracy: 0.398999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1809/1809 [35:14<00:00,  1.16s/it, accuracy=0.426, cost=2.91]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 20, avg loss: 3.074743, avg accuracy: 0.406972\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(epoch):\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    pbar = tqdm(\n",
    "        range(0, len(X), batch_size), desc = 'minibatch loop')\n",
    "    for k in pbar:\n",
    "        index = min(k+batch_size, len(X))\n",
    "        batch_x, seq_x = pad_sentence_batch(X[k: index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(Y[k: index], PAD)\n",
    "        predicted, accuracy,loss, _ = sess.run([model.predicting_ids, \n",
    "                                                model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += accuracy\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    total_loss /= (len(X) / batch_size)\n",
    "    total_accuracy /= (len(X) / batch_size)\n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(i+1, total_loss, total_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_topic = 'isu najib razak mahathir'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_x, _ = pad_sentence_batch(X[:1], PAD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[28, 134, 253, 112]]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_topic_idx = str_idx([test_topic.split()], dictionary)\n",
    "batch_test, _ = pad_sentence_batch(test_topic_idx, PAD)\n",
    "batch_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'looks like javascript is disabled in your browser . malaysiakini requires javascript to run normally . click here to enable javascript in your browser . malaysiakini requires javascript to run normally . click here to enable javascript in your browser . malaysiakini requires javascript to run normally . click here to enable javascript in your browser . malaysiakini requires javascript to run normally . click here to enable javascript in your browser . click here to enable portfolio baharu , tan sri dr . wan azizah wan ismail sebagai pengerusi pakatan harapan , tun dr . mahathir mohamad dan naib presiden pkr , datuk seri dr . wan azizah wan ismail sebagai pengerusi pakatan harapan , tun dr . mahathir mohamad dan naib presiden pkr , datuk seri dr . wan azizah wan ismail sebagai timbalan perdana menteri , tan sri dr . wan azizah wan ismail sebagai timbalan perdana'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_test = sess.run(model.predicting_ids, feed_dict = {model.X: batch_test})[0]\n",
    "' '.join([rev_dictionary[i] for i in predict_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}