Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division, print_function\n",
    "from gensim import models\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers import Dense, Dropout, Reshape, Flatten, concatenate, Input, Embedding\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.models import Sequential\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Model\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import collections\n",
    "import re\n",
    "import string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('imdb_labelled.tsv', header = None, delimiter='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.columns = ['Text', 'Label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A very, very, very slow-moving, aimless movie ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Not sure who was more lost - the flat characte...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Attempting artiness with black &amp; white and cle...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Very little music or anything to speak of.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The best scene in the movie was when Gerardo i...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  Label\n",
       "0  A very, very, very slow-moving, aimless movie ...      0\n",
       "1  Not sure who was more lost - the flat characte...      0\n",
       "2  Attempting artiness with black & white and cle...      0\n",
       "3       Very little music or anything to speak of.        0\n",
       "4  The best scene in the movie was when Gerardo i...      1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.Label.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(748, 2)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos = []\n",
    "neg = []\n",
    "for l in data.Label:\n",
    "    if l == 0:\n",
    "        pos.append(0)\n",
    "        neg.append(1)\n",
    "    elif l == 1:\n",
    "        pos.append(1)\n",
    "        neg.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Pos']= pos\n",
    "data['Neg']= neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Label</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Neg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A very, very, very slow-moving, aimless movie ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Not sure who was more lost - the flat characte...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Attempting artiness with black &amp; white and cle...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Very little music or anything to speak of.</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The best scene in the movie was when Gerardo i...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  Label  Pos  Neg\n",
       "0  A very, very, very slow-moving, aimless movie ...      0    0    1\n",
       "1  Not sure who was more lost - the flat characte...      0    0    1\n",
       "2  Attempting artiness with black & white and cle...      0    0    1\n",
       "3       Very little music or anything to speak of.        0    0    1\n",
       "4  The best scene in the movie was when Gerardo i...      1    1    0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_punct(text):\n",
    "    text_nopunct = ''\n",
    "    text_nopunct = re.sub('['+string.punctuation+']', '', text)\n",
    "    return text_nopunct\n",
    "\n",
    "data['Text_Clean'] = data['Text'].apply(lambda x: remove_punct(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import word_tokenize, WordNetLemmatizer\n",
    "tokens = [word_tokenize(sen) for sen in data.Text_Clean]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lower_token(tokens): \n",
    "    return [w.lower() for w in tokens]    \n",
    "    \n",
    "lower_tokens = [lower_token(token) for token in tokens] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stoplist = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stop_words(tokens):\n",
    "    return [word for word in tokens if word not in stoplist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_words = [remove_stop_words(sen) for sen in lower_tokens] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = [' '.join(sen) for sen in filtered_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Text_Final'] = result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['tokens'] = filtered_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[['Text_Final', 'tokens', 'Label', 'Pos', 'Neg']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text_Final</th>\n",
       "      <th>tokens</th>\n",
       "      <th>Label</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Neg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>slowmoving aimless movie distressed drifting y...</td>\n",
       "      <td>[slowmoving, aimless, movie, distressed, drift...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sure lost flat characters audience nearly half...</td>\n",
       "      <td>[sure, lost, flat, characters, audience, nearl...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>attempting artiness black white clever camera ...</td>\n",
       "      <td>[attempting, artiness, black, white, clever, c...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>little music anything speak</td>\n",
       "      <td>[little, music, anything, speak]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          Text_Final  \\\n",
       "0  slowmoving aimless movie distressed drifting y...   \n",
       "1  sure lost flat characters audience nearly half...   \n",
       "2  attempting artiness black white clever camera ...   \n",
       "3                        little music anything speak   \n",
       "\n",
       "                                              tokens  Label  Pos  Neg  \n",
       "0  [slowmoving, aimless, movie, distressed, drift...      0    0    1  \n",
       "1  [sure, lost, flat, characters, audience, nearl...      0    0    1  \n",
       "2  [attempting, artiness, black, white, clever, c...      0    0    1  \n",
       "3                   [little, music, anything, speak]      0    0    1  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[:4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split data into test and train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train, data_test = train_test_split(data, test_size=0.10, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7218 words total, with a vocabulary size of 2881\n",
      "Max sentence length is 789\n"
     ]
    }
   ],
   "source": [
    "all_training_words = [word for tokens in data_train[\"tokens\"] for word in tokens]\n",
    "training_sentence_lengths = [len(tokens) for tokens in data_train[\"tokens\"]]\n",
    "TRAINING_VOCAB = sorted(list(set(all_training_words)))\n",
    "print(\"%s words total, with a vocabulary size of %s\" % (len(all_training_words), len(TRAINING_VOCAB)))\n",
    "print(\"Max sentence length is %s\" % max(training_sentence_lengths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "580 words total, with a vocabulary size of 457\n",
      "Max sentence length is 24\n"
     ]
    }
   ],
   "source": [
    "all_test_words = [word for tokens in data_test[\"tokens\"] for word in tokens]\n",
    "test_sentence_lengths = [len(tokens) for tokens in data_test[\"tokens\"]]\n",
    "TEST_VOCAB = sorted(list(set(all_test_words)))\n",
    "print(\"%s words total, with a vocabulary size of %s\" % (len(all_test_words), len(TEST_VOCAB)))\n",
    "print(\"Max sentence length is %s\" % max(test_sentence_lengths))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Google News Word2Vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/saadarshad/anaconda3/envs/amna_bot/lib/python3.6/site-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
      "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
     ]
    }
   ],
   "source": [
    "word2vec_path = 'GoogleNews-vectors-negative300.bin.gz'\n",
    "word2vec = models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_average_word2vec(tokens_list, vector, generate_missing=False, k=300):\n",
    "    if len(tokens_list)<1:\n",
    "        return np.zeros(k)\n",
    "    if generate_missing:\n",
    "        vectorized = [vector[word] if word in vector else np.random.rand(k) for word in tokens_list]\n",
    "    else:\n",
    "        vectorized = [vector[word] if word in vector else np.zeros(k) for word in tokens_list]\n",
    "    length = len(vectorized)\n",
    "    summed = np.sum(vectorized, axis=0)\n",
    "    averaged = np.divide(summed, length)\n",
    "    return averaged\n",
    "\n",
    "def get_word2vec_embeddings(vectors, clean_comments, generate_missing=False):\n",
    "    embeddings = clean_comments['tokens'].apply(lambda x: get_average_word2vec(x, vectors, \n",
    "                                                                                generate_missing=generate_missing))\n",
    "    return list(embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_embeddings = get_word2vec_embeddings(word2vec, data_train, generate_missing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 50\n",
    "EMBEDDING_DIM = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize and Pad sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2881 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(num_words=len(TRAINING_VOCAB), lower=True, char_level=False)\n",
    "tokenizer.fit_on_texts(data_train[\"Text_Final\"].tolist())\n",
    "training_sequences = tokenizer.texts_to_sequences(data_train[\"Text_Final\"].tolist())\n",
    "\n",
    "train_word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(train_word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_cnn_data = pad_sequences(training_sequences, maxlen=MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2882, 300)\n"
     ]
    }
   ],
   "source": [
    "train_embedding_weights = np.zeros((len(train_word_index)+1, EMBEDDING_DIM))\n",
    "for word,index in train_word_index.items():\n",
    "    train_embedding_weights[index,:] = word2vec[word] if word in word2vec else np.random.rand(EMBEDDING_DIM)\n",
    "print(train_embedding_weights.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sequences = tokenizer.texts_to_sequences(data_test[\"Text_Final\"].tolist())\n",
    "test_cnn_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define RNN-LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names = ['Pos', 'Neg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = data_train[label_names].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = train_cnn_data\n",
    "y_tr = y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recurrent_nn(embeddings, max_sequence_length, num_words, embedding_dim, labels_index):\n",
    "    \n",
    "    embedding_layer = Embedding(num_words,\n",
    "                            embedding_dim,\n",
    "                            weights=[embeddings],\n",
    "                            input_length=max_sequence_length,\n",
    "                            trainable=False)\n",
    "    \n",
    "    sequence_input = Input(shape=(max_sequence_length,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(sequence_input)\n",
    "\n",
    "#     lstm = LSTM(256, dropout=0.2, recurrent_dropout=0.2, return_sequences=True)(embedded_sequences)\n",
    "    lstm = LSTM(256)(embedded_sequences)\n",
    "    \n",
    "    x = Dense(128, activation='relu')(lstm)\n",
    "    x = Dropout(0.2)(x)\n",
    "    preds = Dense(labels_index, activation='sigmoid')(x)\n",
    "\n",
    "    model = Model(sequence_input, preds)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam',\n",
    "                  metrics=['acc'])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_13 (InputLayer)        (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "embedding_13 (Embedding)     (None, 50, 300)           864600    \n",
      "_________________________________________________________________\n",
      "lstm_19 (LSTM)               (None, 256)               570368    \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 2)                 258       \n",
      "=================================================================\n",
      "Total params: 1,468,122\n",
      "Trainable params: 603,522\n",
      "Non-trainable params: 864,600\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = recurrent_nn(train_embedding_weights, MAX_SEQUENCE_LENGTH, len(train_word_index)+1, EMBEDDING_DIM, \n",
    "                len(list(label_names)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train RNN-LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 5\n",
    "batch_size = 34"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 605 samples, validate on 68 samples\n",
      "Epoch 1/5\n",
      "605/605 [==============================] - 5s 8ms/step - loss: 0.0847 - acc: 0.9752 - val_loss: 0.5045 - val_acc: 0.8676\n",
      "Epoch 2/5\n",
      "605/605 [==============================] - 5s 8ms/step - loss: 0.0697 - acc: 0.9769 - val_loss: 0.4728 - val_acc: 0.8529\n",
      "Epoch 3/5\n",
      "605/605 [==============================] - 5s 8ms/step - loss: 0.0325 - acc: 0.9884 - val_loss: 0.4687 - val_acc: 0.8382\n",
      "Epoch 4/5\n",
      "605/605 [==============================] - 5s 8ms/step - loss: 0.0223 - acc: 0.9917 - val_loss: 0.5192 - val_acc: 0.8456\n",
      "Epoch 5/5\n",
      "605/605 [==============================] - 5s 8ms/step - loss: 0.0144 - acc: 0.9950 - val_loss: 0.5782 - val_acc: 0.8382\n"
     ]
    }
   ],
   "source": [
    "hist = model.fit(x_train, y_tr, epochs=num_epochs, validation_split=0.1, shuffle=True, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test RNN-LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "75/75 [==============================] - 0s 2ms/step\n"
     ]
    }
   ],
   "source": [
    "predictions = model.predict(test_cnn_data, batch_size=1024, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [1, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_labels=[]\n",
    "for p in predictions:\n",
    "    prediction_labels.append(labels[np.argmax(p)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(data_test.Label==prediction_labels)/len(prediction_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}