fasttext-text-classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '..')\n",
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all = pd.read_csv('./data/data_all.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "def clean_str(string):\n",
    "    \"\"\"\n",
    "    Tokenization/string cleaning for all datasets \n",
    "    Every dataset is lower cased\n",
    "    \"\"\"\n",
    "    string = re.sub(r\"[^A-Za-z0-9(),!?\\'\\`]\", \" \", string)     \n",
    "    string = re.sub(r\"\\'s\", \" \\'s\", string) \n",
    "    string = re.sub(r\"\\'ve\", \" \\'ve\", string) \n",
    "    string = re.sub(r\"n\\'t\", \" n\\'t\", string) \n",
    "    string = re.sub(r\"\\'re\", \" \\'re\", string) \n",
    "    string = re.sub(r\"\\'d\", \" \\'d\", string) \n",
    "    string = re.sub(r\"\\'ll\", \" \\'ll\", string) \n",
    "    string = re.sub(r\",\", \" , \", string) \n",
    "    string = re.sub(r\"!\", \" ! \", string) \n",
    "    string = re.sub(r\"\\(\", \" \\( \", string) \n",
    "    string = re.sub(r\"\\)\", \" \\) \", string) \n",
    "    string = re.sub(r\"\\?\", \" \\? \", string) \n",
    "    string = re.sub(r\"\\s{2,}\", \" \", string)    \n",
    "    return string.strip().lower().split()\n",
    "\n",
    "\n",
    "def review_to_words(raw_review):\n",
    "    review_text = BeautifulSoup(raw_review, \"lxml\").get_text()  \n",
    "    words = clean_str(review_text)\n",
    "    return(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = []\n",
    "labels = []\n",
    "for idx in range(data_all.review.shape[0]):\n",
    "    text = review_to_words(data_all.review[idx])\n",
    "    texts.append(text)\n",
    "    labels.append(data_all.sentiment[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sun\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 114767 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "MAX_NB_WORDS = 20000\n",
    "\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(texts)\n",
    "sequences = tokenizer.texts_to_sequences(texts)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000 train sequences\n",
      "25000 test sequences\n",
      "Average train sequence length: 247\n",
      "Average test sequence length: 241\n",
      "Adding 2-gram features\n",
      "Average train sequence length: 493\n",
      "Average test sequence length: 451\n"
     ]
    }
   ],
   "source": [
    "def create_ngram_set(input_list, ngram_value=2):\n",
    "    \"\"\"\n",
    "    Extract a set of n-grams from a list of integers.\n",
    "    >>> create_ngram_set([1, 4, 9, 4, 1, 4], ngram_value=2)\n",
    "    {(4, 9), (4, 1), (1, 4), (9, 4)}\n",
    "    >>> create_ngram_set([1, 4, 9, 4, 1, 4], ngram_value=3)\n",
    "    [(1, 4, 9), (4, 9, 4), (9, 4, 1), (4, 1, 4)]\n",
    "    \"\"\"\n",
    "    return set(zip(*[input_list[i:] for i in range(ngram_value)]))\n",
    "\n",
    "\n",
    "def add_ngram(sequences, token_indice, ngram_range=2):\n",
    "    \"\"\"\n",
    "    Augment the input list of list (sequences) by appending n-grams values.\n",
    "    Example: adding bi-gram\n",
    "    >>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]\n",
    "    >>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017}\n",
    "    >>> add_ngram(sequences, token_indice, ngram_range=2)\n",
    "    [[1, 3, 4, 5, 1337, 2017], [1, 3, 7, 9, 2, 1337, 42]]\n",
    "    Example: adding tri-gram\n",
    "    >>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]\n",
    "    >>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017, (7, 9, 2): 2018}\n",
    "    >>> add_ngram(sequences, token_indice, ngram_range=3)\n",
    "    [[1, 3, 4, 5, 1337, 2017], [1, 3, 7, 9, 2, 1337, 42, 2018]]\n",
    "    \"\"\"\n",
    "    new_sequences = []\n",
    "    for input_list in sequences:\n",
    "        new_list = input_list[:]\n",
    "        for ngram_value in range(2, ngram_range + 1):\n",
    "            for i in range(len(new_list) - ngram_value + 1):\n",
    "                ngram = tuple(new_list[i:i + ngram_value])\n",
    "                if ngram in token_indice:\n",
    "                    new_list.append(token_indice[ngram])\n",
    "        new_sequences.append(new_list)\n",
    "\n",
    "    return new_sequences\n",
    "\n",
    "# Set parameters:\n",
    "# ngram_range = 2 will add bi-grams features\n",
    "ngram_range = 2\n",
    "max_features = 20000\n",
    "\n",
    "x_train = sequences[:25000]\n",
    "x_test = sequences[25000:]\n",
    "\n",
    "print(len(x_train), 'train sequences')\n",
    "print(len(x_test), 'test sequences')\n",
    "print('Average train sequence length: {}'.format(\n",
    "    np.mean(list(map(len, x_train)), dtype=int)))\n",
    "print('Average test sequence length: {}'.format(\n",
    "    np.mean(list(map(len, x_test)), dtype=int)))\n",
    "\n",
    "if ngram_range > 1:\n",
    "    print('Adding {}-gram features'.format(ngram_range))\n",
    "    # Create set of unique n-gram from the training set.\n",
    "    ngram_set = set()\n",
    "    for input_list in x_train:\n",
    "        for i in range(2, ngram_range + 1):\n",
    "            set_of_ngram = create_ngram_set(input_list, ngram_value=i)\n",
    "            ngram_set.update(set_of_ngram)\n",
    "\n",
    "    # Dictionary mapping n-gram token to a unique integer.\n",
    "    # Integer values are greater than max_features in order\n",
    "    # to avoid collision with existing features.\n",
    "    start_index = max_features + 1\n",
    "    token_indice = {v: k + start_index for k, v in enumerate(ngram_set)}\n",
    "    indice_token = {token_indice[k]: k for k in token_indice}\n",
    "\n",
    "    # max_features is the highest integer that could be found in the dataset.\n",
    "    max_features = np.max(list(indice_token.keys())) + 1\n",
    "\n",
    "    # Augmenting x_train and x_test with n-grams features\n",
    "    x_train = add_ngram(x_train, token_indice, ngram_range)\n",
    "    x_test = add_ngram(x_test, token_indice, ngram_range)\n",
    "    print('Average train sequence length: {}'.format(\n",
    "        np.mean(list(map(len, x_train)), dtype=int)))\n",
    "    print('Average test sequence length: {}'.format(\n",
    "        np.mean(list(map(len, x_test)), dtype=int)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1120634\n",
      "1140635\n"
     ]
    }
   ],
   "source": [
    "print(len(ngram_set))\n",
    "print(max_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pad sequences (samples x time)\n",
      "x_train shape: (25000, 400)\n",
      "x_test shape: (25000, 400)\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing import sequence\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "maxlen = 400\n",
    "\n",
    "print('Pad sequences (samples x time)')\n",
    "x_train = sequence.pad_sequences(x_train, maxlen=maxlen)\n",
    "x_test = sequence.pad_sequences(x_test, maxlen=maxlen)\n",
    "labels = to_categorical(np.asarray(labels))\n",
    "y_train = labels[:25000]\n",
    "y_test = labels[25000:]\n",
    "print('x_train shape:', x_train.shape)\n",
    "print('x_test shape:', x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 400, 50)           57031750  \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_1 ( (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2)                 102       \n",
      "=================================================================\n",
      "Total params: 57,031,852\n",
      "Trainable params: 57,031,852\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/15\n",
      "25000/25000 [==============================] - 55s 2ms/step - loss: 0.5850 - acc: 0.7750 - val_loss: 0.4351 - val_acc: 0.8527\n",
      "Epoch 2/15\n",
      "25000/25000 [==============================] - 50s 2ms/step - loss: 0.2911 - acc: 0.9208 - val_loss: 0.3038 - val_acc: 0.8888\n",
      "Epoch 3/15\n",
      "25000/25000 [==============================] - 51s 2ms/step - loss: 0.1487 - acc: 0.9667 - val_loss: 0.2627 - val_acc: 0.8985\n",
      "Epoch 4/15\n",
      "25000/25000 [==============================] - 51s 2ms/step - loss: 0.0815 - acc: 0.9853 - val_loss: 0.2459 - val_acc: 0.9030\n",
      "Epoch 5/15\n",
      "25000/25000 [==============================] - 51s 2ms/step - loss: 0.0463 - acc: 0.9938 - val_loss: 0.2388 - val_acc: 0.9029\n",
      "Epoch 6/15\n",
      "25000/25000 [==============================] - 51s 2ms/step - loss: 0.0270 - acc: 0.9974 - val_loss: 0.2370 - val_acc: 0.9050\n",
      "Epoch 7/15\n",
      "25000/25000 [==============================] - 51s 2ms/step - loss: 0.0163 - acc: 0.9989 - val_loss: 0.2378 - val_acc: 0.9040\n",
      "Epoch 8/15\n",
      "25000/25000 [==============================] - 51s 2ms/step - loss: 0.0100 - acc: 0.9995 - val_loss: 0.2414 - val_acc: 0.9049\n",
      "Epoch 9/15\n",
      "25000/25000 [==============================] - 51s 2ms/step - loss: 0.0064 - acc: 0.9997 - val_loss: 0.2443 - val_acc: 0.9054\n",
      "Epoch 10/15\n",
      "25000/25000 [==============================] - 50s 2ms/step - loss: 0.0041 - acc: 0.9998 - val_loss: 0.2503 - val_acc: 0.9053\n",
      "Epoch 11/15\n",
      "25000/25000 [==============================] - 50s 2ms/step - loss: 0.0028 - acc: 0.9999 - val_loss: 0.2556 - val_acc: 0.9056\n",
      "Epoch 12/15\n",
      "25000/25000 [==============================] - 50s 2ms/step - loss: 0.0019 - acc: 0.9999 - val_loss: 0.2625 - val_acc: 0.9052\n",
      "Epoch 13/15\n",
      "25000/25000 [==============================] - 50s 2ms/step - loss: 0.0013 - acc: 1.0000 - val_loss: 0.2690 - val_acc: 0.9055\n",
      "Epoch 14/15\n",
      "25000/25000 [==============================] - 51s 2ms/step - loss: 8.9012e-04 - acc: 1.0000 - val_loss: 0.2777 - val_acc: 0.9046\n",
      "Epoch 15/15\n",
      "25000/25000 [==============================] - 51s 2ms/step - loss: 6.3463e-04 - acc: 1.0000 - val_loss: 0.2851 - val_acc: 0.9048\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x19e37459320>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import GlobalAveragePooling1D\n",
    "\n",
    "batch_size = 32\n",
    "embedding_dims = 50\n",
    "epochs = 5\n",
    "\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "\n",
    "# we start off with an efficient embedding layer which maps\n",
    "# our vocab indices into embedding_dims dimensions\n",
    "model.add(Embedding(max_features,\n",
    "                    embedding_dims,\n",
    "                    input_length=maxlen))\n",
    "\n",
    "# we add a GlobalAveragePooling1D, which will average the embeddings\n",
    "# of all words in the document\n",
    "model.add(GlobalAveragePooling1D())\n",
    "\n",
    "# We project onto a single unit output layer, and squash it with a sigmoid:\n",
    "model.add(Dense(2, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=15,\n",
    "          validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 400, 10)           11406350  \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_1 ( (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2)                 22        \n",
      "=================================================================\n",
      "Total params: 11,406,372\n",
      "Trainable params: 11,406,372\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/10\n",
      "25000/25000 [==============================] - 14s 570us/step - loss: 0.6589 - acc: 0.7194 - val_loss: 0.5955 - val_acc: 0.8000\n",
      "Epoch 2/10\n",
      "25000/25000 [==============================] - 12s 477us/step - loss: 0.4937 - acc: 0.8706 - val_loss: 0.4520 - val_acc: 0.8498\n",
      "Epoch 3/10\n",
      "25000/25000 [==============================] - 12s 476us/step - loss: 0.3387 - acc: 0.9213 - val_loss: 0.3671 - val_acc: 0.8762\n",
      "Epoch 4/10\n",
      "25000/25000 [==============================] - 12s 477us/step - loss: 0.2390 - acc: 0.9496 - val_loss: 0.3192 - val_acc: 0.8858\n",
      "Epoch 5/10\n",
      "25000/25000 [==============================] - 12s 477us/step - loss: 0.1735 - acc: 0.9658 - val_loss: 0.2901 - val_acc: 0.8930\n",
      "Epoch 6/10\n",
      "25000/25000 [==============================] - 12s 477us/step - loss: 0.1280 - acc: 0.9760 - val_loss: 0.2714 - val_acc: 0.8944\n",
      "Epoch 7/10\n",
      "25000/25000 [==============================] - 12s 475us/step - loss: 0.0950 - acc: 0.9838 - val_loss: 0.2584 - val_acc: 0.8992\n",
      "Epoch 8/10\n",
      "25000/25000 [==============================] - 12s 476us/step - loss: 0.0705 - acc: 0.9895 - val_loss: 0.2482 - val_acc: 0.9017\n",
      "Epoch 9/10\n",
      "25000/25000 [==============================] - 12s 476us/step - loss: 0.0523 - acc: 0.9927 - val_loss: 0.2432 - val_acc: 0.9034\n",
      "Epoch 10/10\n",
      "25000/25000 [==============================] - 12s 476us/step - loss: 0.0388 - acc: 0.9955 - val_loss: 0.2382 - val_acc: 0.9041\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2255f781358>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import GlobalAveragePooling1D\n",
    "\n",
    "batch_size = 32\n",
    "embedding_dims = 10\n",
    "epochs = 5\n",
    "\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "\n",
    "# we start off with an efficient embedding layer which maps\n",
    "# our vocab indices into embedding_dims dimensions\n",
    "model.add(Embedding(max_features,\n",
    "                    embedding_dims,\n",
    "                    input_length=maxlen))\n",
    "\n",
    "# we add a GlobalAveragePooling1D, which will average the embeddings\n",
    "# of all words in the document\n",
    "model.add(GlobalAveragePooling1D())\n",
    "\n",
    "# We project onto a single unit output layer, and squash it with a sigmoid:\n",
    "model.add(Dense(2, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=10,\n",
    "          validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 400, 20)           22812700  \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_3 ( (None, 20)                0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 2)                 42        \n",
      "=================================================================\n",
      "Total params: 22,812,742\n",
      "Trainable params: 22,812,742\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/15\n",
      "25000/25000 [==============================] - 23s 910us/step - loss: 0.6364 - acc: 0.7368 - val_loss: 0.5359 - val_acc: 0.8253\n",
      "Epoch 2/15\n",
      "25000/25000 [==============================] - 21s 856us/step - loss: 0.4082 - acc: 0.8913 - val_loss: 0.3784 - val_acc: 0.8716\n",
      "Epoch 3/15\n",
      "25000/25000 [==============================] - 22s 869us/step - loss: 0.2484 - acc: 0.9432 - val_loss: 0.3105 - val_acc: 0.8865\n",
      "Epoch 4/15\n",
      "25000/25000 [==============================] - 22s 873us/step - loss: 0.1606 - acc: 0.9668 - val_loss: 0.2770 - val_acc: 0.8951\n",
      "Epoch 5/15\n",
      "25000/25000 [==============================] - 22s 871us/step - loss: 0.1071 - acc: 0.9802 - val_loss: 0.2585 - val_acc: 0.8991\n",
      "Epoch 6/15\n",
      "25000/25000 [==============================] - 22s 870us/step - loss: 0.0722 - acc: 0.9884 - val_loss: 0.2466 - val_acc: 0.9025\n",
      "Epoch 7/15\n",
      "25000/25000 [==============================] - 22s 871us/step - loss: 0.0489 - acc: 0.9934 - val_loss: 0.2399 - val_acc: 0.9038\n",
      "Epoch 8/15\n",
      "25000/25000 [==============================] - 22s 863us/step - loss: 0.0333 - acc: 0.9966 - val_loss: 0.2383 - val_acc: 0.9022\n",
      "Epoch 9/15\n",
      "25000/25000 [==============================] - 21s 853us/step - loss: 0.0228 - acc: 0.9979 - val_loss: 0.2363 - val_acc: 0.9038\n",
      "Epoch 10/15\n",
      "25000/25000 [==============================] - 21s 854us/step - loss: 0.0156 - acc: 0.9988 - val_loss: 0.2367 - val_acc: 0.9060\n",
      "Epoch 11/15\n",
      "25000/25000 [==============================] - 21s 853us/step - loss: 0.0108 - acc: 0.9993 - val_loss: 0.2382 - val_acc: 0.9060\n",
      "Epoch 12/15\n",
      "25000/25000 [==============================] - 21s 852us/step - loss: 0.0075 - acc: 0.9996 - val_loss: 0.2413 - val_acc: 0.9062\n",
      "Epoch 13/15\n",
      "25000/25000 [==============================] - 21s 850us/step - loss: 0.0053 - acc: 0.9997 - val_loss: 0.2460 - val_acc: 0.9063\n",
      "Epoch 14/15\n",
      "25000/25000 [==============================] - 21s 856us/step - loss: 0.0037 - acc: 0.9998 - val_loss: 0.2514 - val_acc: 0.9048\n",
      "Epoch 15/15\n",
      "25000/25000 [==============================] - 21s 853us/step - loss: 0.0026 - acc: 0.9999 - val_loss: 0.2563 - val_acc: 0.9056\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x22582dc6b70>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import GlobalAveragePooling1D\n",
    "\n",
    "batch_size = 32\n",
    "embedding_dims = 20\n",
    "epochs = 5\n",
    "\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "\n",
    "# we start off with an efficient embedding layer which maps\n",
    "# our vocab indices into embedding_dims dimensions\n",
    "model.add(Embedding(max_features,\n",
    "                    embedding_dims,\n",
    "                    input_length=maxlen))\n",
    "\n",
    "# we add a GlobalAveragePooling1D, which will average the embeddings\n",
    "# of all words in the document\n",
    "model.add(GlobalAveragePooling1D())\n",
    "\n",
    "# We project onto a single unit output layer, and squash it with a sigmoid:\n",
    "model.add(Dense(2, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=15,\n",
    "          validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}