DatasetProcessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Going to work off of the Keras API to create our own datasets. Kinda. \n",
    "from six.moves import cPickle\n",
    "import numpy as np, pandas as pd\n",
    "import glob\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# datapath = '/media/mike/tera/data/nlp/kaggle_imdb/'\n",
    "path = '/home/mike/.keras/datasets/imdb_full.pkl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(path, 'rb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'tuple'>\n"
     ]
    }
   ],
   "source": [
    "data = cPickle.load(f)\n",
    "\n",
    "f.close()\n",
    "print(type(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train, labels_train), (x_test, labels_test) = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 118M\r\n",
      "drwxrwxr-x 2 mike mike 4.0K Mar 28 17:13 \u001b[0m\u001b[01;34m.\u001b[0m/\r\n",
      "drwxrwxr-x 4 mike mike 4.0K Jan 27 22:58 \u001b[01;34m..\u001b[0m/\r\n",
      "-rw-rw-r-- 1 mike mike  63M Mar 15 18:58 imdb_full.pkl\r\n",
      "-rw-rw-r-- 1 mike mike  17M Mar 28 12:27 imdb.npz\r\n",
      "-rw-rw-r-- 1 mike mike 1.6M Mar 28 12:28 imdb_word_index.json\r\n",
      "-rw-rw-r-- 1 mike mike 2.3M Mar 27 11:06 imdb_word_index.pkl\r\n",
      "-rw-rw-r-- 1 mike mike  11M Mar 28 17:13 mnist.npz\r\n",
      "-rw-rw-r-x 1 mike mike  15M Dec 22 18:01 \u001b[01;32mmnist.pkl.gz\u001b[0m*\r\n",
      "-rw-rw-r-- 1 mike mike 587K Mar 15 22:50 nietzsche.txt\r\n",
      "-rw-rw-r-- 1 mike mike 8.8M Mar 27 11:17 reuters.pkl\r\n"
     ]
    }
   ],
   "source": [
    "ls -lah /home/mike/.keras/datasets/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "datapath = '/media/mike/tera/data/nlp/aclImdb/train/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12500\n",
      "['/media/mike/tera/data/nlp/aclImdb/train/pos/5382_7.txt', '/media/mike/tera/data/nlp/aclImdb/train/pos/6095_7.txt', '/media/mike/tera/data/nlp/aclImdb/train/pos/5764_7.txt', '/media/mike/tera/data/nlp/aclImdb/train/pos/1873_8.txt', '/media/mike/tera/data/nlp/aclImdb/train/pos/6516_10.txt']\n"
     ]
    }
   ],
   "source": [
    "fn_pos = glob.glob(datapath + 'pos/*.txt')\n",
    "print(len(fn_pos))\n",
    "print(fn_pos[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fn = fn_pos[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(fn, 'r') as f:\n",
    "    data = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"After watching the Next Action Star reality TV series, I was pleased to see the winners' movie right away. I was leery of such a showcase of new talent, but I was pleasantly surprised and thrilled. Billy Zane, of course, was his usual great self, but Corinne and Sean held their own beside him. It was also nice to see Jared and Jeanne (also from the competition) in their cameo roles. Sean's character, not Billy's, is the hunted, and his frustration at discovering new rules in the game is well played. Corinne walks the tightrope well between her character liking Sean's and only being in it for the money. I loved how the game was played right to the last second. And then beyond! Not a great movie, but an entertaining one all the way and a great showcase for two folks on their first time out of the gate.\""
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5382_7'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.basename(fn)[:-4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "datapath = '/media/mike/tera/data/nlp/aclImdb/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/media/mike/tera/data/nlp/aclImdb/train/pos/*.txt\n",
      "12500\n",
      "train_pos\n",
      "/media/mike/tera/data/nlp/aclImdb/train/neg/*.txt\n",
      "12500\n",
      "train_neg\n",
      "/media/mike/tera/data/nlp/aclImdb/test/pos/*.txt\n",
      "12500\n",
      "test_pos\n",
      "/media/mike/tera/data/nlp/aclImdb/test/neg/*.txt\n",
      "12500\n",
      "test_neg\n"
     ]
    }
   ],
   "source": [
    "datadict = {}\n",
    "for tset in ['train', 'test']:\n",
    "    for pset in ['pos', 'neg']:\n",
    "        path = datapath + '{}/{}/*.txt'.format(tset, pset)\n",
    "        print(path)\n",
    "        fnames = glob.glob(path)\n",
    "        print(len(fnames))\n",
    "        datas = []\n",
    "        for fn in fnames:\n",
    "            with open(fn, 'r') as f:\n",
    "                data = f.read()\n",
    "                datas.append(data)\n",
    "        key = '{}_{}'.format(tset, pset)\n",
    "        datadict.update({key: datas})\n",
    "        print(key)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12500"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(datadict['train_pos'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"After watching the Next Action Star reality TV series, I was pleased to see the winners' movie right away. I was leery of such a showcase of new talent, but I was pleasantly surprised and thrilled. Billy Zane, of course, was his usual great self, but Corinne and Sean held their own beside him. It was also nice to see Jared and Jeanne (also from the competition) in their cameo roles. Sean's character, not Billy's, is the hunted, and his frustration at discovering new rules in the game is well played. Corinne walks the tightrope well between her character liking Sean's and only being in it for the money. I loved how the game was played right to the last second. And then beyond! Not a great movie, but an entertaining one all the way and a great showcase for two folks on their first time out of the gate.\""
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datadict['train_pos'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = datadict['train_pos'] + datadict['train_neg']\n",
    "y_train = list(np.ones(len(datadict['train_pos']))) + list(np.zeros(len(datadict['train_neg'])))\n",
    "test = datadict['train_pos'] + datadict['train_neg']\n",
    "y_test = list(np.ones(len(datadict['test_pos']))) + list(np.zeros(len(datadict['test_neg'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>After watching the Next Action Star reality TV...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I'm a bit conflicted over this. The show is on...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I originally reviewed this film on Amazon abou...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The violent and rebel twenty-five years old sa...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>hello. i just watched this movie earlier today...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  sentiment\n",
       "0  After watching the Next Action Star reality TV...        1.0\n",
       "1  I'm a bit conflicted over this. The show is on...        1.0\n",
       "2  I originally reviewed this film on Amazon abou...        1.0\n",
       "3  The violent and rebel twenty-five years old sa...        1.0\n",
       "4  hello. i just watched this movie earlier today...        1.0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.DataFrame(list(zip(train, y_train)), columns=['review', 'sentiment'])\n",
    "df_test = pd.DataFrame(list(zip(test, y_test)), columns=['review', 'sentiment'])\n",
    "\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_train.to_csv(datapath + 'imdb_train.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_test.to_csv(datapath + 'imdb_test.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "datapath = '/home/mike/data/sentiment/kaggle_rotten/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PhraseId</th>\n",
       "      <th>SentenceId</th>\n",
       "      <th>Phrase</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>A series of escapades demonstrating the adage ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>A series of escapades demonstrating the adage ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>A series</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>series</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>of escapades demonstrating the adage that what...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>of</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>escapades demonstrating the adage that what is...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>escapades</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>demonstrating the adage that what is good for ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PhraseId  SentenceId                                             Phrase  \\\n",
       "0         1           1  A series of escapades demonstrating the adage ...   \n",
       "1         2           1  A series of escapades demonstrating the adage ...   \n",
       "2         3           1                                           A series   \n",
       "3         4           1                                                  A   \n",
       "4         5           1                                             series   \n",
       "5         6           1  of escapades demonstrating the adage that what...   \n",
       "6         7           1                                                 of   \n",
       "7         8           1  escapades demonstrating the adage that what is...   \n",
       "8         9           1                                          escapades   \n",
       "9        10           1  demonstrating the adage that what is good for ...   \n",
       "\n",
       "   Sentiment  \n",
       "0          1  \n",
       "1          2  \n",
       "2          2  \n",
       "3          2  \n",
       "4          2  \n",
       "5          2  \n",
       "6          2  \n",
       "7          2  \n",
       "8          2  \n",
       "9          2  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(datapath + 'train.tsv', sep='\\t')\n",
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   1,    2,    3, ..., 8542, 8543, 8544])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['SentenceId'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PhraseId</th>\n",
       "      <th>Phrase</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SentenceId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>63</td>\n",
       "      <td>63</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40</td>\n",
       "      <td>40</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>34</td>\n",
       "      <td>34</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>44</td>\n",
       "      <td>44</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>39</td>\n",
       "      <td>39</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>45</td>\n",
       "      <td>45</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8514</th>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8515</th>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8516</th>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8517</th>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8518</th>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8519</th>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8520</th>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8521</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8522</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8523</th>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8524</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8525</th>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8526</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8527</th>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8528</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8529</th>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8531</th>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8532</th>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8533</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8534</th>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8535</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8536</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8537</th>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8538</th>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8539</th>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8540</th>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8541</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8542</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8543</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8544</th>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8529 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            PhraseId  Phrase  Sentiment\n",
       "SentenceId                             \n",
       "1                 63      63         63\n",
       "2                 18      18         18\n",
       "3                 35      35         35\n",
       "4                 40      40         40\n",
       "5                 10      10         10\n",
       "6                 32      32         32\n",
       "7                 15      15         15\n",
       "8                 34      34         34\n",
       "9                 12      12         12\n",
       "10                12      12         12\n",
       "11                35      35         35\n",
       "12                46      46         46\n",
       "13                11      11         11\n",
       "14                44      44         44\n",
       "15                16      16         16\n",
       "16                35      35         35\n",
       "17                15      15         15\n",
       "18                43      43         43\n",
       "19                20      20         20\n",
       "20                15      15         15\n",
       "21                12      12         12\n",
       "22                56      56         56\n",
       "23                28      28         28\n",
       "24                39      39         39\n",
       "25                16      16         16\n",
       "26                23      23         23\n",
       "27                16      16         16\n",
       "28                45      45         45\n",
       "29                24      24         24\n",
       "30                29      29         29\n",
       "...              ...     ...        ...\n",
       "8514              14      14         14\n",
       "8515              23      23         23\n",
       "8516              19      19         19\n",
       "8517              20      20         20\n",
       "8518              13      13         13\n",
       "8519              21      21         21\n",
       "8520              27      27         27\n",
       "8521               6       6          6\n",
       "8522               4       4          4\n",
       "8523              18      18         18\n",
       "8524              12      12         12\n",
       "8525              31      31         31\n",
       "8526              16      16         16\n",
       "8527              22      22         22\n",
       "8528              16      16         16\n",
       "8529              18      18         18\n",
       "8531              13      13         13\n",
       "8532              27      27         27\n",
       "8533              24      24         24\n",
       "8534              11      11         11\n",
       "8535               3       3          3\n",
       "8536              16      16         16\n",
       "8537              19      19         19\n",
       "8538              14      14         14\n",
       "8539              21      21         21\n",
       "8540              13      13         13\n",
       "8541              24      24         24\n",
       "8542              10      10         10\n",
       "8543               8       8          8\n",
       "8544              21      21         21\n",
       "\n",
       "[8529 rows x 3 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.groupby('SentenceId').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'Phrase': 'This quiet , introspective and entertaining independent is worth seeking .',\n",
       " 'PhraseId': 64,\n",
       " 'SentenceId': 2,\n",
       " 'Sentiment': 4}"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask = data['SentenceId'] == 2\n",
    "subidx = data[mask].index[0]\n",
    "print(subidx)\n",
    "dict(data.iloc[subidx])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RangeIndex(start=0, stop=156060, step=1)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = data.rename(columns = {'Sentiment': 'sentiment5', 'Phrase': 'review'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reviews = []\n",
    "for i in data['SentenceId'].unique():\n",
    "    mask = data['SentenceId'] == i\n",
    "    subidx = data[mask].index[0]\n",
    "    reviews.append(dict(data.iloc[subidx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PhraseId</th>\n",
       "      <th>SentenceId</th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>A series of escapades demonstrating the adage ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64</td>\n",
       "      <td>2</td>\n",
       "      <td>This quiet , introspective and entertaining in...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>82</td>\n",
       "      <td>3</td>\n",
       "      <td>Even fans of Ismail Merchant 's work , I suspe...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>117</td>\n",
       "      <td>4</td>\n",
       "      <td>A positively thrilling combination of ethnogra...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>157</td>\n",
       "      <td>5</td>\n",
       "      <td>Aggressive self-glorification and a manipulati...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PhraseId  SentenceId                                             review  \\\n",
       "0         1           1  A series of escapades demonstrating the adage ...   \n",
       "1        64           2  This quiet , introspective and entertaining in...   \n",
       "2        82           3  Even fans of Ismail Merchant 's work , I suspe...   \n",
       "3       117           4  A positively thrilling combination of ethnogra...   \n",
       "4       157           5  Aggressive self-glorification and a manipulati...   \n",
       "\n",
       "   sentiment5  \n",
       "0           1  \n",
       "1           4  \n",
       "2           1  \n",
       "3           3  \n",
       "4           1  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_reviews = pd.DataFrame(reviews)\n",
    "df_reviews.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_reviews['sentiment'] = df_reviews['sentiment5'].map({0:0, 1:0, 2:2, 3:1, 4:1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6874\n"
     ]
    }
   ],
   "source": [
    "df_reviews = df_reviews[df_reviews['sentiment'] < 2]\n",
    "print(len(df_reviews))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_reviews.to_csv(datapath + 'rotten.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.52400349141693336"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_reviews['sentiment'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}