unit08-large-language-models/8.2-bag-of-words/8.2-part1-bag-of-words-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "eed46618-9b66-425c-b7a0-f606d4593e2d",
   "metadata": {},
   "source": [
    "# 8.2 Training a Bag-of-words Model\n",
    "\n",
    "## Part 1: Preparing the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07784975-91ec-4a0b-8761-0f2a1790a2c9",
   "metadata": {},
   "source": [
    "- Large Movie Review Dataset, https://ai.stanford.edu/~amaas/data/sentiment/\n",
    "\n",
    "> This is a dataset for binary sentiment classification containing substantially more data than previous benchmark datasets. We provide a set of 25,000 highly polar movie reviews for training, and 25,000 for testing. There is additional unlabeled data for use as well. Raw text and already processed bag of words formats are provided. See the README file contained in the release for more details.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "44be15f9-ca14-4c93-914b-41c66c2eb318",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch    : 1.13.1\n",
      "lightning: 1.9.2\n",
      "pandas   : 1.5.3\n",
      "\n",
      "conda environment: dl-fundamentals\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -p torch,lightning,pandas --conda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c26e275e-fce8-436e-a77a-bf3a95f31e3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install datasets\n",
    "\n",
    "import os.path as op\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from local_dataset_utilities import download_dataset, load_dataset_into_to_dataframe, partition_dataset\n",
    "from local_dataset_utilities import IMDBDataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d98f84c6-761a-46cb-a566-ce98779dfb90",
   "metadata": {},
   "source": [
    "# 1 Loading the dataset into DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cd96d52b-94bd-4554-9a8c-ff9af76e09a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 50000/50000 [00:21<00:00, 2358.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class distribution:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I went and saw this movie last night after bei...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Actor turned director Bill Paxton follows up h...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>As a recreational golfer with some knowledge o...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I saw this film in a sneak preview, and it is ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Bill Paxton has taken the true story of the 19...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  label\n",
       "0  I went and saw this movie last night after bei...      1\n",
       "0  Actor turned director Bill Paxton follows up h...      1\n",
       "0  As a recreational golfer with some knowledge o...      1\n",
       "0  I saw this film in a sneak preview, and it is ...      1\n",
       "0  Bill Paxton has taken the true story of the 19...      1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "download_dataset()\n",
    "\n",
    "df = load_dataset_into_to_dataframe()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0ec41419-b265-45f9-b1c9-f56aba6fe2f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "partition_dataset(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "35142019-6dd5-437e-9be4-6a391dc46699",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>34995</th>\n",
       "      <td>0</td>\n",
       "      <td>Frank Capra's creativity must have been just a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34996</th>\n",
       "      <td>0</td>\n",
       "      <td>Just saw the film tonight in a preview and it'...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34997</th>\n",
       "      <td>0</td>\n",
       "      <td>If you love Japanese monster movies, you'll lo...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34998</th>\n",
       "      <td>0</td>\n",
       "      <td>Because it came from HBO and based on the IMDb...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34999</th>\n",
       "      <td>0</td>\n",
       "      <td>WARNING!!! SOME POSSIBLE PLOT SPOILERS, AS IF ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       index                                               text  label\n",
       "34995      0  Frank Capra's creativity must have been just a...      0\n",
       "34996      0  Just saw the film tonight in a preview and it'...      0\n",
       "34997      0  If you love Japanese monster movies, you'll lo...      1\n",
       "34998      0  Because it came from HBO and based on the IMDb...      0\n",
       "34999      0  WARNING!!! SOME POSSIBLE PLOT SPOILERS, AS IF ...      0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(\"train.csv\")\n",
    "df_train.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b790d928-40ca-46f8-a265-05ebe0405a2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([17452, 17548])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.bincount(df_train['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ed5a9e2c-8853-47f8-80fb-b6a1952b68fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4995</th>\n",
       "      <td>0</td>\n",
       "      <td>The Matador is a strange film. Its main charac...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4996</th>\n",
       "      <td>0</td>\n",
       "      <td>Not bad performances. Whoopi plays the wise/wa...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4997</th>\n",
       "      <td>0</td>\n",
       "      <td>I was surprised when I saw this film. I'd hear...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4998</th>\n",
       "      <td>0</td>\n",
       "      <td>When great director/actor combinations are tal...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999</th>\n",
       "      <td>0</td>\n",
       "      <td>This show is non Stop hilarity. the first joke...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      index                                               text  label\n",
       "4995      0  The Matador is a strange film. Its main charac...      1\n",
       "4996      0  Not bad performances. Whoopi plays the wise/wa...      0\n",
       "4997      0  I was surprised when I saw this film. I'd hear...      0\n",
       "4998      0  When great director/actor combinations are tal...      0\n",
       "4999      0  This show is non Stop hilarity. the first joke...      1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val = pd.read_csv(\"val.csv\")\n",
    "df_val.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cfc434c6-085e-4896-a349-71a41a0a9ad7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2542, 2458])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.bincount(df_val['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1233e282-89a5-45b9-9a77-63836e95815b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>0</td>\n",
       "      <td>Every generation fully believes it is living i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>0</td>\n",
       "      <td>Possibly the most brilliant thing about Che: P...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>0</td>\n",
       "      <td>I was unsure of this movie before renting and ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>0</td>\n",
       "      <td>Just got out of an advance screening, and wow ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>0</td>\n",
       "      <td>I sense out there a mix of confusion and varyi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      index                                               text  label\n",
       "9995      0  Every generation fully believes it is living i...      0\n",
       "9996      0  Possibly the most brilliant thing about Che: P...      1\n",
       "9997      0  I was unsure of this movie before renting and ...      1\n",
       "9998      0  Just got out of an advance screening, and wow ...      1\n",
       "9999      0  I sense out there a mix of confusion and varyi...      1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = pd.read_csv(\"test.csv\")\n",
    "df_test.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7660e59a-568b-4dbc-8223-f860505dffdf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5006, 4994])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.bincount(df_test['label'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b01ee00f-6ac3-40c5-85d2-f5f43566d9a7",
   "metadata": {},
   "source": [
    "## 2) Bag-of-Words Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "31d3ac5a-fc26-42df-89b6-70eba4133215",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install scikit-learn\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e9d2f340-d5a9-4512-b540-09f92b312ded",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer(max_features=10000, stop_words=&#x27;english&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CountVectorizer</label><div class=\"sk-toggleable__content\"><pre>CountVectorizer(max_features=10000, stop_words=&#x27;english&#x27;)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "CountVectorizer(max_features=10000, stop_words='english')"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = CountVectorizer(lowercase=True, max_features=10_000, stop_words=\"english\")\n",
    "\n",
    "cv.fit(df_train[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "06fd1820-bcd6-41a9-b777-3f5821338d4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'when': 9774,\n",
       " 'we': 9718,\n",
       " 'started': 8484,\n",
       " 'watching': 9705,\n",
       " 'this': 9018,\n",
       " 'series': 7924,\n",
       " 'on': 6249,\n",
       " 'cable': 1335,\n",
       " 'had': 4084,\n",
       " 'no': 6087,\n",
       " 'idea': 4457,\n",
       " 'how': 4399,\n",
       " 'it': 4795,\n",
       " 'would': 9916,\n",
       " 'be': 885,\n",
       " 'even': 3182,\n",
       " 'you': 9971,\n",
       " 'hate': 4164,\n",
       " 'character': 1551,\n",
       " 'hold': 4315,\n",
       " 'back': 790,\n",
       " 'because': 905,\n",
       " 'they': 9002,\n",
       " 'are': 587,\n",
       " 'so': 8266,\n",
       " 'beautifully': 902,\n",
       " 'developed': 2559,\n",
       " 'can': 1371,\n",
       " 'almost': 409,\n",
       " 'understand': 9358,\n",
       " 'why': 9800,\n",
       " 'react': 7173,\n",
       " 'to': 9102,\n",
       " 'frustration': 3725,\n",
       " 'fear': 3419,\n",
       " 'greed': 4002,\n",
       " 'or': 6283,\n",
       " 'temptation': 8931,\n",
       " 'the': 8976,\n",
       " 'way': 9714,\n",
       " 'do': 2720,\n",
       " 'as': 642,\n",
       " 'if': 4473,\n",
       " 'viewer': 9556,\n",
       " 'is': 4781,\n",
       " 'experiencing': 3266,\n",
       " 'one': 6251,\n",
       " 'of': 6212,\n",
       " 'christopher': 1661,\n",
       " 'learning': 5167,\n",
       " 'br': 1170,\n",
       " 'put': 7041,\n",
       " 'up': 9441,\n",
       " 'with': 9856,\n",
       " 'abuse': 186,\n",
       " 'her': 4241,\n",
       " 'physically': 6598,\n",
       " 'and': 470,\n",
       " 'emotionally': 3031,\n",
       " 'but': 1316,\n",
       " 'just': 4934,\n",
       " 'have': 4174,\n",
       " 'read': 7176,\n",
       " 'newspaper': 6058,\n",
       " 'see': 7855,\n",
       " 'many': 5516,\n",
       " 'women': 9874,\n",
       " 'tolerate': 9116,\n",
       " 'such': 8686,\n",
       " 'behavior': 932,\n",
       " 'has': 4161,\n",
       " 'dream': 2811,\n",
       " 'house': 4394,\n",
       " 'endless': 3060,\n",
       " 'supply': 8743,\n",
       " 'expensive': 3262,\n",
       " 'things': 9008,\n",
       " 'sure': 8755,\n",
       " 'she': 8000,\n",
       " 'give': 3879,\n",
       " 'for': 3616,\n",
       " 'loving': 5393,\n",
       " 'faithful': 3352,\n",
       " 'husband': 4439,\n",
       " 'maybe': 5603,\n",
       " 'not': 6125,\n",
       " 'that': 8974,\n",
       " 'watch': 9699,\n",
       " 'doesn': 2733,\n",
       " 'matter': 5592,\n",
       " 'times': 9084,\n",
       " 'an': 464,\n",
       " 'episode': 3125,\n",
       " 'find': 3506,\n",
       " 'something': 8296,\n",
       " 'missed': 5775,\n",
       " 'first': 3526,\n",
       " 'five': 3538,\n",
       " 'episodes': 3126,\n",
       " 'out': 6318,\n",
       " 'sequence': 7917,\n",
       " 'season': 7833,\n",
       " 'late': 5119,\n",
       " 'night': 6072,\n",
       " 'commercials': 1872,\n",
       " 'all': 392,\n",
       " 'language': 5100,\n",
       " 'reruns': 7397,\n",
       " 'movie': 5902,\n",
       " 'network': 6046,\n",
       " 'whenever': 9775,\n",
       " 're': 7167,\n",
       " 'there': 8996,\n",
       " 've': 9509,\n",
       " 'been': 913,\n",
       " 'totally': 9154,\n",
       " 'spoiled': 8408,\n",
       " 'now': 6150,\n",
       " 'also': 416,\n",
       " 'love': 5387,\n",
       " 'around': 614,\n",
       " 'my': 5951,\n",
       " 'neck': 6012,\n",
       " 'favorite': 3411,\n",
       " 'johnny': 4877,\n",
       " 'boy': 1164,\n",
       " 'these': 9001,\n",
       " 'entered': 3099,\n",
       " 'our': 6316,\n",
       " 'family': 3367,\n",
       " 'sign': 8104,\n",
       " 'should': 8062,\n",
       " 'get': 3845,\n",
       " 'life': 5238,\n",
       " 'will': 9817,\n",
       " 'ends': 3062,\n",
       " 'dvd': 2892,\n",
       " 'collection': 1818,\n",
       " 'in': 4539,\n",
       " 'steve': 8535,\n",
       " 'biko': 1004,\n",
       " 'was': 9690,\n",
       " 'black': 1032,\n",
       " 'who': 9791,\n",
       " 'tried': 9249,\n",
       " 'resist': 7410,\n",
       " 'white': 9789,\n",
       " 'minority': 5755,\n",
       " 'south': 8339,\n",
       " 'africa': 319,\n",
       " 'much': 5911,\n",
       " 'same': 7697,\n",
       " 'gandhi': 3775,\n",
       " 'british': 1227,\n",
       " 'empire': 3039,\n",
       " 'india': 4576,\n",
       " 'richard': 7492,\n",
       " 'attenborough': 712,\n",
       " 'film': 3491,\n",
       " 'cry': 2241,\n",
       " 'freedom': 3693,\n",
       " 'about': 170,\n",
       " 'donald': 2751,\n",
       " 'woods': 9888,\n",
       " 'liberal': 5228,\n",
       " 'editor': 2950,\n",
       " 'his': 4294,\n",
       " 'trying': 9282,\n",
       " 'tell': 8924,\n",
       " 'story': 8572,\n",
       " 'jarring': 4830,\n",
       " 'point': 6696,\n",
       " 'view': 9554,\n",
       " 'switch': 8815,\n",
       " 'after': 321,\n",
       " 'dies': 2594,\n",
       " 'prison': 6890,\n",
       " 'from': 3716,\n",
       " 'at': 691,\n",
       " 'hands': 4113,\n",
       " 'african': 320,\n",
       " 'police': 6710,\n",
       " 'played': 6663,\n",
       " 'by': 1329,\n",
       " 'kevin': 4977,\n",
       " 'kline': 5025,\n",
       " 'must': 5943,\n",
       " 'choose': 1637,\n",
       " 'whether': 9780,\n",
       " 'right': 7513,\n",
       " 'thing': 9007,\n",
       " 'flee': 3560,\n",
       " 'country': 2122,\n",
       " 'books': 1120,\n",
       " 'allow': 403,\n",
       " 'wife': 9810,\n",
       " 'penelope': 6518,\n",
       " 'pressure': 6856,\n",
       " 'him': 4281,\n",
       " 'into': 4726,\n",
       " 'forgetting': 3633,\n",
       " 'case': 1455,\n",
       " 'vain': 9480,\n",
       " 'what': 9768,\n",
       " 'begins': 925,\n",
       " 'changing': 1542,\n",
       " 'friendship': 3709,\n",
       " 'between': 986,\n",
       " 'standard': 8464,\n",
       " 'numbers': 6161,\n",
       " 'escape': 3146,\n",
       " 'over': 6339,\n",
       " 'border': 1127,\n",
       " 'yarn': 9953,\n",
       " 'death': 2375,\n",
       " 'oscar': 6306,\n",
       " 'nominated': 6098,\n",
       " 'denzel': 2480,\n",
       " 'washington': 9693,\n",
       " 'good': 3929,\n",
       " 'only': 6256,\n",
       " 'fourth': 3662,\n",
       " 'wrong': 9939,\n",
       " 'tries': 9251,\n",
       " 'depict': 2488,\n",
       " 'struggles': 8619,\n",
       " 'focusing': 3593,\n",
       " 'more': 5859,\n",
       " 'trials': 9241,\n",
       " 'than': 8969,\n",
       " 'half': 4094,\n",
       " 'served': 7931,\n",
       " 'topic': 9138,\n",
       " 'better': 983,\n",
       " 'rise': 7530,\n",
       " 'instead': 4666,\n",
       " 'beginning': 924,\n",
       " 'where': 9776,\n",
       " 'perhaps': 6542,\n",
       " 'actor': 240,\n",
       " 'leading': 5159,\n",
       " 'role': 7578,\n",
       " 'hour': 4392,\n",
       " 'wasn': 9694,\n",
       " 'exactly': 3208,\n",
       " 'big': 995,\n",
       " 'box': 1160,\n",
       " 'office': 6224,\n",
       " 'tremendous': 9236,\n",
       " 'flop': 3577,\n",
       " 'anyway': 538,\n",
       " 'politics': 6721,\n",
       " 'aside': 649,\n",
       " 'still': 8543,\n",
       " 'entertains': 3108,\n",
       " 'sends': 7891,\n",
       " 'message': 5684,\n",
       " 'two': 9311,\n",
       " 'albeit': 370,\n",
       " 'pg': 6571,\n",
       " 'fashion': 3393,\n",
       " 'stars': 8481,\n",
       " 'short': 8054,\n",
       " 'comment': 1864,\n",
       " 'flick': 3566,\n",
       " 'go': 3906,\n",
       " 'pick': 6602,\n",
       " 'chances': 1536,\n",
       " 'going': 3918,\n",
       " 'positively': 6764,\n",
       " 'surprised': 8764,\n",
       " 'diversity': 2711,\n",
       " 'elements': 2995,\n",
       " 'superbly': 8733,\n",
       " 'explored': 3290,\n",
       " 'criminal': 2197,\n",
       " 'thriller': 9042,\n",
       " 'claiming': 1701,\n",
       " 'pushing': 7040,\n",
       " 'room': 7599,\n",
       " 'every': 3190,\n",
       " 'possible': 6773,\n",
       " 'wont': 9884,\n",
       " 'push': 7037,\n",
       " 'your': 9976,\n",
       " 'nerves': 6040,\n",
       " 'edge': 2940,\n",
       " 'thumbs': 9060,\n",
       " 'serious': 7925,\n",
       " 'horror': 4379,\n",
       " 'fan': 3369,\n",
       " 'certain': 1517,\n",
       " 'marketing': 5536,\n",
       " 'used': 9463,\n",
       " 'sell': 7879,\n",
       " 'movies': 5904,\n",
       " 'especially': 3150,\n",
       " 'really': 7198,\n",
       " 'bad': 801,\n",
       " 'ones': 6252,\n",
       " 'wouldn': 9917,\n",
       " 'call': 1346,\n",
       " 'assumed': 677,\n",
       " 'ripping': 7528,\n",
       " 'off': 6213,\n",
       " 'cannibal': 1381,\n",
       " 'zombi': 9993,\n",
       " 'jungle': 4928,\n",
       " 'holocaust': 4328,\n",
       " 'unfortunately': 9387,\n",
       " 'completely': 1913,\n",
       " 'very': 9533,\n",
       " 'hardcore': 4136,\n",
       " 'realized': 7195,\n",
       " 'saw': 7743,\n",
       " 'odd': 6207,\n",
       " 'actual': 245,\n",
       " 'minor': 5754,\n",
       " 'warning': 9680,\n",
       " 'notice': 6134,\n",
       " 'part': 6444,\n",
       " 'though': 9025,\n",
       " 'he': 4186,\n",
       " 'daring': 2330,\n",
       " 'us': 9459,\n",
       " 'catch': 1472,\n",
       " 'group': 4035,\n",
       " 'scientists': 7787,\n",
       " 'whom': 9796,\n",
       " 'pretty': 6863,\n",
       " 'led': 5175,\n",
       " 'sea': 7823,\n",
       " 'captain': 1397,\n",
       " 'penchant': 6517,\n",
       " 'beach': 886,\n",
       " 'search': 7829,\n",
       " 'mutated': 5947,\n",
       " 'native': 5989,\n",
       " 'killing': 4999,\n",
       " 'villagers': 9568,\n",
       " 'due': 2865,\n",
       " 'nuclear': 6157,\n",
       " 'bomb': 1102,\n",
       " 'supposedly': 8752,\n",
       " 'island': 4785,\n",
       " 'radiation': 7091,\n",
       " 'turned': 9294,\n",
       " 'last': 5114,\n",
       " 'man': 5487,\n",
       " 'rapist': 7146,\n",
       " 'killer': 4997,\n",
       " 'writer': 9934,\n",
       " 'george': 3832,\n",
       " 'here': 4245,\n",
       " 'succeeds': 8681,\n",
       " 'keeping': 4965,\n",
       " 'clothes': 1770,\n",
       " 'sex': 7952,\n",
       " 'scenes': 7768,\n",
       " 'walk': 9642,\n",
       " 'nude': 6158,\n",
       " 'strange': 8580,\n",
       " 'asks': 653,\n",
       " 'rape': 7140,\n",
       " 'turns': 9297,\n",
       " 'down': 2774,\n",
       " 'well': 9750,\n",
       " 'chicks': 1616,\n",
       " 'each': 2902,\n",
       " 'other': 6310,\n",
       " 'naturally': 5992,\n",
       " 'scene': 7766,\n",
       " 'chick': 1614,\n",
       " 'toss': 9151,\n",
       " 'another': 512,\n",
       " 'finger': 3512,\n",
       " 'herself': 4254,\n",
       " 'anything': 536,\n",
       " 'know': 5038,\n",
       " 'rest': 7428,\n",
       " 'insane': 4638,\n",
       " 'oh': 6232,\n",
       " 'am': 426,\n",
       " 'kidding': 4987,\n",
       " 'ton': 9122,\n",
       " 'like': 5255,\n",
       " 'pays': 6499,\n",
       " 'guys': 4073,\n",
       " 'tag': 8846,\n",
       " 'team': 8898,\n",
       " 'taking': 8852,\n",
       " 'top': 9137,\n",
       " 'use': 9462,\n",
       " 'cuts': 2280,\n",
       " 'himself': 4282,\n",
       " 'refuses': 7271,\n",
       " 'advances': 293,\n",
       " 'starts': 8488,\n",
       " 'crying': 2242,\n",
       " 'being': 936,\n",
       " 'gentleman': 3826,\n",
       " 'reluctantly': 7323,\n",
       " 'lets': 5216,\n",
       " 'pleasure': 6675,\n",
       " 'front': 3717,\n",
       " 'crew': 2192,\n",
       " 'members': 5652,\n",
       " 'honestly': 4344,\n",
       " 'waiting': 9636,\n",
       " 'pizza': 6637,\n",
       " 'guy': 4072,\n",
       " 'show': 8070,\n",
       " 'ask': 650,\n",
       " 'any': 531,\n",
       " 'pay': 6495,\n",
       " 'happens': 4128,\n",
       " 'before': 917,\n",
       " 'conduct': 1949,\n",
       " 'their': 8983,\n",
       " 'research': 7401,\n",
       " 'wait': 9634,\n",
       " 'thought': 9026,\n",
       " 'zombie': 9994,\n",
       " 'enter': 3098,\n",
       " 'until': 9436,\n",
       " 'mark': 5532,\n",
       " 'time': 9081,\n",
       " 'everyone': 3193,\n",
       " 'else': 3009,\n",
       " 'plenty': 6677,\n",
       " 'found': 3659,\n",
       " 'myself': 5954,\n",
       " 'hitting': 4306,\n",
       " 'fast': 3397,\n",
       " 'forward': 3655,\n",
       " 'few': 3459,\n",
       " 'several': 7948,\n",
       " 'splatter': 8401,\n",
       " 'porn': 6741,\n",
       " 'don': 2750,\n",
       " 'think': 9009,\n",
       " 'does': 2732,\n",
       " 'justice': 4935,\n",
       " 'guess': 4050,\n",
       " 'woman': 9873,\n",
       " 'me': 5611,\n",
       " 'talking': 8863,\n",
       " 'say': 7745,\n",
       " 'plot': 6681,\n",
       " 'less': 5207,\n",
       " 'hairy': 4092,\n",
       " 'funny': 3747,\n",
       " 'could': 2112,\n",
       " 'worked': 9894,\n",
       " 'decent': 2389,\n",
       " 'atomic': 698,\n",
       " 'bombing': 1104,\n",
       " 'bitter': 1029,\n",
       " 'shakes': 7973,\n",
       " 'head': 4187,\n",
       " 'walks': 9647,\n",
       " 'away': 769,\n",
       " 'couple': 2127,\n",
       " 'makes': 5476,\n",
       " 'wonder': 9876,\n",
       " 'disgusted': 2674,\n",
       " 'feel': 3434,\n",
       " 'sound': 8329,\n",
       " 'quality': 7052,\n",
       " 'already': 414,\n",
       " 'guessed': 4051,\n",
       " 'production': 6911,\n",
       " 'shot': 8059,\n",
       " 'including': 4558,\n",
       " 'erotic': 3141,\n",
       " 'nights': 6077,\n",
       " 'living': 5311,\n",
       " 'dead': 2362,\n",
       " 'which': 9781,\n",
       " 'sports': 8421,\n",
       " 'most': 5876,\n",
       " 'cast': 1464,\n",
       " 'said': 7681,\n",
       " 'done': 2752,\n",
       " 'wanted': 9664,\n",
       " 'vacation': 9476,\n",
       " 'paycheck': 6496,\n",
       " 'suddenly': 8693,\n",
       " 'weird': 9745,\n",
       " 'during': 2884,\n",
       " 'while': 9782,\n",
       " 'speaking': 8361,\n",
       " 'italian': 4796,\n",
       " 'seems': 7866,\n",
       " 'recorded': 7231,\n",
       " 'english': 3078,\n",
       " 'dialogue': 2576,\n",
       " 'people': 6524,\n",
       " 'clearly': 1728,\n",
       " 'hear': 4197,\n",
       " 'someone': 8295,\n",
       " 'background': 793,\n",
       " 'yes': 9965,\n",
       " 'wonderful': 9878,\n",
       " 'slightly': 8218,\n",
       " 'amusing': 462,\n",
       " 'score': 7792,\n",
       " 'couldn': 2113,\n",
       " 'save': 7738,\n",
       " 'sfx': 7960,\n",
       " 'were': 9756,\n",
       " 'minimal': 5748,\n",
       " 'best': 977,\n",
       " 'consisted': 2000,\n",
       " 'some': 8291,\n",
       " 'blood': 1068,\n",
       " 'violent': 9578,\n",
       " 'bright': 1217,\n",
       " 'label': 5066,\n",
       " 'cover': 2139,\n",
       " 'ploy': 6685,\n",
       " 'presented': 6848,\n",
       " 'widescreen': 9805,\n",
       " '85': 149,\n",
       " 'aspect': 655,\n",
       " 'ratio': 7157,\n",
       " 'watched': 9701,\n",
       " 'region': 7278,\n",
       " 'rated': 7151,\n",
       " 'version': 9529,\n",
       " 'running': 7650,\n",
       " 'released': 7308,\n",
       " '2005': 103,\n",
       " 'exploitation': 3284,\n",
       " 'digital': 2607,\n",
       " 'apparently': 551,\n",
       " 'doubt': 2769,\n",
       " 'different': 2600,\n",
       " 'shouldn': 8065,\n",
       " '25': 115,\n",
       " '00': 0,\n",
       " 'copy': 2084,\n",
       " 'recommend': 7226,\n",
       " 'nothing': 6133,\n",
       " 'pretend': 6859,\n",
       " 'exist': 3242,\n",
       " 'quote': 7076,\n",
       " 'civilians': 1695,\n",
       " 'luck': 5408,\n",
       " 'too': 9131,\n",
       " 'monsters': 5838,\n",
       " 'extras': 3315,\n",
       " 'original': 6297,\n",
       " 'trailer': 9188,\n",
       " 'shots': 8061,\n",
       " 'kills': 5001,\n",
       " 'make': 5473,\n",
       " 'look': 5354,\n",
       " 'interesting': 4704,\n",
       " 'trailers': 9189,\n",
       " 'ss': 8437,\n",
       " 'hell': 4224,\n",
       " 'camp': 1362,\n",
       " 'informative': 4614,\n",
       " 'interview': 4721,\n",
       " 'bottom': 1150,\n",
       " 'line': 5276,\n",
       " 'lame': 5086,\n",
       " 'porno': 6742,\n",
       " 'weaker': 9720,\n",
       " 'either': 2981,\n",
       " 'real': 7184,\n",
       " 'rating': 7155,\n",
       " '10': 3,\n",
       " 'molly': 5817,\n",
       " 'www': 9949,\n",
       " 'com': 1835,\n",
       " 'robert': 7557,\n",
       " 'cummings': 2255,\n",
       " 'day': 2356,\n",
       " 'jean': 4838,\n",
       " 'star': 8472,\n",
       " 'beautiful': 901,\n",
       " '1940': 30,\n",
       " 'starring': 8480,\n",
       " 'billie': 1007,\n",
       " 'burke': 1297,\n",
       " '15': 12,\n",
       " 'minutes': 5759,\n",
       " 'looks': 5357,\n",
       " 'playboy': 6662,\n",
       " 'desire': 2525,\n",
       " 'sisters': 8153,\n",
       " 'katherine': 4952,\n",
       " 'helen': 4221,\n",
       " 'likes': 5259,\n",
       " 'fix': 3539,\n",
       " 'cars': 1447,\n",
       " 'blonde': 1066,\n",
       " 'social': 8271,\n",
       " 'butterfly': 1322,\n",
       " 'arrives': 623,\n",
       " 'town': 9167,\n",
       " 'believing': 948,\n",
       " 'party': 6456,\n",
       " 'decides': 2394,\n",
       " 'attend': 713,\n",
       " 'since': 8133,\n",
       " 'given': 3880,\n",
       " 'friend': 3706,\n",
       " 'mother': 5879,\n",
       " 'dress': 2817,\n",
       " 'connect': 1971,\n",
       " 'sees': 7868,\n",
       " 'dinner': 2617,\n",
       " 'left': 5178,\n",
       " 'club': 1775,\n",
       " 'becomes': 908,\n",
       " 'terribly': 8952,\n",
       " 'drunk': 2849,\n",
       " 'ride': 7502,\n",
       " 'car': 1405,\n",
       " 'won': 9875,\n",
       " 'let': 5213,\n",
       " 'drive': 2831,\n",
       " 'walking': 9646,\n",
       " 'awhile': 775,\n",
       " 'breaking': 1193,\n",
       " 'shoe': 8044,\n",
       " 'gets': 3846,\n",
       " 'drives': 2836,\n",
       " 'passes': 6464,\n",
       " 'takes': 8851,\n",
       " 'wheel': 9772,\n",
       " 'accidentally': 204,\n",
       " 'remember': 7338,\n",
       " 'take': 8848,\n",
       " 'blame': 1039,\n",
       " 'sister': 8152,\n",
       " 'shoes': 8045,\n",
       " 'plus': 6688,\n",
       " 'manner': 5506,\n",
       " 'realize': 7194,\n",
       " 'isn': 4787,\n",
       " 'telling': 8925,\n",
       " 'truth': 9278,\n",
       " 'convicted': 2062,\n",
       " 'goes': 3917,\n",
       " 'marries': 5544,\n",
       " 'leaves': 5173,\n",
       " 'america': 444,\n",
       " 'list': 5292,\n",
       " 'playing': 6667,\n",
       " 'taylor': 8890,\n",
       " 'mgm': 5698,\n",
       " 'handsome': 4114,\n",
       " 'amiable': 447,\n",
       " 'dazzling': 2360,\n",
       " 'actress': 242,\n",
       " 'mine': 5743,\n",
       " 'constantly': 2008,\n",
       " 'didn': 2588,\n",
       " 'great': 3995,\n",
       " 'face': 3324,\n",
       " 'voice': 9607,\n",
       " 'determined': 2554,\n",
       " 'sympathetic': 8830,\n",
       " 'lovely': 5389,\n",
       " 'lousy': 5385,\n",
       " 'highly': 4272,\n",
       " 'recommended': 7228,\n",
       " 'little': 5304,\n",
       " 'gem': 3806,\n",
       " 'dark': 2331,\n",
       " 'overlooked': 6350,\n",
       " 'known': 5041,\n",
       " 'early': 2910,\n",
       " '80': 147,\n",
       " 'deserves': 2519,\n",
       " 'audience': 728,\n",
       " 'never': 6049,\n",
       " 'damn': 2307,\n",
       " 'shame': 7980,\n",
       " 'seen': 7867,\n",
       " 'compared': 1891,\n",
       " 'others': 6311,\n",
       " 'gotten': 3951,\n",
       " 'bigger': 997,\n",
       " 'name': 5969,\n",
       " 'years': 9959,\n",
       " 'notably': 6127,\n",
       " 'those': 9023,\n",
       " 'comparisons': 1895,\n",
       " 'bit': 1021,\n",
       " 'similar': 8117,\n",
       " 'enough': 3093,\n",
       " 'slipped': 8222,\n",
       " 'through': 9049,\n",
       " 'without': 9859,\n",
       " 'acceptance': 196,\n",
       " 'remake': 7332,\n",
       " 'breathe': 1199,\n",
       " 'new': 6052,\n",
       " 'unless': 9407,\n",
       " 'drained': 2796,\n",
       " 'remakes': 7333,\n",
       " 'days': 2358,\n",
       " 'work': 9893,\n",
       " 'lesser': 5208,\n",
       " 'films': 3498,\n",
       " 'except': 3219,\n",
       " 'awful': 772,\n",
       " 'ghost': 3850,\n",
       " 'ship': 8031,\n",
       " 'opening': 6261,\n",
       " 'falling': 3358,\n",
       " 'utter': 9473,\n",
       " 'crap': 2157,\n",
       " 'happen': 4124,\n",
       " 'fall': 3356,\n",
       " 'lot': 5377,\n",
       " 'haven': 4175,\n",
       " 'bring': 1222,\n",
       " 'course': 2132,\n",
       " 'got': 3948,\n",
       " 'eyes': 3321,\n",
       " 'anyways': 539,\n",
       " 'fans': 3373,\n",
       " 'cause': 1485,\n",
       " 'made': 5445,\n",
       " 'creepy': 2191,\n",
       " 'setting': 7939,\n",
       " 'fairly': 3348,\n",
       " 'acting': 233,\n",
       " 'campy': 1370,\n",
       " 'want': 9663,\n",
       " 'nudity': 6159,\n",
       " 'gore': 3941,\n",
       " 'sorry': 8321,\n",
       " 'nonetheless': 6103,\n",
       " 'solid': 8283,\n",
       " 'enjoy': 3085,\n",
       " 'grave': 3988,\n",
       " 'robber': 7552,\n",
       " 'sitting': 8160,\n",
       " 'cell': 1502,\n",
       " 'awaiting': 761,\n",
       " 'execution': 3237,\n",
       " 'visited': 9593,\n",
       " 'monk': 5828,\n",
       " 'wishing': 9851,\n",
       " 'words': 9891,\n",
       " 'horrible': 4373,\n",
       " 'lead': 5156,\n",
       " 'reluctant': 7322,\n",
       " 'tongue': 9126,\n",
       " 'drink': 2827,\n",
       " 'young': 9972,\n",
       " 'soon': 8308,\n",
       " 'full': 3732,\n",
       " 'undead': 9348,\n",
       " 'bump': 1291,\n",
       " 'york': 9970,\n",
       " 'filmed': 3492,\n",
       " 'brought': 1246,\n",
       " 'spirit': 8395,\n",
       " 'andy': 476,\n",
       " 'lurking': 5423,\n",
       " 'comedies': 1849,\n",
       " 'come': 1844,\n",
       " 'often': 6231,\n",
       " 'its': 4801,\n",
       " 'rate': 7150,\n",
       " 'dominic': 2748,\n",
       " 'plays': 6668,\n",
       " 'arthur': 629,\n",
       " 'blake': 1038,\n",
       " 'ron': 7595,\n",
       " 'father': 3404,\n",
       " 'statement': 8492,\n",
       " 'getting': 3847,\n",
       " 'involved': 4756,\n",
       " 'tale': 8854,\n",
       " 'both': 1143,\n",
       " 'men': 5657,\n",
       " 'having': 4176,\n",
       " 'grand': 3970,\n",
       " 'old': 6237,\n",
       " 'shows': 8078,\n",
       " 'equally': 3130,\n",
       " 'music': 5936,\n",
       " 'jeff': 4841,\n",
       " 'grace': 3958,\n",
       " 'excellent': 3216,\n",
       " 'effects': 2962,\n",
       " 'perfect': 6531,\n",
       " 'sort': 8322,\n",
       " 'silliness': 8113,\n",
       " 'deal': 2366,\n",
       " 'fun': 3735,\n",
       " 'trouble': 9268,\n",
       " 'throws': 9054,\n",
       " 'net': 6044,\n",
       " 'wide': 9802,\n",
       " 'result': 7435,\n",
       " 'needed': 6015,\n",
       " 'alien': 386,\n",
       " 'body': 1093,\n",
       " 'mix': 5792,\n",
       " 'theaters': 8978,\n",
       " 'later': 5121,\n",
       " 'll': 5314,\n",
       " 'worth': 9912,\n",
       " 'liked': 5257,\n",
       " 'script': 7816,\n",
       " 'may': 5602,\n",
       " 'changed': 1540,\n",
       " 'reason': 7201,\n",
       " 'rodney': 7572,\n",
       " 'dangerfield': 2321,\n",
       " 'jackie': 4810,\n",
       " 'mason': 5569,\n",
       " 'although': 421,\n",
       " 'did': 2587,\n",
       " 'alot': 413,\n",
       " 'kept': 4974,\n",
       " 'flaw': 3556,\n",
       " 'dan': 2311,\n",
       " 'bill': 1005,\n",
       " 'murray': 5932,\n",
       " 'carl': 1424,\n",
       " 'quit': 7074,\n",
       " 'job': 4869,\n",
       " 'assistant': 669,\n",
       " 'joined': 4880,\n",
       " 'military': 5726,\n",
       " 'warner': 9679,\n",
       " 'bros': 1242,\n",
       " 'ii': 4480,\n",
       " 'try': 9281,\n",
       " 'seeing': 7858,\n",
       " 'anyone': 535,\n",
       " 'possibly': 6774,\n",
       " 'disappointed': 2644,\n",
       " 'fact': 3330,\n",
       " 'director': 2628,\n",
       " 'cube': 2248,\n",
       " 'comedy': 1850,\n",
       " 'imdb': 4501,\n",
       " 'spell': 8376,\n",
       " 'seem': 7862,\n",
       " 'word': 9890,\n",
       " 'reminiscent': 7347,\n",
       " 'builds': 1280,\n",
       " 'slowly': 8228,\n",
       " 'then': 8991,\n",
       " 'gradually': 3961,\n",
       " 'explanation': 3277,\n",
       " 'mainly': 5467,\n",
       " 'set': 7937,\n",
       " 'respects': 7422,\n",
       " 'probably': 6897,\n",
       " 'commented': 1868,\n",
       " 'masterpiece': 5578,\n",
       " 'spanish': 8352,\n",
       " 'cinema': 1677,\n",
       " 'masters': 5580,\n",
       " 'piece': 6612,\n",
       " 'them': 8986,\n",
       " 'long': 5350,\n",
       " 'ago': 338,\n",
       " 'midnight': 5714,\n",
       " 'cowboy': 2146,\n",
       " 'les': 5203,\n",
       " 'du': 2852,\n",
       " 'realistic': 7189,\n",
       " 'non': 6101,\n",
       " 'spot': 8422,\n",
       " 'trainspotting': 9194,\n",
       " 'hard': 4135,\n",
       " 'place': 6638,\n",
       " 'humour': 4420,\n",
       " 'obviously': 6193,\n",
       " 'dramatic': 2800,\n",
       " 'sense': 7895,\n",
       " 'diamond': 2578,\n",
       " 'resurrection': 7440,\n",
       " 'neo': 6033,\n",
       " 'realism': 7188,\n",
       " 'mixed': 5793,\n",
       " 'ken': 4970,\n",
       " 'discover': 2658,\n",
       " 'modern': 5810,\n",
       " 'tv': 9299,\n",
       " 'classic': 1713,\n",
       " 'bob': 1090,\n",
       " 'girlfriend': 3876,\n",
       " 'named': 5970,\n",
       " 'alicia': 385,\n",
       " 'married': 5543,\n",
       " 'bud': 1264,\n",
       " 'owen': 6366,\n",
       " 'works': 9898,\n",
       " 'jealous': 4836,\n",
       " 'hanging': 4117,\n",
       " 'hangs': 4118,\n",
       " 'secretary': 7845,\n",
       " 'heather': 4211,\n",
       " 'accident': 202,\n",
       " 'prone': 6951,\n",
       " 'kind': 5004,\n",
       " 'lonely': 5347,\n",
       " 'sometimes': 8299,\n",
       " 'wishes': 9850,\n",
       " 'friends': 3708,\n",
       " 'end': 3055,\n",
       " 'looked': 5355,\n",
       " 'finally': 3503,\n",
       " 'went': 9755,\n",
       " 'driving': 2837,\n",
       " 'wedding': 9738,\n",
       " 'making': 5478,\n",
       " 'tiny': 9090,\n",
       " 'stuck': 8624,\n",
       " 'middle': 5711,\n",
       " 'nowhere': 6152,\n",
       " 'happened': 4125,\n",
       " 'next': 6059,\n",
       " 'poor': 6729,\n",
       " 'ended': 3057,\n",
       " 'guide': 4056,\n",
       " 'fox': 3663,\n",
       " 'twice': 9304,\n",
       " 'putting': 7043,\n",
       " 'air': 355,\n",
       " 'loved': 5388,\n",
       " 'cool': 2073,\n",
       " 'glasses': 3889,\n",
       " 'hilarious': 4274,\n",
       " 'miss': 5774,\n",
       " 'reading': 7180,\n",
       " 'book': 1119,\n",
       " 'ending': 3058,\n",
       " 'missing': 5779,\n",
       " 'sad': 7672,\n",
       " 'treatment': 9231,\n",
       " 'subject': 8657,\n",
       " 'quite': 7075,\n",
       " 'controversial': 2046,\n",
       " 'comments': 1870,\n",
       " 'distinction': 2693,\n",
       " 'based': 854,\n",
       " 'believe': 944,\n",
       " 'portrayed': 6753,\n",
       " 'basically': 859,\n",
       " 'sequels': 7916,\n",
       " '30': 121,\n",
       " 'values': 9487,\n",
       " 'plan': 6646,\n",
       " 'outer': 6322,\n",
       " 'space': 8344,\n",
       " 'level': 5221,\n",
       " 'glen': 3890,\n",
       " 'glenda': 3891,\n",
       " 'ed': 2937,\n",
       " 'wood': 9886,\n",
       " 'religious': 7321,\n",
       " 'scary': 7762,\n",
       " 'add': 256,\n",
       " 'slightest': 8217,\n",
       " 'might': 5717,\n",
       " 'actually': 246,\n",
       " 'close': 1762,\n",
       " 'future': 3755,\n",
       " 'scarier': 7757,\n",
       " 'reasons': 7205,\n",
       " 'behind': 934,\n",
       " 'code': 1794,\n",
       " 'everything': 3194,\n",
       " 'thief': 9004,\n",
       " 'explain': 3273,\n",
       " 'east': 2921,\n",
       " 'effect': 2959,\n",
       " 'itself': 4802,\n",
       " 'thin': 9006,\n",
       " 'happening': 4126,\n",
       " 'forget': 3630,\n",
       " 'stories': 8570,\n",
       " 'told': 9113,\n",
       " 'god': 3910,\n",
       " 'frightening': 3713,\n",
       " 'wild': 9812,\n",
       " 'rebels': 7210,\n",
       " 'frustrating': 3724,\n",
       " 'least': 5170,\n",
       " 'deals': 2370,\n",
       " 'race': 7082,\n",
       " 'driver': 2834,\n",
       " 'bikers': 1002,\n",
       " 'called': 1347,\n",
       " ...}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "65d50678-3ab0-4075-ab50-3c12e63fd95f",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = cv.transform(df_train[\"text\"])\n",
    "X_val = cv.transform(df_val[\"text\"])\n",
    "X_test = cv.transform(df_test[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "03491130-ad33-4fb7-8ed2-12de9662d400",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(35000, 10000)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "12dd8956-f4db-410b-94c0-78190a8881cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x10000 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 139 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9e42b356-2aad-4447-b714-be0880df3c6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(X_train[0].todense())[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3f185b5b-b7b2-4660-b6df-8c6faee8bf19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9861,  101,   17,    9,    6,    3,    2,    0,    0,    0,    1])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.bincount(np.array(X_train[0].todense())[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "23e6a768-8fa8-444c-8bba-3c3797229a53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0, 0, 0, ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[0].todense().flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "ae181592-134d-4cff-983f-90d6fcae5068",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(35000, 10000)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(X_train.todense()).shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}