fasttext_model_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fasttext for text classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fasttext\n",
    "from preprocessing import *\n",
    "from helpers import *\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = load_tweets(full = True)\n",
    "tweets = remove_tags(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i dunno justin read my mention or not . only j...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>because your logic is so dumb , i won't even c...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just put casper in a box !  looved the battle ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thanks sir &gt; &gt; don't trip lil mama ... just ke...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visiting my brother tmr is the bestest birthda...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity\n",
       "0  i dunno justin read my mention or not . only j...         1\n",
       "1  because your logic is so dumb , i won't even c...         1\n",
       "2  just put casper in a box !  looved the battle ...         1\n",
       "3  thanks sir > > don't trip lil mama ... just ke...         1\n",
       "4  visiting my brother tmr is the bestest birthda...         1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Finding the best preprocessing for fasttext\n",
    "\n",
    "_We will first try to find the preprocessing that works best for fasttext before tuning hyperparameters._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Without preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_basic, test_basic = train_test_split(tweets, test_size = 0.2, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "construct_fasttext_input(train_basic, 'tweets_basic_train.csv')\n",
    "construct_fasttext_input(test_basic, 'tweets_basic_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = fasttext.train_supervised(input = 'tweets_basic_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1966637, 0.861019089948984, 0.861019089948984)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.test('tweets_basic_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(491660, 0.8354899727453932, 0.8354899727453932)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.test('tweets_basic_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Removing punctuation & neutral stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk_stopwords = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "negative_stopwords = set([\"above\", \"against\", \"ain\", \"aren't\", \"but\", \"couldn\", \"couldn't\", \"didn\", \"didn't\", \"doesn't\", \"don\", \"don't\", \"hadn\", \"hadn't\",\n",
    "\"hasn't\", \"haven\", \"haven't\", \"isn\", \"isn't\", \"mightn\", \"mightn't\", \"mustn\", \"mustn't\", \"needn\", \"needn't\", \"no\",\n",
    "\"nor\", \"not\", \"over\", \"shan\", \"shan't\", \"shouldn\", \"shouldn't\", \"t\", \"under\", \"wasn\", \"wasn't\", \"weren\", \"weren't\",\n",
    "\"won\", \"won't\", \"wouldn\", \"wouldn't\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = nltk_stopwords - negative_stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a',\n",
       " 'about',\n",
       " 'after',\n",
       " 'again',\n",
       " 'all',\n",
       " 'am',\n",
       " 'an',\n",
       " 'and',\n",
       " 'any',\n",
       " 'are',\n",
       " 'aren',\n",
       " 'as',\n",
       " 'at',\n",
       " 'be',\n",
       " 'because',\n",
       " 'been',\n",
       " 'before',\n",
       " 'being',\n",
       " 'below',\n",
       " 'between',\n",
       " 'both',\n",
       " 'by',\n",
       " 'can',\n",
       " 'd',\n",
       " 'did',\n",
       " 'do',\n",
       " 'does',\n",
       " 'doesn',\n",
       " 'doing',\n",
       " 'down',\n",
       " 'during',\n",
       " 'each',\n",
       " 'few',\n",
       " 'for',\n",
       " 'from',\n",
       " 'further',\n",
       " 'had',\n",
       " 'has',\n",
       " 'hasn',\n",
       " 'have',\n",
       " 'having',\n",
       " 'he',\n",
       " 'her',\n",
       " 'here',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'him',\n",
       " 'himself',\n",
       " 'his',\n",
       " 'how',\n",
       " 'i',\n",
       " 'if',\n",
       " 'in',\n",
       " 'into',\n",
       " 'is',\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'just',\n",
       " 'll',\n",
       " 'm',\n",
       " 'ma',\n",
       " 'me',\n",
       " 'more',\n",
       " 'most',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'now',\n",
       " 'o',\n",
       " 'of',\n",
       " 'off',\n",
       " 'on',\n",
       " 'once',\n",
       " 'only',\n",
       " 'or',\n",
       " 'other',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'out',\n",
       " 'own',\n",
       " 're',\n",
       " 's',\n",
       " 'same',\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'so',\n",
       " 'some',\n",
       " 'such',\n",
       " 'than',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'the',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'them',\n",
       " 'themselves',\n",
       " 'then',\n",
       " 'there',\n",
       " 'these',\n",
       " 'they',\n",
       " 'this',\n",
       " 'those',\n",
       " 'through',\n",
       " 'to',\n",
       " 'too',\n",
       " 'until',\n",
       " 'up',\n",
       " 've',\n",
       " 'very',\n",
       " 'was',\n",
       " 'we',\n",
       " 'were',\n",
       " 'what',\n",
       " 'when',\n",
       " 'where',\n",
       " 'which',\n",
       " 'while',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'why',\n",
       " 'will',\n",
       " 'with',\n",
       " 'y',\n",
       " 'you',\n",
       " \"you'd\",\n",
       " \"you'll\",\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_neutral_stopwords(tweets):\n",
    "    tweets_2 = tokenize_tweets(tweets, stop_words = False, stemming = False)\n",
    "    tweets_2['tokens_stopwords'] = tweets_2['tokens'].copy().apply(lambda tokens: [token for token in tokens if token not in stopwords])\n",
    "    tweets_2['tweet'] = tweets_2['tokens_stopwords'].copy().apply(lambda tokens: ' '.join(tokens))\n",
    "    \n",
    "    return tweets_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_2 = remove_neutral_stopwords(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "      <th>tokens</th>\n",
       "      <th>tokens_stopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dunno justin read mention not justin god knows...</td>\n",
       "      <td>1</td>\n",
       "      <td>[i, dunno, justin, read, my, mention, or, not,...</td>\n",
       "      <td>[dunno, justin, read, mention, not, justin, go...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>logic dumb wo nt even crop name photo tsk</td>\n",
       "      <td>1</td>\n",
       "      <td>[because, your, logic, is, so, dumb, i, wo, nt...</td>\n",
       "      <td>[logic, dumb, wo, nt, even, crop, name, photo,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>put casper box looved battle crakkbitch</td>\n",
       "      <td>1</td>\n",
       "      <td>[just, put, casper, in, a, box, looved, the, b...</td>\n",
       "      <td>[put, casper, box, looved, battle, crakkbitch]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thanks sir nt trip lil mama keep doin ya thang</td>\n",
       "      <td>1</td>\n",
       "      <td>[thanks, sir, do, nt, trip, lil, mama, just, k...</td>\n",
       "      <td>[thanks, sir, nt, trip, lil, mama, keep, doin,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visiting brother tmr bestest birthday gift eve...</td>\n",
       "      <td>1</td>\n",
       "      <td>[visiting, my, brother, tmr, is, the, bestest,...</td>\n",
       "      <td>[visiting, brother, tmr, bestest, birthday, gi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity  \\\n",
       "0  dunno justin read mention not justin god knows...         1   \n",
       "1          logic dumb wo nt even crop name photo tsk         1   \n",
       "2            put casper box looved battle crakkbitch         1   \n",
       "3     thanks sir nt trip lil mama keep doin ya thang         1   \n",
       "4  visiting brother tmr bestest birthday gift eve...         1   \n",
       "\n",
       "                                              tokens  \\\n",
       "0  [i, dunno, justin, read, my, mention, or, not,...   \n",
       "1  [because, your, logic, is, so, dumb, i, wo, nt...   \n",
       "2  [just, put, casper, in, a, box, looved, the, b...   \n",
       "3  [thanks, sir, do, nt, trip, lil, mama, just, k...   \n",
       "4  [visiting, my, brother, tmr, is, the, bestest,...   \n",
       "\n",
       "                                    tokens_stopwords  \n",
       "0  [dunno, justin, read, mention, not, justin, go...  \n",
       "1  [logic, dumb, wo, nt, even, crop, name, photo,...  \n",
       "2     [put, casper, box, looved, battle, crakkbitch]  \n",
       "3  [thanks, sir, nt, trip, lil, mama, keep, doin,...  \n",
       "4  [visiting, brother, tmr, bestest, birthday, gi...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_wo_stopwords, test_wo_stopwords = train_test_split(tweets_2, test_size = 0.2, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "construct_fasttext_input(train_wo_stopwords, 'tweets_wo_stopwords_train.csv')\n",
    "construct_fasttext_input(test_wo_stopwords, 'tweets_wo_stopwords_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_2 = fasttext.train_supervised(input = 'tweets_wo_stopwords_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1966637, 0.9063716384874281, 0.9063716384874281)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_2.test('tweets_wo_stopwords_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(491660, 0.8110584550298987, 0.8110584550298987)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_2.test('tweets_wo_stopwords_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Conclusion: Removing neutral stopwords decreases performance.***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. Adding word stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stem_tweets(tweets):\n",
    "    tweets_2 = tokenize_tweets(tweets, stop_words = False, stemming = True)\n",
    "    tweets_2['tweet'] = tweets_2['tokens'].copy().apply(lambda tokens: ' '.join(tokens))\n",
    "    \n",
    "    return tweets_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_3 = stem_tweets(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i dunno justin read my mention or not onli jus...</td>\n",
       "      <td>1</td>\n",
       "      <td>[i, dunno, justin, read, my, mention, or, not,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>becaus your logic is so dumb i wo nt even crop...</td>\n",
       "      <td>1</td>\n",
       "      <td>[becaus, your, logic, is, so, dumb, i, wo, nt,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just put casper in a box loov the battl crakkb...</td>\n",
       "      <td>1</td>\n",
       "      <td>[just, put, casper, in, a, box, loov, the, bat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thank sir do nt trip lil mama just keep doin y...</td>\n",
       "      <td>1</td>\n",
       "      <td>[thank, sir, do, nt, trip, lil, mama, just, ke...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visit my brother tmr is the bestest birthday g...</td>\n",
       "      <td>1</td>\n",
       "      <td>[visit, my, brother, tmr, is, the, bestest, bi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity  \\\n",
       "0  i dunno justin read my mention or not onli jus...         1   \n",
       "1  becaus your logic is so dumb i wo nt even crop...         1   \n",
       "2  just put casper in a box loov the battl crakkb...         1   \n",
       "3  thank sir do nt trip lil mama just keep doin y...         1   \n",
       "4  visit my brother tmr is the bestest birthday g...         1   \n",
       "\n",
       "                                              tokens  \n",
       "0  [i, dunno, justin, read, my, mention, or, not,...  \n",
       "1  [becaus, your, logic, is, so, dumb, i, wo, nt,...  \n",
       "2  [just, put, casper, in, a, box, loov, the, bat...  \n",
       "3  [thank, sir, do, nt, trip, lil, mama, just, ke...  \n",
       "4  [visit, my, brother, tmr, is, the, bestest, bi...  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stemming, test_stemming = train_test_split(tweets_3, test_size = 0.2, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "construct_fasttext_input(train_stemming, 'tweets_stemming_train.csv')\n",
    "construct_fasttext_input(test_stemming, 'tweets_stemming_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_3 = fasttext.train_supervised(input = 'tweets_stemming_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1966637, 0.8445407057835279, 0.8445407057835279)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_3.test('tweets_stemming_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(491660, 0.8157730952284099, 0.8157730952284099)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_3.test('tweets_stemming_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Conclusion: Stemming did not increase performance.***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4. Removing punctuation / special characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def remove_punctuations_hashtags_mentions(tweets):\n",
    "    tweets_2 = tweets.copy()\n",
    "    tweets_2['tweet'] = tweets_2['tweet'].copy().apply(lambda tweet: ' '.join(re.sub(\"(@[A-Za-z0-9]+)|(#[A-Za-z0-9]+)|(\\w+:\\/\\/\\S+)|[\\.\\,\\!\\?\\:\\;\\-\\=\\<\\>]\", \" \", tweet).split()))\n",
    "    \n",
    "    return tweets_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_4 = remove_punctuations_hashtags_mentions(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i dunno justin read my mention or not only jus...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>because your logic is so dumb i won't even cro...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just put casper in a box looved the battle</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thanks sir don't trip lil mama just keep doin ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visiting my brother tmr is the bestest birthda...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity\n",
       "0  i dunno justin read my mention or not only jus...         1\n",
       "1  because your logic is so dumb i won't even cro...         1\n",
       "2         just put casper in a box looved the battle         1\n",
       "3  thanks sir don't trip lil mama just keep doin ...         1\n",
       "4  visiting my brother tmr is the bestest birthda...         1"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_4.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_chars, test_chars = train_test_split(tweets_4, test_size = 0.2, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "construct_fasttext_input(train_chars, 'tweets_chars_train.csv')\n",
    "construct_fasttext_input(test_chars, 'tweets_chars_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4 = fasttext.train_supervised(input = 'tweets_chars_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1966637, 0.8539832211028268, 0.8539832211028268)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_4.test('tweets_chars_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(491660, 0.8295305699060326, 0.8295305699060326)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_4.test('tweets_chars_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Conclusion: removing special chars such as punctuation, hashtags and mentions did not increase performance.***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5. Replacing smileys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This dictionary was taken from the following GitHub\n",
    "# https://github.com/charlesmalafosse/FastText-sentiment-analysis-for-tweets/blob/master/betsentiment_sentiment_analysis_fasttext.py\n",
    "def replace_smiley(tweet):\n",
    "    smileys = {\n",
    "        \":‑)\":\"smile\",\n",
    "        \":-]\":\"smile\",\n",
    "        \":-3\":\"smile\",\n",
    "        \":->\":\"smile\",\n",
    "        \"8-)\":\"smile\",\n",
    "        \":-}\":\"smile\",\n",
    "        \":)\":\"smile\",\n",
    "        \":]\":\"smile\",\n",
    "        \":3\":\"smile\",\n",
    "        \":>\":\"smile\",\n",
    "        \"8)\":\"smile\",\n",
    "        \":}\":\"smile\",\n",
    "        \":o)\":\"smile\",\n",
    "        \":c)\":\"smile\",\n",
    "        \":^)\":\"smile\",\n",
    "        \"=]\":\"smile\",\n",
    "        \"=)\":\"smile\",\n",
    "        \":-))\":\"smile\",\n",
    "        \":‑D\":\"smile\",\n",
    "        \"8‑D\":\"smile\",\n",
    "        \"x‑D\":\"smile\",\n",
    "        \"X‑D\":\"smile\",\n",
    "        \":D\":\"smile\",\n",
    "        \"8D\":\"smile\",\n",
    "        \"xD\":\"smile\",\n",
    "        \"XD\":\"smile\",\n",
    "        \":‑(\":\"sad\",\n",
    "        \":‑c\":\"sad\",\n",
    "        \":‑<\":\"sad\",\n",
    "        \":‑[\":\"sad\",\n",
    "        \":(\":\"sad\",\n",
    "        \":c\":\"sad\",\n",
    "        \":<\":\"sad\",\n",
    "        \":[\":\"sad\",\n",
    "        \":-||\":\"sad\",\n",
    "        \">:[\":\"sad\",\n",
    "        \":{\":\"sad\",\n",
    "        \":@\":\"sad\",\n",
    "        \">:(\":\"sad\",\n",
    "        \":'‑(\":\"sad\",\n",
    "        \":'(\":\"sad\",\n",
    "        \"<3\":\"love\"\n",
    "        }\n",
    "\n",
    "    for smiley, val in smileys.items():\n",
    "        tweet = tweet.replace(smiley, val)\n",
    "    return tweet\n",
    "\n",
    "\n",
    "def replace_smileys_tweets(tweets):\n",
    "    tweets_2 = tweets.copy()\n",
    "    tweets_2['tweet'] = tweets_2['tweet'].copy().apply(lambda tweet: replace_smiley(tweet))\n",
    "    \n",
    "    return tweets_2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_5 = replace_smileys_tweets(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i dunno justin read my mention or not . only j...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>because your logic is so dumb , i won't even c...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just put casper in a box !  looved the battle ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thanks sir &gt; &gt; don't trip lil mama ... just ke...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visiting my brother tmr is the bestest birthda...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity\n",
       "0  i dunno justin read my mention or not . only j...         1\n",
       "1  because your logic is so dumb , i won't even c...         1\n",
       "2  just put casper in a box !  looved the battle ...         1\n",
       "3  thanks sir > > don't trip lil mama ... just ke...         1\n",
       "4  visiting my brother tmr is the bestest birthda...         1"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_5.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_smileys, test_smileys = train_test_split(tweets_5, test_size = 0.2, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "construct_fasttext_input(train_smileys, 'tweets_smileys_train.csv')\n",
    "construct_fasttext_input(test_smileys, 'tweets_smileys_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_5 = fasttext.train_supervised(input = 'tweets_smileys_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1966637, 0.8604058603595884, 0.8604058603595884)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_5.test('tweets_smileys_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(491660, 0.8351197982345523, 0.8351197982345523)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_5.test('tweets_smileys_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Conclusion:Perfomances are similar than without replacing smileys.***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Finding the best hyperparameters for fasttext\n",
    "\n",
    "_We will now tune the hyperparameters to increase the performance of fasttext. As seen in the previous keeping raw tweets got us the best performance so we won't take any preprocessing for the task._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, validation_test = train_test_split(tweets_5, test_size = 0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation, test = train_test_split(validation_test, test_size = 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train contains 1720807 samples\n",
      "validation contains 368745 samples\n",
      "test contains 368745 samples\n"
     ]
    }
   ],
   "source": [
    "print('train contains {x} samples'.format(x = len(train)))\n",
    "print('validation contains {x} samples'.format(x = len(validation)))\n",
    "print('test contains {x} samples'.format(x = len(test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_We will use train for training the fasttext model, validation for perform test to find the best hyperparameters, and test to evaluate the final model._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "construct_fasttext_input(train, 'tweets_train.csv')\n",
    "construct_fasttext_input(validation, 'tweets_validation.csv')\n",
    "construct_fasttext_input(test, 'tweets_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Here are the hyperparameters that we will tune and their previous default values:_\n",
    "\n",
    " - **minCount:**           minimal number of word occurrences [1]\n",
    " - **wordNgrams:**         max length of word ngram [1]\n",
    " - **lr:**              learning rate [0.1]\n",
    " - **dim:**               size of word vectors [100]\n",
    " - **ws:**                 size of the context window [5]\n",
    " - **epoch:**              number of epochs [5]\n",
    " - **neg:**                number of negatives sampled [5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 125, 'ws': 1, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8662625933910968\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 125, 'ws': 1, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8683453334960474\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 125, 'ws': 1, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8688443233128584\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 125, 'ws': 2, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8662436100828486\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 125, 'ws': 2, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8683399096936908\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 125, 'ws': 2, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8688226281034319\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 125, 'ws': 3, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8661785244545689\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 125, 'ws': 3, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8683155025830859\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 125, 'ws': 3, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8686897449456942\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 150, 'ws': 1, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8661893720592821\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 150, 'ws': 1, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8683317739901558\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 150, 'ws': 1, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8689202565458515\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 150, 'ws': 2, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8662544576875618\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 150, 'ws': 2, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.868350757298404\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 150, 'ws': 2, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8689148327434948\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 150, 'ws': 3, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8661080150239325\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 150, 'ws': 3, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8683182144842642\n",
      "Using params: {'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 150, 'ws': 3, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8688931375340683\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 125, 'ws': 1, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8647140978182755\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 125, 'ws': 1, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8672578611235406\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 125, 'ws': 1, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8677134605214986\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 125, 'ws': 2, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8647466406324154\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 125, 'ws': 2, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8672687087282539\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 125, 'ws': 2, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8677432914344602\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 125, 'ws': 3, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8646978264112056\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 125, 'ws': 3, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8672605730247189\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 125, 'ws': 3, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.867835496074523\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 150, 'ws': 1, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8646544359923525\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 150, 'ws': 1, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8673798966765651\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 150, 'ws': 1, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8678544793827713\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 150, 'ws': 2, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8647791834465552\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 150, 'ws': 2, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8673148110482855\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 150, 'ws': 2, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8678273603709881\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 150, 'ws': 3, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8648415571736566\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 150, 'ws': 3, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8672551492223624\n",
      "Using params: {'minCount': 1, 'wordNgrams': 4, 'lr': 0.01, 'dim': 150, 'ws': 3, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.867767698545065\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 125, 'ws': 1, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8627994955863808\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 125, 'ws': 1, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8656497037247962\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 125, 'ws': 1, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8665744620266037\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 125, 'ws': 2, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8629350906452968\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 125, 'ws': 2, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8656280085153697\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 125, 'ws': 2, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8666422595560618\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 125, 'ws': 3, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8629540739535452\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 125, 'ws': 3, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8655846180965165\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 125, 'ws': 3, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8666558190619534\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 150, 'ws': 1, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8628401741040557\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 150, 'ws': 1, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8657554678707508\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 150, 'ws': 1, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.866653107160775\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 150, 'ws': 2, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.8629432263488318\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 150, 'ws': 2, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8656605513295096\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 150, 'ws': 2, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.8665609025207122\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 150, 'ws': 3, 'epoch': 7, 'bucket': 2000000} - Accuracy = 0.862918819238227\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 150, 'ws': 3, 'epoch': 7, 'bucket': 3000000} - Accuracy = 0.8656551275271529\n",
      "Using params: {'minCount': 1, 'wordNgrams': 5, 'lr': 0.01, 'dim': 150, 'ws': 3, 'epoch': 7, 'bucket': 4000000} - Accuracy = 0.86667751427138\n",
      "best accuracy = 0.8689202565458515\n",
      "{'minCount': 1, 'wordNgrams': 3, 'lr': 0.01, 'dim': 150, 'ws': 1, 'epoch': 7, 'bucket': 4000000}\n"
     ]
    }
   ],
   "source": [
    "minCount_val = [1] #, 3, 5]\n",
    "wordNgrams_val = [3, 4, 5]\n",
    "lr_val = [0.01]\n",
    "dim_val = [125, 150]\n",
    "ws_val = [1, 2, 3]\n",
    "epoch_val = [7]\n",
    "bucket_val = [2000000, 3000000, 4000000]\n",
    "#neg_val = [3, 5, 7, 9]\n",
    "\n",
    "accuracy = 0\n",
    "best_params = None\n",
    "\n",
    "best_params = {}\n",
    "for minCount in minCount_val:\n",
    "    for wordNgrams in wordNgrams_val:\n",
    "        for lr in lr_val:\n",
    "            for dim in dim_val:\n",
    "                for ws in ws_val:\n",
    "                    for epoch in epoch_val:\n",
    "                        for bucket in bucket_val:\n",
    "                            model = fasttext.train_supervised(input = 'tweets_train.csv',\n",
    "                                                             minCount = minCount,\n",
    "                                                             wordNgrams = wordNgrams,\n",
    "                                                             lr = lr,\n",
    "                                                             dim = dim,\n",
    "                                                             ws = ws,\n",
    "                                                             epoch = epoch,\n",
    "                                                             bucket = bucket)\n",
    "                            \n",
    "                            new_accuracy = model.test('tweets_validation.csv')\n",
    "                            \n",
    "                            params = {'minCount': minCount,\n",
    "                                      'wordNgrams': wordNgrams,\n",
    "                                      'lr': lr,\n",
    "                                      'dim': dim,\n",
    "                                      'ws': ws,\n",
    "                                      'epoch': epoch,\n",
    "                                      'bucket':bucket,\n",
    "                                     }\n",
    "                            print('Using params: {p} - Accuracy = {a}'.format(p = params, a = new_accuracy[1]))\n",
    "                            \n",
    "                            if new_accuracy[1] > accuracy:\n",
    "                                accuracy = new_accuracy[1]\n",
    "                                best_params = params\n",
    "            \n",
    "print('best accuracy = {a}'.format(a = accuracy))    \n",
    "print(best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using params: {'epoch': 7, 'bucket': 10000000} - Accuracy = 0.8710084204531587\n"
     ]
    }
   ],
   "source": [
    "epoch_val = [7]\n",
    "\n",
    "bucket_val = [10000000, 11000000]\n",
    "\n",
    "accuracy = 0\n",
    "best_params = None\n",
    "\n",
    "best_params = {}\n",
    "for epoch in epoch_val:\n",
    "    for bucket in bucket_val:\n",
    "        model = fasttext.train_supervised(input = 'tweets_train.csv',\n",
    "                                                   minCount = 1,\n",
    "                                                   wordNgrams = 3,\n",
    "                                                   lr = 0.01,\n",
    "                                                   dim = 200,\n",
    "                                                   ws = 1,\n",
    "                                                   epoch = epoch,\n",
    "                                                   bucket = bucket)\n",
    "                            \n",
    "        new_accuracy = model.test('tweets_validation.csv')\n",
    "                            \n",
    "        params = {'epoch': epoch,\n",
    "                  'bucket':bucket,\n",
    "                 }\n",
    "        print('Using params: {p} - Accuracy = {a}'.format(p = params, a = new_accuracy[1]))\n",
    "                            \n",
    "        if new_accuracy[1] > accuracy:\n",
    "            accuracy = new_accuracy[1]\n",
    "            best_params = params\n",
    "            \n",
    "print('best accuracy = {a}'.format(a = accuracy))    \n",
    "print(best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model = fasttext.train_supervised(input = 'tweets_train.csv',\n",
    "                                                   minCount = 1,\n",
    "                                                   wordNgrams = 3,\n",
    "                                                   lr = 0.01,\n",
    "                                                   dim = 200,\n",
    "                                                   ws = 1,\n",
    "                                                   epoch = 7,\n",
    "                                                   bucket = 10000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuracy of our best model is (368745, 0.8732321794193819, 0.8732321794193819).\n"
     ]
    }
   ],
   "source": [
    "accuracy = best_model.test('tweets_test.csv')\n",
    "print('The accuracy of our best model is {a}.'.format(a = accuracy))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}