Airline_Sentiment_Analysis_Without_Vectorization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "INsvANBUQ_Nn"
   },
   "outputs": [],
   "source": [
    "\n",
    "#importing libraries for model evaluation and algorithms\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import roc_curve,auc\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn import metrics\n",
    "from IPython.display import display\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk import pos_tag\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.wordnet import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 445
    },
    "id": "DoU4sypRQ_Gj",
    "outputId": "4c802395-68b5-484f-b11f-483664fd2809"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>airline_sentiment</th>\n",
       "      <th>airline_sentiment_confidence</th>\n",
       "      <th>negativereason</th>\n",
       "      <th>negativereason_confidence</th>\n",
       "      <th>airline</th>\n",
       "      <th>airline_sentiment_gold</th>\n",
       "      <th>name</th>\n",
       "      <th>negativereason_gold</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>tweet_coord</th>\n",
       "      <th>tweet_created</th>\n",
       "      <th>tweet_location</th>\n",
       "      <th>user_timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>570306133677760513</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cairdin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica What @dhepburn said.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:35:52 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eastern Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>570301130888122368</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.3486</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica plus you've added commercials t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:59 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>570301083672813571</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.6837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yvonnalynn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica I didn't today... Must mean I n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:48 -0800</td>\n",
       "      <td>Lets Play</td>\n",
       "      <td>Central Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>570301031407624196</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Bad Flight</td>\n",
       "      <td>0.7033</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica it's really aggressive to blast...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:36 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>570300817074462722</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Can't Tell</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica and it's a really big bad thing...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:14:45 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id  ...               user_timezone\n",
       "0  570306133677760513  ...  Eastern Time (US & Canada)\n",
       "1  570301130888122368  ...  Pacific Time (US & Canada)\n",
       "2  570301083672813571  ...  Central Time (US & Canada)\n",
       "3  570301031407624196  ...  Pacific Time (US & Canada)\n",
       "4  570300817074462722  ...  Pacific Time (US & Canada)\n",
       "\n",
       "[5 rows x 15 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reading the dataframe\n",
    "df=pd.read_csv('Tweets.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "EdR5LRQ8UT2Q"
   },
   "outputs": [],
   "source": [
    "#train test split\n",
    "data_train, data_test= train_test_split(df,test_size = 0.2, random_state = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "qeMhkpU3Q_V3"
   },
   "outputs": [],
   "source": [
    "#defining train labels i.e-dependent variable\n",
    "train_labels=[label for label in data_train['airline_sentiment']]   \n",
    "#word tokenization of train and test tweets\n",
    "train_tweets=[word_tokenize(tweet) for tweet in data_train['text']]\n",
    "test_tweets=[word_tokenize(tweet) for tweet in data_test['text']]\n",
    "#list of stopwords and punctuations\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "stops = set(stopwords.words('english'))\n",
    "punctuations = list(string.punctuation)\n",
    "#stops include stopwords and punctuations which would later be eliminated\n",
    "stops.update(punctuations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "pBB6viQxQ_TD"
   },
   "outputs": [],
   "source": [
    "# now we will create our list of documents from the files\n",
    "# each entry of the list is a tuple\n",
    "# tuple contains 1.all words of a particular document 2.Label of that document\n",
    "train_docs=[]\n",
    "for j in range(0,len(train_tweets)):\n",
    "    train_docs.append((train_tweets[j],train_labels[j]))\n",
    "#to get the simple pos(part of speech) tag\n",
    "from nltk.corpus import wordnet\n",
    "def get_simple_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "SIJF3OYoQ_La"
   },
   "outputs": [],
   "source": [
    "#to get the pos tag for a word\n",
    "from nltk import pos_tag\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "# now we are going to clean our data \n",
    "# we will remove stopwords and punctuations and lemmatize each document\n",
    "lemmatizer=WordNetLemmatizer()\n",
    "def clean(words):\n",
    "    output=[]\n",
    "    for word in words:\n",
    "        if word.lower() not in stops:\n",
    "            pos=pos_tag(word)\n",
    "            clean_word=lemmatizer.lemmatize(word,pos=get_simple_pos(pos[0][1]))\n",
    "            output.append(clean_word.lower())\n",
    "    return output\n",
    "train_docs=[(clean(doc),label) for doc,label in train_docs]\n",
    "test_docs=[ clean(doc) for doc in test_tweets]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "67KWKlSEXNeW",
    "outputId": "f7d0f226-e8eb-4c4c-b856-0faa2377b512"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['americanair',\n",
       "  'car',\n",
       "  'gng',\n",
       "  'dfw',\n",
       "  'pulled',\n",
       "  '1hr',\n",
       "  'ago',\n",
       "  'icy',\n",
       "  'road',\n",
       "  'on-hold',\n",
       "  'aa',\n",
       "  'since',\n",
       "  '1hr',\n",
       "  'ca',\n",
       "  \"n't\",\n",
       "  'reach',\n",
       "  'arpt',\n",
       "  'aa2450',\n",
       "  'wat',\n",
       "  '2'],\n",
       " 'negative')"
      ]
     },
     "execution_count": 28,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Lets look at how each of the values of train_docs look\n",
    "train_docs[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "BwzLp5O2Q_J8"
   },
   "outputs": [],
   "source": [
    "#creating a list of all words that are in the documents after cleaning\n",
    "all_words = []\n",
    "for doc in train_docs:\n",
    "    all_words += doc[0]\n",
    "#to find the 2500 most frequent words which will act as features\n",
    "import nltk\n",
    "freq = nltk.FreqDist(all_words)\n",
    "common = freq.most_common(2500)\n",
    "features = [i[0] for i in common]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "FeL5aCnYQ_E9"
   },
   "outputs": [],
   "source": [
    "#Here I am going to return a dictionary which contains true and false tags for the top 2500 words in each line of text on the basis of whether\n",
    "# they are present in that document \n",
    "def get_feature_dict(words):\n",
    "    current_features = {}\n",
    "    words_set = set(words)\n",
    "    for w in features:\n",
    "        current_features[w] = w in words_set\n",
    "    return current_features\n",
    "training_data = [(get_feature_dict(doc),category) for doc, category in train_docs]\n",
    "testing_data = [get_feature_dict(doc) for doc  in test_docs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DT2aR-5zLa5e",
    "outputId": "dae7574d-18f2-4dc8-f5a0-978be11e7fc8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'united': False,\n",
       "  'flight': True,\n",
       "  'usairways': False,\n",
       "  'americanair': True,\n",
       "  'southwestair': False,\n",
       "  'jetblue': False,\n",
       "  \"n't\": False,\n",
       "  \"'s\": False,\n",
       "  'get': False,\n",
       "  'http': False,\n",
       "  'hour': False,\n",
       "  'thanks': False,\n",
       "  'cancelled': False,\n",
       "  '...': False,\n",
       "  'service': False,\n",
       "  'customer': False,\n",
       "  'time': False,\n",
       "  'help': True,\n",
       "  'call': False,\n",
       "  'bag': False,\n",
       "  'plane': False,\n",
       "  \"'m\": False,\n",
       "  'amp': False,\n",
       "  'flights': False,\n",
       "  'us': False,\n",
       "  'hold': False,\n",
       "  '2': False,\n",
       "  'would': False,\n",
       "  'still': False,\n",
       "  'thank': False,\n",
       "  'please': True,\n",
       "  'need': False,\n",
       "  'make': False,\n",
       "  'one': False,\n",
       "  'back': False,\n",
       "  'ca': False,\n",
       "  'gate': False,\n",
       "  'delayed': False,\n",
       "  'flightled': False,\n",
       "  'virginamerica': False,\n",
       "  'airline': False,\n",
       "  'say': False,\n",
       "  'got': False,\n",
       "  'seat': False,\n",
       "  \"'ve\": False,\n",
       "  \"''\": False,\n",
       "  '``': False,\n",
       "  'phone': False,\n",
       "  'delay': False,\n",
       "  'like': False,\n",
       "  'late': False,\n",
       "  'guy': True,\n",
       "  'today': False,\n",
       "  'agent': False,\n",
       "  'take': False,\n",
       "  'fly': False,\n",
       "  'know': False,\n",
       "  'ticket': False,\n",
       "  'waiting': False,\n",
       "  'airport': False,\n",
       "  'trying': False,\n",
       "  'change': False,\n",
       "  'way': False,\n",
       "  'minute': False,\n",
       "  'going': False,\n",
       "  'great': False,\n",
       "  'could': False,\n",
       "  'u': False,\n",
       "  'go': False,\n",
       "  'day': True,\n",
       "  'want': False,\n",
       "  'wait': False,\n",
       "  'min': False,\n",
       "  '3': False,\n",
       "  \"'re\": False,\n",
       "  'last': False,\n",
       "  'never': False,\n",
       "  'flying': False,\n",
       "  'hr': False,\n",
       "  'good': False,\n",
       "  'tomorrow': False,\n",
       "  'really': False,\n",
       "  'people': False,\n",
       "  'weather': False,\n",
       "  '4': False,\n",
       "  'home': False,\n",
       "  'even': False,\n",
       "  'new': False,\n",
       "  'another': False,\n",
       "  'work': False,\n",
       "  'see': False,\n",
       "  'give': False,\n",
       "  'check': False,\n",
       "  'dm': False,\n",
       "  'issue': True,\n",
       "  'told': False,\n",
       "  'first': False,\n",
       "  'luggage': False,\n",
       "  'love': False,\n",
       "  'travel': False,\n",
       "  \"'ll\": False,\n",
       "  'worst': False,\n",
       "  'crew': False,\n",
       "  'getting': False,\n",
       "  'lost': False,\n",
       "  'next': False,\n",
       "  'someone': False,\n",
       "  'let': False,\n",
       "  'ever': False,\n",
       "  'much': False,\n",
       "  'yes': False,\n",
       "  'number': False,\n",
       "  'due': False,\n",
       "  'email': True,\n",
       "  'baggage': False,\n",
       "  'trip': False,\n",
       "  'two': False,\n",
       "  'flighted': False,\n",
       "  'reservation': False,\n",
       "  'days': False,\n",
       "  'experience': False,\n",
       "  'line': False,\n",
       "  '”': False,\n",
       "  'sitting': False,\n",
       "  'right': False,\n",
       "  'online': False,\n",
       "  'response': False,\n",
       "  'staff': False,\n",
       "  'week': False,\n",
       "  'aa': False,\n",
       "  '“': False,\n",
       "  'jfk': False,\n",
       "  'best': False,\n",
       "  'sent': False,\n",
       "  'connection': False,\n",
       "  'left': False,\n",
       "  'pay': False,\n",
       "  'book': False,\n",
       "  'boarding': False,\n",
       "  'better': False,\n",
       "  'bad': False,\n",
       "  'passengers': False,\n",
       "  '1': False,\n",
       "  'long': False,\n",
       "  'since': False,\n",
       "  'booked': False,\n",
       "  'already': False,\n",
       "  'morning': False,\n",
       "  'flightr': False,\n",
       "  'miss': False,\n",
       "  'stuck': False,\n",
       "  'think': False,\n",
       "  'wo': False,\n",
       "  'happen': False,\n",
       "  'sure': False,\n",
       "  'nothing': False,\n",
       "  'well': False,\n",
       "  'find': False,\n",
       "  'keep': False,\n",
       "  'care': False,\n",
       "  'system': False,\n",
       "  'tell': False,\n",
       "  'website': False,\n",
       "  '5': False,\n",
       "  'problem': False,\n",
       "  'booking': False,\n",
       "  \"'d\": False,\n",
       "  'problems': False,\n",
       "  'come': False,\n",
       "  'hope': False,\n",
       "  'attendant': False,\n",
       "  'yet': False,\n",
       "  'fleek': False,\n",
       "  'info': False,\n",
       "  'fleet': False,\n",
       "  'mile': False,\n",
       "  'hotel': False,\n",
       "  'dfw': False,\n",
       "  'night': False,\n",
       "  'tonight': False,\n",
       "  'put': False,\n",
       "  'refund': False,\n",
       "  'use': False,\n",
       "  'rude': False,\n",
       "  'flt': False,\n",
       "  'tried': False,\n",
       "  'year': False,\n",
       "  'pilot': False,\n",
       "  'air': False,\n",
       "  'done': False,\n",
       "  '’': False,\n",
       "  'rebooked': True,\n",
       "  'look': False,\n",
       "  'ago': False,\n",
       "  'fail': False,\n",
       "  'answer': False,\n",
       "  'follow': False,\n",
       "  'point': False,\n",
       "  'nice': False,\n",
       "  'also': False,\n",
       "  'lax': False,\n",
       "  'missed': False,\n",
       "  'thing': False,\n",
       "  'finally': False,\n",
       "  'awesome': False,\n",
       "  'credit': False,\n",
       "  'free': False,\n",
       "  'checked': False,\n",
       "  'class': False,\n",
       "  'talk': False,\n",
       "  'wifi': False,\n",
       "  'pass': False,\n",
       "  'board': False,\n",
       "  'able': False,\n",
       "  'anything': False,\n",
       "  'business': False,\n",
       "  'working': False,\n",
       "  'anyone': False,\n",
       "  'employee': False,\n",
       "  'rep': False,\n",
       "  'voucher': False,\n",
       "  'every': False,\n",
       "  '10': False,\n",
       "  'delta': False,\n",
       "  'team': False,\n",
       "  'app': False,\n",
       "  'sfo': False,\n",
       "  '30': False,\n",
       "  'leave': False,\n",
       "  'status': False,\n",
       "  'many': False,\n",
       "  'update': False,\n",
       "  'rebook': False,\n",
       "  'sit': False,\n",
       "  'always': False,\n",
       "  'show': False,\n",
       "  'appreciate': False,\n",
       "  '6': False,\n",
       "  'available': False,\n",
       "  'without': False,\n",
       "  'missing': False,\n",
       "  'thx': False,\n",
       "  'looking': False,\n",
       "  'phl': False,\n",
       "  'making': False,\n",
       "  'yesterday': False,\n",
       "  'via': False,\n",
       "  'gt': False,\n",
       "  'person': False,\n",
       "  'name': False,\n",
       "  'speak': False,\n",
       "  'sorry': False,\n",
       "  'ok': False,\n",
       "  'fix': False,\n",
       "  'almost': False,\n",
       "  'disappointed': False,\n",
       "  'understand': False,\n",
       "  '1st': False,\n",
       "  'terrible': False,\n",
       "  'helpful': False,\n",
       "  'departure': False,\n",
       "  'early': False,\n",
       "  'money': False,\n",
       "  'extra': False,\n",
       "  'send': True,\n",
       "  'instead': False,\n",
       "  'earlier': False,\n",
       "  'upgrade': False,\n",
       "  'ground': False,\n",
       "  'supposed': False,\n",
       "  'claim': False,\n",
       "  'amazing': False,\n",
       "  'card': False,\n",
       "  'southwest': False,\n",
       "  'stop': False,\n",
       "  'try': False,\n",
       "  'suck': False,\n",
       "  'site': False,\n",
       "  'unacceptable': False,\n",
       "  'ord': False,\n",
       "  'contact': False,\n",
       "  'took': False,\n",
       "  'w/': False,\n",
       "  'mean': False,\n",
       "  'received': False,\n",
       "  'full': False,\n",
       "  '20': False,\n",
       "  'something': False,\n",
       "  'rt': False,\n",
       "  'boston': False,\n",
       "  'chance': False,\n",
       "  'different': False,\n",
       "  'happy': False,\n",
       "  'hi': False,\n",
       "  'dca': False,\n",
       "  'start': False,\n",
       "  'though': False,\n",
       "  'member': False,\n",
       "  'paid': False,\n",
       "  'job': False,\n",
       "  'oh': False,\n",
       "  'poor': False,\n",
       "  'add': False,\n",
       "  'company': False,\n",
       "  'destinationdragons': False,\n",
       "  'vegas': False,\n",
       "  'chicago': False,\n",
       "  'tweet': False,\n",
       "  'direct': False,\n",
       "  'ridiculous': False,\n",
       "  'charge': False,\n",
       "  'open': False,\n",
       "  'least': False,\n",
       "  'hang': False,\n",
       "  'clt': False,\n",
       "  'bos': False,\n",
       "  'connecting': False,\n",
       "  'wife': False,\n",
       "  'policy': False,\n",
       "  'reply': False,\n",
       "  'denver': False,\n",
       "  'hung': False,\n",
       "  'family': False,\n",
       "  'ewr': False,\n",
       "  'snow': False,\n",
       "  'account': False,\n",
       "  'twitter': False,\n",
       "  'cause': False,\n",
       "  'month': False,\n",
       "  'message': False,\n",
       "  'actually': False,\n",
       "  'question': False,\n",
       "  'maybe': False,\n",
       "  'might': False,\n",
       "  'tarmac': False,\n",
       "  'american': False,\n",
       "  'desk': False,\n",
       "  'old': False,\n",
       "  'reason': False,\n",
       "  'everyone': False,\n",
       "  'dallas': False,\n",
       "  'ur': False,\n",
       "  'found': False,\n",
       "  'handle': False,\n",
       "  'horrible': False,\n",
       "  'enough': False,\n",
       "  'landed': False,\n",
       "  'far': False,\n",
       "  'half': False,\n",
       "  'fee': False,\n",
       "  'lot': False,\n",
       "  'stranded': False,\n",
       "  'leaving': False,\n",
       "  '45': False,\n",
       "  'bc': False,\n",
       "  'past': False,\n",
       "  'food': False,\n",
       "  'kid': False,\n",
       "  '--': False,\n",
       "  'frustrated': False,\n",
       "  'complaint': False,\n",
       "  'needs': False,\n",
       "  'hey': False,\n",
       "  'wrong': True,\n",
       "  'using': False,\n",
       "  'seriously': False,\n",
       "  'together': False,\n",
       "  'three': False,\n",
       "  '7': False,\n",
       "  '8': False,\n",
       "  'confirmation': False,\n",
       "  'big': False,\n",
       "  'link': False,\n",
       "  'vacation': False,\n",
       "  'san': False,\n",
       "  'away': False,\n",
       "  'w': False,\n",
       "  'longer': False,\n",
       "  'row': False,\n",
       "  'ua': False,\n",
       "  'asked': False,\n",
       "  'terminal': False,\n",
       "  'airways': False,\n",
       "  'plan': False,\n",
       "  'soon': False,\n",
       "  'mechanical': False,\n",
       "  'runway': False,\n",
       "  'traveling': False,\n",
       "  'cool': False,\n",
       "  'little': False,\n",
       "  'return': False,\n",
       "  'used': False,\n",
       "  'iad': False,\n",
       "  'apology': False,\n",
       "  'guess': False,\n",
       "  'option': False,\n",
       "  'pls': False,\n",
       "  'price': False,\n",
       "  'newark': False,\n",
       "  'worse': False,\n",
       "  '40': False,\n",
       "  'around': False,\n",
       "  'coming': False,\n",
       "  'twice': False,\n",
       "  '50': False,\n",
       "  'destination': False,\n",
       "  'may': False,\n",
       "  'scheduled': False,\n",
       "  'possible': False,\n",
       "  'broken': False,\n",
       "  'stay': False,\n",
       "  'went': False,\n",
       "  'le': False,\n",
       "  'cost': False,\n",
       "  'cust': False,\n",
       "  'nyc': False,\n",
       "  'changed': False,\n",
       "  'thought': False,\n",
       "  'friend': False,\n",
       "  'charlotte': False,\n",
       "  'car': False,\n",
       "  'international': False,\n",
       "  'awful': False,\n",
       "  'seems': False,\n",
       "  'deal': False,\n",
       "  'error': False,\n",
       "  'waited': False,\n",
       "  'real': False,\n",
       "  'second': False,\n",
       "  'landing': False,\n",
       "  'lol': False,\n",
       "  'offer': False,\n",
       "  'life': False,\n",
       "  'believe': False,\n",
       "  'options': False,\n",
       "  'else': False,\n",
       "  'room': False,\n",
       "  'frustrating': False,\n",
       "  'arrived': False,\n",
       "  'lack': False,\n",
       "  'im': False,\n",
       "  'computer': False,\n",
       "  'city': False,\n",
       "  'pick': False,\n",
       "  'idea': False,\n",
       "  'run': False,\n",
       "  'jet': False,\n",
       "  '200': False,\n",
       "  'assistance': False,\n",
       "  'philly': False,\n",
       "  'heard': False,\n",
       "  'everything': False,\n",
       "  '25': False,\n",
       "  '1/2': False,\n",
       "  'arrive': False,\n",
       "  '15': False,\n",
       "  'iah': False,\n",
       "  'quick': False,\n",
       "  'thru': False,\n",
       "  'standby': False,\n",
       "  'maintenance': False,\n",
       "  'telling': False,\n",
       "  'fare': False,\n",
       "  'given': False,\n",
       "  'watch': False,\n",
       "  'place': False,\n",
       "  'needed': False,\n",
       "  'waste': False,\n",
       "  'award': False,\n",
       "  'plus': False,\n",
       "  'reach': False,\n",
       "  'entire': False,\n",
       "  'dc': False,\n",
       "  'feel': False,\n",
       "  'lga': False,\n",
       "  'imaginedragons': False,\n",
       "  'reflight': False,\n",
       "  'request': False,\n",
       "  'address': True,\n",
       "  'word': False,\n",
       "  'hear': False,\n",
       "  'fll': False,\n",
       "  'date': False,\n",
       "  'den': False,\n",
       "  'loyal': False,\n",
       "  'phx': False,\n",
       "  'human': False,\n",
       "  '24': False,\n",
       "  'na': False,\n",
       "  'blue': False,\n",
       "  'end': False,\n",
       "  'process': False,\n",
       "  'checking': False,\n",
       "  'glad': False,\n",
       "  'houston': False,\n",
       "  'passenger': False,\n",
       "  'swa': False,\n",
       "  'busy': False,\n",
       "  'hard': False,\n",
       "  'club': False,\n",
       "  'wow': False,\n",
       "  'case': False,\n",
       "  'wish': False,\n",
       "  'information': False,\n",
       "  'group': False,\n",
       "  'ask': False,\n",
       "  'route': False,\n",
       "  'expect': False,\n",
       "  'usair': False,\n",
       "  'sw': False,\n",
       "  'moved': False,\n",
       "  'monday': False,\n",
       "  'form': False,\n",
       "  'la': False,\n",
       "  'joke': False,\n",
       "  'airlines': False,\n",
       "  'wtf': False,\n",
       "  'empty': False,\n",
       "  'hoping': False,\n",
       "  'miami': False,\n",
       "  \"y'all\": False,\n",
       "  'hopefully': False,\n",
       "  'forward': False,\n",
       "  'world': False,\n",
       "  'flew': False,\n",
       "  '2nd': False,\n",
       "  'asking': False,\n",
       "  'sleep': False,\n",
       "  'carry': False,\n",
       "  'tv': False,\n",
       "  'tsa': False,\n",
       "  'confirmed': False,\n",
       "  'leg': False,\n",
       "  'bought': False,\n",
       "  'unitedairlines': False,\n",
       "  '100': False,\n",
       "  'helping': False,\n",
       "  '9': False,\n",
       "  'bring': False,\n",
       "  'super': False,\n",
       "  'support': False,\n",
       "  'part': False,\n",
       "  'okay': False,\n",
       "  'detail': False,\n",
       "  'provide': False,\n",
       "  'beyond': False,\n",
       "  'sunday': False,\n",
       "  'treat': False,\n",
       "  'buy': False,\n",
       "  'yeah': False,\n",
       "  'mco': False,\n",
       "  'traveler': False,\n",
       "  'asap': False,\n",
       "  'access': False,\n",
       "  'counting': False,\n",
       "  'seem': False,\n",
       "  'pretty': False,\n",
       "  'worry': False,\n",
       "  '800': False,\n",
       "  'lose': False,\n",
       "  'storm': False,\n",
       "  'las': False,\n",
       "  'communication': False,\n",
       "  'ceo': False,\n",
       "  'either': False,\n",
       "  'hate': False,\n",
       "  'flightlations': False,\n",
       "  'read': False,\n",
       "  'drive': False,\n",
       "  'boarded': False,\n",
       "  'lt': False,\n",
       "  'original': False,\n",
       "  'spent': False,\n",
       "  'fees': False,\n",
       "  'bwi': False,\n",
       "  'counter': False,\n",
       "  'live': False,\n",
       "  '11': False,\n",
       "  'set': False,\n",
       "  'completely': False,\n",
       "  'holding': False,\n",
       "  'cant': False,\n",
       "  'future': False,\n",
       "  'ready': False,\n",
       "  'whole': False,\n",
       "  'figure': False,\n",
       "  'supervisor': False,\n",
       "  'lounge': False,\n",
       "  'luv': False,\n",
       "  'charged': False,\n",
       "  '1.5': False,\n",
       "  'correct': False,\n",
       "  'following': False,\n",
       "  'situation': False,\n",
       "  'r': False,\n",
       "  'middle': False,\n",
       "  'companion': False,\n",
       "  'rock': False,\n",
       "  'apparently': False,\n",
       "  'flown': False,\n",
       "  'huge': False,\n",
       "  'fine': False,\n",
       "  'losing': False,\n",
       "  'report': False,\n",
       "  '12': False,\n",
       "  'record': False,\n",
       "  'probably': False,\n",
       "  'must': False,\n",
       "  '..': False,\n",
       "  'automated': False,\n",
       "  'atl': False,\n",
       "  'currently': False,\n",
       "  'fault': False,\n",
       "  'nashville': False,\n",
       "  'updates': False,\n",
       "  'friends': False,\n",
       "  'checkin': False,\n",
       "  'flighting': False,\n",
       "  'land': False,\n",
       "  'rather': False,\n",
       "  'tuesday': False,\n",
       "  'weekend': False,\n",
       "  'dont': False,\n",
       "  'matter': False,\n",
       "  'unable': False,\n",
       "  'respond': False,\n",
       "  'platinum': False,\n",
       "  'atlanta': False,\n",
       "  'priority': False,\n",
       "  'appreciated': False,\n",
       "  'rdu': False,\n",
       "  'fact': False,\n",
       "  'allow': False,\n",
       "  'arrival': False,\n",
       "  'b': False,\n",
       "  'mom': False,\n",
       "  'switch': False,\n",
       "  'kind': False,\n",
       "  'gon': False,\n",
       "  'baby': False,\n",
       "  'disconnected': False,\n",
       "  'delivered': False,\n",
       "  'country': False,\n",
       "  'friday': False,\n",
       "  '2015': False,\n",
       "  'drop': False,\n",
       "  'upset': False,\n",
       "  'husband': False,\n",
       "  'share': False,\n",
       "  'dividend': False,\n",
       "  'catch': False,\n",
       "  'non': False,\n",
       "  'layover': False,\n",
       "  'kudos': False,\n",
       "  'virgin': False,\n",
       "  'explain': False,\n",
       "  'friendly': False,\n",
       "  'crazy': False,\n",
       "  'save': False,\n",
       "  'safety': False,\n",
       "  'sending': False,\n",
       "  'mobile': False,\n",
       "  'drink': False,\n",
       "  'tag': False,\n",
       "  'running': False,\n",
       "  'luck': False,\n",
       "  'knew': False,\n",
       "  'treated': False,\n",
       "  'front': False,\n",
       "  'giving': False,\n",
       "  'sad': False,\n",
       "  'reschedule': False,\n",
       "  'high': False,\n",
       "  'inconvenience': False,\n",
       "  'top': False,\n",
       "  'compensation': False,\n",
       "  'orlando': False,\n",
       "  'helped': False,\n",
       "  'pm': False,\n",
       "  'allowed': False,\n",
       "  'small': False,\n",
       "  'fun': False,\n",
       "  'absolutely': False,\n",
       "  'several': False,\n",
       "  'showing': False,\n",
       "  'confirm': False,\n",
       "  'extremely': False,\n",
       "  'letter': False,\n",
       "  'id': False,\n",
       "  'totally': False,\n",
       "  'flightlation': False,\n",
       "  'sign': False,\n",
       "  'lie': False,\n",
       "  'saw': False,\n",
       "  'seen': False,\n",
       "  'zero': False,\n",
       "  'customerservice': False,\n",
       "  'ppl': False,\n",
       "  'winter': False,\n",
       "  'cold': False,\n",
       "  'offered': False,\n",
       "  'birthday': False,\n",
       "  'true': False,\n",
       "  'yr': False,\n",
       "  'folks': False,\n",
       "  'purchase': False,\n",
       "  'window': False,\n",
       "  'tix': False,\n",
       "  'item': False,\n",
       "  'answering': False,\n",
       "  'held': False,\n",
       "  'aircraft': False,\n",
       "  'schedule': False,\n",
       "  'sense': False,\n",
       "  'page': False,\n",
       "  'worth': False,\n",
       "  'multiple': False,\n",
       "  'complete': False,\n",
       "  'meal': False,\n",
       "  'conf': False,\n",
       "  'b/c': False,\n",
       "  'shit': False,\n",
       "  'evening': False,\n",
       "  'story': False,\n",
       "  'unfortunately': False,\n",
       "  'door': False,\n",
       "  'austin': False,\n",
       "  'cabin': False,\n",
       "  'plz': False,\n",
       "  'easy': False,\n",
       "  'water': False,\n",
       "  'dealing': False,\n",
       "  'overnight': False,\n",
       "  'yall': False,\n",
       "  'mileage': False,\n",
       "  'note': False,\n",
       "  'load': False,\n",
       "  'fixed': False,\n",
       "  'course': False,\n",
       "  'short': False,\n",
       "  'overhead': False,\n",
       "  'advisory': False,\n",
       "  'four': False,\n",
       "  'behind': False,\n",
       "  'clothes': False,\n",
       "  'nope': False,\n",
       "  'bna': False,\n",
       "  'offering': False,\n",
       "  'gold': False,\n",
       "  '1k': False,\n",
       "  'trouble': False,\n",
       "  'americanairlines': False,\n",
       "  'son': False,\n",
       "  'news': False,\n",
       "  'submitted': False,\n",
       "  'control': False,\n",
       "  'excuse': False,\n",
       "  'attitude': False,\n",
       "  'wonderful': False,\n",
       "  'despite': False,\n",
       "  'area': False,\n",
       "  'turn': False,\n",
       "  '3rd': False,\n",
       "  'started': False,\n",
       "  'mine': False,\n",
       "  'rescheduled': False,\n",
       "  'spend': False,\n",
       "  'usairwaysfail': False,\n",
       "  'changing': False,\n",
       "  'literally': False,\n",
       "  'receipt': False,\n",
       "  'round': False,\n",
       "  'ny': False,\n",
       "  'code': False,\n",
       "  'choice': False,\n",
       "  'n': False,\n",
       "  'social': False,\n",
       "  '2.5': False,\n",
       "  'passbook': False,\n",
       "  'mind': False,\n",
       "  'move': False,\n",
       "  'afternoon': False,\n",
       "  'anyway': False,\n",
       "  'kept': False,\n",
       "  'hand': False,\n",
       "  'fl': False,\n",
       "  'list': False,\n",
       "  'others': False,\n",
       "  'connect': False,\n",
       "  'medium': False,\n",
       "  'ride': False,\n",
       "  'win': False,\n",
       "  'excellent': False,\n",
       "  'followed': False,\n",
       "  'forced': False,\n",
       "  'feedback': False,\n",
       "  'concern': False,\n",
       "  'mistake': False,\n",
       "  'select': False,\n",
       "  'hello': False,\n",
       "  'bumped': False,\n",
       "  'close': False,\n",
       "  'explanation': False,\n",
       "  'takeoff': False,\n",
       "  'child': False,\n",
       "  'march': False,\n",
       "  'excited': False,\n",
       "  'feb': False,\n",
       "  'airway': False,\n",
       "  'party': False,\n",
       "  'frequent': False,\n",
       "  'program': False,\n",
       "  'haha': False,\n",
       "  'domestic': False,\n",
       "  'fair': False,\n",
       "  'learn': False,\n",
       "  'man': False,\n",
       "  'transfer': False,\n",
       "  'anymore': False,\n",
       "  'damage': False,\n",
       "  'depart': False,\n",
       "  'spoke': False,\n",
       "  'stuff': False,\n",
       "  'check-in': False,\n",
       "  'neveragain': False,\n",
       "  'mess': False,\n",
       "  'airplane': False,\n",
       "  'saturday': False,\n",
       "  'web': False,\n",
       "  'refuse': False,\n",
       "  'letting': False,\n",
       "  'bank': False,\n",
       "  'assist': False,\n",
       "  'space': False,\n",
       "  'kidding': False,\n",
       "  'meeting': False,\n",
       "  'phoenix': False,\n",
       "  'funeral': False,\n",
       "  'arriving': False,\n",
       "  'earn': False,\n",
       "  'south': False,\n",
       "  'daughter': False,\n",
       "  'expected': False,\n",
       "  'bird': False,\n",
       "  'avgeek': False,\n",
       "  '90': False,\n",
       "  'security': False,\n",
       "  'cc': False,\n",
       "  'receive': False,\n",
       "  'ice': False,\n",
       "  'mia': False,\n",
       "  'head': False,\n",
       "  'office': False,\n",
       "  '1hr': False,\n",
       "  'failure': False,\n",
       "  'bit': False,\n",
       "  'except': False,\n",
       "  'warm': False,\n",
       "  'flightd': False,\n",
       "  'america': False,\n",
       "  'visit': False,\n",
       "  'charging': False,\n",
       "  'dropped': False,\n",
       "  'sister': False,\n",
       "  'state': False,\n",
       "  'fan': False,\n",
       "  'file': False,\n",
       "  'notice': False,\n",
       "  'funny': False,\n",
       "  'c': False,\n",
       "  'responding': False,\n",
       "  'lady': False,\n",
       "  'however': False,\n",
       "  '75': False,\n",
       "  'useless': False,\n",
       "  'enjoy': False,\n",
       "  '2+': False,\n",
       "  'photo': False,\n",
       "  'svc': False,\n",
       "  'department': False,\n",
       "  'power': False,\n",
       "  'relation': False,\n",
       "  'cater': False,\n",
       "  'volume': False,\n",
       "  'reward': False,\n",
       "  'total': False,\n",
       "  'tho': False,\n",
       "  'tmrw': False,\n",
       "  'seating': False,\n",
       "  'god': False,\n",
       "  'fit': False,\n",
       "  'anywhere': False,\n",
       "  'bus': False,\n",
       "  'cut': False,\n",
       "  'onto': False,\n",
       "  'unhappy': False,\n",
       "  'rule': False,\n",
       "  'btw': False,\n",
       "  'denied': False,\n",
       "  'w/o': False,\n",
       "  'regarding': False,\n",
       "  'filed': False,\n",
       "  'wall': False,\n",
       "  'lovely': False,\n",
       "  'usually': False,\n",
       "  'mexico': False,\n",
       "  'emails': False,\n",
       "  'major': False,\n",
       "  'worked': False,\n",
       "  'ruined': False,\n",
       "  'moving': False,\n",
       "  'train': False,\n",
       "  'post': False,\n",
       "  'updated': False,\n",
       "  'tired': False,\n",
       "  'resolved': False,\n",
       "  'fantastic': False,\n",
       "  'center': False,\n",
       "  'none': False,\n",
       "  'impossible': False,\n",
       "  'bother': False,\n",
       "  'difference': False,\n",
       "  'view': False,\n",
       "  'closed': False,\n",
       "  'unhelpful': False,\n",
       "  'per': False,\n",
       "  'oscars': False,\n",
       "  'arrives': False,\n",
       "  'current': False,\n",
       "  'representative': False,\n",
       "  'philadelphia': False,\n",
       "  'main': False,\n",
       "  'premier': False,\n",
       "  'step': False,\n",
       "  'damn': False,\n",
       "  'fucking': False,\n",
       "  'especially': False,\n",
       "  'sort': False,\n",
       "  'crossed': False,\n",
       "  'sick': False,\n",
       "  'forgot': False,\n",
       "  'notification': False,\n",
       "  'gone': False,\n",
       "  'delivery': False,\n",
       "  'promise': False,\n",
       "  'entertainment': False,\n",
       "  'willing': False,\n",
       "  'children': False,\n",
       "  'imagine': False,\n",
       "  'order': False,\n",
       "  'fyi': False,\n",
       "  'keeping': False,\n",
       "  'clear': False,\n",
       "  'answered': False,\n",
       "  'count': False,\n",
       "  'favorite': False,\n",
       "  'screen': False,\n",
       "  'badcustomerservice': False,\n",
       "  'cover': False,\n",
       "  'captain': False,\n",
       "  'emailed': False,\n",
       "  'iphone': False,\n",
       "  'starting': False,\n",
       "  'provided': False,\n",
       "  'dfwairport': False,\n",
       "  'phlairport': False,\n",
       "  'standing': False,\n",
       "  'shot': False,\n",
       "  'disappointing': False,\n",
       "  'expensive': False,\n",
       "  'loved': False,\n",
       "  'uk': False,\n",
       "  'angry': False,\n",
       "  'walk': False,\n",
       "  'turned': False,\n",
       "  'pathetic': False,\n",
       "  'snack': False,\n",
       "  'listening': False,\n",
       "  'bin': False,\n",
       "  'alone': False,\n",
       "  'compensate': False,\n",
       "  'nonstop': False,\n",
       "  'special': False,\n",
       "  'merger': False,\n",
       "  'resolve': False,\n",
       "  'rental': False,\n",
       "  'ta': False,\n",
       "  'resolution': False,\n",
       "  'acceptable': False,\n",
       "  'smh': False,\n",
       "  'suitcase': False,\n",
       "  'equipment': False,\n",
       "  'wonder': False,\n",
       "  'touch': False,\n",
       "  'planned': False,\n",
       "  'trust': False,\n",
       "  'screw': False,\n",
       "  'condition': False,\n",
       "  'callback': False,\n",
       "  'added': False,\n",
       "  ...},\n",
       " 'negative')"
      ]
     },
     "execution_count": 31,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Lets look at how each of the training data sample looks like\n",
    "training_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "Jvkw6LjDQ-88"
   },
   "outputs": [],
   "source": [
    "#importing Naive Bayes Classifier from nltk\n",
    "from nltk import NaiveBayesClassifier as nbayes\n",
    "#training naive bayes on training data and predicting on the testing_data\n",
    "clf=nbayes.train(training_data)\n",
    "predicted_labels=[clf.classify(test) for test in testing_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 623
    },
    "id": "Bx85sihtQ-7X",
    "outputId": "73e9650a-8b16-40b9-daae-192a06b0ea0a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Text(69.0, 0.5, 'Predicted Sentiment'), Text(0.5, 69.0, 'Actual Sentiment')]"
      ]
     },
     "execution_count": 33,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkMAAAJNCAYAAAAyM3HrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd5wdZdn/8c+VBglJSOiSBAgIIkoRIgIqVREQBBWRJuVB86igiAjSFFDxhz4iYEGMgIBSBARBpCOggpTQu0R6AEEghRKSXa7fHzPBDWY3J5vMOSeZz9vXvPbMPe3eZcxe+517ZiIzkSRJqqs+re6AJElSK1kMSZKkWrMYkiRJtWYxJEmSas1iSJIk1ZrFkCRJqrV+re5AdyLCe/4lSbWSmdHM4w18335N+137+p0/6/F7i4jTgG2B5zPzvV3avwLsC3QCf8rMg8v2Q4F9yvavZuaVZftWwIlAX+CUzDx2Tn1r22IIYNF19m11F7SQmHbXzwF4YeqMFvdEC4ulh/QH4JU33mxxT7SwGLxI7S/WnA78DDhzZkNEbAZsD6ydmW9ExDJl+xrAzsB7gOWBayJitXKznwMfBZ4GbouISzLzgZ4O3NbFkCRJqofM/EtErPS25i8Bx2bmG+U6z5ft2wPnlu2PRcQEYP1y2YTMfBQgIs4t1+2xGKp9GSpJUm1Fn+ZNvbMa8OGIuCUiboiI95ftI4Cnuqz3dNnWXXuPLIYkSVLlImJsRIzvMo1tYLN+wBLABsBBwHkRMd/HVXmZTJKkupr/dUW3MnMcMG4uN3sauDCLF6neGhFvAksBE4FRXdYbWbbRQ3u3TIYkSVK7+gOwGUA5QHoA8G/gEmDniFgkIkYDqwK3ArcBq0bE6IgYQDHI+pI5HcRkSJKkuur9WJ75LiLOATYFloqIp4EjgdOA0yLiPmA6sGeZEt0fEedRDIzuAPbNzM5yP/sBV1LcWn9aZt4/p2NbDEmSpJbLzF26WbR7N+sfAxwzm/bLgMvm5tgWQ5Ik1VUTxwy1s/bJxyRJklrAZEiSpLpqozFDreRPQZIk1ZrJkCRJdeWYIcBkSJIk1ZzFkCRJqjUvk0mSVFcOoAZMhiRJUs2ZDEmSVFcOoAZMhiRJUs2ZDEmSVFeOGQJMhiRJUs2ZDEmSVFeOGQJMhiRJUs2ZDEmSVFeOGQJMhiRJUs2ZDEmSVFeOGQJMhiRJUs2ZDEmSVFeOGQJMhiRJUs2ZDEmSVFcmQ4DJkCRJqjmLIUmSVGteJpMkqa76eGs9mAxJkqSaMxmSJKmuHEANmAxJkqSaMxmSJKmufB0HYDIkSZJqzmRIkqS6cswQYDIkSZJqzmRIkqS6cswQYDIkSZJqzmRIkqS6cswQYDIkSZJqzmRIkqS6cswQYDIkSZJqzmRIkqS6cswQYDIkSZJqzmJIkiTVmpfJJEmqKwdQAyZDkiSp5kyGJEmqKwdQAyZDkiSp5kyGJEmqK8cMASZDkiSp5kyGJEmqK8cMASZDkiSp5kyGJEmqK5MhwGRIkiTVnMmQJEl15d1kgMmQJEmqOZMhSZLqyjFDgMmQJEmqOZMhSZLqyjFDgMmQJEmqOYshSZJUa14mkySprhxADZgMSZKkmjMZkiSprhxADZgMSZKkmrMYkiSppiKiaVMDfTktIp6PiPtms+zAiMiIWKqcj4j4SURMiIh7ImLdLuvuGRGPlNOejfwcLIYkSVI7OB3Y6u2NETEK2BJ4skvz1sCq5TQW+EW57hLAkcAHgPWBIyNi+JwObDEkSVJNtVMylJl/AV6azaLjgYOB7NK2PXBmFm4GhkXEO4CPAVdn5kuZ+TJwNbMpsN7OYkiSJFUuIsZGxPgu09gGttkemJiZd79t0QjgqS7zT5dt3bX3yLvJJEmqqybeTJaZ44Bxja4fEYOAwygukVXKZEiSJLWjVYDRwN0R8TgwErgjIpYDJgKjuqw7smzrrr1HFkOSJNVUO40ZervMvDczl8nMlTJzJYpLXutm5nPAJcAe5V1lGwCTM/NZ4Epgy4gYXg6c3rJs65HFkCRJarmIOAf4O/CuiHg6IvbpYfXLgEeBCcCvgC8DZOZLwHeB28rpO2VbjxwzJElSTfUmsalKZu4yh+UrdfmcwL7drHcacNrcHNtkSJIk1ZrJkCRJNdVOyVArmQxJkqRasxiSJEm15mUySZJqystkBZMhSZJUaxZDbejkI3fjiWv/H+PPP2yW9i/tvAl3XXgEt19wOMfsvz0ASyy+GFeM+yov3Hgcx3/zM7Os/753j+K28w7jvouP5LiDd2xa/9Xevn/0EXz8Ix9m9522n6X9/HPPYpdPbctun/kEPz/xRwBMnjSJ/cbuxUc+NIbjfvC9VnRXC5izf3smn/nkduy4w7ac9ZszADj+uB/yqe22ZqdPfYID99+PqVOmtLiXeks0cWpjFkNt6Dd/vJnt9/35LG0bj1mVbTddk/U/eyzr7XgMJ5x5LQDT3pjBd066lEOPv+i/9vOTwz7Lvt89m/dufzSrrLA0W35wjab0X+1tm+124Mc//eUsbbffdgt/u+HPnHHuhZx1/iXs+rm9ARiwyAC+8KWvsO/XDmpFV7WAmfDIP7jo9+dz5tnnce4Ff+CvN1zPk08+wQYbbsR5F/2R8y68hBVWXInTTmn49VRSU1gMtaEb7/gnL01+bZa2sZ/5MD/69dVMn9EBwAsvvwLAa9Omc9NdjzLtjRmzrL/cUkMZstii3Hrv4wCcfemtbLfpWtV3Xm1vnXXHMHTxxWdp+8MFv2P3vT7PgAEDABi+xJIADBw4iLXft95b7VJPHnv0Ud675loMHDiQfv36sd6Y9/Pna65mw40+RL9+xRDVNddem+f/9VyLe6qZ2vl1HM1UaTFUvjNk94j4djm/QkSsX+UxF1bvXHEZPvi+VfjLmd/gqlP2Z701Vuhx/eWXGcbE5ye9NT/xX5NYfplhVXdTC6gnn3ycu++8nS/ssTP7fmFPHrz/3lZ3SQugVVZdlTvvGM+kSS/z+uuv87e/3sC/nnt2lnUuvuj3bPShjVvUQ2n2qr6b7CTgTWBz4DvAVOD3wPsrPu5Cp1/fPiyx+GJsvMePGPOeFfntD/+Hd297VKu7pYVEZ2cnU6ZMZtwZ5/Dg/ffyrUMO5PxLrmz7v+bUXlZeeRX2+p8v8OWx+zBw4CDetfq76dO371vLTxl3Mv369mObbbdrYS/Vlf8fL1RdDH0gM9eNiDsBMvPliOg2b4+IscDYivu0QJr4r0n84dq7ABh//xO8+Way1PDB/Lu8XPZ2zzw/iRFdkqARyw7jmS5JkdTVMsssyyabfYSIYI33rkVEHyZNepnhw5dodde0gNnhUzuyw6eKGzZ+euKPWXbZ5QC45A8X8tcbruPkU073F7DaTtVjhmZERF8gASJiaYqkaLYyc1xmjsnMMRX3a4Hzx+vvYZP3rwbAO1dYhgH9+3VbCAE89+8pTH11GuuvuRIAu267PpfecE8zuqoF0Ic33YI7xt8KwJNPPE5HxwyGDRve4l5pQfTSiy8C8Oyzz3DdNVez9TbbcuPf/soZvz6VE376CwYOHNjiHqorxwwVonjxa0U7j9gN+CywLnAGsCNwRGae38C2ueg6s30h7ULvjP+3Fx9eb1WWGjaY51+awndPvoyzL72VXx61G2u9ayTTZ3Ry6PEXccNt/wDgoT8dzZDFFmVA/35Mnvoa23755zz06HOsu8YKjDt6dwYu0p+rbnyAA34wxx/7QmvaXcXdeS9MnTGHNRd+Rx72De4cfxuTJk1iiSWXZJ//3ZetPr4d3z/6Wzzyj4fo368/+33tG6y3/gYAfHrbj/Lqq6/QMWMGg4cM5fifj2P0yu9s8XfReksP6Q/AK290+/ddLf3PnrsxedIk+vXrx9cPOoQPbLAhn9hmS2ZMn87iw4q0es211ubwbx/d4p62n8GL9CEzm1o1LPG5s6srAt7mpd/s2rYVUaXFEEBErA5sQfGUgWsz88EGt6ttMaT5z2JI85vFkOa3VhRDS+5xTtOKoRfP3KVti6FKxwxFxE+AczPz53NcWZIkqQWqHjN0O3BERPwzIn4UEY4FkiSpXfgEaqDiYigzz8jMbShupX8Y+EFEPFLlMSVJkuZGs95a/05gdWBFoKExQ5IkqVrtfpdXs1T9BOoflknQd4D7gDGZ6dO2JElS26g6GfonsGFm/rvi40iSpLlkMlSopBiKiNUz8yHgNmCFiJjlRVqZeUcVx5UkSZpbVSVDX6d4rcZxs1mWFO8qkyRJarlKiqHMnPl+sa0zc1rXZRGxaBXHlCRJc8fLZIWqnzN0U4NtkiRJLVHVmKHlgBHAwIh4H/953NJQYFAVx5QkSXPJYAiobszQx4C9gJHAj7u0TwUOq+iYkiRJc62qMUNnAGdExKcz8/dVHEOSJM0bxwwVqrpMtntm/hZYKSK+/vblmfnj2WwmSZLUdFVdJlus/Dq4ov1LkqR5ZDJUqOoy2S/Lr0dXsX9JkqT5pRnvJhsaEf0j4tqIeCEidq/ymJIkqTER0bSpnVX9nKEtM3MKsC3wOMXb6w+q+JiSJEkNq/pFrTP3/3Hg/Myc3O7VoSRJdeHv5ELVxdClEfEQ8DrwpYhYGpg2h20kSZKaptLLZJl5CLARMCYzZwCvAttXeUxJktSgaOLUxipNhiKiP7A7sHEZxd0AnFzlMSVJkuZG1ZfJfgH0B04q5z9Xtn2+4uNKkqQ5cMxQoepi6P2ZuXaX+T9HxN0VH1OSJKlhVd9a3xkRq8yciYiVgc6KjylJktSwqpOhg4DrIuLRcn4lYO+KjylJkhrgZbJC1cnQjcAvgTeBl8rPf6/4mJIkSQ2rOhk6E5gCfLec3xX4DfCZio8rSZLmwGSoUHUx9N7MXKPL/HUR8UDFx5QkSWpY1ZfJ7oiIDWbORMQHgPEVH1OSJDXChy4C1SdD6wE3RcST5fwKwMMRcS+QmblWxceXJEnqUdXF0FYV71+SJPWSY4YKlRZDmflElfuXJEmaV1UnQ5IkqU2ZDBWqHkAtSZLU1kyGJEmqKZOhgsmQJEmqNZMhSZJqymSoYDIkSZJqzWRIkqS6MhgCTIYkSVLNWQxJkqRa8zKZJEk15QDqgsmQJEmqNZMhSZJqymSoYDIkSZJqzWJIkqSaimjeNOe+xGkR8XxE3Nel7f8i4qGIuCciLoqIYV2WHRoREyLi4Yj4WJf2rcq2CRFxSCM/B4shSZLUDk4Htnpb29XAezNzLeAfwKEAEbEGsDPwnnKbkyKib0T0BX4ObA2sAexSrtsjxwxJklRT7TRmKDP/EhErva3tqi6zNwM7lp+3B87NzDeAxyJiArB+uWxCZj4KEBHnlus+0NOxTYYkSdKC4H+Ay8vPI4Cnuix7umzrrr1HJkOSJNVUM4OhiBgLjO3SNC4zxzW47eFAB3BWFX2zGJIkSZUrC5+Gip+uImIvYFtgi8zMsnkiMKrLaiPLNnpo75bFkCRJNdVOY4ZmJyK2Ag4GNsnM17osugQ4OyJ+DCwPrArcSvHq2VUjYjRFEbQzsOucjmMxJEmSWi4izgE2BZaKiKeBIynuHlsEuLos3G7OzC9m5v0RcR7FwOgOYN/M7Cz3sx9wJdAXOC0z75/TsS2GJEmqqXYKhjJzl9k0n9rD+scAx8ym/TLgsrk5tneTSZKkWjMZkiSppvr0aaNoqIVMhiRJUq1ZDEmSpFrzMpkkSTXVTgOoW8lkSJIk1ZrJkCRJNdXuD11sFpMhSZJUayZDkiTVlMFQwWRIkiTVmsmQJEk15ZihgsmQJEmqNZMhSZJqymSoYDIkSZJqzWRIkqSaMhgqmAxJkqRaMxmSJKmmHDNUMBmSJEm1ZjIkSVJNGQwVTIYkSVKtWQxJkqRa8zKZJEk15QDqgsmQJEmqNZMhSZJqymCoYDIkSZJqzWRIkqSacsxQwWRIkiTVmsmQJEk1ZTBUMBmSJEm1ZjIkSVJNOWao0NbF0LS7ft7qLmghs/SQ/q3ughYygxcxYJcWdG1dDEmSpOoYDBXauhh6fuqMVndBC4llykTo3qemtrgnWlisOWoIAK9Nzxb3RAuLQQOsTFqlrYshSZJUHccMFbzYLUmSas1kSJKkmjIYKpgMSZKkWrMYkiRJteZlMkmSasoB1AWTIUmSVGsmQ5Ik1ZTBUMFkSJIk1ZrJkCRJNeWYoYLJkCRJqjWTIUmSaspkqGAyJEmSas1kSJKkmjIYKpgMSZKkWjMZkiSpphwzVDAZkiRJtWYyJElSTRkMFUyGJElSrZkMSZJUU44ZKpgMSZKkWrMYkiRJteZlMkmSasqrZAWTIUmSVGsmQ5Ik1VQfoyHAZEiSJNWcyZAkSTVlMFQwGZIkSbVmMiRJUk350MWCyZAkSWq5iDgtIp6PiPu6tC0REVdHxCPl1+Fle0TETyJiQkTcExHrdtlmz3L9RyJiz0aObTEkSVJN9YnmTQ04HdjqbW2HANdm5qrAteU8wNbAquU0FvgFFMUTcCTwAWB94MiZBVSPP4eGuidJklShzPwL8NLbmrcHzig/nwHs0KX9zCzcDAyLiHcAHwOuzsyXMvNl4Gr+u8D6L44ZkiSpppo5ZigixlKkODONy8xxc9hs2cx8tvz8HLBs+XkE8FSX9Z4u27pr75HFkCRJqlxZ+Myp+Olp+4yInI9deouXySRJqqmI5k299K/y8hfl1+fL9onAqC7rjSzbumvvkcWQJElqV5cAM+8I2xO4uEv7HuVdZRsAk8vLaVcCW0bE8HLg9JZlW4+8TCZJUk0F7fOcoYg4B9gUWCoinqa4K+xY4LyI2Ad4AtipXP0yYBtgAvAasDdAZr4UEd8FbivX+05mvn1Q9n+xGJIkSS2Xmbt0s2iL2aybwL7d7Oc04LS5ObaXySRJUq3NsRiKiM800iZJkhYsbfbQxZZpJBk6tME2SZKkBU63Y4YiYmuKwUkjIuInXRYNBTqq7pgkSaqWL2ot9DSA+hlgPPAJ4PYu7VOBA6rslCRJUrN0Wwxl5t3A3RFxdmbOaGKfJElSExgMFRq5tX79iDgKWLFcPyjualu5yo5JkiQ1QyPF0KkUl8VuBzqr7Y4kSWqWPkZDQGPF0OTMvLzynkiSJLVAI8XQdRHxf8CFwBszGzPzjsp6JUmSKmcwVGikGPpA+XVMl7YENp//3ZEkSWquORZDmblZMzoiSZKay+cMFRp5HceyEXFqRFxezq9Rvj1WkiRpgdfI6zhOB64Eli/n/wF8raoOSZKk5oho3tTOGimGlsrM84A3ATKzA2+xlyRJC4lGBlC/GhFLUgyaJiI2ACZX2itJklQ5nzNUaKQY+jpwCbBKRNwILA3sWGmvJEmSmqSRu8nuiIhNgHdRvIrjYd9VJkmSFhZzLIYioi+wDbBSuf6WEUFm/rjivkmSpAp5kazQyGWyPwLTgHspB1FLkiQtLBophkZm5lqV90SSJDWVD10sNHJr/eURsWXlPZEkSWqBRpKhm4GLIqIPMIPiEmNm5tBKeyZJkirVx2AIaKwY+jGwIXBvZmbF/ZEkSWqqRoqhp4D7LIQkSVq4OGao0Egx9Chwffmi1jdmNnprvSRJWhg0Ugw9Vk4DykmSJC0EDIYKjTyB+uhmdESSJKkVui2GIuKEzPxaRPyR8iWtXWXmJyrtmSRJqpRjhgo9JUO/Kb/+qBkdkSRJaoVui6HMvL38uE5mnth1WUTsD9xQZcckSVK1fM5QoZEnUO85m7a95nM/JEmSWqKnMUO7ALsCoyPiki6LhgAvVd0xSZJULccMFXoaM3QT8CywFHBcl/apwD1VdkqSJKlZehoz9ATwBMWrOCRJkhZKcxwzFBGfiohHImJyREyJiKkRMaUZnZMkSdWJJk7trJEnUP8Q2C4zH6y6M5IkSc3WSDH0LwshSZIWPn0cQA00VgyNj4jfAX9g1he1XlhZryRJkpqkkWJoKPAasGWXtgQshiRJWoAZDBUaeVHr3s3oiCRJUis0cjfZahFxbUTcV86vFRFHVN81SZJUpYho2tTOGnkdx6+AQ4EZAJl5D7BzlZ2SJElqlkbGDA3KzFvfVtV1VNQfSZLUJG0e2DRNI8nQvyNiFYpB00TEjhSv6ZAkSVrgNZIM7QuMA1aPiInAY8BulfZKkiRVzucMFRq5m+xR4CMRsRjQJzOnVt8tzfT9o4/gpr/ewPAlluA35138VvsF557FheedQ5++fdjoQxvz5f2/wbPPTGS3HbdjhRVXAuA9a67NQYcd2aKeq11Nn/4G3/raF5gxYzqdnZ1suPEW7LzXF/npD47kgXvuYNBigwHY7+CjGP3OdwFw313j+fVJx9HR0cHQxYfx3eN/1cpvQW3s8cce5eBvHPDW/MSnn+JL+32Ve+66i8cffwyAqVOnMmTIEM77/cXd7UZqqm6LoYjYDrinfGErwIHApyPiCWD/zHysGR2su22224FP77Qr3zvy0Lfa7rjtFv56w585/dwLGTBgAC+/9OJby0aMHMXp5/gIKHWvf/8BHHXcyQwcOIiOjhkcsf8+rLv+BwHYY+z+bLjJR2ZZ/9VXpvKrE4/liGN/ytLLvoPJL7/Uim5rAbHS6JXfKnI6OzvZcvON2XyLj7L75/Z6a53j/u9YBg8e3KIeqiuDoUJPY4aOAV4AiIhtgd2B/wEuAU6uvmsCWGfdMQxdfPFZ2i664HfsvtfnGTBgAADDl1iyFV3TAioiGDhwEACdHR10dHT0+BbFv157OR/48OYsvew7AFh8+BLN6KYWArfc/HdGjhrF8suPeKstM7nqisvZapttW9gzaVY9FUOZma+Vnz8FnJqZt2fmKcDSjR4gIgZGxLvmpZOa1VNPPs49d97OF/bYmf2+sCcP3n/vW8uenTiRvXf9NPt9YU/uvvP2FvZS7ayzs5MDx+7C/3z6o6y93gas9u41ATj7tJM44POf5dcnHceM6dMBeObpJ3l16hS+/fWxHPTF3bj+qktb2XUtQK68/E9s/bai547bx7PkkkuyYnk5X63lc4YKPRVDERGDI6IPsAVwbZdlizay8/JS213AFeX8OhFxSW87q0JnZydTpkxm3Bnn8OX9D+TbhxxIZrLkUkvz+z9dw6/P/j37ff1gjj78YF595ZVWd1dtqG/fvhw37hzG/e5yHnnoPp58bAK7f34/fnL67/nhSb/hlSmTuejc04HifPvnIw9y2DEn8q0f/Izzf3sKzzz1RM8HUO3NmDGdG67/Mx/dcqtZ2q+47FJTIbWdnoqhEygKmfHAg5k5HiAi3kfjt9YfBawPTALIzLuA0d2tHBFjI2J8RIxvcP+1tPQyy7LJZh8hIljjvWsR0YdJk15mwIABLD5sGACrv/s9LD9yFE89+XhrO6u2ttjgIbx3nTHcedtNDF9yaSKC/gMGsNlWn2DCQ/cDsOTSy7DOmA1ZdOBAhi4+nDXWXJfHH/1Hi3uudve3v/6F1d/9HpZcaqm32jo6Orj2mqv52FbbtLBn6qpPE6d21m3/MvM0YBNgH6Drmfsc0Oj7ymZk5uS377qHY47LzDGZOabB/dfSxptuwR3jbwXgyScep6NjBsOGDefll1+is7MTKO7gePrJJ1h+xMhWdlVtaPKkl3n1leKm0DfemMY9t9/CiFEr8fKLLwDFmI5bb7yeUaNXAWD9jTblofvuorOzgzemvc4jD93HyBW6/ZtGAuCKy/7EVtt8fJa2W26+idErr8yyyy3Xol5Js9fjrfWZORGY+La2uXng4v0RsSvQNyJWBb4K3DTXvayxIw/7BneNv41Jkybxya03Z5//3ZePb/9J/t/R3+JzO21P/379OfyoY4gI7r5jPKec/DP69etHn+jDNw77NkMXH9bqb0Ft5uUX/83PfngknZ2dZCYbbfIRxmy4MUce+L9MmfwymTB6ldUYe8BhAIxccTTrvH8jvv75nYk+ffjINjuwwuh3tvi7UDt7/bXXuPnvN3HEkd+Zpf2Kyy9jq60/3s1WUutEZrdBzbzvPGIQcDiwZdl0JfC9zJzWwLb5/NQZlfVN9bLMkP4A3PuUj8nS/LHmqCEAvDa9un9DVS+DBgSZ2dSRxl/9w0NNO4F/ssPqbTuKupEnUM+L1TPzcIqCSJIkqe309NDFHh8mkpmNPHntuIhYDrgA+F1m3jeX/ZMkSRXp07ZZTXP1lAzdTjHYOYAVgJfLz8OAJ+nhrrCZMnOzshjaCfhlRAylKIq+N68dlyRJmh96uptsdGauDFwDbJeZS2XmksC2wFWNHiAzn8vMnwBfpLhV/9vz2GdJkjQf9InmTe2skVv/N8jMy2bOZOblwEaN7Dwi3h0RR0XEvcBPKe4k815vSZLUNhoZQP1MRBwB/Lac3w14psH9nwb8DvhYZja6jSRJaoJ2f01GszSSDO1C8S6yi4ALy8+7NLLzzNwwM0+wEJIkST2JiAMi4v6IuC8izomIRSNidETcEhETIuJ3ETGgXHeRcn5CuXyleTn2HJOh8q6x/SNiscx8tcFv6LzM3Km8PNb1GQZR7DLX6l13JUnS/NIuY3kiYgTFg5nXyMzXI+I8YGeKN2Acn5nnRsTJFG/F+EX59eXMfGdE7Az8APhsb48/x2QoIjaKiAeAB8v5tSPipDlstn/5dVtguy7TzHlJkqSu+gEDI6IfMIjiPaibUzyeB+AMYIfy8/blPOXyLWIervk1cpnseOBjwIsAmXk3sHFPG3R5ZceXM/OJrhPw5d52VpIkzT8RzZt6Ur7+60cUj+55FphM8YifSZnZUa72NDCi/DwCeKrctqNcf8ne/hwaepFsZj71tqbOBvf/0dm0bd3gtpIkaSEREWMjYnyXaWyXZcMp0p7RwPLAYsBWzepbI3eTPRURGwEZEf0pLoE92NMGEfEligRo5Yi4p8uiIcCNve2sJEmaf/o08W6yzBwHjOtm8UeAxzLzBYCIuBD4IDAsIvqV6c9I/vPy+InAKODp8rLa4pRXsHqjkWToi8C+FJHURGAd5nyp62yKsUGXMOuYofUyc/fedlaSJC2UngQ2iIhB5difLYAHgOuAHct19gQuLj9fUs5TLv9zzsOb5xtJht6Vmbt1bYiID9JDwpOZkymu3+1Srr8MsCgwOCIGZ+aTve2wJEmaPxoaK9MEmXlLRFwA3IkQkJgAAB2gSURBVAF0AHdSpEh/As6NiO+VbaeWm5wK/CYiJgAvUdx51muNFEM/BdZtoO2/RMR2wI8prv89D6xIcYntPXPXTUmStDDLzCOBI9/W/Ciw/mzWnQZ8Zn4du6e31m9I8dqNpSPi610WDQX6Nrj/7wEbANdk5vsiYjPAy2SSJKlt9JQMDQAGl+sM6dI+hf9cv5uTGZn5YkT0iYg+mXldRJzQy75KkqT5yLdxFLothjLzBuCGiDi9fD5Qb0yKiMHAX4CzIuJ5oKGnWEuSJDVDI2OnTomIYTNnImJ4RFzZ4P63B14HDgCuAP6JT6CWJKkt9Ilo2tTOGhlAvVRmTpo5k5kvl3eHzdHb3mV2RrcrSpIktUgjxdCbEbHCzNvhI2JFZn35arciYups1p0MjAcOzMxH56azkiRp/mnzwKZpGimGDgf+FhE3ULx1/sPA2J43ecsJFO8SObvcdmdgFYrnCJwGbDqX/ZUkSZqv5lgMZeYVEbEuxS3yAF/LzH83uP9PZObaXebHRcRdmfnNiDhsbjsrSZLmnz4mQ0APA6gjYvXy67rACsAz5bRC2daI1yJip5m31kfETsC0clmvH5stSZI0v/SUDB0IfAE4bjbLEti8gf3vBpwInFRuczOwe0QMBPabu65KkqT5qd3v8mqWnp4z9IXy62a93Xk5QLq7W+n/1tv9SpIkzS89vY7jUz1tmJkXzmnnEbEa8Atg2cx8b0SsRTGO6Htz3VNJkjRfGQwVerpMNjPRWYbiHWV/Luc3A24C5lgMAb8CDgJ+CZCZ90TE2RTvLJMkSWq5ni6T7Q0QEVcBa2Tms+X8O4DTG9z/oMy8NWYtPTt611VJkjQ/eTdZoZHXcYyaWQiV/kVxd1kj/h0Rq1DeORYROwLP9ryJJElS8zTy0MVry3eRnVPOfxa4psH97wuMA1aPiInAYxR3mEmSpBYLjIagsYcu7hcRnwQ2LpvGZeZFDe5/IvBr4DpgCWAKsCfwnV70VZIkab5rJBmC4vUZUzPzmogYFBFDMnNqA9tdDEwqt3+mt52UJEmqyhyLoYj4AsW7yJageK/YCOBkYIsG9j8yM7eapx5KkqRKOIC60MgA6n2BD1Jc4iIzH6G43b4RN0XEmr3smyRJUuUauUz2RmZOn3l7fET0o/H3in0I2CsiHgPeoHhzfWbmWr3prCRJmn9MhgqNFEM3lG+YHxgRHwW+DPyxwf1v3eueSZIkNUEjxdA3gc8D9wL/C1wGnNLIzjPzid53TZIkVSl8Hwcwh2IoIvoC92fm6hSv1pAkSVqo9FgMZWZnRDwcEStk5pPN6pQkSaqeY4YKjVwmGw7cHxG3Aq/ObMzMT1TWK0mSpCZppBj6VuW9kCRJTeeQoUK3xVBELAp8EXgnxeDpUzPTN85LkqSFSk/J0BnADOCvFLfIrwHs34xOSZKk6vUxGgJ6LobWyMw1ASLiVODW5nRJkiSpeXoqhmbM/JCZHT6LQJKkhYt3kxV6KobWjogp5eegeAL1FP7zSo2hlfdOkiSpYt0WQ5nZt5kdkSRJzeVFn0Ijb62XJElaaFkMSZKkWmvkoYuSJGkh1Aevk4HJkCRJqjmTIUmSasoB1AWTIUmSVGsmQ5Ik1ZQPXSyYDEmSpFozGZIkqaZ8UWvBZEiSJNWayZAkSTVlMFQwGZIkSbVmMiRJUk05ZqhgMiRJkmrNZEiSpJoyGCqYDEmSpFozGZIkqaZMRAr+HCRJUq1ZDEmSpFrzMpkkSTUVjqAGTIYkSVLNmQxJklRT5kIFkyFJklRrJkOSJNWUr+MomAxJkqRaMxmSJKmmzIUKJkOSJKnWTIYkSaophwwVTIYkSVKtWQxJklRTEdG0qYG+DIuICyLioYh4MCI2jIglIuLqiHik/Dq8XDci4icRMSEi7omIdefl52AxJEmS2sGJwBWZuTqwNvAgcAhwbWauClxbzgNsDaxaTmOBX8zLgS2GJEmqqT5NnHoSEYsDGwOnAmTm9MycBGwPnFGudgawQ/l5e+DMLNwMDIuId8zLz0GSJKlSETE2IsZ3mcZ2WTwaeAH4dUTcGRGnRMRiwLKZ+Wy5znPAsuXnEcBTXbZ/umzrFe8mkySpppr51vrMHAeM62ZxP2Bd4CuZeUtEnMh/LonN3D4jIqvom8mQJElqtaeBpzPzlnL+Aori6F8zL3+VX58vl08ERnXZfmTZ1isWQ5IkqaUy8zngqYh4V9m0BfAAcAmwZ9m2J3Bx+fkSYI/yrrINgMldLqfNNS+TSZJUU232zMWvAGdFxADgUWBvitDmvIjYB3gC2Klc9zJgG2AC8Fq5bq9ZDEmSpJbLzLuAMbNZtMVs1k1g3/l17LYuhpYZ0r/VXdBCZs1RQ1rdBS1kBg1os7+tpbnQzAHU7cwxQ5IkqdbaOhl6fUYld9Cphgb2L/768ZzS/DLznHpg4ist7okWFmuMGNz0Y5qIFPw5SJKkWmvrZEiSJFXHMUMFkyFJklRrJkOSJNWUuVDBZEiSJNWayZAkSTXlkKGCyZAkSao1kyFJkmqqj6OGAJMhSZJUcyZDkiTVlGOGCiZDkiSp1iyGJElSrXmZTJKkmgoHUAMmQ5IkqeZMhiRJqikHUBdMhiRJUq2ZDEmSVFM+dLFgMiRJkmrNZEiSpJpyzFDBZEiSJNWayZAkSTVlMlQwGZIkSbVmMiRJUk35BOqCyZAkSao1kyFJkmqqj8EQYDIkSZJqzmRIkqSacsxQwWRIkiTVmsWQJEmqNS+TSZJUUz50sWAyJEmSas1kSJKkmnIAdcFkSJIk1ZrJkCRJNeVDFwsmQ5IkqdZMhiRJqinHDBVMhiRJUq2ZDEmSVFM+Z6hgMiRJkmrNZEiSpJoyGCqYDEmSpFozGZIkqab6OGgIMBmSJEk1ZzIkSVJNmQsVTIYkSVKtWQxJkqRa8zKZJEl15XUywGRIkiTVnMmQJEk15YtaCyZDkiSp1kyGJEmqKZ+5WDAZkiRJtWYyJElSTRkMFUyGJElSrZkMSZJUV0ZDgMmQJEmqOZMhSZJqyucMFUyGJElSrVkMSZJUUxHNmxrrT/SNiDsj4tJyfnRE3BIREyLidxExoGxfpJyfUC5faV5+DhZDkiSpXewPPNhl/gfA8Zn5TuBlYJ+yfR/g5bL9+HK9XrMYkiSppqKJ0xz7EjES+DhwSjkfwObABeUqZwA7lJ+3L+cpl29Rrt8rFkOSJKlyETE2IsZ3mca+bZUTgIOBN8v5JYFJmdlRzj8NjCg/jwCeAiiXTy7X7xXvJpMkSZXLzHHAuNkti4htgecz8/aI2LSpHcNiSJKk+mqfO+s/CHwiIrYBFgWGAicCwyKiX5n+jAQmlutPBEYBT0dEP2Bx4MXeHtzLZJIkqaUy89DMHJmZKwE7A3/OzN2A64Ady9X2BC4uP19SzlMu/3NmZm+PbzIkSVJNLQAPXfwmcG5EfA+4Ezi1bD8V+E1ETABeoiiges1iSJIktY3MvB64vvz8KLD+bNaZBnxmfh3TYkiSpJrq/c3oCxfHDEmSpFozGZIkqaYMhgomQ5IkqdZMhiRJqiujIcBkSJIk1ZzJkCRJNbUAPGeoKUyGJElSrZkMSZJUUz5nqGAyJEmSas1kSJKkmjIYKpgMSZKkWjMZkiSproyGAJMhSZJUcxZDkiSp1rxMJklSTfnQxYLJkCRJqjWToQXMlClTOPrbRzBhwj+ICI7+7vdZZJFF+d53jmT6G2/Qt19fDjviKNZca61Wd1ULiNmdU7/9zRk88dhjAEydOpUhQ4Zw3oUXt7inakfTp7/B4ft/nhnTp9PZ2clGm2zBLnt/iZ/+8Gj++fADJMnyI1fkq4cczcCBg7j4vN9y9WUX0bdvX4YuPpyvHHwkyyy3fKu/jdryoYuFyMxW92G2IiJfn9GefWulIw79JuuuN4ZP7fgZZkyfzuvTpnHQgV/jc3vsyYc+vAl//csNnH7aKZx6+m9a3dW2MrB/8f94z6n/NrtzaujQoW8t/9EPj2Xw4MF88cv7tbCX7WfmOfXAxFda3JPWykymTXudgQMH0dExg0O/sg+f/8o3GLXiygxabDAAp/38OBYfvgSf3nVv7r3zNlZ793tZZNGBXH7x+dx313gOOvIHLf4u2sMaIwaTmU0tTx545tWm/aO4xvKLtW3p5WWyBcjUqVO5/fbb+OSndwSg/4ABDB06lCB45ZVXAXhl6lSWXnqZVnZTC5DuzqmZMpOrrrycrT++bau6qDYXEQwcOAiAzo4OOjs7COKtQigzmT79DaKMINZ83/tZZNGBALxrjTV58YXnW9NxAcWd9c2a2pmXyRYgE59+muHDl+Dbhx/Kww8/xBrveQ8HH3I4Bx9yGF8auw8//tEPePPNNznzrHNb3VUtILo7pwYNKn653XH7eJZccklWXHGl1nZUba2zs5MD/3c3npv4FFvvsBOrrbEmAD/5wZHcfsuNjFpxZfb+0gH/td01l/2BdT/wwWZ3V/ovlSRDETE1IqbMZpoaEVN62G5sRIyPiPFV9GtB19nZwUMPPsBndt6F837/BwYOHMhpp4zjvN+dw0HfPJSrrr2Bg755KEd96/BWd1ULiO7OqZkuv+xSttrGVEg969u3Lyecci6nnH8Fjzx0P088NgGAr37zaE47/0pGrjCav1131SzbXH/1n5jw8AN88rN7tKLLmsloCKioGMrMIZk5dDbTkMwc2sN24zJzTGaOqaJfC7pll12OZZddjrXWWhuAj265FQ89+AB/vPgitvjolgBs+bGtue/ee1rZTS1AujunADo6Orj2mqvZaqttWtlFLUAGDx7CmuuM4c5bb3qrrW/fvnx48y35+1+ufavt7ttv4YLfnsphx5xA/wEDWtFVaRZNGTMUEctExAozp2Ycc2G01NJLs+xyy/H4Y48CcMvNf2flVVZh6WWWYfxttwJw6y03s4KXNNSg7s4pgFv+fhOjR6/Mssst18ouqs1NnvQyr7wyFYA33pjGXbffzPKjVuTZiU8CxZihW2/6CyNWGA3Ao488xEk/PobDjjmBYcOXaFm/VYgm/q+dVTpmKCI+ARwHLA88D6wIPAi8p8rjLswOOexbHPrNbzBjxgxGjhzFd773/9h0sy344bHfp7OjgwGLLMK3j/pOq7upBcjszimAKy6/jK22+XiLe6d29/KLL3DisUfy5pud5JvJBzf9KGM2+DCHfXUfXnvtVchkpVVW44sHHArA6SefwLTXX+OHRx0MwNLLLsfhx5zQym9BqvbW+oi4G9gcuCYz3xcRmwG7Z+Y+DWzrrfWab7y1XvObt9ZrfmvFrfUPP/da0/5RfNdyg9o2Hqr6MtmMzHwR6BMRfTLzOsDxQJIkqW1UfWv9pIgYDPwFOCsingderfiYkiSpAW0b1TRZ1cnQ9sBrwAHAFcA/ge0qPqYkSVLDKkuGIqIvcGlmbga8CZxR1bEkSVIvGA0BFSZDmdkJvBkRi1d1DEmSpHlV9ZihV4B7I+JquowVysyvVnxcSZKkhlRdDF1YTl15b7MkSW2g3R+G2CxVF0PDMvPErg0RsX/Fx5QkSWpY1XeT7Tmbtr0qPqYkSWpARPOmdlZJMhQRuwC7AqMj4pIui4YAL1VxTEmSpN6o6jLZTcCzwFIU7yabaSrgK9UlSWoDbR7YNE0lxVBmPgE8AWxYxf4lSZLml6rfWj+V/9w9NgDoD7yamUOrPK4kSWqA0RBQcTGUmUNmfo6IoHg9xwZVHlOSJGluVH032Vuy8AfgY806piRJ6l408X/trOrLZJ/qMtsHGANMq/KYkiRJc6Pqhy52fUN9B/A4xaUySZLUYu3+/J9mqXrM0N5V7l+SJGleVTpmKCJWi4hrI+K+cn6tiDiiymNKkqTGRBOndlb1AOpfAYcCMwAy8x5g54qPKUmS1LCqxwwNysxbY9aLkh0VH1OSJDWi3SObJqk6Gfp3RKxC+eDFiNiR4jUdkiRJbaHqZGhfYBywekRMBB4Ddqv4mJIkSQ2ruhiaCPwauA5YApgC7Al8p+LjSpKkOWj3hyE2S9XF0MXAJOAO4JmKjyVJkjTXqi6GRmbmVhUfQ5Ik9YIPXSxUPYD6pohYs+JjSJIk9VrVydCHgL0i4jHgDYqb+DIz16r4uJIkaQ4MhgpVF0NbV7x/SZKkeVL1u8meqHL/kiSp9xwzVKh6zJAkSVJbq/oymSRJaltGQ2AyJEmSas5kSJKkmnLMUMFkSJIk1ZrFkCRJNRVNnHrsR8SoiLguIh6IiPsjYv+yfYmIuDoiHim/Di/bIyJ+EhETIuKeiFh3Xn4OFkOSJKnVOoADM3MNYANg34hYAzgEuDYzVwWuLeeheI7hquU0FvjFvBzcYkiSpJqKaN7Uk8x8NjPvKD9PBR4ERgDbA2eUq50B7FB+3h44Mws3A8Mi4h29/TlYDEmSpLYRESsB7wNuAZbNzGfLRc8By5afRwBPddns6bKtVyyGJElS5SJibESM7zKNnc06g4HfA1/LzCldl2VmAllF37y1XpKkmoomPnQxM8cB47rtS0R/ikLorMy8sGz+V0S8IzOfLS+DPV+2TwRGddl8ZNnWKyZDkiSppSIigFOBBzPzx10WXQLsWX7eE7i4S/se5V1lGwCTu1xOm2smQ5Ik1VX7PHTxg8DngHsj4q6y7TDgWOC8iNgHeALYqVx2GbANMAF4Ddh7Xg5uMSRJkloqM/9G96XZFrNZP4F959fxLYYkSaqp9gmGWssxQ5IkqdZMhiRJqilf1FowGZIkSbVmMiRJUk018zlD7cxkSJIk1ZrJkCRJdWUwBJgMSZKkmjMZkiSppgyGCiZDkiSp1kyGJEmqKZ8zVDAZkiRJtWYxJEmSas3LZJIk1ZQPXSyYDEmSpFozGZIkqaYcQF0wGZIkSbVmMSRJkmrNYkiSJNWaY4YkSaopxwwVTIYkSVKtmQxJklRTPmeoYDIkSZJqzWRIkqSacsxQwWRIkiTVmsmQJEk1ZTBUMBmSJEm1ZjIkSVJdGQ0BJkOSJKnmLIYkSVKteZlMkqSa8qGLBZMhSZJUayZDkiTVlA9dLJgMSZKkWjMZkiSppgyGCiZDkiSp1kyGJEmqK6MhwGRIkiTVnMmQJEk15XOGCiZDkiSp1kyGJEmqKZ8zVIjMbHUfZisi2rNjkiRVJDObWp5M66Bpv2sX7de+1+TathhS4yJibGaOa3U/tHDwfNL85jmldueYoYXD2FZ3QAsVzyfNb55TamsWQ5IkqdYshiRJUq1ZDC0cvBav+cnzSfOb55TamgOoJUlSrZkMSZKkWrMYWoBFxBcjYo/y814RsXyXZadExBqt650WBhExLCK+3GV++Yi4oJV90oIpIlaKiF17ue0r87s/UldeJltIRMT1wDcyc3yr+6KFR0SsBFyame9tcVe0gIuITSn+jdp2Nsv6ZWZHD9u+kpmDq+yf6s1kqEXKv5IeioizIuLBiLggIgZFxBYRcWdE3BsRp0XEIuX6x0bEAxFxT0T8qGw7KiK+ERE7AmOAsyLirogYGBHXR8SYMj36vy7H3SsiflZ+3j0ibi23+WVE9G3Fz0K9V55HD0bEryLi/oi4qvzvv0pEXBERt0fEXyNi9XL9VSLi5vL8+t7Mv7gjYnBEXBsRd5TLti8PcSywSnmO/F95vPvKbW6OiPd06cvMc26x8ty9tTyXt397v7Xg6MU5dnr5b9LM7WemOscCHy7PpQPKf4suiYg/A9f2cA5K1ctMpxZMwEpAAh8s508DjgCeAlYr284EvgYsCTzMf5K8YeXXoyj+0gK4HhjTZf/XUxRISwMTurRfDnwIeDfwR6B/2X4SsEerfy5OvTqPOoB1yvnzgN2Ba4FVy7YPAH8uP18K7FJ+/iLwSvm5HzC0/LwUMAGIcv/3ve1495WfDwCOLj+/A3i4/Px9YPeZ5yrwD2CxVv+snJp2jp0O7Nhl+5nn2KYUKePM9r2Ap4ElejoHu+7DyamqyWSotZ7KzBvLz78FtgAey8x/lG1nABsDk4FpwKkR8SngtUYPkJkvAI9GxAYRsSSwOnBjeaz1gNsi4q5yfuX58D2p+R7LzLvKz7dT/PLaCDi//G/7S4piBWBD4Pzy89ld9hHA9yPiHuAaYASw7ByOex4wMwHYCZg5lmhL4JDy2NcDiwIrzPV3pXYyN+fY3Lg6M18qP/fmHJTmC99a31pvH7A1iSIFmnWlzI6IWJ+iYNkR2A/YfC6Ocy7FL6uHgIsyMyMigDMy89Be9Vzt5I0unzspfoFMysx15mIfu1GkiOtl5oyIeJyiiOlWZk6MiBcjYi3gsxRJExS/1D6dmQ/PxfHV3ubmHOugHIIREX2AAT3s99Uun+f6HJTmF5Oh1lohIjYsP+8KjAdWioh3lm2fA26IiMHA4pl5GcWlibVns6+pwJBujnMRsD2wC0VhBEXEvWNELAMQEUtExIrz+g2pLUwBHouIzwBEYeY5czPw6fLzzl22WRx4vvwltBkw81zo6bwC+B1wMMX5eU/ZdiXwlbLgJiLeN6/fkNpOT+fY4xSpM8AngP7l5zmdS92dg1LlLIZa62Fg34h4EBgOHA/sTRE93wu8CZxM8Q/IpWV8/Dfg67PZ1+nAyTMHUHddkJkvAw8CK2bmrWXbAxRjlK4q93s1vYu51Z52A/aJiLuB+ymKYSjGoH29/G/+TopLsABnAWPK824PihSRzHwRuDEi7us6EL+LCyiKqvO6tH2X4hfgPRFxfzmvhU9359ivgE3K9g35T/pzD9AZEXdHxAGz2d9sz0GpGby1vkXCW5bVAhHx/9u7nxCryjiM498naZNG0KKwPzQWIziWjYgSVDJKDbWojGbRUFDYH2kx4cYQ2lgEEUVQKVjMQoiRItvEEDMQzSaIkhKSolVBmBASFQhDKf5a/N5jxzuOZ8bbXBfn+azuPfc973vmLO787nvOeZ8rgNlyqfRR8mZqP7VjZq3me4bM2mUDsLdcwvoT2H6Jj8fM7JLzzJCZmZm1mu8ZMjMzs1ZzMWRmZmat5mLIzMzMWs3FkFkPSdomKaocp4a2O8vTXxc71tkcuo7t10qaLI84/yDp0y7HuK72flzSwMX2t8AxuzovZmadXAyZ9dYouVbU6ALa7gSW4p/+y2QMwu0RMQDs7qKvJ4GzxVBEPF3WsFpKS3VezKylXAyZ9UhZSfwu4Clqqz9LWibpjbKw4XeSxiQ9TxYZM5JmSruTtX1GJB0orx+Q9FVJiP9MUlOe00oyIBOA2srRSNol6XA5jpfKtvlSy0fIMOCJarFPleT66niVSfffl+PaVD7/SdKDtb/99dqYO8r2odL2kKQfJU2UVY7nnBczs265GDLrnYeAqRLE+7ukKrLgWTL4cjAi1gETEfE2cBzYEhFbGvr9ArgjItaTcSsvNLTfR4b+zkh6sbrMJWkY6Ac2AYPABkmbyz79wL6IWEuuT/RIRBwiI2Qei4jBiJjtGGc5mWS+loxieAW4F3iYnJ2CLAz/ioiNwEbgGUmrymfryVmgATJE+M5FnhczswXxootmvTMKvFVef1DefwPcA+yPiNMAtRTvhboB+FDSSjIU8+cLNY6IaUk3A/cB9wNHJN1Kps0PA0dK0xVkEfQL508tb/IPMFVeHwX+LrlTR2v7DwPryiwTZD5Vf9n364g4BqBMRu8jCz8zs/+ViyGzHpB0NbAVuE1SAMuAkLRrEd3UV0itp3m/A7wZEZ9IGgL2NHaUBddB4KCkSWAzmTb/akS823HsfcxNLT8n/24ep+K/VV3PVH1ExBlJ1XePgLGImO4Yc+g8Y/r7ysyWhC+TmfXGCPB+RNwUEX0RcSM5g3M3GZK7oyoQSuEEc1O+f5O0RtJl5KWmylXAr+X1E00HImlr9TSWpCuBW8jZn2lge7m3CUnXS7qmobumJPIm08Bzki4vY66WtHyJxzQzO4d/aZn1xijwWse2j8v2MWA1mfJ+ikz93gu8B0xJOl7uj9kNTAInyHt1VpR+9gAfSfoD+BxYxYVV+WSnyR9E4xFxGEDSGuDLjC7jJPA4OSsznwPAfkmzZEL5Yo2Tl7++LXlpJ4BtDft0nhczs644m8zMzMxazZfJzMzMrNVcDJmZmVmruRgyMzOzVnMxZGZmZq3mYsjMzMxazcWQmZmZtZqLITMzM2s1F0NmZmbWav8Cfykb4oIwC7wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x720 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#to plot a confusion matrix\n",
    "cm=confusion_matrix(data_test['airline_sentiment'],predicted_labels)\n",
    "cm1 = pd.DataFrame(cm , index = ['positive','negative','neutral'] , columns = ['positive','negative','neutral'])\n",
    "plt.figure(figsize = (10,10))\n",
    "ax=sns.heatmap(cm1,cmap= \"Blues\", linecolor = 'black' , linewidth = 1 , annot = True, fmt='')\n",
    "ax.set(xlabel=\"Actual Sentiment\", ylabel = \"Predicted Sentiment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "YROuzeMaQ-qC"
   },
   "outputs": [],
   "source": [
    "#Here I have considered the positive tweets to be our positive class and the rest as negative class and hence evaluated the below paramters\n",
    "accuracy=metrics.accuracy_score(data_test['airline_sentiment'],predicted_labels)\n",
    "TP=cm[0,0]\n",
    "TN=cm[1,1]+cm[1,2]+cm[2,1]+cm[2,2]\n",
    "FP=cm[0,1]+cm[0,2]\n",
    "FN=cm[1,0]+cm[2,0]\n",
    "precision=TP/(TP+FP)\n",
    "recall=TP/(TP+FN)\n",
    "f1_score=2*precision*recall/(precision+recall)\n",
    "specificity=(TN)/(TN+FP)\n",
    "FPR=1-specificity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 80
    },
    "id": "lT6wQCSot4B8",
    "outputId": "1373ccf8-cc79-4c75-a10a-99deb89a83c8"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1_Score</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>FPR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Model evaluations</th>\n",
       "      <td>0.78</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Accuracy  Precision  Recall  F1_Score  Specificity   FPR\n",
       "Model evaluations      0.78       0.86    0.87      0.87         0.77  0.23"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#displaying the evaluation parameters\n",
    "from IPython.display import display\n",
    "out,a={},[]\n",
    "out['Accuracy']=accuracy\n",
    "out['Precision']=precision\n",
    "out['Recall']=recall\n",
    "out['F1_Score']=f1_score\n",
    "out['Specificity']=specificity\n",
    "out['FPR']=FPR\n",
    "a.append(out)\n",
    "Class_Algo_dt1= round(pd.DataFrame(a,index=['Model evaluations']),2)\n",
    "display(Class_Algo_dt1)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Airline Sentiment Analysis Without Vectorization.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}