candidate_classifier_NLP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "autoscroll": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "import nltk\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, VectorizerMixin\n",
    "from utils import *\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "pd.set_option('max_colwidth', 120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note: you can add other languages that Spacy supports, or download\n",
    "# larger models for english that Spacy offers. \n",
    "nlp = spacy.load('en') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS\n",
    "\n",
    "def stop_word_removal(li):    \n",
    "    return [l for l in li if l not in ENGLISH_STOP_WORDS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "autoscroll": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "from utils import clean_html\n",
    "from sklearn.feature_extraction.text import strip_accents_unicode\n",
    "\n",
    "\n",
    "def clean_twitter(s):\n",
    "    \"\"\" Cleans Twitter specific issues \n",
    "    \n",
    "    Can you think of what else you might need to add here?\n",
    "    \"\"\"\n",
    "    # MAKE A FUNCTION THAT REMOVES NON ENGLISH TWEETS\n",
    "    # MAKE A FUNCTION THAT REMOVES URLS\n",
    "    s = re.sub(r\"http\\S+\", \"\", s)\n",
    "    s = re.sub(r\"https\\S+\", \"\", s)\n",
    "    s = re.sub(r\"www.\\S+\", \"\", s)\n",
    "\n",
    "    # MAKE A FUNCTION THAT REMOVES \"RT\"\n",
    "    s = re.sub(r\"RT\", \"\", s)\n",
    "\n",
    "    #s = sub(r'@\\w+', '', s) #remove @ mentions from tweets    \n",
    "    return s\n",
    "\n",
    "def preprocessor(s):\n",
    "    \"\"\" For all basic string cleanup. \n",
    "    \n",
    "    Think of what you can add to this to improve things. What is\n",
    "    specific to your goal, how can you transform the text. Add tokens,\n",
    "    remove things, unify things. \n",
    "    \"\"\"\n",
    "    s = clean_html(s)\n",
    "    s = strip_accents_unicode(s.lower())\n",
    "    s = clean_twitter(s)\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'die migrant'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "def cool_tokenizer(sent):\n",
    "    \"\"\" Idea from Travis in class: adds a token to the end with nsubj and root verb!\"\"\"\n",
    "    doc = nlp(sent)\n",
    "    tokens = sorted(doc, key = lambda t: t.dep_)\n",
    "    return ' '.join([t.lemma_ for t in tokens if t.dep_ in ['nsubj', 'ROOT']])\n",
    "\n",
    "cool_tokenizer('a migrant died in crossing the river')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['migrant:nsubj', 'die:ROOT', 'cross:pcomp', 'river:dobj']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langdetect.lang_detect_exception import LangDetectException\n",
    "from langdetect import detect\n",
    "\n",
    "def dep_tokenizer(sent):\n",
    "    \"\"\" A simple version of tokenzing with the dependencies.\n",
    "    \n",
    "    Note: this is monolingual! Also, it still doesn't take into \n",
    "    account correlations!\n",
    "    \"\"\"\n",
    "    doc = nlp(sent)\n",
    "    tokens = [t for t in doc if not t.is_stop and t.dep_ not in ['punct', '']]\n",
    "    return [':'.join([t.lemma_,t.dep_]) for t in tokens]\n",
    "\n",
    "dep_tokenizer('a migrant died in crossing the river')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "autoscroll": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS\n",
    "\n",
    "def analyzer(s, ngram_range = (2,4)):\n",
    "    \"\"\" Does everything to turn raw documents into tokens.  \n",
    "    \n",
    "    Note: None of the above tokenizers are implemented!\n",
    "    \"\"\"\n",
    "    s = preprocessor(s)\n",
    "    pattern = re.compile(r\"(?u)\\b\\w\\w+\\b\")\n",
    "    unigrams = pattern.findall(s)\n",
    "    unigrams = [u for u in unigrams if u not in ENGLISH_STOP_WORDS]\n",
    "    tokens = ngrammer(unigrams, ngram_range)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "autoscroll": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "X = pd.read_csv('kaggle/data/train.csv').tweet\n",
    "y = pd.read_csv('kaggle/train.csv').label\n",
    "\n",
    "cutoff = 2050\n",
    "X_train, X_test, y_train, y_test = X[0:cutoff], X[cutoff:], y[0:cutoff], y[cutoff:]\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((150,), (150,))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "autoscroll": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "def create_vectors(X_train, X_test, analyzer = analyzer):\n",
    "    \"\"\" Just a small helper function that applies the SKLearn Vectorizer with our analyzer \"\"\"\n",
    "    idx = X_train.shape[0]\n",
    "    X = pd.concat([X_train, X_test])\n",
    "    vectorizer = TfidfVectorizer(analyzer=analyzer).fit(X)\n",
    "    vector = vectorizer.transform(X)\n",
    "    return vector[0:idx], vector[idx:], vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "autoscroll": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "V_train, V_test, vectorizer = create_vectors(X_train, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "autoscroll": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.25693255734337556"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Naive Bayes\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import recall_score, precision_score, f1_score, average_precision_score, roc_auc_score\n",
    "\n",
    "model = MultinomialNB(class_prior=[0.5,0.5])\n",
    "model.fit(V_train, y_train)\n",
    "preds = model.predict_proba(V_test)[:,1]\n",
    "roc_auc_score(y_test, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "autoscroll": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Linear Support Vector Classifier\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "model = LinearSVC(tol = 10e-7, max_iter = -1)\n",
    "model.fit(V_train, y_train)\n",
    "preds = model.decision_function(V_test)\n",
    "roc_auc_score(y_test, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7603560424512152"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Radial Support Vector Classifier\n",
    "from sklearn import svm\n",
    "\n",
    "model = svm.SVC(kernel='rbf', tol = 10e-7, max_iter = -1)\n",
    "model.fit(V_train, y_train)\n",
    "preds = model.decision_function(V_test)\n",
    "roc_auc_score(y_test, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GLM with regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Look at your false predictions!\n",
    "false_pos, false_neg = get_errors(X_test, y_test, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submission!\n",
    "\n",
    "Here you can make the submission required for Kaggle. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('kaggle/test.csv')\n",
    "X_sub, id_sub = test_df.tweet, test_df.id\n",
    "V_train, V_test, _ = create_vectors(X, X_sub)\n",
    "model.fit(V_train, y)\n",
    "preds = model.decision_function(V_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_submission_csv(preds, id_sub, 'kaggle/submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "name": "Untitled.ipynb"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}