modelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Couldn't import dot_parser, loading of dot files will not be possible.\n"
     ]
    }
   ],
   "source": [
    "# Imports/setup\n",
    "\n",
    "import numpy as np\n",
    "import pandas\n",
    "import spacy\n",
    "import re\n",
    "import math\n",
    "import jellyfish\n",
    "from collections import Counter\n",
    "from bs4 import UnicodeDammit\n",
    "from itertools import permutations\n",
    "\n",
    "from gensim import corpora\n",
    "from gensim.models.ldamodel import LdaModel\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import DefaultTagger, UnigramTagger, BigramTagger, TrigramTagger\n",
    "from textblob import TextBlob\n",
    "\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "nlp = spacy.load(\"en\")\n",
    "stops = set(stopwords.words(\"english\"))\n",
    "\n",
    "question_tokens = set([\"why\", \"how\", \"what\", \"when\", \"which\", \"who\", \"whose\", \"whom\"])\n",
    "common_question_tokens = set([\"why\", \"how\", \"what\", \"when\", \"which\", \"who\"])\n",
    "\n",
    "common_q_token_pairs = [(\"why\", \"why\"), (\"how\", \"how\"), (\"what\", \"what\"), (\"when\", \"when\"), (\"which\", \"which\"), (\"who\", \"who\")]\n",
    "common_q_token_pairs.extend(\n",
    "    list(permutations(list(common_question_tokens), 2))\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Feature engineering methods\n",
    "\n",
    "def count_grams_full(df, n):\n",
    "    c = Counter()\n",
    "    for (idx, row) in list(df.iterrows()):\n",
    "        q_tokens = remove_punc(str(row[\"question1\"])).lower().split() + remove_punc(str(row[\"question2\"])).lower().split()\n",
    "        c.update(map(lambda x: '_'.join(x), zip(*[q_tokens[i:] for i in range(n)])))\n",
    "        \n",
    "    return c\n",
    "\n",
    "\n",
    "def count_grams(input_list, n):\n",
    "    \"\"\" Returns a count of n-grams \"\"\"\n",
    "    return Counter(map(lambda x: '_'.join(x), zip(*[input_list[i:] for i in range(n)])))\n",
    "\n",
    "\n",
    "def set_overlap_score_model(questions_df):\n",
    "    \"\"\"\n",
    "    appending column of set overlap percentage, where each set\n",
    "    is a set of tokens excluding stop-words and punctuation,\n",
    "    and in lemmatised form\n",
    "    \"\"\"\n",
    "    def set_overlap_score(row):\n",
    "        set1, set2 = \\\n",
    "            (set([w.lemma_.lower() for w in row[\"cleaned_question1_words\"]]),\n",
    "             set([w.lemma_.lower() for w in row[\"cleaned_question2_words\"]]))\n",
    "        return 0.0 if not len(set1.union(set2)) else 1.0 * len(set1.intersection(set2)) / len(set1.union(set2))\n",
    "    questions_df[\"cleaned_question1_words\"] = questions_df[\"question1\"].map(clean_statement)\n",
    "    questions_df[\"cleaned_question2_words\"] = questions_df[\"question2\"].map(clean_statement)\n",
    "\n",
    "    questions_df[\"score\"] = questions_df.apply(set_overlap_score, axis=1)\n",
    "\n",
    "    return questions_df\n",
    "\n",
    "\n",
    "def remove_punc(s):\n",
    "    return re.sub(r'[^\\w\\s]', '', UnicodeDammit(str(s)).markup)\n",
    "\n",
    "\n",
    "def clean_statement(s):\n",
    "    \"\"\"\n",
    "    Remove punctuation, stop words and standardise casing\n",
    "    words, and return remaining tokens\n",
    "    \"\"\"\n",
    "\n",
    "    # Remove punctuation\n",
    "    s = remove_punc(s)\n",
    "    sentence = nlp(s)\n",
    "    sentence_with_stop_checks = [(sentence[i], sentence[i].is_stop) for i in range(len(sentence))]\n",
    "\n",
    "    return sorted([w for (w, stop_bool) in sentence_with_stop_checks if not stop_bool])\n",
    "\n",
    "\n",
    "def construct_doc_list(df):\n",
    "    \"\"\"\n",
    "    Take the question pairs DF and return a list of 2 docs per\n",
    "    row with the cleaned up sentence\n",
    "    \"\"\"\n",
    "    for index, row in df.iterrows():\n",
    "        q1, q2 = row[\"question1\"], row[\"question2\"]\n",
    "        q1_tokens, q2_tokens = clean_statement(q1), clean_statement(q2)\n",
    "\n",
    "        q1_doc = [w.lemma_.lower() for w in q1_tokens]\n",
    "        q2_doc = [w.lemma_.lower() for w in q2_tokens]\n",
    "\n",
    "        yield q1_doc\n",
    "        yield q2_doc\n",
    "\n",
    "\n",
    "def train_lda(n_topics, id2word_dictionary=None, documents=None, corpus=None):\n",
    "    \"\"\"\n",
    "    Training method for LDA. documents is a list of lists of words/tokens\n",
    "    documents is used to construct a dictionary and a corpus from which the\n",
    "    topics for LDA are inferred\n",
    "    \"\"\"\n",
    "    # Construct dictionary of words if it's not passed\n",
    "    if not id2word_dictionary:\n",
    "        id2word_dictionary = corpora.Dictionary(documents)\n",
    "\n",
    "    word2idx_dictionary = dict([(w, idx) for (idx, w) in id2word_dictionary.items()])\n",
    "\n",
    "    # Construct corpus for model\n",
    "    if documents and not corpus:\n",
    "        corpus = [id2word_dictionary.doc2bow(document) for document in documents]\n",
    "\n",
    "    # Cluster the documents into topics using LDA. number of topics is given\n",
    "    # by n_topics\n",
    "    lda_model = LdaModel(corpus=corpus,\n",
    "                         id2word=id2word_dictionary,\n",
    "                         num_topics=n_topics,\n",
    "                         update_every=1,\n",
    "                         chunksize=10000,\n",
    "                         passes=1)\n",
    "\n",
    "    \"\"\"\n",
    "    Default value for topn (number of top words to show by probability) is 10.\n",
    "    A high enough value should return the words covering most or all of the\n",
    "    probability mass\n",
    "    \"\"\"\n",
    "    topics = [lda_model.show_topic(idx, topn=50000)\n",
    "              for idx in range(0, n_topics)]\n",
    "\n",
    "    return lda_model, id2word_dictionary, word2idx_dictionary, topics\n",
    "\n",
    "\n",
    "# If a word appears only once, we ignore it completely (likely a typo)\n",
    "# Epsilon defines a smoothing constant, which makes the effect of extremely rare words smaller\n",
    "def get_weight(count, eps=10000, min_count=2):\n",
    "    if count < min_count:\n",
    "        return 0\n",
    "    else:\n",
    "        return 1.0 / (count + eps)\n",
    "\n",
    "    \n",
    "def tfidf_word_match_share(row, weights):\n",
    "    q1words = {}\n",
    "    q2words = {}\n",
    "    for word in remove_punc(row['question1']).lower().split():\n",
    "        if word not in stops:\n",
    "            q1words[word] = 1\n",
    "    for word in remove_punc(row['question2']).lower().split():\n",
    "        if word not in stops:\n",
    "            q2words[word] = 1\n",
    "    if len(q1words) == 0 or len(q2words) == 0:\n",
    "        # The computer-generated chaff includes a few questions that are nothing but stopwords\n",
    "        return 0\n",
    "    \n",
    "    shared_weights = [weights.get(w, 0) for w in q1words.keys() if w in q2words] + [weights.get(w, 0) for w in q2words.keys() if w in q1words]\n",
    "    total_weights = [weights.get(w, 0) for w in q1words] + [weights.get(w, 0) for w in q2words]\n",
    "    \n",
    "    R = np.sum(shared_weights) / float(np.sum(total_weights))\n",
    "    return R\n",
    "\n",
    "\n",
    "def tf_idf_ngrams_match(row, weights, n=2):\n",
    "    q1_ngrams = {}\n",
    "    q2_ngrams = {}\n",
    "    \n",
    "    q1_words, q2_words = (\n",
    "        count_grams(remove_punc(str(row[\"question1\"])).lower().split(), n),\n",
    "        count_grams(remove_punc(str(row[\"question2\"])).lower().split(), n)\n",
    "    )\n",
    "    \n",
    "    shared_weights = [weights.get(w, 0) for w in q1_words.keys() if w in q2_words] +\\\n",
    "                     [weights.get(w, 0) for w in q2_words.keys() if w in q1_words]\n",
    "    total_weights = [weights.get(w, 0) for w in q1_words] + [weights.get(w, 0) for w in q2_words]\n",
    "    \n",
    "    R = np.sum(shared_weights) / float(np.sum(total_weights))\n",
    "    return R\n",
    "\n",
    "\n",
    "def weighted_token_overlap_score(row):\n",
    "    cleaned_question1_words = clean_statement(row[\"question1\"])\n",
    "    cleaned_question2_words = clean_statement(row[\"question2\"])\n",
    "    \n",
    "    set1, set2 = \\\n",
    "            (set([w.lemma_.lower() for w in cleaned_question1_words]),\n",
    "             set([w.lemma_.lower() for w in cleaned_question2_words]))\n",
    "        \n",
    "    return (1.0 * len(set1.intersection(set2)) / (len(set1.union(set2)) or 1)) * \\\n",
    "            (\n",
    "                min(len(str(row[\"question1\"])), len(str(row[\"question2\"]))) / \n",
    "                (1.0 * max(len(str(row[\"question1\"])), len(str(row[\"question2\"]))))\n",
    "            )\n",
    "    \n",
    "def stops_ratios(row):\n",
    "    q1_tokens = [t.lower() for t in remove_punc(row[\"question1\"]).split()]\n",
    "    q2_tokens = [t.lower() for t in remove_punc(row[\"question2\"]).split()]\n",
    "    q1_stops = set([t for t in q1_tokens if t in stops])\n",
    "    q2_stops = set([t for t in q2_tokens if t in stops])\n",
    "    return (\n",
    "        float(len(q1_stops.intersection(q2_stops))) / (len(q1_stops.union(q2_stops)) or 1.0),\n",
    "        float(len(q1_stops)) / (len(q1_tokens) or 1.0),\n",
    "        float(len(q2_stops)) / (len(q2_tokens) or 1.0),\n",
    "        math.fabs(float(len(q1_stops)) / (len(q1_tokens) or 1.0) - float(len(q2_stops)) / (len(q2_tokens) or 1.0))\n",
    "    )\n",
    "\n",
    "def question_tokens_ratio(row):\n",
    "    q1_quest_tokens = set([t.lower() for t in remove_punc(row[\"question1\"]) if t.lower() in question_tokens])\n",
    "    q2_quest_tokens = set([t.lower() for t in remove_punc(row[\"question2\"]) if t.lower() in question_tokens])\n",
    "    return (\n",
    "        float(len(q1_quest_tokens.intersection(q2_quest_tokens))) / (len(q1_quest_tokens.union(q2_quest_tokens)) or 1.0)\n",
    "    )\n",
    "\n",
    "\n",
    "def noun_phrase_overlap(row):\n",
    "    q1_doc = nlp(UnicodeDammit(str(row[\"question1\"])).markup)\n",
    "    q2_doc = nlp(UnicodeDammit(str(row[\"question2\"])).markup)\n",
    "    q1_np = set([noun_p.text for noun_p in q1_doc.noun_chunks])\n",
    "    q2_np = set([noun_p.text for noun_p in q2_doc.noun_chunks])\n",
    "    return len(q1_np.intersection(q2_np)) / (float(len(q1_np.union(q2_np))) or 1.0)\n",
    "\n",
    "\n",
    "def shared_ngrams(row, n):\n",
    "    \"\"\"Ratio of shared n-grams to total n-grams across both questions\"\"\"\n",
    "    \n",
    "    q1_words, q2_words = count_grams(str(row[\"question1\"]), n), count_grams(str(row[\"question2\"]), n)\n",
    "    if len(q1_words) == 0 or len(q2_words) == 0:\n",
    "        return 0.0\n",
    "        \n",
    "    shared_words = [w for w in q1_words if w in q2_words]\n",
    "    R = (2 * len(shared_words)) / float(len(q1_words) + len(q2_words))\n",
    "    return R"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# All features method\n",
    "# def features(row, lda_model, word2idx_dict, n_lda_topics=10):\n",
    "def features(df, lda_model, word2idx_dict, n_lda_topics=10, word_weights={}, naive_bayes_models={}):\n",
    "    \"\"\"\n",
    "    More features to implement:\n",
    "    - TF-IDF or similar scheme string similarity (with and without stopwords)\n",
    "    - Better LDA model by incorporating children, synonyms, related concepts, subtrees\n",
    "    - Difference in lengths between both questions, ratio of lengths\n",
    "        - for full original questions\n",
    "        - noun phrases\n",
    "        - after filtering stopwords\n",
    "    - Number of sentences in both questions, ratios, difference in number\n",
    "    - Stop words in both questions, stopq1/len(q1), stopq2/len(q2), stopq1.intersect(stopq2),...\n",
    "\n",
    "    - Common Bigrams/Trigrams\n",
    "    - Country specific features: countries or locations mentioned in both questions\n",
    "    - More features from LDA model topic probability vectors\n",
    "        - Appending both vectors\n",
    "        - [p1/p2 for (p1,p2) in zip(vector_1, vector2)]\n",
    "        - cosine sim\n",
    "    - Features specific to each question separately\n",
    "        - Length of q1\n",
    "        - Length of q2\n",
    "        - # sentences in q1\n",
    "        - # sentences in q2\n",
    "        - # words in q1\n",
    "        - # words in q2\n",
    "    - Question tokens in both questions (why, how, when, what, ..): count in each q, set intersection, difference, etc\n",
    "    - Naive encoding of question word rules as boolean vars\n",
    "        - Is \"what\" in q1 and in q2?\n",
    "        - Is \"what\" in q1 and \"how\" in q2?\n",
    "        ... repeat for the most common 6 tokens \"why\", \"how\", \"what\", \"when\", \"which\", \"who\"\n",
    "    - Sentiment analysis (see https://textblob.readthedocs.io/en/dev/quickstart.html#sentiment-analysis)\n",
    "    - Use full difference document vectors from spacy instead of just cosine and dot products (consider removing dot product)\n",
    "    - Consider using different means of lemmatising other than spacy's (ex: singular form in textblob)\n",
    "    - Various string similarity measures: Jaccard, Jaro Winkler, Levenstein distance\n",
    "    - TF-IDF features for bigrams and trigrams\n",
    "    \"\"\"\n",
    "\n",
    "    features_col = pandas.Series([[]], index=np.arange(df.shape[0]))\n",
    "\n",
    "    for (idx, row) in list(df.iterrows()):\n",
    "        q1, q2 = row[\"question1\"], row[\"question2\"]\n",
    "        q1_no_punc, q2_no_punc = remove_punc(q1), remove_punc(q2)\n",
    "        q1_tokens, q2_tokens = clean_statement(q1), clean_statement(q2)\n",
    "        tf_idf_sim = tfidf_word_match_share(row, word_weights[1])\n",
    "\n",
    "        # LDA related features\n",
    "        q1_lda_doc = [w.lemma_.lower() for w in q1_tokens]\n",
    "        q2_lda_doc = [w.lemma_.lower() for w in q2_tokens]\n",
    "        q1_topic_probs = dict(\n",
    "            lda_model.get_document_topics(Counter([word2idx_dict[w] for w in q1_lda_doc if w in word2idx_dict]).items())\n",
    "        )\n",
    "        q2_topic_probs = dict(\n",
    "            lda_model.get_document_topics(Counter([word2idx_dict[w] for w in q2_lda_doc if w in word2idx_dict]).items())\n",
    "        )\n",
    "\n",
    "        q1_topic_probs = [(t, q1_topic_probs[t]) if t in q1_topic_probs else (t, 0.0) for t in range(n_lda_topics)]\n",
    "        q2_topic_probs = [(t, q2_topic_probs[t]) if t in q2_topic_probs else (t, 0.0) for t in range(n_lda_topics)]\n",
    "\n",
    "        q1_topic_vector = np.array([prob for (topic, prob) in q1_topic_probs])\n",
    "        q2_topic_vector = np.array([prob for (topic, prob) in q2_topic_probs])\n",
    "        diff_topic_vector = q1_topic_vector - q2_topic_vector\n",
    "\n",
    "        q1_doc = nlp(UnicodeDammit(' '.join([w.lemma_.lower() for w in q1_tokens])).markup) if q1_tokens else None\n",
    "        q2_doc = nlp(UnicodeDammit(' '.join([w.lemma_.lower() for w in q2_tokens])).markup) if q2_tokens else None\n",
    "\n",
    "        q1_vector, q2_vector = (\n",
    "            q1_doc.vector if q1_doc and q1_doc.has_vector else None,\n",
    "            q2_doc.vector if q2_doc and q2_doc.has_vector else None\n",
    "        )\n",
    "        \n",
    "        if q1_vector is not None and q2_vector is not None:\n",
    "            diff_vector = (q1_vector - q2_vector) / (np.linalg.norm(q1_vector) * np.linalg.norm(q2_vector))\n",
    "        else:\n",
    "            diff_vector = [1] * 300\n",
    "\n",
    "        q1_tokens_set = set(q1_tokens)\n",
    "        q2_tokens_set = set(q2_tokens)\n",
    "\n",
    "        token_overlap_ratio = (\n",
    "            0.0 if not len(q1_tokens_set.union(q2_tokens_set))\n",
    "            else 1.0 * float(len(q1_tokens_set.intersection(q2_tokens_set))) / len(q1_tokens_set.union(q2_tokens_set))\n",
    "        )\n",
    "        \n",
    "        # Weighted TF-IDF sim\n",
    "        wt_token_overlap_score = weighted_token_overlap_score(row)\n",
    "        \n",
    "        # Stop word occurrence\n",
    "        (stops_ratio, stops_ratio_q1, stops_ratio_q2, stops_diff) = stops_ratios(row)\n",
    "        \n",
    "        # Question token pair vars\n",
    "        q_token_vars = []\n",
    "        for (q_token_q1, q_token_q2) in sorted(common_q_token_pairs):\n",
    "            q_token_vars.append(\n",
    "                float(q_token_q1 in (str(q1).lower() or '') and q_token_q2 in (str(q2).lower() or ''))\n",
    "            )\n",
    "            \n",
    "        # Basic Sentiment analysis\n",
    "        q1_text_blob = TextBlob(q1_no_punc.lower()) \n",
    "        q2_text_blob = TextBlob(q2_no_punc.lower())\n",
    "        q1_polarity = q1_text_blob.polarity\n",
    "        q2_polarity = q2_text_blob.polarity\n",
    "        q1_subjectivity = q1_text_blob.subjectivity\n",
    "        q2_subjectivity = q2_text_blob.subjectivity\n",
    "        polarity_abs_diff = math.fabs(q1_polarity - q2_polarity)\n",
    "        subjectivity_abs_diff = math.fabs(q1_subjectivity - q2_subjectivity)\n",
    "        q1_bigger_subjectivity = float(q1_subjectivity > q2_subjectivity)\n",
    "        equal_subjectivity = float(q1_subjectivity == q2_subjectivity)\n",
    "        q1_bigger_polarity = float(q1_polarity > q2_polarity)\n",
    "        equal_polarity = float(q1_polarity == q2_polarity)\n",
    "        opposite_polarity = float(np.sign(q1_polarity) != np.sign(q2_polarity))\n",
    "        \n",
    "        # Noun phrases\n",
    "        n_phrase_overlap = noun_phrase_overlap(row)\n",
    "        q1_doc = nlp(UnicodeDammit(str(q1)).markup)\n",
    "        q2_doc = nlp(UnicodeDammit(str(q2)).markup)\n",
    "        q1_np = set([noun_p.text for noun_p in q1_doc.noun_chunks])\n",
    "        q2_np = set([noun_p.text for noun_p in q2_doc.noun_chunks])\n",
    "        \n",
    "        # name similarity metrics\n",
    "        jaro_winkler_sim = jellyfish.jaro_winkler(UnicodeDammit(str(q1)).markup.lower(), UnicodeDammit(str(q2)).markup.lower()),\n",
    "        levenshtein_dist = jellyfish.levenshtein_distance(UnicodeDammit(str(q1)).markup.lower(), UnicodeDammit(str(q2)).markup.lower()),\n",
    "        hamming_dist = jellyfish.hamming_distance(UnicodeDammit(str(q1)).markup.lower(), UnicodeDammit(str(q2)).markup.lower())\n",
    "        \n",
    "        # n-gram analysis\n",
    "        shared_n_gram_vars = []\n",
    "        for n in range(2, 8):\n",
    "            shared_n_gram_vars.append(shared_ngrams(row, n))\n",
    "            \n",
    "        # n-gram TF-IDF sim\n",
    "        two_gram_tfidf_sim = tf_idf_ngrams_match(row, word_weights[2], n=2)\n",
    "        three_gram_tfidf_sim = tf_idf_ngrams_match(row, word_weights[3], n=3)\n",
    "        \n",
    "        # Prob vectors of question classification based on NB models\n",
    "        # trained on the Univ Illinois dataset\n",
    "        fine_grained_nb = naive_bayes_models[\"fine_grained\"]\n",
    "        coarse_grained_nb = naive_bayes_models[\"coarse_grained\"]\n",
    "        \n",
    "        # Fine grained classification model\n",
    "        try:\n",
    "            p_q1_fine_grained_vec = fine_grained_nb.predict_proba([q1_no_punc.lower()])[0]\n",
    "            p_q2_fine_grained_vec = fine_grained_nb.predict_proba([q2_no_punc.lower()])[0]\n",
    "\n",
    "            diff_fine_grained_nb_vec = list(np.abs(p_q1_fine_grained_vec - p_q2_fine_grained_vec))\n",
    "        except ValueError:\n",
    "            diff_fine_grained_nb_vec = [1.0] * len(fine_grained_nb.classes_)\n",
    "        \n",
    "        # Coarse grained classification model\n",
    "        try:\n",
    "            p_q1_coarse_grained_vec = coarse_grained_nb.predict_proba([q1_no_punc.lower()])[0]\n",
    "            p_q2_coarse_grained_vec = coarse_grained_nb.predict_proba([q2_no_punc.lower()])[0]\n",
    "\n",
    "            diff_coarse_grained_nb_vec = list(np.abs(p_q1_coarse_grained_vec - p_q2_coarse_grained_vec))\n",
    "        except ValueError:\n",
    "            diff_coarse_grained_nb_vec = [1.0] * len(coarse_grained_nb.classes_)\n",
    "        \n",
    "        if q1_vector is not None and q2_vector is not None:\n",
    "            dot_product = q1_vector.dot(q2_vector) \n",
    "            cosine_sim = cosine_similarity(q1_vector, q2_vector)[0][0]\n",
    "            euclidean_dist = np.linalg.norm(q1_vector - q2_vector)\n",
    "            euclidean_lda_probs_dist = np.linalg.norm(diff_topic_vector)\n",
    "        else:\n",
    "            dot_product = cosine_sim = 0.0\n",
    "            euclidean_dist = euclidean_lda_probs_dist = 100.0 # Not a very good hack\n",
    "            \n",
    "        if type(jaro_winkler_sim) == tuple:\n",
    "            jaro_winkler_sim = jaro_winkler_sim[0]\n",
    "            \n",
    "        feature_list = [\n",
    "            token_overlap_ratio,\n",
    "            float(token_overlap_ratio == 0),\n",
    "            float(token_overlap_ratio == 1),\n",
    "            dot_product,\n",
    "            cosine_sim,\n",
    "#             euclidean_dist,\n",
    "#             euclidean_lda_probs_dist,\n",
    "            tf_idf_sim,\n",
    "            wt_token_overlap_score,\n",
    "            stops_ratio,\n",
    "            stops_ratio_q1,\n",
    "            stops_ratio_q2,\n",
    "            stops_diff,\n",
    "            q1_polarity,\n",
    "            q2_polarity,\n",
    "            q1_subjectivity,\n",
    "            q2_subjectivity,\n",
    "            q1_bigger_subjectivity,\n",
    "            equal_subjectivity,\n",
    "            q1_bigger_polarity,\n",
    "            equal_polarity,\n",
    "#             polarity_abs_diff,\n",
    "            subjectivity_abs_diff,\n",
    "            opposite_polarity,\n",
    "            len(q1_no_punc),\n",
    "            len(q2_no_punc),\n",
    "            len(q1_lda_doc),\n",
    "            len(q2_lda_doc),\n",
    "            n_phrase_overlap,\n",
    "            len(q1_np),\n",
    "            len(q2_np),\n",
    "            two_gram_tfidf_sim,\n",
    "            three_gram_tfidf_sim,\n",
    "            jaro_winkler_sim,\n",
    "            1 / float(hamming_dist or 0.0)\n",
    "        ]\n",
    "        feature_list.extend(q_token_vars)\n",
    "        feature_list.extend(shared_n_gram_vars)\n",
    "        feature_list.extend(diff_fine_grained_nb_vec)\n",
    "        feature_list.extend(diff_coarse_grained_nb_vec)\n",
    "#         feature_list.extend(list(diff_vector))  # Wasn't a good feature. Could be the way it was constructed\n",
    "        feature_list.extend(list(diff_topic_vector))\n",
    "\n",
    "        # return feature_list\n",
    "        features_col[idx] = feature_list\n",
    "\n",
    "    df[\"features\"] = features_col\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "def train_naive_bayes(documents):\n",
    "    X = pandas.DataFrame(documents.text.apply(lambda x: unicode(x, errors='replace')))\n",
    "    y = documents.target\n",
    "\n",
    "    model = Pipeline([\n",
    "        ('count', CountVectorizer(ngram_range=(1, 3), min_df=1)),\n",
    "        ('tfidf', TfidfTransformer()),\n",
    "        ('clf',   MultinomialNB(alpha=0.1)),\n",
    "    ])\n",
    "\n",
    "    model.fit(X.text, y)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Random Forest model\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "\n",
    "from sklearn.cross_validation import train_test_split, StratifiedKFold\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_curve, auc, confusion_matrix\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "\n",
    "def predict_rf(row, model):\n",
    "    \"\"\"\n",
    "    Assumes row object has a `features` column\n",
    "    with the same features as those on which\n",
    "    `model` was trained\n",
    "    \"\"\"\n",
    "    return float(model.predict_proba(np.array(row[\"features\"]))[0][1])\n",
    "\n",
    "class RandomForestModel():\n",
    "    n_trees = 500\n",
    "    # test_size = 0.3\n",
    "    rf_max_features = None\n",
    "    folds = 10\n",
    "    \n",
    "    def grid_search(self, training_df):\n",
    "        model = RandomForestClassifier(n_jobs=-1, max_features= 'sqrt', n_estimators=self.n_trees, oob_score = True) \n",
    "\n",
    "        param_grid = { \n",
    "            'n_estimators': range(400, 750, 50),\n",
    "            'max_features': ['auto', 'sqrt', 'log2'],\n",
    "            'class_weight': ['subsample', 'auto'],\n",
    "#             \"criterion\": [\"gini\", \"entropy\"],\n",
    "            'max_depth': [None, 10, 30, 50]\n",
    "        }\n",
    "        \n",
    "        featureMatrix, labelVector = training_df[\"features\"], training_df[\"label\"]\n",
    "        featureMatrix = np.array([list(f) for f in featureMatrix])\n",
    "        featureMatrix = np.nan_to_num(featureMatrix)\n",
    "        labelVector = np.array(list(labelVector))\n",
    "        labelVector = np.nan_to_num(labelVector)\n",
    "\n",
    "        CV_model = GridSearchCV(estimator=model, param_grid=param_grid, cv=5, verbose=1)\n",
    "        CV_model.fit(featureMatrix, labelVector)\n",
    "        return CV_model.best_params_\n",
    "\n",
    "\n",
    "    def train(self, training_df, cv=True):\n",
    "        \"\"\"\n",
    "        Expects a `features` column which holds a\n",
    "        list of floats to be used as features for\n",
    "        the classifier and an integer `label` column\n",
    "        encoding the output to be predicted\n",
    "        \"\"\"\n",
    "        featureMatrix, labelVector = training_df[\"features\"], training_df[\"label\"]\n",
    "        featureMatrix = np.array([list(f) for f in featureMatrix])\n",
    "        featureMatrix = np.nan_to_num(featureMatrix)\n",
    "        labelVector = np.array(list(labelVector))\n",
    "        labelVector = np.nan_to_num(labelVector)\n",
    "\n",
    "        auc_list = []\n",
    "        logloss_list = []\n",
    "\n",
    "        if cv:\n",
    "            idx = 1\n",
    "            for train, test in StratifiedKFold(labelVector, self.folds):\n",
    "                print \"Starting Cross Validation Fold {}\".format(idx)\n",
    "\n",
    "                x_train, y_train = featureMatrix[train], labelVector[train]\n",
    "                x_test, y_test = featureMatrix[test], labelVector[test]\n",
    "                x_train = np.asarray(x_train)\n",
    "                y_train = np.asarray(y_train)\n",
    "                x_test = np.asarray(x_test)\n",
    "                y_test = np.asarray(y_test)\n",
    "\n",
    "                model = RandomForestClassifier(n_estimators=self.n_trees, max_features='sqrt', max_depth=None, class_weight=\"auto\")\\\n",
    "                    if self.rf_max_features else RandomForestClassifier(n_estimators=self.n_trees, class_weight=\"auto\")\n",
    "\n",
    "                model.fit(x_train, y_train)\n",
    "\n",
    "                predictions = model.predict_proba(x_test)[:, 1]\n",
    "                fprArray, tprArray, thres = roc_curve(y_test, predictions)\n",
    "                roc_auc = auc(fprArray, tprArray)\n",
    "                logloss = binary_logloss(y_test, predictions)\n",
    "                auc_list.append(roc_auc)\n",
    "                logloss_list.append(logloss_list)\n",
    "\n",
    "                print \"CV Fold result: AUC is {auc} and Log Loss is {loss}\".format(auc=roc_auc, loss=logloss)\n",
    "                print \"#########\"\n",
    "\n",
    "                idx += 1\n",
    "            \n",
    "            # Just for fast testing\n",
    "            return\n",
    "\n",
    "            model = RandomForestClassifier(n_estimators=self.n_trees, max_features=self.rf_max_features, class_weight=\"auto\")\\\n",
    "                if self.rf_max_features else RandomForestClassifier(n_estimators=self.n_trees, class_weight=\"auto\")\n",
    "\n",
    "            roc_auc = np.mean(auc_list)\n",
    "            logloss = np.mean(logloss_list)\n",
    "            print \"<======================================>\"\n",
    "            print \"Finished cross validation experiments!\"\n",
    "            print \"Average AUC is {auc} and average Log Loss is {loss}\".format(auc=roc_auc, loss=logloss)\n",
    "            print \"Starting full model training!\"\n",
    "\n",
    "            model.fit(featureMatrix, labelVector)\n",
    "\n",
    "            return {'model': model, 'roc_auc': roc_auc, 'logloss': logloss}\n",
    "        else:\n",
    "            model = RandomForestClassifier(n_estimators=self.n_trees, max_features=self.rf_max_features, class_weight=\"auto\")\\\n",
    "                if self.rf_max_features else RandomForestClassifier(n_estimators=self.n_trees, class_weight=\"auto\")\n",
    "\n",
    "            model.fit(featureMatrix, labelVector)\n",
    "\n",
    "            return {'model': model}\n",
    "\n",
    "    def compute_precision_scores(self, y_pred, y_true, prob_thresholds):\n",
    "        \"\"\"\n",
    "        Compute precision scores at different probability thresholds\n",
    "        This allows us to pick a probability threshold for the classifier\n",
    "        given a desired precision score\n",
    "            pr = tpr  /  (tpr + fpr)\n",
    "        returns: list((precision_score, prob_thres))\n",
    "        \"\"\"\n",
    "        precisions = []\n",
    "        for prob_thres in prob_thresholds:\n",
    "            flagged_idxes = filter(lambda idx: y_pred[idx] >= prob_thres, range(len(y_pred)))\n",
    "            true_flagged_idxes = filter(lambda idx: y_pred[idx] >= prob_thres and y_true[idx] == 1, range(len(y_pred)))\n",
    "            precision = (len(true_flagged_idxes) / float(len(flagged_idxes))) if len(flagged_idxes) else 0.0\n",
    "            precisions.append((precision, prob_thres))\n",
    "\n",
    "        return sorted(precisions, key=lambda (prec, prob): prec)\n",
    "\n",
    "    def compute_accuracy_scores(self, y_pred, y_true, prob_thresholds):\n",
    "        \"\"\"\n",
    "        Compute accuracy scores at different probability thresholds\n",
    "        This allows us to pick a probability threshold for the classifier\n",
    "        given a desired precision score\n",
    "        returns: list((accuracy_score, prob_thres))\n",
    "        \"\"\"\n",
    "        accuracy_scores = []\n",
    "        for prob_thres in prob_thresholds:\n",
    "            correct_predicted_data_points = filter(lambda prob_idx:\n",
    "                                                   (y_pred[prob_idx] >= prob_thres and y_true[prob_idx] == 1) or\n",
    "                                                   (y_pred[prob_idx] < prob_thres and y_true[prob_idx] == 0),\n",
    "                                                   range(len(y_pred)))\n",
    "            accuracy = len(correct_predicted_data_points) / float(len(y_true)) if len(y_true) else 0.0\n",
    "            accuracy_scores.append((accuracy, prob_thres))\n",
    "\n",
    "        return sorted(accuracy_scores, key=lambda (acc, prob): acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# XgBoost model\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "\n",
    "from sklearn.cross_validation import train_test_split, StratifiedKFold\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_curve, auc, confusion_matrix\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "\n",
    "def binary_logloss(act, pred):\n",
    "    \"\"\"\n",
    "    act and pred are vectors of actual class\n",
    "    and prediction probability of class 1,\n",
    "    respectively\n",
    "    \"\"\"\n",
    "    epsilon = 1e-15\n",
    "    pred = sp.maximum(epsilon, pred)\n",
    "    pred = sp.minimum(1 - epsilon, pred)\n",
    "    ll = sum(act * sp.log(pred) + sp.subtract(1, act) * sp.log(sp.subtract(1, pred)))\n",
    "    ll = ll * -1.0 / len(act)\n",
    "    return ll\n",
    "\n",
    "\n",
    "def xgboost_eval(act, pred):\n",
    "    return 'error', binary_logloss(act, pred)\n",
    "\n",
    "\n",
    "def predict_xgboost(row, model):\n",
    "    \"\"\"\n",
    "    Assumes row object has a `features` column\n",
    "    with the same features as those on which\n",
    "    `model` was trained\n",
    "    \"\"\"\n",
    "    return float(model.predict(np.array(row[\"features\"]))[0])\n",
    "\n",
    "\n",
    "class XgBoostModel():\n",
    "    n_boost_rounds = 2000\n",
    "    max_depth = 5\n",
    "    objective = 'binary:logistic'\n",
    "    eval_metric = \"logloss\"\n",
    "    early_stopping_rounds = 70\n",
    "    folds = 10\n",
    "    learning_rate = 0.1\n",
    "    scale_pos_weight = 1\n",
    "    gamma = 0.1\n",
    "\n",
    "    def train(self, training_df, cv=True):\n",
    "        \"\"\"\n",
    "        Expects a `features` column which holds a\n",
    "        list of floats to be used as features for\n",
    "        the classifier and an integer `label` column\n",
    "        encoding the output to be predicted\n",
    "        \"\"\"\n",
    "        featureMatrix, labelVector = training_df[\"features\"], training_df[\"label\"]\n",
    "        featureMatrix = np.array([list(f) for f in featureMatrix])\n",
    "        featureMatrix = np.nan_to_num(featureMatrix)\n",
    "        labelVector = np.array(list(labelVector))\n",
    "        labelVector = np.nan_to_num(labelVector)\n",
    "\n",
    "        auc_list = []\n",
    "        logloss_list = []\n",
    "\n",
    "        if cv:\n",
    "            idx = 1\n",
    "            for train, test in StratifiedKFold(labelVector, self.folds):\n",
    "                print \"Starting Cross Validation Fold {}\".format(idx)\n",
    "\n",
    "                x_train, y_train = featureMatrix[train], labelVector[train]\n",
    "                x_test, y_test = featureMatrix[test], labelVector[test]\n",
    "                x_train = np.asarray(x_train)\n",
    "                y_train = np.asarray(y_train)\n",
    "                x_test = np.asarray(x_test)\n",
    "                y_test = np.asarray(y_test)\n",
    "                \n",
    "                params = {}\n",
    "                params['objective'] = self.objective\n",
    "                params['eval_metric'] = self.eval_metric\n",
    "                params['eta'] = self.learning_rate\n",
    "                params['max_depth'] = self.max_depth\n",
    "                params['scale_pos_weight'] = self.scale_pos_weight\n",
    "                params['gamma'] = self.gamma\n",
    "                params['silent'] = 1\n",
    "\n",
    "                d_train = xgb.DMatrix(x_train, label=y_train)\n",
    "                d_valid = xgb.DMatrix(x_test, label=y_test)\n",
    "                \n",
    "                watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n",
    "                \n",
    "                model = xgb.train(\n",
    "                    params,\n",
    "                    d_train,\n",
    "                    self.n_boost_rounds,\n",
    "                    watchlist,\n",
    "                    early_stopping_rounds=self.early_stopping_rounds\n",
    "                )\n",
    "\n",
    "                x_test_df = pandas.DataFrame(x_test, columns=[\"feature_%s\" % str(i) for i in range(x_test.shape[1])])\n",
    "                predictions = model.predict(xgb.DMatrix(x_test_df))\n",
    "                                \n",
    "                fprArray, tprArray, thres = roc_curve(y_test, predictions)\n",
    "                roc_auc = auc(fprArray, tprArray)\n",
    "                logloss = binary_logloss(y_test, predictions)\n",
    "                auc_list.append(roc_auc)\n",
    "                logloss_list.append(logloss_list)\n",
    "\n",
    "                print \"CV Fold result: AUC is {auc} and Log Loss is {loss}\".format(auc=roc_auc, loss=logloss)\n",
    "                print \"#########\"\n",
    "\n",
    "                idx += 1\n",
    "            \n",
    "            # Just for fast testing\n",
    "            return\n",
    "\n",
    "            roc_auc = np.mean(auc_list)\n",
    "            logloss = np.mean(logloss_list)\n",
    "            print \"<======================================>\"\n",
    "            print \"Finished cross validation experiments!\"\n",
    "            print \"Average AUC is {auc} and average Log Loss is {loss}\".format(auc=roc_auc, loss=logloss)\n",
    "            print \"Starting full model training!\"\n",
    "\n",
    "#             model = xgb.XGBClassifier(max_depth=self.max_depth, n_estimators=self.n_trees)\n",
    "#             model.fit(featureMatrix, labelVector, eval_metric=self.eval_metric)#, early_stopping_rounds=self.early_stopping_rounds)\n",
    "            # make prediction\n",
    "            # preds = model.predict(x_test)\n",
    "\n",
    "            return {'model': model, 'roc_auc': roc_auc, 'logloss': logloss}\n",
    "        else:\n",
    "            model = xgb.XGBClassifier(max_depth=self.max_depth, n_estimators=self.n_trees)\n",
    "            model.fit(featureMatrix, labelVector, eval_metric=self.eval_metric)#, early_stopping_rounds=self.early_stopping_rounds)\n",
    "\n",
    "            return {'model': model}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read data\n",
    "\n",
    "train_path = \"/Users/mohamedabdelbary/Documents/kaggle_quora/train.csv\"\n",
    "uillinois_q_set_path = \"/Users/mohamedabdelbary/Documents/kaggle_quora/uillinois_labelled_question_set.txt\"\n",
    "models_path = \"/Users/mohamedabdelbary/Documents/kaggle_quora/models.pkl\"\n",
    "train_pred_path = \"/Users/mohamedabdelbary/Documents/kaggle_quora/train_preds.csv\"\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas\n",
    "from functools import partial\n",
    "from collections import Counter\n",
    "\n",
    "def read_data(path):\n",
    "    return pandas.read_csv(path)\n",
    "\n",
    "n_sample = 50000\n",
    "full_df = read_data(train_path)\n",
    "rows = np.random.choice(full_df.index.values, n_sample)\n",
    "df = full_df.ix[rows]\n",
    "\n",
    "# df = read_data(train_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "uillinois_q_set_path = \"/Users/mohamedabdelbary/Documents/kaggle_quora/uillinois_labelled_question_set.txt\"\n",
    "def read_uillinois_q_set(path, granular_target=True):\n",
    "    questions = []\n",
    "    with open(path, 'rb') as f:\n",
    "        for rec in f.readlines():\n",
    "            if granular_target:\n",
    "                questions.append({\n",
    "                    'target': rec.split(' ')[0],\n",
    "                    'text': \" \".join(rec.split(' ')[1:]).lower().rstrip()})\n",
    "            else:\n",
    "                questions.append({\n",
    "                    'target': rec.split(' ')[0].split(\":\")[0],\n",
    "                    'text': \" \".join(rec.split(' ')[1:]).lower().rstrip()})\n",
    "\n",
    "    return pandas.DataFrame(questions)\n",
    "\n",
    "# Train Naive Bayes models for classifying the U-Illinois\n",
    "# labelled question set\n",
    "uillinois_q_set_granular_target = read_uillinois_q_set(uillinois_q_set_path)\n",
    "uillinois_q_set_coarse_target = read_uillinois_q_set(uillinois_q_set_path, granular_target=False)\n",
    "\n",
    "nb_uillinois_fine_grained = train_naive_bayes(uillinois_q_set_granular_target)\n",
    "nb_uillinois_coarse_grained = train_naive_bayes(uillinois_q_set_coarse_target)\n",
    "\n",
    "naive_bayes_models = {\n",
    "    \"fine_grained\": nb_uillinois_fine_grained,\n",
    "    \"coarse_grained\": nb_uillinois_coarse_grained}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# n-gram frequency dicts\n",
    "\n",
    "questions = pandas.Series(df['question1'].tolist() + df['question2'].tolist()).astype(str)\n",
    "questions = [remove_punc(q).lower() for q in questions]\n",
    "eps = 500 \n",
    "words = (\" \".join(questions)).lower().split()\n",
    "counts = Counter(words)\n",
    "weights = {word: get_weight(count, eps=eps) for word, count in counts.items()}\n",
    "weights_2gram = count_grams_full(df, 2)\n",
    "weights_3gram = count_grams_full(df, 3)\n",
    "\n",
    "ngram_weights = {1: weights, 2: weights_2gram, 3: weights_3gram}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting LDA modelling!\n"
     ]
    }
   ],
   "source": [
    "# Train topic model\n",
    "n_lda_topics = 20\n",
    "print \"Starting LDA modelling!\"\n",
    "\n",
    "doc_list_lda_train = list(construct_doc_list(df))\n",
    "lda_model, id2word_dictionary, word2idx_dictionary, topics = \\\n",
    "    train_lda(n_lda_topics,\n",
    "              documents=doc_list_lda_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting feature construction!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mohamedabdelbary/.pyenv/versions/2.7.8/lib/python2.7/site-packages/ipykernel/__main__.py:217: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "/Users/mohamedabdelbary/.pyenv/versions/2.7.8/lib/python2.7/site-packages/ipykernel/__main__.py:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "# Feature construction\n",
    "print \"Starting feature construction!\"\n",
    "feature_method = partial(\n",
    "    features,\n",
    "    lda_model=lda_model,\n",
    "    word2idx_dict=word2idx_dictionary,\n",
    "    n_lda_topics=n_lda_topics,\n",
    "    word_weights=ngram_weights,\n",
    "    naive_bayes_models=naive_bayes_models\n",
    "    )\n",
    "df = feature_method(df)\n",
    "df[\"label\"] = df[\"is_duplicate\"].map(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.191243809958\n"
     ]
    }
   ],
   "source": [
    "# Purely for experimenting!! This oversampling process can lead to overfitting\n",
    "# and is generally not very good ML practise\n",
    "pos_train = df[df[\"is_duplicate\"] == 1]\n",
    "neg_train = df[df[\"is_duplicate\"] == 0]\n",
    "\n",
    "# Now we oversample the negative class\n",
    "# There is likely a much more elegant way to do this...\n",
    "p = 0.165\n",
    "scale = ((float(len(pos_train)) / (len(pos_train) + len(neg_train))) / p) - 1\n",
    "while scale > 1:\n",
    "    neg_train = pandas.concat([neg_train, neg_train])\n",
    "    scale -=1\n",
    "neg_train = pandas.concat([neg_train, neg_train[:int(scale * len(neg_train))]])\n",
    "print len(pos_train) / float(len(pos_train) + len(neg_train))\n",
    "\n",
    "df_resampled = pandas.concat([pos_train, neg_train])\n",
    "y_train = (np.zeros(len(pos_train)) + 1).tolist() + np.zeros(len(neg_train)).tolist()\n",
    "del pos_train, neg_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Cross Validation Fold 1\n",
      "CV Fold result: AUC is 0.878075179217 and Log Loss is 0.42181305973\n",
      "#########\n",
      "Starting Cross Validation Fold 2\n",
      "CV Fold result: AUC is 0.880067221197 and Log Loss is 0.421521908041\n",
      "#########\n",
      "Starting Cross Validation Fold 3\n",
      "CV Fold result: AUC is 0.875200850929 and Log Loss is 0.42839037419\n",
      "#########\n",
      "Starting Cross Validation Fold 4\n",
      "CV Fold result: AUC is 0.875666245591 and Log Loss is 0.428394372034\n",
      "#########\n",
      "Starting Cross Validation Fold 5\n",
      "CV Fold result: AUC is 0.875203251196 and Log Loss is 0.42802626049\n",
      "#########\n",
      "Starting Cross Validation Fold 6\n",
      "CV Fold result: AUC is 0.880795016497 and Log Loss is 0.424856869741\n",
      "#########\n",
      "Starting Cross Validation Fold 7\n",
      "CV Fold result: AUC is 0.867136553257 and Log Loss is 0.434197482051\n",
      "#########\n",
      "Starting Cross Validation Fold 8\n",
      "CV Fold result: AUC is 0.874072468181 and Log Loss is 0.427934633326\n",
      "#########\n",
      "Starting Cross Validation Fold 9\n",
      "CV Fold result: AUC is 0.880467380026 and Log Loss is 0.422286222041\n",
      "#########\n",
      "Starting Cross Validation Fold 10\n",
      "CV Fold result: AUC is 0.875559092333 and Log Loss is 0.428137515502\n",
      "#########\n"
     ]
    }
   ],
   "source": [
    "# Model training\n",
    "# model = XgBoostModel()\n",
    "model = RandomForestModel()\n",
    "model_obj = model.train(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 168 candidates, totalling 840 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 jobs       | elapsed:   40.5s\n",
      "[Parallel(n_jobs=1)]: Done  50 jobs       | elapsed: 48.1min\n",
      "[Parallel(n_jobs=1)]: Done 200 jobs       | elapsed: 147.0min\n",
      "[Parallel(n_jobs=1)]: Done 450 jobs       | elapsed: 363.8min\n",
      "[Parallel(n_jobs=1)]: Done 800 jobs       | elapsed: 657.9min\n",
      "[Parallel(n_jobs=1)]: Done 840 out of 840 | elapsed: 692.0min finished\n"
     ]
    }
   ],
   "source": [
    "# CV experiment\n",
    "g = RandomForestModel()\n",
    "best_params = g.grid_search(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_features': 'sqrt', 'n_estimators': 700, 'max_depth': None, 'class_weight': 'auto'}\n"
     ]
    }
   ],
   "source": [
    "print best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['HUM:ind']\n",
      "['HUM:gr']\n",
      "['DESC:desc']\n",
      "['NUM:date']\n",
      "['LOC:other']\n"
     ]
    }
   ],
   "source": [
    "print nb_uillinois_fine_grained.predict([uillinois_q_set_granular_target[\"text\"][0]])\n",
    "print nb_uillinois_fine_grained.predict([uillinois_q_set_granular_target[\"text\"][1]])\n",
    "print nb_uillinois_fine_grained.predict([uillinois_q_set_granular_target[\"text\"][2]])\n",
    "print nb_uillinois_fine_grained.predict([uillinois_q_set_granular_target[\"text\"][3]])\n",
    "print nb_uillinois_fine_grained.predict([uillinois_q_set_granular_target[\"text\"][4]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "what football coach 's story was told in the movie run to daylight ?\n",
      "what businesses in alaska were affected by the exxon valdez oil spill ?\n",
      "what does lloyd 's lutine bell announce ?\n",
      "when did the chernobyl nuclear accident occur ?\n",
      "where did the maya people live ?\n"
     ]
    }
   ],
   "source": [
    "print uillinois_q_set_granular_target[\"text\"][0]\n",
    "print uillinois_q_set_granular_target[\"text\"][1]\n",
    "print uillinois_q_set_granular_target[\"text\"][2]\n",
    "print uillinois_q_set_granular_target[\"text\"][3]\n",
    "print uillinois_q_set_granular_target[\"text\"][4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}