02_preprocessing_text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.corpus import stopwords\n",
    "from textblob import TextBlob as tb\n",
    "from contractions import CONTRACTION_MAP\n",
    "stopwords = stopwords.words('english')\n",
    "stopwords.remove('no')\n",
    "stopwords.remove('not')\n",
    "stopwords.append('sa')\n",
    "stopwords.append('na')\n",
    "stopwords.append('ng')\n",
    "stopwords.append('ko')\n",
    "stopwords.append('ba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('datasets/final_dataset.csv',index_col=None,header=0)\n",
    "n = data.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submission_title</th>\n",
       "      <th>submission_text</th>\n",
       "      <th>author</th>\n",
       "      <th>body</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit_name_prefixed</th>\n",
       "      <th>replies</th>\n",
       "      <th>length</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4xury6</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Ryvaeus</td>\n",
       "      <td>No questions, just wanted to thank you for pos...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>245</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Somebody_over_there</td>\n",
       "      <td>Holy shit thank you so much!     \\nAs someone ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>8</td>\n",
       "      <td>393</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>PotatoMurderer</td>\n",
       "      <td>&gt; people there are nice and friendly; super no...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>0</td>\n",
       "      <td>507</td>\n",
       "      <td>commenting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4y5kmd</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>mazikeen24</td>\n",
       "      <td>Thanks for posting this! Have you tried Yaz or...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>16</td>\n",
       "      <td>238</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d4y8tvk</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>a_reverse_giraffe</td>\n",
       "      <td>As a minor, did you need to show your parents ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>55</td>\n",
       "      <td>questioning</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                   submission_title  \\\n",
       "0  d4xury6  I got my contraceptive implant, Implanon, last...   \n",
       "1  d4yi9y9  I got my contraceptive implant, Implanon, last...   \n",
       "2  d4yhm4h  I got my contraceptive implant, Implanon, last...   \n",
       "3  d4y5kmd  I got my contraceptive implant, Implanon, last...   \n",
       "4  d4y8tvk  I got my contraceptive implant, Implanon, last...   \n",
       "\n",
       "                                     submission_text               author  \\\n",
       "0  **UPDATED September 26, 2018; scroll down**\\n\\...              Ryvaeus   \n",
       "1  **UPDATED September 26, 2018; scroll down**\\n\\...  Somebody_over_there   \n",
       "2  **UPDATED September 26, 2018; scroll down**\\n\\...       PotatoMurderer   \n",
       "3  **UPDATED September 26, 2018; scroll down**\\n\\...           mazikeen24   \n",
       "4  **UPDATED September 26, 2018; scroll down**\\n\\...    a_reverse_giraffe   \n",
       "\n",
       "                                                body  depth  parent_id  score  \\\n",
       "0  No questions, just wanted to thank you for pos...      0  t3_4r2tl4     31   \n",
       "1  Holy shit thank you so much!     \\nAs someone ...      0  t3_4r2tl4      7   \n",
       "2  > people there are nice and friendly; super no...      0  t3_4r2tl4      7   \n",
       "3  Thanks for posting this! Have you tried Yaz or...      0  t3_4r2tl4      3   \n",
       "4  As a minor, did you need to show your parents ...      0  t3_4r2tl4      3   \n",
       "\n",
       "  subreddit_name_prefixed  replies  length       labels  \n",
       "0           r/Philippines        4     245   supporting  \n",
       "1           r/Philippines        8     393   supporting  \n",
       "2           r/Philippines        0     507   commenting  \n",
       "3           r/Philippines       16     238   supporting  \n",
       "4           r/Philippines        4      55  questioning  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(df):\n",
    "    \"\"\"\n",
    "    Function for cleaning text based on class jupyter notebook on text manipulation\n",
    "    \"\"\"\n",
    "    text = df.copy()\n",
    "    text['cleaned'] = text['body'].fillna('')\n",
    "    text['cleaned'] = text['cleaned'].str.lower()\n",
    "    text['cleaned'] = text['cleaned'].str.replace(r'([-])', ' ')\n",
    "    text['cleaned'] = text['cleaned'].str.replace(r'([^A-Z a-z\\'])|', '')\n",
    "    text['cleaned'] = text['cleaned'].apply(lambda x: ' '.join([w for w in x.split() if w not in (stopwords)]))\n",
    "    \n",
    "    text['cleaned_sub'] = text['submission_text'].fillna('')\n",
    "    text['cleaned_sub'] = text['cleaned_sub'].str.lower()\n",
    "    text['cleaned_sub'] = text['cleaned_sub'].str.replace(r'([-])', ' ')\n",
    "    text['cleaned_sub'] = text['cleaned_sub'].str.replace(r'([^A-Z a-z\\'])|', '')\n",
    "    text['cleaned_sub'] = text['cleaned_sub'].apply(lambda x: ' '.join([w for w in x.split() if w not in (stopwords)]))\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean = clean_text(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expand_contractions(text, contraction_mapping=CONTRACTION_MAP):\n",
    "    \n",
    "    contractions_pattern = re.compile('({})'.format('|'.join(contraction_mapping.keys())), \n",
    "                                      flags=re.IGNORECASE|re.DOTALL)\n",
    "    def expand_match(contraction):\n",
    "        match = contraction.group(0)\n",
    "        first_char = match[0]\n",
    "        expanded_contraction = contraction_mapping.get(match)\\\n",
    "                                if contraction_mapping.get(match)\\\n",
    "                                else contraction_mapping.get(match.lower())                       \n",
    "        expanded_contraction = first_char+expanded_contraction[1:]\n",
    "        return expanded_contraction\n",
    "        \n",
    "    expanded_text = contractions_pattern.sub(expand_match, text)\n",
    "    expanded_text = re.sub(\"'\", \"\", expanded_text)\n",
    "    return expanded_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submission_title</th>\n",
       "      <th>submission_text</th>\n",
       "      <th>author</th>\n",
       "      <th>body</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit_name_prefixed</th>\n",
       "      <th>replies</th>\n",
       "      <th>length</th>\n",
       "      <th>labels</th>\n",
       "      <th>cleaned</th>\n",
       "      <th>cleaned_sub</th>\n",
       "      <th>expand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4xury6</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Ryvaeus</td>\n",
       "      <td>No questions, just wanted to thank you for pos...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>245</td>\n",
       "      <td>supporting</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Somebody_over_there</td>\n",
       "      <td>Holy shit thank you so much!     \\nAs someone ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>8</td>\n",
       "      <td>393</td>\n",
       "      <td>supporting</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>PotatoMurderer</td>\n",
       "      <td>&gt; people there are nice and friendly; super no...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>0</td>\n",
       "      <td>507</td>\n",
       "      <td>commenting</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4y5kmd</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>mazikeen24</td>\n",
       "      <td>Thanks for posting this! Have you tried Yaz or...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>16</td>\n",
       "      <td>238</td>\n",
       "      <td>supporting</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d4y8tvk</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>a_reverse_giraffe</td>\n",
       "      <td>As a minor, did you need to show your parents ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>55</td>\n",
       "      <td>questioning</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                   submission_title  \\\n",
       "0  d4xury6  I got my contraceptive implant, Implanon, last...   \n",
       "1  d4yi9y9  I got my contraceptive implant, Implanon, last...   \n",
       "2  d4yhm4h  I got my contraceptive implant, Implanon, last...   \n",
       "3  d4y5kmd  I got my contraceptive implant, Implanon, last...   \n",
       "4  d4y8tvk  I got my contraceptive implant, Implanon, last...   \n",
       "\n",
       "                                     submission_text               author  \\\n",
       "0  **UPDATED September 26, 2018; scroll down**\\n\\...              Ryvaeus   \n",
       "1  **UPDATED September 26, 2018; scroll down**\\n\\...  Somebody_over_there   \n",
       "2  **UPDATED September 26, 2018; scroll down**\\n\\...       PotatoMurderer   \n",
       "3  **UPDATED September 26, 2018; scroll down**\\n\\...           mazikeen24   \n",
       "4  **UPDATED September 26, 2018; scroll down**\\n\\...    a_reverse_giraffe   \n",
       "\n",
       "                                                body  depth  parent_id  score  \\\n",
       "0  No questions, just wanted to thank you for pos...      0  t3_4r2tl4     31   \n",
       "1  Holy shit thank you so much!     \\nAs someone ...      0  t3_4r2tl4      7   \n",
       "2  > people there are nice and friendly; super no...      0  t3_4r2tl4      7   \n",
       "3  Thanks for posting this! Have you tried Yaz or...      0  t3_4r2tl4      3   \n",
       "4  As a minor, did you need to show your parents ...      0  t3_4r2tl4      3   \n",
       "\n",
       "  subreddit_name_prefixed  replies  length       labels  \\\n",
       "0           r/Philippines        4     245   supporting   \n",
       "1           r/Philippines        8     393   supporting   \n",
       "2           r/Philippines        0     507   commenting   \n",
       "3           r/Philippines       16     238   supporting   \n",
       "4           r/Philippines        4      55  questioning   \n",
       "\n",
       "                                             cleaned  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                         cleaned_sub  \\\n",
       "0  updated september scroll downhi lengthy post c...   \n",
       "1  updated september scroll downhi lengthy post c...   \n",
       "2  updated september scroll downhi lengthy post c...   \n",
       "3  updated september scroll downhi lengthy post c...   \n",
       "4  updated september scroll downhi lengthy post c...   \n",
       "\n",
       "                                              expand  \n",
       "0  no questions wanted thank posting topic birth ...  \n",
       "1  holy shit thank much someone recently turned s...  \n",
       "2  people nice friendly super non judgmental espe...  \n",
       "3  thanks posting tried yaz oral contraceptives p...  \n",
       "4                    minor need show parents consent  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expand_list = []\n",
    "for i in range(n):\n",
    "    expand_list.append(expand_contractions(clean.loc[i]['cleaned']))\n",
    "clean['expand'] = expand_list\n",
    "clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submission_title</th>\n",
       "      <th>submission_text</th>\n",
       "      <th>author</th>\n",
       "      <th>body</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit_name_prefixed</th>\n",
       "      <th>replies</th>\n",
       "      <th>length</th>\n",
       "      <th>labels</th>\n",
       "      <th>cleaned</th>\n",
       "      <th>cleaned_sub</th>\n",
       "      <th>expand</th>\n",
       "      <th>expand_sub</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4xury6</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Ryvaeus</td>\n",
       "      <td>No questions, just wanted to thank you for pos...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>245</td>\n",
       "      <td>supporting</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Somebody_over_there</td>\n",
       "      <td>Holy shit thank you so much!     \\nAs someone ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>8</td>\n",
       "      <td>393</td>\n",
       "      <td>supporting</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>PotatoMurderer</td>\n",
       "      <td>&gt; people there are nice and friendly; super no...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>0</td>\n",
       "      <td>507</td>\n",
       "      <td>commenting</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4y5kmd</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>mazikeen24</td>\n",
       "      <td>Thanks for posting this! Have you tried Yaz or...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>16</td>\n",
       "      <td>238</td>\n",
       "      <td>supporting</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d4y8tvk</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>a_reverse_giraffe</td>\n",
       "      <td>As a minor, did you need to show your parents ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>55</td>\n",
       "      <td>questioning</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                   submission_title  \\\n",
       "0  d4xury6  I got my contraceptive implant, Implanon, last...   \n",
       "1  d4yi9y9  I got my contraceptive implant, Implanon, last...   \n",
       "2  d4yhm4h  I got my contraceptive implant, Implanon, last...   \n",
       "3  d4y5kmd  I got my contraceptive implant, Implanon, last...   \n",
       "4  d4y8tvk  I got my contraceptive implant, Implanon, last...   \n",
       "\n",
       "                                     submission_text               author  \\\n",
       "0  **UPDATED September 26, 2018; scroll down**\\n\\...              Ryvaeus   \n",
       "1  **UPDATED September 26, 2018; scroll down**\\n\\...  Somebody_over_there   \n",
       "2  **UPDATED September 26, 2018; scroll down**\\n\\...       PotatoMurderer   \n",
       "3  **UPDATED September 26, 2018; scroll down**\\n\\...           mazikeen24   \n",
       "4  **UPDATED September 26, 2018; scroll down**\\n\\...    a_reverse_giraffe   \n",
       "\n",
       "                                                body  depth  parent_id  score  \\\n",
       "0  No questions, just wanted to thank you for pos...      0  t3_4r2tl4     31   \n",
       "1  Holy shit thank you so much!     \\nAs someone ...      0  t3_4r2tl4      7   \n",
       "2  > people there are nice and friendly; super no...      0  t3_4r2tl4      7   \n",
       "3  Thanks for posting this! Have you tried Yaz or...      0  t3_4r2tl4      3   \n",
       "4  As a minor, did you need to show your parents ...      0  t3_4r2tl4      3   \n",
       "\n",
       "  subreddit_name_prefixed  replies  length       labels  \\\n",
       "0           r/Philippines        4     245   supporting   \n",
       "1           r/Philippines        8     393   supporting   \n",
       "2           r/Philippines        0     507   commenting   \n",
       "3           r/Philippines       16     238   supporting   \n",
       "4           r/Philippines        4      55  questioning   \n",
       "\n",
       "                                             cleaned  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                         cleaned_sub  \\\n",
       "0  updated september scroll downhi lengthy post c...   \n",
       "1  updated september scroll downhi lengthy post c...   \n",
       "2  updated september scroll downhi lengthy post c...   \n",
       "3  updated september scroll downhi lengthy post c...   \n",
       "4  updated september scroll downhi lengthy post c...   \n",
       "\n",
       "                                              expand  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                          expand_sub  \n",
       "0  updated september scroll downhi lengthy post c...  \n",
       "1  updated september scroll downhi lengthy post c...  \n",
       "2  updated september scroll downhi lengthy post c...  \n",
       "3  updated september scroll downhi lengthy post c...  \n",
       "4  updated september scroll downhi lengthy post c...  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expand_list = []\n",
    "for i in range(n):\n",
    "    expand_list.append(expand_contractions(clean.loc[i]['cleaned_sub']))\n",
    "clean['expand_sub'] = expand_list\n",
    "clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submission_title</th>\n",
       "      <th>submission_text</th>\n",
       "      <th>author</th>\n",
       "      <th>body</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit_name_prefixed</th>\n",
       "      <th>replies</th>\n",
       "      <th>length</th>\n",
       "      <th>labels</th>\n",
       "      <th>cleaned</th>\n",
       "      <th>cleaned_sub</th>\n",
       "      <th>expand</th>\n",
       "      <th>expand_sub</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4xury6</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Ryvaeus</td>\n",
       "      <td>No questions, just wanted to thank you for pos...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>245</td>\n",
       "      <td>supporting</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Somebody_over_there</td>\n",
       "      <td>Holy shit thank you so much!     \\nAs someone ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>8</td>\n",
       "      <td>393</td>\n",
       "      <td>supporting</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>PotatoMurderer</td>\n",
       "      <td>&gt; people there are nice and friendly; super no...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>0</td>\n",
       "      <td>507</td>\n",
       "      <td>commenting</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4y5kmd</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>mazikeen24</td>\n",
       "      <td>Thanks for posting this! Have you tried Yaz or...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>16</td>\n",
       "      <td>238</td>\n",
       "      <td>supporting</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d4y8tvk</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>a_reverse_giraffe</td>\n",
       "      <td>As a minor, did you need to show your parents ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>55</td>\n",
       "      <td>questioning</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                   submission_title  \\\n",
       "0  d4xury6  I got my contraceptive implant, Implanon, last...   \n",
       "1  d4yi9y9  I got my contraceptive implant, Implanon, last...   \n",
       "2  d4yhm4h  I got my contraceptive implant, Implanon, last...   \n",
       "3  d4y5kmd  I got my contraceptive implant, Implanon, last...   \n",
       "4  d4y8tvk  I got my contraceptive implant, Implanon, last...   \n",
       "\n",
       "                                     submission_text               author  \\\n",
       "0  **UPDATED September 26, 2018; scroll down**\\n\\...              Ryvaeus   \n",
       "1  **UPDATED September 26, 2018; scroll down**\\n\\...  Somebody_over_there   \n",
       "2  **UPDATED September 26, 2018; scroll down**\\n\\...       PotatoMurderer   \n",
       "3  **UPDATED September 26, 2018; scroll down**\\n\\...           mazikeen24   \n",
       "4  **UPDATED September 26, 2018; scroll down**\\n\\...    a_reverse_giraffe   \n",
       "\n",
       "                                                body  depth  parent_id  score  \\\n",
       "0  No questions, just wanted to thank you for pos...      0  t3_4r2tl4     31   \n",
       "1  Holy shit thank you so much!     \\nAs someone ...      0  t3_4r2tl4      7   \n",
       "2  > people there are nice and friendly; super no...      0  t3_4r2tl4      7   \n",
       "3  Thanks for posting this! Have you tried Yaz or...      0  t3_4r2tl4      3   \n",
       "4  As a minor, did you need to show your parents ...      0  t3_4r2tl4      3   \n",
       "\n",
       "  subreddit_name_prefixed  replies  length       labels  \\\n",
       "0           r/Philippines        4     245   supporting   \n",
       "1           r/Philippines        8     393   supporting   \n",
       "2           r/Philippines        0     507   commenting   \n",
       "3           r/Philippines       16     238   supporting   \n",
       "4           r/Philippines        4      55  questioning   \n",
       "\n",
       "                                             cleaned  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                         cleaned_sub  \\\n",
       "0  updated september scroll downhi lengthy post c...   \n",
       "1  updated september scroll downhi lengthy post c...   \n",
       "2  updated september scroll downhi lengthy post c...   \n",
       "3  updated september scroll downhi lengthy post c...   \n",
       "4  updated september scroll downhi lengthy post c...   \n",
       "\n",
       "                                              expand  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                          expand_sub  \n",
       "0  updated september scroll downhi lengthy post c...  \n",
       "1  updated september scroll downhi lengthy post c...  \n",
       "2  updated september scroll downhi lengthy post c...  \n",
       "3  updated september scroll downhi lengthy post c...  \n",
       "4  updated september scroll downhi lengthy post c...  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cleaning again\n",
    "\n",
    "def clean_text_2(clean):\n",
    "    text = clean.copy()\n",
    "    text['expand'] = text['expand'].str.replace(r'([\\'])', '')\n",
    "    text['expand'] = text['expand'].apply(lambda x: ' '.join([w for w in x.split() if w not in (stopwords)]))\n",
    "    \n",
    "    text['expand_sub'] = text['expand_sub'].str.replace(r'([\\'])', '')\n",
    "    text['expand_sub'] = text['expand_sub'].apply(lambda x: ' '.join([w for w in x.split() if w not in (stopwords)]))\n",
    "    return text\n",
    "\n",
    "expand = clean_text_2(clean)\n",
    "expand.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lemmatizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submission_title</th>\n",
       "      <th>submission_text</th>\n",
       "      <th>author</th>\n",
       "      <th>body</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit_name_prefixed</th>\n",
       "      <th>replies</th>\n",
       "      <th>length</th>\n",
       "      <th>labels</th>\n",
       "      <th>cleaned</th>\n",
       "      <th>cleaned_sub</th>\n",
       "      <th>expand</th>\n",
       "      <th>expand_sub</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4xury6</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Ryvaeus</td>\n",
       "      <td>No questions, just wanted to thank you for pos...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>245</td>\n",
       "      <td>supporting</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Somebody_over_there</td>\n",
       "      <td>Holy shit thank you so much!     \\nAs someone ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>8</td>\n",
       "      <td>393</td>\n",
       "      <td>supporting</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>PotatoMurderer</td>\n",
       "      <td>&gt; people there are nice and friendly; super no...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>0</td>\n",
       "      <td>507</td>\n",
       "      <td>commenting</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4y5kmd</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>mazikeen24</td>\n",
       "      <td>Thanks for posting this! Have you tried Yaz or...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>16</td>\n",
       "      <td>238</td>\n",
       "      <td>supporting</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d4y8tvk</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>a_reverse_giraffe</td>\n",
       "      <td>As a minor, did you need to show your parents ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>55</td>\n",
       "      <td>questioning</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                   submission_title  \\\n",
       "0  d4xury6  I got my contraceptive implant, Implanon, last...   \n",
       "1  d4yi9y9  I got my contraceptive implant, Implanon, last...   \n",
       "2  d4yhm4h  I got my contraceptive implant, Implanon, last...   \n",
       "3  d4y5kmd  I got my contraceptive implant, Implanon, last...   \n",
       "4  d4y8tvk  I got my contraceptive implant, Implanon, last...   \n",
       "\n",
       "                                     submission_text               author  \\\n",
       "0  **UPDATED September 26, 2018; scroll down**\\n\\...              Ryvaeus   \n",
       "1  **UPDATED September 26, 2018; scroll down**\\n\\...  Somebody_over_there   \n",
       "2  **UPDATED September 26, 2018; scroll down**\\n\\...       PotatoMurderer   \n",
       "3  **UPDATED September 26, 2018; scroll down**\\n\\...           mazikeen24   \n",
       "4  **UPDATED September 26, 2018; scroll down**\\n\\...    a_reverse_giraffe   \n",
       "\n",
       "                                                body  depth  parent_id  score  \\\n",
       "0  No questions, just wanted to thank you for pos...      0  t3_4r2tl4     31   \n",
       "1  Holy shit thank you so much!     \\nAs someone ...      0  t3_4r2tl4      7   \n",
       "2  > people there are nice and friendly; super no...      0  t3_4r2tl4      7   \n",
       "3  Thanks for posting this! Have you tried Yaz or...      0  t3_4r2tl4      3   \n",
       "4  As a minor, did you need to show your parents ...      0  t3_4r2tl4      3   \n",
       "\n",
       "  subreddit_name_prefixed  replies  length       labels  \\\n",
       "0           r/Philippines        4     245   supporting   \n",
       "1           r/Philippines        8     393   supporting   \n",
       "2           r/Philippines        0     507   commenting   \n",
       "3           r/Philippines       16     238   supporting   \n",
       "4           r/Philippines        4      55  questioning   \n",
       "\n",
       "                                             cleaned  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                         cleaned_sub  \\\n",
       "0  updated september scroll downhi lengthy post c...   \n",
       "1  updated september scroll downhi lengthy post c...   \n",
       "2  updated september scroll downhi lengthy post c...   \n",
       "3  updated september scroll downhi lengthy post c...   \n",
       "4  updated september scroll downhi lengthy post c...   \n",
       "\n",
       "                                              expand  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                          expand_sub  \n",
       "0  updated september scroll downhi lengthy post c...  \n",
       "1  updated september scroll downhi lengthy post c...  \n",
       "2  updated september scroll downhi lengthy post c...  \n",
       "3  updated september scroll downhi lengthy post c...  \n",
       "4  updated september scroll downhi lengthy post c...  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expand.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cleaned\n",
    "def lemmatize(clean):\n",
    "    to_lemm = clean.copy()\n",
    "    to_lemm['lemmatized'] = ''\n",
    "    for i in range(n):\n",
    "        lemmatized = ''\n",
    "        if( len(to_lemm['expand'][i]) > 0 ):\n",
    "            blob = tb(to_lemm['expand'][i]).sentences[0]\n",
    "            tags = blob.tags\n",
    "            for j in range(0,len(tags)):\n",
    "                if tags[j][1].startswith('NN'):\n",
    "                    lemmatized += tags[j][0].lemmatize('n')\n",
    "                    lemmatized += ' '\n",
    "                elif tags[j][1].startswith('VB'):\n",
    "                    lemmatized += tags[j][0].lemmatize('v')\n",
    "                    lemmatized += ' '\n",
    "                elif tags[j][1].startswith('JJ'):\n",
    "                    lemmatized += tags[j][0].lemmatize('a')\n",
    "                    lemmatized += ' '\n",
    "                else:\n",
    "                    lemmatized += tags[j][0]\n",
    "                    lemmatized += ' '\n",
    "        to_lemm['lemmatized'][i] = lemmatized.strip()\n",
    "    return to_lemm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\limri\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\ipykernel_launcher.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submission_title</th>\n",
       "      <th>submission_text</th>\n",
       "      <th>author</th>\n",
       "      <th>body</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit_name_prefixed</th>\n",
       "      <th>replies</th>\n",
       "      <th>length</th>\n",
       "      <th>labels</th>\n",
       "      <th>cleaned</th>\n",
       "      <th>cleaned_sub</th>\n",
       "      <th>expand</th>\n",
       "      <th>expand_sub</th>\n",
       "      <th>lemmatized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4xury6</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Ryvaeus</td>\n",
       "      <td>No questions, just wanted to thank you for pos...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>245</td>\n",
       "      <td>supporting</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>no question want thank post topic birth contro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Somebody_over_there</td>\n",
       "      <td>Holy shit thank you so much!     \\nAs someone ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>8</td>\n",
       "      <td>393</td>\n",
       "      <td>supporting</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>holy shit thank much someone recently turn sup...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>PotatoMurderer</td>\n",
       "      <td>&gt; people there are nice and friendly; super no...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>0</td>\n",
       "      <td>507</td>\n",
       "      <td>commenting</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4y5kmd</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>mazikeen24</td>\n",
       "      <td>Thanks for posting this! Have you tried Yaz or...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>16</td>\n",
       "      <td>238</td>\n",
       "      <td>supporting</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>thanks post tried yaz oral contraceptive prior...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d4y8tvk</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>a_reverse_giraffe</td>\n",
       "      <td>As a minor, did you need to show your parents ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>55</td>\n",
       "      <td>questioning</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>minor need show parent consent</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                   submission_title  \\\n",
       "0  d4xury6  I got my contraceptive implant, Implanon, last...   \n",
       "1  d4yi9y9  I got my contraceptive implant, Implanon, last...   \n",
       "2  d4yhm4h  I got my contraceptive implant, Implanon, last...   \n",
       "3  d4y5kmd  I got my contraceptive implant, Implanon, last...   \n",
       "4  d4y8tvk  I got my contraceptive implant, Implanon, last...   \n",
       "\n",
       "                                     submission_text               author  \\\n",
       "0  **UPDATED September 26, 2018; scroll down**\\n\\...              Ryvaeus   \n",
       "1  **UPDATED September 26, 2018; scroll down**\\n\\...  Somebody_over_there   \n",
       "2  **UPDATED September 26, 2018; scroll down**\\n\\...       PotatoMurderer   \n",
       "3  **UPDATED September 26, 2018; scroll down**\\n\\...           mazikeen24   \n",
       "4  **UPDATED September 26, 2018; scroll down**\\n\\...    a_reverse_giraffe   \n",
       "\n",
       "                                                body  depth  parent_id  score  \\\n",
       "0  No questions, just wanted to thank you for pos...      0  t3_4r2tl4     31   \n",
       "1  Holy shit thank you so much!     \\nAs someone ...      0  t3_4r2tl4      7   \n",
       "2  > people there are nice and friendly; super no...      0  t3_4r2tl4      7   \n",
       "3  Thanks for posting this! Have you tried Yaz or...      0  t3_4r2tl4      3   \n",
       "4  As a minor, did you need to show your parents ...      0  t3_4r2tl4      3   \n",
       "\n",
       "  subreddit_name_prefixed  replies  length       labels  \\\n",
       "0           r/Philippines        4     245   supporting   \n",
       "1           r/Philippines        8     393   supporting   \n",
       "2           r/Philippines        0     507   commenting   \n",
       "3           r/Philippines       16     238   supporting   \n",
       "4           r/Philippines        4      55  questioning   \n",
       "\n",
       "                                             cleaned  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                         cleaned_sub  \\\n",
       "0  updated september scroll downhi lengthy post c...   \n",
       "1  updated september scroll downhi lengthy post c...   \n",
       "2  updated september scroll downhi lengthy post c...   \n",
       "3  updated september scroll downhi lengthy post c...   \n",
       "4  updated september scroll downhi lengthy post c...   \n",
       "\n",
       "                                              expand  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                          expand_sub  \\\n",
       "0  updated september scroll downhi lengthy post c...   \n",
       "1  updated september scroll downhi lengthy post c...   \n",
       "2  updated september scroll downhi lengthy post c...   \n",
       "3  updated september scroll downhi lengthy post c...   \n",
       "4  updated september scroll downhi lengthy post c...   \n",
       "\n",
       "                                          lemmatized  \n",
       "0  no question want thank post topic birth contro...  \n",
       "1  holy shit thank much someone recently turn sup...  \n",
       "2  people nice friendly super non judgmental espe...  \n",
       "3  thanks post tried yaz oral contraceptive prior...  \n",
       "4                     minor need show parent consent  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemm = lemmatize(expand)\n",
    "lemm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cleaned_sub\n",
    "def lemmatize_2(clean):\n",
    "    to_lemm = clean.copy()\n",
    "    to_lemm['lemm_sub'] = ''\n",
    "    for i in range(n):\n",
    "        lemmatized = ''\n",
    "        if( len(to_lemm['expand_sub'][i]) > 0 ):\n",
    "            blob = tb(to_lemm['expand_sub'][i]).sentences[0]\n",
    "            tags = blob.tags\n",
    "            for j in range(0,len(tags)):\n",
    "                if tags[j][1].startswith('NN'):\n",
    "                    lemmatized += tags[j][0].lemmatize('n')\n",
    "                    lemmatized += ' '\n",
    "                elif tags[j][1].startswith('VB'):\n",
    "                    lemmatized += tags[j][0].lemmatize('v')\n",
    "                    lemmatized += ' '\n",
    "                elif tags[j][1].startswith('JJ'):\n",
    "                    lemmatized += tags[j][0].lemmatize('a')\n",
    "                    lemmatized += ' '\n",
    "                else:\n",
    "                    lemmatized += tags[j][0]\n",
    "                    lemmatized += ' '\n",
    "        to_lemm['lemm_sub'][i] = lemmatized.strip()\n",
    "    return to_lemm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\limri\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\ipykernel_launcher.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submission_title</th>\n",
       "      <th>submission_text</th>\n",
       "      <th>author</th>\n",
       "      <th>body</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit_name_prefixed</th>\n",
       "      <th>replies</th>\n",
       "      <th>length</th>\n",
       "      <th>labels</th>\n",
       "      <th>cleaned</th>\n",
       "      <th>cleaned_sub</th>\n",
       "      <th>expand</th>\n",
       "      <th>expand_sub</th>\n",
       "      <th>lemmatized</th>\n",
       "      <th>lemm_sub</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4xury6</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Ryvaeus</td>\n",
       "      <td>No questions, just wanted to thank you for pos...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>245</td>\n",
       "      <td>supporting</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>no question want thank post topic birth contro...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Somebody_over_there</td>\n",
       "      <td>Holy shit thank you so much!     \\nAs someone ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>8</td>\n",
       "      <td>393</td>\n",
       "      <td>supporting</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>holy shit thank much someone recently turn sup...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>PotatoMurderer</td>\n",
       "      <td>&gt; people there are nice and friendly; super no...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>0</td>\n",
       "      <td>507</td>\n",
       "      <td>commenting</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4y5kmd</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>mazikeen24</td>\n",
       "      <td>Thanks for posting this! Have you tried Yaz or...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>16</td>\n",
       "      <td>238</td>\n",
       "      <td>supporting</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>thanks post tried yaz oral contraceptive prior...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d4y8tvk</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>a_reverse_giraffe</td>\n",
       "      <td>As a minor, did you need to show your parents ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>55</td>\n",
       "      <td>questioning</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>minor need show parent consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                   submission_title  \\\n",
       "0  d4xury6  I got my contraceptive implant, Implanon, last...   \n",
       "1  d4yi9y9  I got my contraceptive implant, Implanon, last...   \n",
       "2  d4yhm4h  I got my contraceptive implant, Implanon, last...   \n",
       "3  d4y5kmd  I got my contraceptive implant, Implanon, last...   \n",
       "4  d4y8tvk  I got my contraceptive implant, Implanon, last...   \n",
       "\n",
       "                                     submission_text               author  \\\n",
       "0  **UPDATED September 26, 2018; scroll down**\\n\\...              Ryvaeus   \n",
       "1  **UPDATED September 26, 2018; scroll down**\\n\\...  Somebody_over_there   \n",
       "2  **UPDATED September 26, 2018; scroll down**\\n\\...       PotatoMurderer   \n",
       "3  **UPDATED September 26, 2018; scroll down**\\n\\...           mazikeen24   \n",
       "4  **UPDATED September 26, 2018; scroll down**\\n\\...    a_reverse_giraffe   \n",
       "\n",
       "                                                body  depth  parent_id  score  \\\n",
       "0  No questions, just wanted to thank you for pos...      0  t3_4r2tl4     31   \n",
       "1  Holy shit thank you so much!     \\nAs someone ...      0  t3_4r2tl4      7   \n",
       "2  > people there are nice and friendly; super no...      0  t3_4r2tl4      7   \n",
       "3  Thanks for posting this! Have you tried Yaz or...      0  t3_4r2tl4      3   \n",
       "4  As a minor, did you need to show your parents ...      0  t3_4r2tl4      3   \n",
       "\n",
       "  subreddit_name_prefixed  replies  length       labels  \\\n",
       "0           r/Philippines        4     245   supporting   \n",
       "1           r/Philippines        8     393   supporting   \n",
       "2           r/Philippines        0     507   commenting   \n",
       "3           r/Philippines       16     238   supporting   \n",
       "4           r/Philippines        4      55  questioning   \n",
       "\n",
       "                                             cleaned  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                         cleaned_sub  \\\n",
       "0  updated september scroll downhi lengthy post c...   \n",
       "1  updated september scroll downhi lengthy post c...   \n",
       "2  updated september scroll downhi lengthy post c...   \n",
       "3  updated september scroll downhi lengthy post c...   \n",
       "4  updated september scroll downhi lengthy post c...   \n",
       "\n",
       "                                              expand  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                          expand_sub  \\\n",
       "0  updated september scroll downhi lengthy post c...   \n",
       "1  updated september scroll downhi lengthy post c...   \n",
       "2  updated september scroll downhi lengthy post c...   \n",
       "3  updated september scroll downhi lengthy post c...   \n",
       "4  updated september scroll downhi lengthy post c...   \n",
       "\n",
       "                                          lemmatized  \\\n",
       "0  no question want thank post topic birth contro...   \n",
       "1  holy shit thank much someone recently turn sup...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks post tried yaz oral contraceptive prior...   \n",
       "4                     minor need show parent consent   \n",
       "\n",
       "                                            lemm_sub  \n",
       "0  updated september scroll downhi lengthy post c...  \n",
       "1  updated september scroll downhi lengthy post c...  \n",
       "2  updated september scroll downhi lengthy post c...  \n",
       "3  updated september scroll downhi lengthy post c...  \n",
       "4  updated september scroll downhi lengthy post c...  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemm = lemmatize_2(lemm)\n",
    "lemm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze_text(lemm):\n",
    "    \"\"\"\n",
    "    Function for analyzing sentiment and subjectivity based on class jupyter notebook on text analysis\n",
    "    \"\"\"\n",
    "    text = lemm.copy()\n",
    "    sentiment = []\n",
    "    subjectivity = []\n",
    "    for i in text['lemmatized']:\n",
    "        blob = tb(i)\n",
    "        sentiment.append(blob.polarity)\n",
    "        subjectivity.append(blob.subjectivity)\n",
    "    text['sentiment'] = sentiment\n",
    "    text['subjectivity'] = subjectivity\n",
    "    sentiment = []\n",
    "    subjectivity = []\n",
    "    for i in text['lemm_sub']:\n",
    "        blob = tb(i)\n",
    "        sentiment.append(blob.polarity)\n",
    "        subjectivity.append(blob.subjectivity)\n",
    "    text['sentiment_sub'] = sentiment\n",
    "    text['subjectivity_sub'] = subjectivity\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submission_title</th>\n",
       "      <th>submission_text</th>\n",
       "      <th>author</th>\n",
       "      <th>body</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit_name_prefixed</th>\n",
       "      <th>replies</th>\n",
       "      <th>...</th>\n",
       "      <th>cleaned</th>\n",
       "      <th>cleaned_sub</th>\n",
       "      <th>expand</th>\n",
       "      <th>expand_sub</th>\n",
       "      <th>lemmatized</th>\n",
       "      <th>lemm_sub</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>subjectivity</th>\n",
       "      <th>sentiment_sub</th>\n",
       "      <th>subjectivity_sub</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4xury6</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Ryvaeus</td>\n",
       "      <td>No questions, just wanted to thank you for pos...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>no questions wanted thank posting topic birth ...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>no question want thank post topic birth contro...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.533718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Somebody_over_there</td>\n",
       "      <td>Holy shit thank you so much!     \\nAs someone ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>holy shit thank much someone recently turned s...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>holy shit thank much someone recently turn sup...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.104167</td>\n",
       "      <td>0.308333</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.533718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>PotatoMurderer</td>\n",
       "      <td>&gt; people there are nice and friendly; super no...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.189815</td>\n",
       "      <td>0.540741</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.533718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4y5kmd</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>mazikeen24</td>\n",
       "      <td>Thanks for posting this! Have you tried Yaz or...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>thanks posting tried yaz oral contraceptives p...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>thanks post tried yaz oral contraceptive prior...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.533718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d4y8tvk</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>a_reverse_giraffe</td>\n",
       "      <td>As a minor, did you need to show your parents ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>minor need show parents consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>minor need show parent consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.533718</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                   submission_title  \\\n",
       "0  d4xury6  I got my contraceptive implant, Implanon, last...   \n",
       "1  d4yi9y9  I got my contraceptive implant, Implanon, last...   \n",
       "2  d4yhm4h  I got my contraceptive implant, Implanon, last...   \n",
       "3  d4y5kmd  I got my contraceptive implant, Implanon, last...   \n",
       "4  d4y8tvk  I got my contraceptive implant, Implanon, last...   \n",
       "\n",
       "                                     submission_text               author  \\\n",
       "0  **UPDATED September 26, 2018; scroll down**\\n\\...              Ryvaeus   \n",
       "1  **UPDATED September 26, 2018; scroll down**\\n\\...  Somebody_over_there   \n",
       "2  **UPDATED September 26, 2018; scroll down**\\n\\...       PotatoMurderer   \n",
       "3  **UPDATED September 26, 2018; scroll down**\\n\\...           mazikeen24   \n",
       "4  **UPDATED September 26, 2018; scroll down**\\n\\...    a_reverse_giraffe   \n",
       "\n",
       "                                                body  depth  parent_id  score  \\\n",
       "0  No questions, just wanted to thank you for pos...      0  t3_4r2tl4     31   \n",
       "1  Holy shit thank you so much!     \\nAs someone ...      0  t3_4r2tl4      7   \n",
       "2  > people there are nice and friendly; super no...      0  t3_4r2tl4      7   \n",
       "3  Thanks for posting this! Have you tried Yaz or...      0  t3_4r2tl4      3   \n",
       "4  As a minor, did you need to show your parents ...      0  t3_4r2tl4      3   \n",
       "\n",
       "  subreddit_name_prefixed  replies  ...  \\\n",
       "0           r/Philippines        4  ...   \n",
       "1           r/Philippines        8  ...   \n",
       "2           r/Philippines        0  ...   \n",
       "3           r/Philippines       16  ...   \n",
       "4           r/Philippines        4  ...   \n",
       "\n",
       "                                             cleaned  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                         cleaned_sub  \\\n",
       "0  updated september scroll downhi lengthy post c...   \n",
       "1  updated september scroll downhi lengthy post c...   \n",
       "2  updated september scroll downhi lengthy post c...   \n",
       "3  updated september scroll downhi lengthy post c...   \n",
       "4  updated september scroll downhi lengthy post c...   \n",
       "\n",
       "                                              expand  \\\n",
       "0  no questions wanted thank posting topic birth ...   \n",
       "1  holy shit thank much someone recently turned s...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks posting tried yaz oral contraceptives p...   \n",
       "4                    minor need show parents consent   \n",
       "\n",
       "                                          expand_sub  \\\n",
       "0  updated september scroll downhi lengthy post c...   \n",
       "1  updated september scroll downhi lengthy post c...   \n",
       "2  updated september scroll downhi lengthy post c...   \n",
       "3  updated september scroll downhi lengthy post c...   \n",
       "4  updated september scroll downhi lengthy post c...   \n",
       "\n",
       "                                          lemmatized  \\\n",
       "0  no question want thank post topic birth contro...   \n",
       "1  holy shit thank much someone recently turn sup...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks post tried yaz oral contraceptive prior...   \n",
       "4                     minor need show parent consent   \n",
       "\n",
       "                                            lemm_sub sentiment subjectivity  \\\n",
       "0  updated september scroll downhi lengthy post c...  0.333333     0.416667   \n",
       "1  updated september scroll downhi lengthy post c...  0.104167     0.308333   \n",
       "2  updated september scroll downhi lengthy post c...  0.189815     0.540741   \n",
       "3  updated september scroll downhi lengthy post c...  0.133333     0.133333   \n",
       "4  updated september scroll downhi lengthy post c... -0.050000     0.200000   \n",
       "\n",
       "   sentiment_sub  subjectivity_sub  \n",
       "0       0.109876          0.533718  \n",
       "1       0.109876          0.533718  \n",
       "2       0.109876          0.533718  \n",
       "3       0.109876          0.533718  \n",
       "4       0.109876          0.533718  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent = analyze_text(lemm)\n",
    "sent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = sent[['id','depth','parent_id','score','lemmatized','lemm_sub','sentiment','subjectivity','sentiment_sub','subjectivity_sub','labels']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>lemmatized</th>\n",
       "      <th>lemm_sub</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>subjectivity</th>\n",
       "      <th>sentiment_sub</th>\n",
       "      <th>subjectivity_sub</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4xury6</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>no question want thank post topic birth contro...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.533718</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>holy shit thank much someone recently turn sup...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.104167</td>\n",
       "      <td>0.308333</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.533718</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.189815</td>\n",
       "      <td>0.540741</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.533718</td>\n",
       "      <td>commenting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4y5kmd</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>thanks post tried yaz oral contraceptive prior...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.533718</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d4y8tvk</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>3</td>\n",
       "      <td>minor need show parent consent</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.533718</td>\n",
       "      <td>questioning</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  depth  parent_id  score  \\\n",
       "0  d4xury6      0  t3_4r2tl4     31   \n",
       "1  d4yi9y9      0  t3_4r2tl4      7   \n",
       "2  d4yhm4h      0  t3_4r2tl4      7   \n",
       "3  d4y5kmd      0  t3_4r2tl4      3   \n",
       "4  d4y8tvk      0  t3_4r2tl4      3   \n",
       "\n",
       "                                          lemmatized  \\\n",
       "0  no question want thank post topic birth contro...   \n",
       "1  holy shit thank much someone recently turn sup...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "3  thanks post tried yaz oral contraceptive prior...   \n",
       "4                     minor need show parent consent   \n",
       "\n",
       "                                            lemm_sub  sentiment  subjectivity  \\\n",
       "0  updated september scroll downhi lengthy post c...   0.333333      0.416667   \n",
       "1  updated september scroll downhi lengthy post c...   0.104167      0.308333   \n",
       "2  updated september scroll downhi lengthy post c...   0.189815      0.540741   \n",
       "3  updated september scroll downhi lengthy post c...   0.133333      0.133333   \n",
       "4  updated september scroll downhi lengthy post c...  -0.050000      0.200000   \n",
       "\n",
       "   sentiment_sub  subjectivity_sub       labels  \n",
       "0       0.109876          0.533718   supporting  \n",
       "1       0.109876          0.533718   supporting  \n",
       "2       0.109876          0.533718   commenting  \n",
       "3       0.109876          0.533718   supporting  \n",
       "4       0.109876          0.533718  questioning  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df.to_csv('datasets/preprocessed_data_2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0b4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}