04_vader_sentiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting vaderSentiment\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/86/9e/c53e1fc61aac5ee490a6ac5e21b1ac04e55a7c2aba647bb8411c9aadf24e/vaderSentiment-3.2.1-py2.py3-none-any.whl (125kB)\n",
      "\u001b[K    100% |████████████████████████████████| 133kB 340kB/s \n",
      "\u001b[31mtensorflow 1.9.0 has requirement setuptools<=39.1.0, but you'll have setuptools 41.0.1 which is incompatible.\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: vaderSentiment\n",
      "Successfully installed vaderSentiment-3.2.1\n",
      "\u001b[33mYou are using pip version 10.0.1, however version 19.1.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip3 install vaderSentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/usr/local/lib/python3.6/site-packages')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>submission_title</th>\n",
       "      <th>submission_text</th>\n",
       "      <th>author</th>\n",
       "      <th>body</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit_name_prefixed</th>\n",
       "      <th>replies</th>\n",
       "      <th>length</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4xury6</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Ryvaeus</td>\n",
       "      <td>No questions, just wanted to thank you for pos...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>4</td>\n",
       "      <td>245</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>I got my contraceptive implant, Implanon, last...</td>\n",
       "      <td>**UPDATED September 26, 2018; scroll down**\\n\\...</td>\n",
       "      <td>Somebody_over_there</td>\n",
       "      <td>Holy shit thank you so much!     \\nAs someone ...</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>r/Philippines</td>\n",
       "      <td>8</td>\n",
       "      <td>393</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                   submission_title  \\\n",
       "0  d4xury6  I got my contraceptive implant, Implanon, last...   \n",
       "1  d4yi9y9  I got my contraceptive implant, Implanon, last...   \n",
       "\n",
       "                                     submission_text               author  \\\n",
       "0  **UPDATED September 26, 2018; scroll down**\\n\\...              Ryvaeus   \n",
       "1  **UPDATED September 26, 2018; scroll down**\\n\\...  Somebody_over_there   \n",
       "\n",
       "                                                body  depth  parent_id  score  \\\n",
       "0  No questions, just wanted to thank you for pos...      0  t3_4r2tl4     31   \n",
       "1  Holy shit thank you so much!     \\nAs someone ...      0  t3_4r2tl4      7   \n",
       "\n",
       "  subreddit_name_prefixed  replies  length      labels  \n",
       "0           r/Philippines        4     245  supporting  \n",
       "1           r/Philippines        8     393  supporting  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analyzer = SentimentIntensityAnalyzer()\n",
    "full_data = pd.read_csv('datasets/final_dataset.csv')\n",
    "data = pd.read_csv('datasets/preprocessed_data_2.csv')\n",
    "full_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>lemmatized</th>\n",
       "      <th>lemm_sub</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sentiment_sub</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>d4xury6</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>no question want thank post topic birth contro...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>holy shit thank much someone recently turn sup...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.104167</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.189815</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>commenting</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0       id  depth  parent_id  score  \\\n",
       "0           0  d4xury6      0  t3_4r2tl4     31   \n",
       "1           1  d4yi9y9      0  t3_4r2tl4      7   \n",
       "2           2  d4yhm4h      0  t3_4r2tl4      7   \n",
       "\n",
       "                                          lemmatized  \\\n",
       "0  no question want thank post topic birth contro...   \n",
       "1  holy shit thank much someone recently turn sup...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "\n",
       "                                            lemm_sub  sentiment  \\\n",
       "0  updated september scroll downhi lengthy post c...   0.333333   \n",
       "1  updated september scroll downhi lengthy post c...   0.104167   \n",
       "2  updated september scroll downhi lengthy post c...   0.189815   \n",
       "\n",
       "   sentiment_sub      labels  \n",
       "0       0.109876  supporting  \n",
       "1       0.109876  supporting  \n",
       "2       0.109876  commenting  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop the subjectivity columns since that is only for TextBlob\n",
    "\n",
    "data.drop('subjectivity', axis=1, inplace=True)\n",
    "data.drop('subjectivity_sub', axis=1, inplace=True)\n",
    "data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.9837\n",
       "1    0.9837\n",
       "2    0.9837\n",
       "3    0.9837\n",
       "4    0.9837\n",
       "Name: sentiment_sub, dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['sentiment_sub'] = full_data.apply(\n",
    "    lambda row : analyzer.polarity_scores(row['submission_text'])['compound'], axis=1\n",
    ")\n",
    "data['sentiment_sub'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.0772\n",
       "1    0.9184\n",
       "2    0.9544\n",
       "3   -0.8341\n",
       "4    0.2263\n",
       "Name: sentiment, dtype: float64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['sentiment'] = full_data.apply(lambda row : analyzer.polarity_scores(row['body'])['compound'], axis=1)\n",
    "data['sentiment'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>depth</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>lemmatized</th>\n",
       "      <th>lemm_sub</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>sentiment_sub</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>d4xury6</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>31</td>\n",
       "      <td>no question want thank post topic birth contro...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.0772</td>\n",
       "      <td>0.9837</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>d4yi9y9</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>holy shit thank much someone recently turn sup...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.9184</td>\n",
       "      <td>0.9837</td>\n",
       "      <td>supporting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>d4yhm4h</td>\n",
       "      <td>0</td>\n",
       "      <td>t3_4r2tl4</td>\n",
       "      <td>7</td>\n",
       "      <td>people nice friendly super non judgmental espe...</td>\n",
       "      <td>updated september scroll downhi lengthy post c...</td>\n",
       "      <td>0.9544</td>\n",
       "      <td>0.9837</td>\n",
       "      <td>commenting</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0       id  depth  parent_id  score  \\\n",
       "0           0  d4xury6      0  t3_4r2tl4     31   \n",
       "1           1  d4yi9y9      0  t3_4r2tl4      7   \n",
       "2           2  d4yhm4h      0  t3_4r2tl4      7   \n",
       "\n",
       "                                          lemmatized  \\\n",
       "0  no question want thank post topic birth contro...   \n",
       "1  holy shit thank much someone recently turn sup...   \n",
       "2  people nice friendly super non judgmental espe...   \n",
       "\n",
       "                                            lemm_sub  sentiment  \\\n",
       "0  updated september scroll downhi lengthy post c...     0.0772   \n",
       "1  updated september scroll downhi lengthy post c...     0.9184   \n",
       "2  updated september scroll downhi lengthy post c...     0.9544   \n",
       "\n",
       "   sentiment_sub      labels  \n",
       "0         0.9837  supporting  \n",
       "1         0.9837  supporting  \n",
       "2         0.9837  commenting  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('datasets/preprocessed_data_vader.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0b4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}