preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing of data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import essay\n",
    "import numpy as np\n",
    "import pickle\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mbti_to_big5(mbti):\n",
    "    # check https://en.wikipedia.org/wiki/Myers%E2%80%93Briggs_Type_Indicator\n",
    "    # in mbti (myers briggs) ther is invrovert vs. extrovert\n",
    "    # which corellates with Extroversion in BIG FIVE\n",
    "    mbti = mbti.lower()\n",
    "    cEXT, cNEU, cAGR, cCON, cOPN = 0,np.NaN,0,0,0\n",
    "    if mbti[0] == \"i\":\n",
    "        cEXT = 0\n",
    "    elif mbti[0] == \"e\":\n",
    "        cEXT = 1\n",
    "\n",
    "    # in mbti (myers briggs) ther is I*N*TUITION vs SENSING\n",
    "    # which corellates with OPENNESS in BIG FIVE\n",
    "    if mbti[1] == \"n\":\n",
    "        cOPN = 1\n",
    "    elif mbti[1] == \"s\":\n",
    "        cOPN = 0   \n",
    "\n",
    "    # in mbti (myers briggs) ther is THINKER vs FEELER\n",
    "    # which corellates with AGREEABLENESS in BIG FIVE\n",
    "    if mbti[2] == \"t\":\n",
    "        cAGR = 0\n",
    "    elif mbti[2] == \"f\":\n",
    "        cAGR = 1\n",
    "\n",
    "    # in mbti (myers briggs) ther is JUDGER vs PERCEIVER\n",
    "    # which corellates with CONSCIENTIOUSNESS in BIG FIVE (worst corellation)\n",
    "    # especially bec. orderlyness corellates with conscientiousness\n",
    "    if mbti[3] == \"p\":\n",
    "        cCON = 0\n",
    "    elif mbti[3] == \"j\":\n",
    "        cCON = 1\n",
    "\n",
    "    return cEXT, cNEU, cAGR, cCON, cOPN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_unemotional_scentences(emotional_words, text_as_one_string):\n",
    "    reduced_s = \"\"\n",
    "    scentences = re.split('(?<=[.!?]) +', text_as_one_string)\n",
    "    for s in scentences:\n",
    "        if any(e in s for e in emotional_words):\n",
    "            reduced_s = reduced_s + s + \" \"\n",
    "        else:\n",
    "            pass\n",
    "    return reduced_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# simply put every row of our read dataframe into a list of \n",
    "# the object \"Essay\"\n",
    "# remove data from list substract\n",
    "def create_essays(df, subtract=None):\n",
    "    essays = []\n",
    "    for index, row in df.iterrows():\n",
    "        essays.append(essay.Essay(row.TEXT, row.cEXT, row.cNEU, row.cAGR, row.cCON, row.cOPN))  \n",
    "\n",
    "    # remove scentences which do not contain emotionally charged words \n",
    "    # from the emotional lexicon\n",
    "    if subtract != None:\n",
    "        for x in essays:\n",
    "            x.filtered_text = remove_unemotional_scentences(emotional_words, x.clean_text)\n",
    "\n",
    "    return essays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the essays from paper\n",
    "## scientific gold standard\n",
    "## https://sentic.net/deep-learning-based-personality-detection.pdf\n",
    "### (scientific gold standard \"stream of counsciousness\" essays labeled with personality traits of the big five)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TEXT</th>\n",
       "      <th>cEXT</th>\n",
       "      <th>cNEU</th>\n",
       "      <th>cAGR</th>\n",
       "      <th>cCON</th>\n",
       "      <th>cOPN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Well, right now I just woke up from a mid-day ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Well, here we go with the stream of consciousn...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>An open keyboard and buttons to push. The thin...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I can't believe it!  It's really happening!  M...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Well, here I go with the good old stream of co...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2462</th>\n",
       "      <td>I'm home. wanted to go to bed but remembe...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2463</th>\n",
       "      <td>Stream of consiousnesssskdj. How do you s...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2464</th>\n",
       "      <td>It is Wednesday, December 8th and a lot has be...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2465</th>\n",
       "      <td>Man this week has been hellish. Anyways, now i...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2466</th>\n",
       "      <td>I have just gotten off the phone with brady. I...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2467 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   TEXT  cEXT  cNEU  cAGR  \\\n",
       "0     Well, right now I just woke up from a mid-day ...     0     1     1   \n",
       "1     Well, here we go with the stream of consciousn...     0     0     1   \n",
       "2     An open keyboard and buttons to push. The thin...     0     1     0   \n",
       "3     I can't believe it!  It's really happening!  M...     1     0     1   \n",
       "4     Well, here I go with the good old stream of co...     1     0     1   \n",
       "...                                                 ...   ...   ...   ...   \n",
       "2462       I'm home. wanted to go to bed but remembe...     0     1     0   \n",
       "2463       Stream of consiousnesssskdj. How do you s...     1     1     0   \n",
       "2464  It is Wednesday, December 8th and a lot has be...     0     0     1   \n",
       "2465  Man this week has been hellish. Anyways, now i...     0     1     0   \n",
       "2466  I have just gotten off the phone with brady. I...     0     1     1   \n",
       "\n",
       "      cCON  cOPN  \n",
       "0        0     1  \n",
       "1        0     0  \n",
       "2        1     1  \n",
       "3        1     0  \n",
       "4        0     1  \n",
       "...    ...   ...  \n",
       "2462     1     0  \n",
       "2463     0     1  \n",
       "2464     0     0  \n",
       "2465     0     1  \n",
       "2466     0     1  \n",
       "\n",
       "[2467 rows x 6 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we read in the data from \"essays.csv\" and \n",
    "# \"essays.csv\" contains all essays with classification \n",
    "df_essays = pd.read_csv('data/training/essays.csv', encoding='cp1252', delimiter=',', quotechar='\"')\n",
    "\n",
    "# for every essay, we replace the personalitiy categories \n",
    "# of the essay wich are \"y\" and \"n\" with \"1\" and \"0\" \n",
    "for e in df_essays.columns[2:7]:\n",
    "    df_essays[e] = df_essays[e].replace('n', '0')\n",
    "    df_essays[e] = df_essays[e].replace('y', '1')\n",
    "    # not sure if we need this line: furter investigation possible:\n",
    "    df_essays[e] = pd.to_numeric(df_essays[e])\n",
    "\n",
    "df_essays = df_essays[[\"TEXT\", \"cEXT\", \"cNEU\", \"cAGR\", \"cCON\", \"cOPN\"]]\n",
    "df_essays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the MBTI kaggle dataset\n",
    "## https://www.kaggle.com/datasnaek/mbti-type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TEXT</th>\n",
       "      <th>cEXT</th>\n",
       "      <th>cNEU</th>\n",
       "      <th>cAGR</th>\n",
       "      <th>cCON</th>\n",
       "      <th>cOPN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>'http://www.youtube.com/watch?v=qsXHcwe3krw ht...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'I'm finding the lack of me in these posts ver...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>'Good one  _____   https://www.youtube.com/wat...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>'Dear INTP,   I enjoyed our conversation the o...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>'You're fired. That's another silly misconcept...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8670</th>\n",
       "      <td>'https://www.youtube.com/watch?v=t8edHB_h908 I...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8671</th>\n",
       "      <td>'So...if this thread already exists someplace ...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8672</th>\n",
       "      <td>'So many questions when i do these things.  I ...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8673</th>\n",
       "      <td>'I am very conflicted right now when it comes ...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8674</th>\n",
       "      <td>'It has been too long since I have been on per...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8675 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   TEXT  cEXT  cNEU  cAGR  \\\n",
       "0     'http://www.youtube.com/watch?v=qsXHcwe3krw ht...     0   NaN     1   \n",
       "1     'I'm finding the lack of me in these posts ver...     1   NaN     0   \n",
       "2     'Good one  _____   https://www.youtube.com/wat...     0   NaN     0   \n",
       "3     'Dear INTP,   I enjoyed our conversation the o...     0   NaN     0   \n",
       "4     'You're fired. That's another silly misconcept...     1   NaN     0   \n",
       "...                                                 ...   ...   ...   ...   \n",
       "8670  'https://www.youtube.com/watch?v=t8edHB_h908 I...     0   NaN     1   \n",
       "8671  'So...if this thread already exists someplace ...     1   NaN     1   \n",
       "8672  'So many questions when i do these things.  I ...     0   NaN     0   \n",
       "8673  'I am very conflicted right now when it comes ...     0   NaN     1   \n",
       "8674  'It has been too long since I have been on per...     0   NaN     1   \n",
       "\n",
       "      cCON  cOPN  \n",
       "0        1     1  \n",
       "1        0     1  \n",
       "2        0     1  \n",
       "3        1     1  \n",
       "4        1     1  \n",
       "...    ...   ...  \n",
       "8670     0     0  \n",
       "8671     0     1  \n",
       "8672     0     1  \n",
       "8673     0     1  \n",
       "8674     0     1  \n",
       "\n",
       "[8675 rows x 6 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_kaggle = pd.read_csv('data/training/mbti_1.csv',  skiprows=0 )\n",
    "\n",
    "df_kaggle[\"cEXT\"] =   df_kaggle.apply(lambda x: mbti_to_big5(x.type)[0], 1)\n",
    "df_kaggle[\"cNEU\"] =   df_kaggle.apply(lambda x: mbti_to_big5(x.type)[1], 1)\n",
    "df_kaggle[\"cAGR\"] =   df_kaggle.apply(lambda x: mbti_to_big5(x.type)[2], 1)\n",
    "df_kaggle[\"cCON\"] =   df_kaggle.apply(lambda x: mbti_to_big5(x.type)[3], 1)\n",
    "df_kaggle[\"cOPN\"] =   df_kaggle.apply(lambda x: mbti_to_big5(x.type)[4], 1)\n",
    "\n",
    "df_kaggle = df_kaggle[[\"posts\", \"cEXT\", \"cNEU\", \"cAGR\", \"cCON\", \"cOPN\"]]\n",
    "df_kaggle.columns = [\"TEXT\", \"cEXT\", \"cNEU\", \"cAGR\", \"cCON\", \"cOPN\"]\n",
    "\n",
    "# remove som fancy ||| things\n",
    "df_kaggle[\"TEXT\"] = df_kaggle.apply(lambda x: x.TEXT.replace(\"|||\", \" \")[:], 1)\n",
    "\n",
    "\n",
    "df_kaggle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Reddit Dataset\n",
    "## Matej Gjurković  https://peopleswksh.github.io/pdf/PEOPLES12.pdf\n",
    "### received on request\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\python\\python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3050: DtypeWarning: Columns (0,2) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n"
     ]
    }
   ],
   "source": [
    "# the file is kinda huge. thanks Matej\n",
    "file = 'data/training/typed_comments.csv'\n",
    "df_reddit = pd.read_csv(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TEXT</th>\n",
       "      <th>cEXT</th>\n",
       "      <th>cNEU</th>\n",
       "      <th>cAGR</th>\n",
       "      <th>cCON</th>\n",
       "      <th>cOPN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Okay, this has been educational. I was not awa...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>This depends on *what MBTI means to you*.The *...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>O witty! Gotta do the TYPE_MENTION PIT maneuve...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I'm glad it's interesting! :)So I had to spend...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Yeah, I don't *like* to dismiss a cool new ide...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78217</th>\n",
       "      <td>People post videos, memes, and curse in PPD pr...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78218</th>\n",
       "      <td>Well first off, I'm not a libertarian in the s...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78219</th>\n",
       "      <td>Yeah, I am kinda ending Westphalian nation sta...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78220</th>\n",
       "      <td>Hey  No problem, was just a heads up for the f...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78221</th>\n",
       "      <td>Most of your list I agree with in principle, i...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>78222 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    TEXT  cEXT  cNEU  cAGR  \\\n",
       "0      Okay, this has been educational. I was not awa...     1   NaN     0   \n",
       "1      This depends on *what MBTI means to you*.The *...     1   NaN     0   \n",
       "2      O witty! Gotta do the TYPE_MENTION PIT maneuve...     1   NaN     0   \n",
       "3      I'm glad it's interesting! :)So I had to spend...     1   NaN     0   \n",
       "4      Yeah, I don't *like* to dismiss a cool new ide...     1   NaN     0   \n",
       "...                                                  ...   ...   ...   ...   \n",
       "78217  People post videos, memes, and curse in PPD pr...     1   NaN     1   \n",
       "78218  Well first off, I'm not a libertarian in the s...     1   NaN     1   \n",
       "78219  Yeah, I am kinda ending Westphalian nation sta...     1   NaN     1   \n",
       "78220  Hey  No problem, was just a heads up for the f...     1   NaN     1   \n",
       "78221  Most of your list I agree with in principle, i...     1   NaN     1   \n",
       "\n",
       "       cCON  cOPN  \n",
       "0         0     1  \n",
       "1         0     1  \n",
       "2         0     1  \n",
       "3         0     1  \n",
       "4         0     1  \n",
       "...     ...   ...  \n",
       "78217     1     0  \n",
       "78218     1     0  \n",
       "78219     1     0  \n",
       "78220     1     0  \n",
       "78221     1     0  \n",
       "\n",
       "[78222 rows x 6 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#remove some rows to to keep longer text (the 420 because it makes avg word count compareable to the rest of the data)\n",
    "df_reddit = df_reddit[df_reddit.word_count > 420]\n",
    "\n",
    "df_reddit[\"cEXT\"] =   df_reddit.apply(lambda x: mbti_to_big5(x.type)[0], 1)\n",
    "df_reddit[\"cNEU\"] =   df_reddit.apply(lambda x: mbti_to_big5(x.type)[1], 1)\n",
    "df_reddit[\"cAGR\"] =   df_reddit.apply(lambda x: mbti_to_big5(x.type)[2], 1)\n",
    "df_reddit[\"cCON\"] =   df_reddit.apply(lambda x: mbti_to_big5(x.type)[3], 1)\n",
    "df_reddit[\"cOPN\"] =   df_reddit.apply(lambda x: mbti_to_big5(x.type)[4], 1)\n",
    "df_reddit = df_reddit[[\"comment\", \"cEXT\", \"cNEU\", \"cAGR\", \"cCON\", \"cOPN\"]]\n",
    "df_reddit.columns = [\"TEXT\", \"cEXT\", \"cNEU\", \"cAGR\", \"cCON\", \"cOPN\"]\n",
    "df_reddit.reset_index(drop=True, inplace=True)\n",
    "df_reddit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Emotional Lexicon to substract from data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# also from \"Emotional_Lexicon.csv\" we read in the data, which is a list of words and \n",
    "# has several categories of emotions. \n",
    "# anger - anticipation - disgust - fear - joy - negative - positive \n",
    "# - sadness - surprise - trust - Charged\n",
    "df_lexicon = pd.read_csv('data/training/Emotion_Lexicon.csv', index_col=0)\n",
    "\n",
    "\n",
    "# some of the words have no emotional category, \n",
    "# so let's remove them as they have no use to us.\n",
    "# can be improved by not even loading them when all columns are 0. maybe later.\n",
    "df_lexicon = df_lexicon[(df_lexicon.T != 0).any()]\n",
    "emotional_words = df_lexicon.index.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Concatinate the datasets for 3 different data to work with and compare to"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Data Base 1 - only esseys.csv - and save as object list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved entries:  2467\n"
     ]
    }
   ],
   "source": [
    "# save preprocessed data by converting into OBJECT essay and save with pickle and removing non emotional scentences\n",
    "essays = create_essays(df_essays, emotional_words)\n",
    "pickle.dump(essays, open( \"essays/essays2467.p\", \"wb\"))\n",
    "print(\"saved entries: \", len(essays))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Data Base 2 - Essay Data and Kaggle data - and save as object list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved entries:  11142\n"
     ]
    }
   ],
   "source": [
    "# concatinate the dataframes:\n",
    "frames  = [df_essays, df_kaggle]\n",
    "essays_kaggle = pd.concat(frames, sort=False)\n",
    "essays_kaggle.reset_index(drop=True)\n",
    "\n",
    "# preprocess data by converting into OBJECT essay and save with pickle and removing non emotional scentences\n",
    "essays_kaggle = create_essays(essays_kaggle, emotional_words)\n",
    "pickle.dump(essays_kaggle, open(\"essays/essays11142.p\", \"wb\"))\n",
    "print(\"saved entries: \", len(essays_kaggle))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Data Base 3 - Essay Data and Kaggle data and Reddit data - and save as object list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved entries:  89364\n"
     ]
    }
   ],
   "source": [
    "# concatinate the dataframes:\n",
    "frames  = [df_essays, df_kaggle, df_reddit]\n",
    "essays_kaggle_reddit = pd.concat(frames, sort=False)\n",
    "essays_kaggle_reddit.reset_index(drop=True)\n",
    "\n",
    "# preprocess data by converting into OBJECT essay and save with pickle and removing non emotional scentences\n",
    "essays_kaggle_reddit = create_essays(essays_kaggle_reddit, emotional_words)\n",
    "pickle.dump(essays_kaggle_reddit, open(\"essays/essays89364.p\", \"wb\"))\n",
    "print(\"saved entries: \", len(essays_kaggle_reddit))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}