Create_training_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Informativeness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('../HULK_new/Counterspeech/Datasets/Pandora/author_profiles.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['author', 'mbti', 'introverted', 'intuitive', 'thinking', 'perceiving',\n",
       "       'gender', 'age', 'enneagram', 'country', 'state', 'type',\n",
       "       'agreeableness', 'openness', 'conscientiousness', 'extraversion',\n",
       "       'neuroticism', 'is_description', 'is_percentile', 'is_score',\n",
       "       'contains_details', 'num_comments', 'en_comments',\n",
       "       'en_comments_percentage', 'region', 'continent', 'country_code',\n",
       "       'enneagram_type', 'enneagram_wing', 'is_native_english_country',\n",
       "       'predicted_test', 'test_name', 'test_scale', '16pers_ta',\n",
       "       'test_result_type', 'is_female', 'is_female_pred', 'is_female_proba'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "intp    2336\n",
       "intj    1847\n",
       "infp    1074\n",
       "infj    1051\n",
       "entp     631\n",
       "enfp     617\n",
       "istp     407\n",
       "entj     320\n",
       "istj     195\n",
       "enfj     163\n",
       "isfp     123\n",
       "isfj     109\n",
       "estp      72\n",
       "esfp      50\n",
       "estj      43\n",
       "esfj      29\n",
       "intx       4\n",
       "infx       4\n",
       "xntp       2\n",
       "xnfp       1\n",
       "inxj       1\n",
       "inxx       1\n",
       "xnfx       1\n",
       "xsfp       1\n",
       "xnxj       1\n",
       "exxp       1\n",
       "Name: mbti, dtype: int64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mbti.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_labels=pd.read_csv('../HULK/Counterspeech/Datasets/Toxicity/test_labels.csv')\n",
    "df_test=pd.read_csv('../HULK/Counterspeech/Datasets/Toxicity/test.csv')\n",
    "\n",
    "\n",
    "\n",
    "df_test=df_test.merge(df_test_labels, left_on='id', right_on='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuple_head=[]\n",
    "for index,row in df_test.iterrows():\n",
    "    tuple_temp=[row['id'], row['comment_text']]\n",
    "    flag=0\n",
    "    for ele in list(df.columns[2:]):\n",
    "        if(row[ele]==1):\n",
    "            tuple_temp.append('toxic')\n",
    "            flag=1\n",
    "            break\n",
    "    if(flag==0):\n",
    "        tuple_temp.append('non_toxic')\n",
    "    tuple_head.append(tuple_temp)\n",
    "df_new_test=pd.DataFrame(tuple_head,columns=['id','text','labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuple_head=[]\n",
    "for index,row in df.iterrows():\n",
    "    tuple_temp=[row['id'], row['comment_text']]\n",
    "    flag=0\n",
    "    for ele in list(df.columns[2:]):\n",
    "        if(row[ele]==1):\n",
    "            tuple_temp.append('toxic')\n",
    "            flag=1\n",
    "            break\n",
    "    if(flag==0):\n",
    "        tuple_temp.append('non_toxic')\n",
    "    tuple_head.append(tuple_temp)\n",
    "df_new=pd.DataFrame(tuple_head,columns=['id','text','labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val = train_test_split(df_new, test_size=0.1,random_state=42,stratify=df_new['labels'])\n",
    "X_train.to_csv('../HULK/Counterspeech/Datasets/Toxicity/Train.csv',index=False)\n",
    "X_val.to_csv('../HULK/Counterspeech/Datasets/Toxicity/Val.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new_test.to_csv('../HULK/Counterspeech/Datasets/Toxicity/Test.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dialect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "list_lines=[]\n",
    "count=0\n",
    "with open('../HULK/Counterspeech/Datasets/Dialect/TwitterAAE-full-v1/twitteraae_all_aa') as tsv:\n",
    "    for line in csv.reader(tsv, dialect=\"excel-tab\"): #You can also use delimiter=\"\\t\" rather than giving a dialect.\n",
    "        if(len(line)<14):\n",
    "            line=line[0:6]+['NA']*4+line[6:10]\n",
    "        if(len(line)>14):\n",
    "            count+=1\n",
    "        else:\n",
    "            list_lines.append(line)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "179"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame(list_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns=['tweet_IDs', 'tweet_timestamps', 'user_ID', 'tweet-longitudes-latitudes', 'temp','tweet','NA','NA','NA','NA','AA', 'Hispanic', 'Other', 'White']\n",
    "df.columns=columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_IDs</th>\n",
       "      <th>tweet_timestamps</th>\n",
       "      <th>user_ID</th>\n",
       "      <th>tweet-longitudes-latitudes</th>\n",
       "      <th>temp</th>\n",
       "      <th>tweet</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>NA</th>\n",
       "      <th>AA</th>\n",
       "      <th>Hispanic</th>\n",
       "      <th>Other</th>\n",
       "      <th>White</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>293846693215096832</td>\n",
       "      <td>Tue Jan 22 22:24:45 +0000 2013</td>\n",
       "      <td>1028920752</td>\n",
       "      <td>[-80.01040975, 32.80108357]</td>\n",
       "      <td>450190027021</td>\n",
       "      <td>Click Clack Motha Fucka I ain't tryin to hear ...</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.894545454545</td>\n",
       "      <td>0.0163636363636</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0890909090909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>288532237186646016</td>\n",
       "      <td>Tue Jan 08 06:27:00 +0000 2013</td>\n",
       "      <td>1040150983</td>\n",
       "      <td>[-87.322161, 41.6006888]</td>\n",
       "      <td>180890106001</td>\n",
       "      <td>Man imissed a called from my bae hella mad -_-...</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.058</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>289899995472224257</td>\n",
       "      <td>Sat Jan 12 01:01:59 +0000 2013</td>\n",
       "      <td>1068611971</td>\n",
       "      <td>[-78.85113963, 42.909513]</td>\n",
       "      <td>360290033021</td>\n",
       "      <td>@devontekthomas OMG I keep sayin boo wen I mea...</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.951111111111</td>\n",
       "      <td>0.0311111111111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0177777777778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>289901379869351936</td>\n",
       "      <td>Sat Jan 12 01:07:29 +0000 2013</td>\n",
       "      <td>1068611971</td>\n",
       "      <td>[-78.8510475, 42.90955088]</td>\n",
       "      <td>360290033021</td>\n",
       "      <td>@devontekthomas I did not mean to say dat</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.957142857143</td>\n",
       "      <td>0.0342857142857</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00857142857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>289902453367910401</td>\n",
       "      <td>Sat Jan 12 01:11:45 +0000 2013</td>\n",
       "      <td>1068611971</td>\n",
       "      <td>[-78.85107559, 42.90955579]</td>\n",
       "      <td>360290033021</td>\n",
       "      <td>@devontekthomas awww u do too</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1147957</th>\n",
       "      <td>411962043273838592</td>\n",
       "      <td>Sat Dec 14 20:52:59 +0000 2013</td>\n",
       "      <td>847779924</td>\n",
       "      <td>[-76.6672756, 39.3247266]</td>\n",
       "      <td>245101511003</td>\n",
       "      <td>S/O to mi #NF @VillayMusic</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1147958</th>\n",
       "      <td>412144961187237888</td>\n",
       "      <td>Sun Dec 15 08:59:50 +0000 2013</td>\n",
       "      <td>847779924</td>\n",
       "      <td>[-76.6691758, 39.3169376]</td>\n",
       "      <td>245101507012</td>\n",
       "      <td>S/O to mi #NF @Wizmiharez</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.075</td>\n",
       "      <td>0.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1147959</th>\n",
       "      <td>413802894128087040</td>\n",
       "      <td>Thu Dec 19 22:47:52 +0000 2013</td>\n",
       "      <td>847779924</td>\n",
       "      <td>[-76.6636053, 39.3265908]</td>\n",
       "      <td>245101505001</td>\n",
       "      <td>S/O to mi #NF @ZolexDe</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.855</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.095</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1147960</th>\n",
       "      <td>412286992790339584</td>\n",
       "      <td>Sun Dec 15 18:24:13 +0000 2013</td>\n",
       "      <td>97004590</td>\n",
       "      <td>[-84.95087723, 32.60678422]</td>\n",
       "      <td>132150102031</td>\n",
       "      <td>@Dj_Fly_Guy u str8</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1147961</th>\n",
       "      <td>409054967132336128</td>\n",
       "      <td>Fri Dec 06 20:21:18 +0000 2013</td>\n",
       "      <td>990654176</td>\n",
       "      <td>[-92.2886571, 34.729104]</td>\n",
       "      <td>051190047002</td>\n",
       "      <td>I SEE HOES TRYNA LEAVE OUT OF 2013 WIT A GOOD ...</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.995384615385</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00461538461538</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1147962 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  tweet_IDs                tweet_timestamps     user_ID  \\\n",
       "0        293846693215096832  Tue Jan 22 22:24:45 +0000 2013  1028920752   \n",
       "1        288532237186646016  Tue Jan 08 06:27:00 +0000 2013  1040150983   \n",
       "2        289899995472224257  Sat Jan 12 01:01:59 +0000 2013  1068611971   \n",
       "3        289901379869351936  Sat Jan 12 01:07:29 +0000 2013  1068611971   \n",
       "4        289902453367910401  Sat Jan 12 01:11:45 +0000 2013  1068611971   \n",
       "...                     ...                             ...         ...   \n",
       "1147957  411962043273838592  Sat Dec 14 20:52:59 +0000 2013   847779924   \n",
       "1147958  412144961187237888  Sun Dec 15 08:59:50 +0000 2013   847779924   \n",
       "1147959  413802894128087040  Thu Dec 19 22:47:52 +0000 2013   847779924   \n",
       "1147960  412286992790339584  Sun Dec 15 18:24:13 +0000 2013    97004590   \n",
       "1147961  409054967132336128  Fri Dec 06 20:21:18 +0000 2013   990654176   \n",
       "\n",
       "          tweet-longitudes-latitudes          temp  \\\n",
       "0        [-80.01040975, 32.80108357]  450190027021   \n",
       "1           [-87.322161, 41.6006888]  180890106001   \n",
       "2          [-78.85113963, 42.909513]  360290033021   \n",
       "3         [-78.8510475, 42.90955088]  360290033021   \n",
       "4        [-78.85107559, 42.90955579]  360290033021   \n",
       "...                              ...           ...   \n",
       "1147957    [-76.6672756, 39.3247266]  245101511003   \n",
       "1147958    [-76.6691758, 39.3169376]  245101507012   \n",
       "1147959    [-76.6636053, 39.3265908]  245101505001   \n",
       "1147960  [-84.95087723, 32.60678422]  132150102031   \n",
       "1147961     [-92.2886571, 34.729104]  051190047002   \n",
       "\n",
       "                                                     tweet  NA  NA  NA  NA  \\\n",
       "0        Click Clack Motha Fucka I ain't tryin to hear ...  NA  NA  NA  NA   \n",
       "1        Man imissed a called from my bae hella mad -_-...  NA  NA  NA  NA   \n",
       "2        @devontekthomas OMG I keep sayin boo wen I mea...  NA  NA  NA  NA   \n",
       "3                @devontekthomas I did not mean to say dat  NA  NA  NA  NA   \n",
       "4                            @devontekthomas awww u do too  NA  NA  NA  NA   \n",
       "...                                                    ...  ..  ..  ..  ..   \n",
       "1147957                         S/O to mi #NF @VillayMusic  NA  NA  NA  NA   \n",
       "1147958                          S/O to mi #NF @Wizmiharez  NA  NA  NA  NA   \n",
       "1147959                             S/O to mi #NF @ZolexDe  NA  NA  NA  NA   \n",
       "1147960                                 @Dj_Fly_Guy u str8  NA  NA  NA  NA   \n",
       "1147961  I SEE HOES TRYNA LEAVE OUT OF 2013 WIT A GOOD ...  NA  NA  NA  NA   \n",
       "\n",
       "                     AA         Hispanic             Other             White  \n",
       "0        0.894545454545  0.0163636363636               0.0   0.0890909090909  \n",
       "1                 0.942            0.058               0.0               0.0  \n",
       "2        0.951111111111  0.0311111111111               0.0   0.0177777777778  \n",
       "3        0.957142857143  0.0342857142857               0.0  0.00857142857143  \n",
       "4                 0.975             0.01               0.0             0.015  \n",
       "...                 ...              ...               ...               ...  \n",
       "1147957            0.87              0.0               0.1              0.03  \n",
       "1147958            0.83            0.005             0.075              0.09  \n",
       "1147959           0.855              0.0             0.095              0.05  \n",
       "1147960            0.85             0.02               0.0              0.13  \n",
       "1147961  0.995384615385              0.0  0.00461538461538               0.0  \n",
       "\n",
       "[1147962 rows x 14 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Emotion\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_pickle(r'../HULK/Counterspeech/Datasets/Emotion/merged_training.pkl')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns=['text','labels']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27383</th>\n",
       "      <td>i feel awful about it too because it s my job ...</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110083</th>\n",
       "      <td>im alone i feel awful</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140764</th>\n",
       "      <td>ive probably mentioned this before but i reall...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100071</th>\n",
       "      <td>i was feeling a little low few days back</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2837</th>\n",
       "      <td>i beleive that i am much more sensitive to oth...</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>that was what i felt when i was finally accept...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36236</th>\n",
       "      <td>i take every day as it comes i m just focussin...</td>\n",
       "      <td>fear</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76229</th>\n",
       "      <td>i just suddenly feel that everything was fake</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131640</th>\n",
       "      <td>im feeling more eager than ever to claw back w...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64703</th>\n",
       "      <td>i give you plenty of attention even when i fee...</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>416809 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text   labels\n",
       "27383   i feel awful about it too because it s my job ...  sadness\n",
       "110083                              im alone i feel awful  sadness\n",
       "140764  ive probably mentioned this before but i reall...      joy\n",
       "100071           i was feeling a little low few days back  sadness\n",
       "2837    i beleive that i am much more sensitive to oth...     love\n",
       "...                                                   ...      ...\n",
       "566     that was what i felt when i was finally accept...      joy\n",
       "36236   i take every day as it comes i m just focussin...     fear\n",
       "76229       i just suddenly feel that everything was fake  sadness\n",
       "131640  im feeling more eager than ever to claw back w...      joy\n",
       "64703   i give you plenty of attention even when i fee...  sadness\n",
       "\n",
       "[416809 rows x 2 columns]"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val_test = train_test_split(df, test_size=0.2,random_state=42,stratify=df['labels'])\n",
    "X_val, X_test = train_test_split(X_val_test, test_size=0.5,random_state=42,stratify=X_val_test['labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_csv('../HULK/Counterspeech/Datasets/Emotion/Train.csv',index=False)\n",
    "X_val.to_csv('../HULK/Counterspeech/Datasets/Emotion/Val.csv',index=False)\n",
    "X_test.to_csv('../HULK/Counterspeech/Datasets/Emotion/Test.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_polite=pd.read_csv('../HULK/Counterspeech/Datasets/Politeness/politeness.tsv',sep='\\t')\n",
    "df_polite.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuple_head=[]\n",
    "for index,row in df_polite[df_polite['split']=='train'].iterrows():\n",
    "    tuple_temp=[row['txt']]\n",
    "    if(row['style']=='P_9'):\n",
    "        tuple_temp.append('polite')\n",
    "    else:\n",
    "        tuple_temp.append('non_polite')\n",
    "    tuple_head.append(tuple_temp)\n",
    "df_polite_train=pd.DataFrame(tuple_head,columns=['text','labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_polite_train.to_csv('../HULK/Counterspeech/Datasets/Politeness/Train.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuple_head=[]\n",
    "for index,row in df_polite[df_polite['split']=='val'].iterrows():\n",
    "    tuple_temp=[row['txt']]\n",
    "    if(row['style']=='P_9'):\n",
    "        tuple_temp.append('polite')\n",
    "    else:\n",
    "        tuple_temp.append('non_polite')\n",
    "    tuple_head.append(tuple_temp)\n",
    "df_polite_val=pd.DataFrame(tuple_head,columns=['text','labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_polite_val.to_csv('../HULK/Counterspeech/Datasets/Politeness/Val.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuple_head=[]\n",
    "for index,row in df_polite[df_polite['split']=='test'].iterrows():\n",
    "    tuple_temp=[row['txt']]\n",
    "    if(row['style']=='P_9'):\n",
    "        tuple_temp.append('polite')\n",
    "    else:\n",
    "        tuple_temp.append('non_polite')\n",
    "    tuple_head.append(tuple_temp)\n",
    "df_polite_test=pd.DataFrame(tuple_head,columns=['text','labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_polite_test.to_csv('../HULK/Counterspeech/Datasets/Politeness/Test.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df_polite_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train=pd.read_csv('../HULK/Counterspeech/Datasets/Humour/train.csv')\n",
    "df_test=pd.read_csv('../HULK/Counterspeech/Datasets/Humour/dev.csv')\n",
    "df_total=pd.read_csv('../HULK/Counterspeech/Datasets/Humour/dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label=[]\n",
    "\n",
    "for index,row in df_train.iterrows():\n",
    "    if(row['humor']==True):\n",
    "        label.append('humor')\n",
    "    else:\n",
    "        label.append('non-humor')\n",
    "        \n",
    "df_train['labels']=label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label=[]\n",
    "\n",
    "for index,row in df_test.iterrows():\n",
    "    if(row['humor']==True):\n",
    "        label.append('humor')\n",
    "    else:\n",
    "        label.append('non-humor')\n",
    "        \n",
    "df_test['labels']=label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_val, X_test = train_test_split(df_test, test_size=0.5,random_state=42,stratify=df_test['labels'])\n",
    "df_train.to_csv('../HULK/Counterspeech/Datasets/Humour/Train.csv',index=False)\n",
    "X_test.to_csv('../HULK/Counterspeech/Datasets/Humour/Test.csv',index=False)\n",
    "X_val.to_csv('../HULK/Counterspeech/Datasets/Humour/Val.csv',index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data generation save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Save the CONAN, Reddit/Gab dataset, create debate dataset \n",
    "### Columns - id, initiator_message, reply_message\n",
    "import json\n",
    "with open('../HULK/Counterspeech/Datasets/Create_debate/selected_arguments.json', 'r') as fp:\n",
    "        dict_urls=json.load(fp)\n",
    "        \n",
    "total_data_sentences=[]\n",
    "for key in dict_urls:\n",
    "    dict_temp=dict_urls[key]['selected_arguments']\n",
    "    \n",
    "    stance=[]\n",
    "    for ele in dict_temp:\n",
    "        stance.append(ele['reply_stance'])\n",
    "    \n",
    "    stance=list(set(stance))\n",
    "    \n",
    "    for ele in dict_temp:\n",
    "        ele['title']=dict_urls[key]['title']\n",
    "        ele['stance_list']=stance\n",
    "        total_data_sentences.append(ele)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_data_sentences[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame(total_data_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test_dev = train_test_split(df, test_size=0.2, random_state=42, shuffle=True)\n",
    "X_test, X_dev = train_test_split(X_test_dev, test_size=0.5, random_state=42, shuffle=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_csv('../HULK/Counterspeech/Datasets/Create_debate/Train.csv')\n",
    "X_dev.to_csv('../HULK/Counterspeech/Datasets/Create_debate/Val.csv')\n",
    "X_test.to_csv('../HULK/Counterspeech/Datasets/Create_debate/Test.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train=pd.read_csv('../HULK_new/Counterspeech/Datasets/Create_debate/Train.csv')\n",
    "X_dev=pd.read_csv('../HULK_new/Counterspeech/Datasets/Create_debate/Val.csv')\n",
    "X_test=pd.read_csv('../HULK_new/Counterspeech/Datasets/Create_debate/Val.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initiator: you finally got something right . i can not handle debating a teenager . i used to have some teenagers at home  happy  somehow in thier short lives , they have accumulated all the wisdom of the universe and have no problem stating it as fact .\n",
      "\n",
      "Reply: you sound like you just gave up because you can not handle debating with me . \" have a great life and please think about what you support when you start voting \" . i thought you gave up :,) . i also never said if i was a teenager or not . i guess you are so desperate to have something against me that you start making up stuff .\n",
      "==================\n",
      "Initiator: it clearly says in the bible and written out that it a sin , and a choice . stop being ignorant .\n",
      "\n",
      "Reply: prove it . post the verses from the king james bible . .\n",
      "==================\n",
      "Initiator: i loathe people who think that their version of logic is the proper one - but enough of these insults . there is one logical reason to engage in intercourse . procreation . homosexuality cannot result in impregnation and thus there is no logical reason to engage in homosexual behavior . \" because we can . \" is not a logical answer . we can do a great many things , but that does not mean that there is a logical reason to do so .\n",
      "\n",
      "Reply: humans are one of the only species to engage in sex for pleasure , as well as reproduction . and on top of this , we can reproduce without sex , which many couples resort to because they can not bear children . therefore , marriage is not only for people intending to have children .\n",
      "==================\n",
      "Initiator: if you have a difference in opinion , how in the world can you expect me , an atheist to take either of you seriously ? srom spouts his version of bullshit , then you spout your bullshit , then some random muslim spouts their bullshit , and then we get loads of others who spout their bullshit . truth is not an opinion you fucking idiot .\n",
      "\n",
      "Reply: but everyone has their opinion on what true and what is not . we just differ in that area . we belong to different denominations .\n",
      "==================\n",
      "Initiator: the whole gun analogy does not work . burglary is rare , something a person can expect to go their entire lives without experiencing if they live in a safe area . sex is common , and pregnancy costs the country : especially unwanted pregnancies where a child is given into care . if you are not an idiot you should know that birth control has other uses . condoms protect against these sometimes fatal , sometimes irritating things we call sti . y know , like aids and syphilis . birth control is not cheap . if i had to pay for the pill i would be spending  money  / week , which is a lot . more than i spend on food .\n",
      "\n",
      "Reply: you have not actually provided a good enough demonstration that his analogy is bad . if burglary is so rare then you only have to spend money one time to protect someone for life . since you do not need to get super fancy with a gun for home protection  money  will be good enough .  money  to prevent burglary is a good deal . birth control is not cheap . if i had to pay for the pill i would be spending  money  / week , which is a lot . more than i spend on food . for the price of the pill , you can pay off the gun in  number  weeks . so , as long as you need the pill for half a year it is actually cheaper to pay for his gun and protect him forever . how is that for actual debating ?  happy \n",
      "==================\n",
      "Initiator: i think it makes sense to let women play any sport with men as long as no special treatment towards either is happening . if a women proves to be as good of player as any of the other guys she should at be considered .\n",
      "\n",
      "Reply: i think that if a woman can be proven to be as good as an nba player , she should be able to play . but since we have not seen this , they should not be able to play . today , there became a new leading scorer in the wnba with somewhere in the  number  , 0 0 0 in scoring , while in the nba , it is in the  number  , 0 0 0 . i know points is not the whole story put the nba is a very physical league compared to the wnba , which is a very physical league as well\n",
      "==================\n",
      "Initiator: nice try prodigee , i found weird that you would be able of actual reply or argument ( even so weak one ) . i have googled your reply  laugh \n",
      "\n",
      "Reply: i am not prodigee -.- why do you keep saying that ? actually reply or your argument is invalid .\n",
      "==================\n",
      "Initiator: prior to isaac newton we did not really understand planetary motion . this is what ptolemy said about planetary motion , i know that i am mortal by nature , and ephemeral ; but when i trace at my pleasure the windings to and fro of the heavenly bodies i no longer touch the earth with my feet : i stand in the presence of zeus himself and take my fill of ambrosia for ptolemy , having lived over a millennia before newton , the movement of the planets was an unknowable mystery only understood by the gods . history is replete with things that we will never be know , that with time were eventually known , and when you resign god or gods to such gaps in human knowledge they dwell in increasingly small gaps . the temptation to assign the cause of things we do not currently understand to the supernatural is an ever present one , and an intellectually perilous one at that .\n",
      "\n",
      "Reply: well they seemed to have had an idea that time worked around the solar system . or at least if they did not they worded it in a way that conveyed it when time was established . why did the writer choose the order for the creation days ? even that took some thought and advanced thinking .\n",
      "==================\n",
      "Initiator: down voted , reason a lie . proof and evidence says so , period . this is not not even disputable , it is fact . the very same fact that makes you an atheist . to dispute this is to dispute all facts . where now is your proof god does not exist ? source : defination of god , all dictionaries .\n",
      "\n",
      "Reply: um . what ? seriously . just , what ? i have read that three times . i can not even figure out what argument you are trying to make that relates to what i said .\n",
      "==================\n",
      "Initiator: if you call copy and pasting original , i d have to agree with you . your idea of copy paste is way more generous than anybody else . if someone talks about a topic you consider it copy and paste if anyone else ever talked about that topic . that is not copying . the bible contains  number  books . how books have you read to confirm your non - beliefs ? that your argument ? more books sort of contradicts the whole point of your debate . what is the threshold ? is  number  the number ? if i have read  number  books i have enough information ? what about  number  ? plus , it is still books i would use to confirm my beliefs . it is funny how you are forced to call it non - beliefs to lie to yourself that you are right .\n",
      "\n",
      "Reply: that your argument ? more books sort of contradicts the whole point of your debate . what is the threshold ? is  number  the number ? if i have read  number  books i have enough information ? what about  number  ? plus , it is still books i would use to confirm my beliefs . it is funny how you are forced to call it non - beliefs to lie to yourself that you are right . you keep telling me about all these sources and you said the bible is just one , i am merely pointing out that it is not one book . you keep claiming more is better . your words , not mine . it does not contradict the point of the debate , because i did not argue that more is better . you did . i just pointed this out . i was poking funny of atheists with the non - belief line . learn the english language , things are not also meant literal .\n",
      "==================\n"
     ]
    }
   ],
   "source": [
    "from Generation.data import *\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"microsoft/DialoGPT-medium\",cache_dir='../HULK_new/Saved_models/')\n",
    "max_length=256\n",
    "\n",
    "def preprocess_func(text):\n",
    "        remove_words=['<allcaps>','</allcaps>','<hashtag>','</hashtag>','<elongated>','<emphasis>','<repeated>','\\'','s']\n",
    "        \n",
    "        \n",
    "        word_list=text_processor.pre_process_doc(text)\n",
    "        word_list=list(filter(lambda a: a not in remove_words, word_list)) \n",
    "        sent=\" \".join(word_list)\n",
    "        sent = re.sub(r\"[<\\*>]\", \" \",sent)\n",
    "        return sent\n",
    "    \n",
    "    \n",
    "def construct_conv(dict_reply_pair):\n",
    "    conv = None\n",
    "    flatten = lambda l: [item for sublist in l for item in sublist]\n",
    "    initiator=preprocess_func(dict_reply_pair['initiator_message'])\n",
    "    reply=preprocess_func(dict_reply_pair['reply_message'])\n",
    "\n",
    "\n",
    "#     conv = list([tokenizer.encode(initiator,truncation=True,max_length=int((max_length/2)-1))+ \n",
    "#                  [tokenizer.eos_token_id] + \n",
    "#                tokenizer.encode(reply,truncation=True,max_length=int((max_length/2)-1))+\n",
    "#                 [tokenizer.eos_token_id]])\n",
    "\n",
    "#     conv = flatten(conv)\n",
    "#     initiator= tokenizer.tokenize(initiator,truncation=True,max_length=int((max_length/2)-1))\n",
    "#     reply = tokenizer.tokenize(reply,truncation=True,max_length=int((max_length/2)-1))\n",
    "    return (initiator,reply)\n",
    "\n",
    "df_sample=X_train.sample(10)\n",
    "\n",
    "\n",
    "for index,row in df_sample.iterrows():\n",
    "    initiator,reply=construct_conv(row)\n",
    "    print(\"Initiator:\",initiator)\n",
    "    print()\n",
    "    print(\"Reply:\",reply)\n",
    "    print(\"==================\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0                                                        4920\n",
       "reply_message        He doesn't bring up a valid point. Hes not que...\n",
       "reply_stance                            Christianity, mostly to blame.\n",
       "initiator_message    Hey wait a minute. He brings up a valid point....\n",
       "initiator_stance                               Islam, mostly to blame.\n",
       "reply_type                                                    disputed\n",
       "title                                  Religion is hurting Humanity.\\n\n",
       "stance_list          ['Islam, mostly to blame.', 'Christianity, mos...\n",
       "Name: 10876, dtype: object"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('../HULK/Counterspeech/Datasets/Reddit/reddit.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reddit Data preparing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "#Data Input and Preprocessing which is data specific\n",
    "data = pd.read_csv('../HULK/Counterspeech/Datasets/Gab/gab.csv')\n",
    "data_mod = pd.DataFrame([[]])\n",
    "\n",
    "for row in range(len(data[\"id\"])):\n",
    "    #print(data[\"id\"][row])\n",
    "    data[\"id\"][row]=(data[\"id\"][row].splitlines())\n",
    "    for num in range(len(data[\"id\"][row])):\n",
    "        strin=''.join(data[\"id\"][row][num].split())\n",
    "        newstr='';\n",
    "        for i in range(len(strin)-2):\n",
    "            newstr+=strin[i+2]\n",
    "        data[\"id\"][row][num]=newstr\n",
    "\n",
    "for row in range(len(data[\"hate_speech_idx\"])):\n",
    "        if data[\"hate_speech_idx\"][row]=='n/a':\n",
    "            data[\"hate_speech_idx\"][row]=\"[0]\"\n",
    "\n",
    "for row in range(len(data[\"hate_speech_idx\"])):\n",
    "        if data[\"hate_speech_idx\"][row]=='n/a':\n",
    "            data[\"hate_speech_idx\"][row]=\"[0]\"\n",
    "\n",
    "for row in range(len(data[\"hate_speech_idx\"])):\n",
    "        if isinstance(data[\"hate_speech_idx\"][row],float):\n",
    "            data[\"hate_speech_idx\"][row]=\"[0]\"  \n",
    "\n",
    "for row in range(len(data[\"response\"])):\n",
    "        if isinstance(data[\"response\"][row],float):\n",
    "            data[\"response\"][row]='This is normal'\n",
    "            \n",
    "for row in range(len(data[\"text\"])):\n",
    "    #print(data[\"id\"][row])\n",
    "    data[\"text\"][row]=(data[\"text\"][row].splitlines())\n",
    "    for num in range(len(data[\"text\"][row])):\n",
    "        strin=' '.join(data[\"text\"][row][num].split())\n",
    "        newstr='';\n",
    "        for i in range(len(strin)-2):\n",
    "            newstr+=strin[i+2]\n",
    "        data[\"text\"][row][num]=newstr\n",
    "    \n",
    "    data[\"hate_speech_idx\"][row]=data[\"hate_speech_idx\"][row].strip('][').split(', ');\n",
    "    data[\"hate_speech_idx\"][row]=list(map(int, data[\"hate_speech_idx\"][row]))\n",
    "    data[\"response\"][row] = data[\"response\"][row].replace(\"\\\"\",\"\\'\")\n",
    "    data[\"response\"][row] = data[\"response\"][row].replace(\"[\\'\",\"\")\n",
    "    data[\"response\"][row] = data[\"response\"][row].replace(\"\\']\",\"\")\n",
    "    data[\"response\"][row] = data[\"response\"][row].split('\\', \\'');\n",
    "    \n",
    "    for idx in data[\"hate_speech_idx\"][row]:\n",
    "        if idx!=0 and idx<=len(data[\"text\"][row]):\n",
    "            bad=idx-1;\n",
    "#            print(bad)\n",
    "            for resp in data[\"response\"][row]:\n",
    "                newrow=pd.Series([data[\"text\"][row][bad],resp])\n",
    "                rowdf=pd.DataFrame([newrow])\n",
    "                data_mod=pd.concat([data_mod,rowdf],ignore_index=True)\n",
    "                #print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_mod=data_mod[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_mod.columns=['initiator_message','reply_message']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_mod=data_mod[(data_mod['reply_message']!='NA') & (data_mod['reply_message']!='n/a') & (data_mod['reply_message']!='N/A') & (data_mod['initiator_message']!='') & (data_mod['reply_message']!='')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test_dev = train_test_split(data_mod, test_size=0.2, random_state=42, shuffle=True)\n",
    "X_test, X_dev = train_test_split(X_test_dev, test_size=0.5, random_state=42, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.iloc[9166]['reply_message']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_csv('../HULK/Counterspeech/Datasets/Gab/Train.csv')\n",
    "X_dev.to_csv('../HULK/Counterspeech/Datasets/Gab/Val.csv')\n",
    "X_test.to_csv('../HULK/Counterspeech/Datasets/Gab/Test.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_json('../HULK/Counterspeech/Datasets/CONAN/CONAN.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=df[df['cn_id'].str.contains(\"EN\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new=df[['cn_id','hateSpeech','counterSpeech']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new.columns=['cn_id','initiator_message','reply_message']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test_dev = train_test_split(df_new, test_size=0.2, random_state=42, shuffle=True)\n",
    "X_test, X_dev = train_test_split(X_test_dev, test_size=0.5, random_state=42, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_csv('../HULK/Counterspeech/Datasets/CONAN/Train.csv')\n",
    "X_dev.to_csv('../HULK/Counterspeech/Datasets/CONAN/Val.csv')\n",
    "X_test.to_csv('../HULK/Counterspeech/Datasets/CONAN/Test.csv')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CONAN dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import neptune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neptune.init()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:.conda-csgen] *",
   "language": "python",
   "name": "conda-env-.conda-csgen-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}