prepare_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "82e80d5c-9323-4b13-9bed-d023f8d2a6bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "bb3e9e36-50c4-4c58-807b-19458537786b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15524</th>\n",
       "      <td>28f9c68b3f066089</td>\n",
       "      <td>I am sorry, but I believe you are erroneously ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19683</th>\n",
       "      <td>33ff474f09c50a93</td>\n",
       "      <td>\"===Review Upholds: \"\"24 3RR Hours Block by Us...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130568</th>\n",
       "      <td>ba76eab138456823</td>\n",
       "      <td>\"\\n\\nOf course deletion is done. The lynch mob...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114804</th>\n",
       "      <td>65ef089c977a0ebe</td>\n",
       "      <td>\"\\n\\n Rename the Article \\n\\nI think to resolv...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63518</th>\n",
       "      <td>a9f81a26404a21b1</td>\n",
       "      <td>\"\\n\\nAgreed. Using \"\"the\"\" is inappropriate in...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      id                                       comment_text  \\\n",
       "15524   28f9c68b3f066089  I am sorry, but I believe you are erroneously ...   \n",
       "19683   33ff474f09c50a93  \"===Review Upholds: \"\"24 3RR Hours Block by Us...   \n",
       "130568  ba76eab138456823  \"\\n\\nOf course deletion is done. The lynch mob...   \n",
       "114804  65ef089c977a0ebe  \"\\n\\n Rename the Article \\n\\nI think to resolv...   \n",
       "63518   a9f81a26404a21b1  \"\\n\\nAgreed. Using \"\"the\"\" is inappropriate in...   \n",
       "\n",
       "        toxic  severe_toxic  obscene  threat  insult  identity_hate  \n",
       "15524       0             0        0       0       0              0  \n",
       "19683       0             0        0       0       0              0  \n",
       "130568      0             0        0       0       0              0  \n",
       "114804      0             0        0       0       0              0  \n",
       "63518       0             0        0       0       0              0  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dir = './dataset/'\n",
    "df = pd.read_csv(data_dir+'train.csv')\n",
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d4eb54cc-7399-434a-ab24-5f72cede901a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id               object\n",
       "comment_text     object\n",
       "toxic             int64\n",
       "severe_toxic      int64\n",
       "obscene           int64\n",
       "threat            int64\n",
       "insult            int64\n",
       "identity_hate     int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b71fc515-a54f-4a89-bca4-fe23638bfa5a",
   "metadata": {},
   "source": [
    "* drop `id` column\n",
    "* compress 6 label columns to 1 column of list, then drop these 6 columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "bf53f884-62db-4e52-92d1-917fd2aaace1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if 'id' in df.columns: df.drop('id', axis = 'columns', inplace=True)\n",
    "label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n",
    "if all(col in df.columns for col in label_cols): \n",
    "    df['labels'] = df[label_cols].apply(lambda x: list(x), axis = 'columns')\n",
    "    df.drop(label_cols, axis='columns', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "52bff577-41f5-40b1-80ce-e70121cf0f3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment_text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        comment_text              labels\n",
       "0  Explanation\\nWhy the edits made under my usern...  [0, 0, 0, 0, 0, 0]\n",
       "1  D'aww! He matches this background colour I'm s...  [0, 0, 0, 0, 0, 0]\n",
       "2  Hey man, I'm really not trying to edit war. It...  [0, 0, 0, 0, 0, 0]\n",
       "3  \"\\nMore\\nI can't make any real suggestions on ...  [0, 0, 0, 0, 0, 0]\n",
       "4  You, sir, are my hero. Any chance you remember...  [0, 0, 0, 0, 0, 0]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "6a96180a-1469-49ba-b0b9-cd1fb944c021",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "comment_text    object\n",
       "labels          object\n",
       "dtype: object"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "8bec87a3-753c-4435-95a3-17de18cb3aa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(data_dir+'processed_train.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}