0_866_tfidf_ridge_simple_baseline.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "id": "f62ef52c",
      "metadata": {
        "id": "f62ef52c",
        "papermill": {
          "duration": 0.033007,
          "end_time": "2022-01-07T09:33:47.396671",
          "exception": false,
          "start_time": "2022-01-07T09:33:47.363664",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "TFIDF_Ridge_simple_baseline\n",
        "\n",
        "\n",
        "Data from [Toxic Comment Classification Challenge](https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "gUWKndrrbbf6",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gUWKndrrbbf6",
        "outputId": "34ba881c-92c2-46e9-816e-c2860a4ccb4d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/bin/kaggle\", line 5, in <module>\n",
            "    from kaggle.cli import main\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/kaggle/__init__.py\", line 23, in <module>\n",
            "    api.authenticate()\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/kaggle/api/kaggle_api_extended.py\", line 166, in authenticate\n",
            "    self.config_file, self.config_dir))\n",
            "OSError: Could not find kaggle.json. Make sure it's located in /root/.kaggle. Or use the environment method.\n"
          ]
        }
      ],
      "source": [
        "!kaggle competitions download -c jigsaw-toxic-comment-classification-challenge"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "919c199d",
      "metadata": {
        "id": "919c199d",
        "papermill": {
          "duration": 0.031704,
          "end_time": "2022-01-07T09:33:47.461193",
          "exception": false,
          "start_time": "2022-01-07T09:33:47.429489",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Import Library"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "355ece36",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:33:47.535588Z",
          "iopub.status.busy": "2022-01-07T09:33:47.531481Z",
          "iopub.status.idle": "2022-01-07T09:33:49.727087Z",
          "shell.execute_reply": "2022-01-07T09:33:49.727710Z",
          "shell.execute_reply.started": "2022-01-05T23:50:42.217865Z"
        },
        "id": "355ece36",
        "papermill": {
          "duration": 2.236963,
          "end_time": "2022-01-07T09:33:49.728070",
          "exception": false,
          "start_time": "2022-01-07T09:33:47.491107",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from tqdm.auto import tqdm\n",
        "from bs4 import BeautifulSoup\n",
        "from collections import defaultdict\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "\n",
        "import re \n",
        "import scipy\n",
        "from scipy import sparse\n",
        "\n",
        "from IPython.display import display\n",
        "from pprint import pprint\n",
        "from matplotlib import pyplot as plt \n",
        "\n",
        "import time\n",
        "import scipy.optimize as optimize\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "pd.options.display.max_colwidth=300\n",
        "pd.options.display.max_columns = 100\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from nltk.tokenize import word_tokenize\n",
        "from sklearn.linear_model import Ridge"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "680a1f42",
      "metadata": {
        "id": "680a1f42",
        "papermill": {
          "duration": 0.029215,
          "end_time": "2022-01-07T09:33:49.788454",
          "exception": false,
          "start_time": "2022-01-07T09:33:49.759239",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Prepare train data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8b61b324",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:33:49.851787Z",
          "iopub.status.busy": "2022-01-07T09:33:49.851100Z",
          "iopub.status.idle": "2022-01-07T09:33:52.126132Z",
          "shell.execute_reply": "2022-01-07T09:33:52.124995Z",
          "shell.execute_reply.started": "2022-01-05T23:50:42.229578Z"
        },
        "id": "8b61b324",
        "papermill": {
          "duration": 2.307503,
          "end_time": "2022-01-07T09:33:52.126324",
          "exception": false,
          "start_time": "2022-01-07T09:33:49.818821",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv(\"../input/jigsaw-toxic-comment-classification-challenge/train.csv\")\n",
        "# df_test = pd.read_csv(\"../input/jigsaw-unintended-bias-in-toxicity-classification/train.csv\")\n",
        "# df_test_label = pd.read_csv(\"../input/jigsaw-toxic-comment-classification-challenge/test_labels.csv\").replace(-1,0)\n",
        "df_sub = pd.read_csv(\"../input/jigsaw-toxic-severity-rating/comments_to_score.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9bf07937",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:33:52.190446Z",
          "iopub.status.busy": "2022-01-07T09:33:52.189379Z",
          "iopub.status.idle": "2022-01-07T09:33:52.211752Z",
          "shell.execute_reply": "2022-01-07T09:33:52.212296Z",
          "shell.execute_reply.started": "2022-01-05T23:50:44.274927Z"
        },
        "id": "9bf07937",
        "outputId": "9b2c01ba-e6b9-4416-f630-a532541aee85",
        "papermill": {
          "duration": 0.056161,
          "end_time": "2022-01-07T09:33:52.212473",
          "exception": false,
          "start_time": "2022-01-07T09:33:52.156312",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0000997932d777bf</td>\n",
              "      <td>Explanation\\nWhy the edits made under my username Hardcore Metallica Fan were reverted? They weren't vandalisms, just closure on some GAs after I voted at New York Dolls FAC. And please don't remove the template from the talk page since I'm retired now.89.205.38.27</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000103f0d9cfb60f</td>\n",
              "      <td>D'aww! He matches this background colour I'm seemingly stuck with. Thanks.  (talk) 21:51, January 11, 2016 (UTC)</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 id  \\\n",
              "0  0000997932d777bf   \n",
              "1  000103f0d9cfb60f   \n",
              "\n",
              "                                                                                                                                                                                                                                                                comment_text  \\\n",
              "0  Explanation\\nWhy the edits made under my username Hardcore Metallica Fan were reverted? They weren't vandalisms, just closure on some GAs after I voted at New York Dolls FAC. And please don't remove the template from the talk page since I'm retired now.89.205.38.27   \n",
              "1                                                                                                                                                           D'aww! He matches this background colour I'm seemingly stuck with. Thanks.  (talk) 21:51, January 11, 2016 (UTC)   \n",
              "\n",
              "   toxic  severe_toxic  obscene  threat  insult  identity_hate  \n",
              "0      0             0        0       0       0              0  \n",
              "1      0             0        0       0       0              0  "
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_train.head(2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8967122f",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:33:52.277397Z",
          "iopub.status.busy": "2022-01-07T09:33:52.276381Z",
          "iopub.status.idle": "2022-01-07T09:33:52.382296Z",
          "shell.execute_reply": "2022-01-07T09:33:52.382851Z",
          "shell.execute_reply.started": "2022-01-05T23:50:44.300006Z"
        },
        "id": "8967122f",
        "outputId": "b99aa2e6-048e-4bcb-de01-1e4a7495876b",
        "papermill": {
          "duration": 0.139929,
          "end_time": "2022-01-07T09:33:52.383017",
          "exception": false,
          "start_time": "2022-01-07T09:33:52.243088",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "      <th>score</th>\n",
              "      <th>y</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0002bcb3da6cb337</td>\n",
              "      <td>COCKSUCKER BEFORE YOU PISS AROUND ON MY WORK</td>\n",
              "      <td>0.32</td>\n",
              "      <td>1.5</td>\n",
              "      <td>0.16</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.64</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.62</td>\n",
              "      <td>2.62</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>0005c987bdfc9d4b</td>\n",
              "      <td>Hey... what is it..\\n@ | talk .\\nWhat is it... an exclusive group of some WP TALIBANS...who are good at destroying, self-appointed purist who GANG UP any one who asks them questions abt their ANTI-SOCIAL and DESTRUCTIVE (non)-contribution at WP?\\n\\nAsk Sityush to clean up his behavior than issue...</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.32</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                  id  \\\n",
              "6   0002bcb3da6cb337   \n",
              "12  0005c987bdfc9d4b   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                   comment_text  \\\n",
              "6                                                                                                                                                                                                                                                                  COCKSUCKER BEFORE YOU PISS AROUND ON MY WORK   \n",
              "12  Hey... what is it..\\n@ | talk .\\nWhat is it... an exclusive group of some WP TALIBANS...who are good at destroying, self-appointed purist who GANG UP any one who asks them questions abt their ANTI-SOCIAL and DESTRUCTIVE (non)-contribution at WP?\\n\\nAsk Sityush to clean up his behavior than issue...   \n",
              "\n",
              "    toxic  severe_toxic  obscene  threat  insult  identity_hate  score     y  \n",
              "6    0.32           1.5     0.16     0.0    0.64            0.0   2.62  2.62  \n",
              "12   0.32           0.0     0.00     0.0    0.00            0.0   0.32  0.32  "
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Create a score that measure how much toxic is a comment\n",
        "cat_mtpl = {'obscene': 0.16, 'toxic': 0.32, 'threat': 1.5, \n",
        "            'insult': 0.64, 'severe_toxic': 1.5, 'identity_hate': 1.5}\n",
        "\n",
        "for category in cat_mtpl:\n",
        "    df_train[category] = df_train[category] * cat_mtpl[category]\n",
        "\n",
        "df_train['score'] = df_train.loc[:, 'toxic':'identity_hate'].sum(axis=1)\n",
        "\n",
        "df_train['y'] = df_train['score']\n",
        "\n",
        "min_len = (df_train['y'] > 0).sum()  # len of toxic comments\n",
        "df_y0_undersample = df_train[df_train['y'] == 0].sample(n=min_len, random_state=201)  # take non toxic comments\n",
        "df_train_new = pd.concat([df_train[df_train['y'] > 0], df_y0_undersample])  # make new df\n",
        "df_train_new.head(2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5b364d99",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:33:52.453531Z",
          "iopub.status.busy": "2022-01-07T09:33:52.452110Z",
          "iopub.status.idle": "2022-01-07T09:33:52.465585Z",
          "shell.execute_reply": "2022-01-07T09:33:52.464736Z",
          "shell.execute_reply.started": "2022-01-05T23:50:44.436049Z"
        },
        "id": "5b364d99",
        "papermill": {
          "duration": 0.05146,
          "end_time": "2022-01-07T09:33:52.465737",
          "exception": false,
          "start_time": "2022-01-07T09:33:52.414277",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "df_train = df_train.rename(columns={'comment_text':'text'})"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "d9ccc1b2",
      "metadata": {
        "id": "d9ccc1b2",
        "papermill": {
          "duration": 0.031118,
          "end_time": "2022-01-07T09:33:52.528004",
          "exception": false,
          "start_time": "2022-01-07T09:33:52.496886",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "<h3>Text Cleaning</h3>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9e0c44d0",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:33:52.593775Z",
          "iopub.status.busy": "2022-01-07T09:33:52.593042Z",
          "iopub.status.idle": "2022-01-07T09:33:52.601720Z",
          "shell.execute_reply": "2022-01-07T09:33:52.601021Z",
          "shell.execute_reply.started": "2022-01-05T23:50:44.457256Z"
        },
        "id": "9e0c44d0",
        "papermill": {
          "duration": 0.04238,
          "end_time": "2022-01-07T09:33:52.601861",
          "exception": false,
          "start_time": "2022-01-07T09:33:52.559481",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "def text_cleaning(text):\n",
        "    '''\n",
        "    Cleans text into a basic form for NLP. Operations include the following:-\n",
        "    1. Remove special charecters like &, #, etc\n",
        "    2. Removes extra spaces\n",
        "    3. Removes embedded URL links\n",
        "    4. Removes HTML tags\n",
        "    5. Removes emojis\n",
        "    \n",
        "    text - Text piece to be cleaned.\n",
        "    '''\n",
        "    template = re.compile(r'https?://\\S+|www\\.\\S+') #Removes website links\n",
        "    text = template.sub(r'', text)\n",
        "    \n",
        "    soup = BeautifulSoup(text, 'lxml') #Removes HTML tags\n",
        "    only_text = soup.get_text()\n",
        "    text = only_text\n",
        "    \n",
        "    emoji_pattern = re.compile(\"[\"\n",
        "                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "                               u\"\\U00002702-\\U000027B0\"\n",
        "                               u\"\\U000024C2-\\U0001F251\"\n",
        "                               \"]+\", flags=re.UNICODE)\n",
        "    text = emoji_pattern.sub(r'', text)\n",
        "    \n",
        "    text = re.sub(r\"[^a-zA-Z\\d]\", \" \", text) #Remove special Charecters\n",
        "    text = re.sub(' +', ' ', text) #Remove Extra Spaces\n",
        "    text = text.strip() # remove spaces at the beginning and at the end of string\n",
        "\n",
        "    return text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8f18867c",
      "metadata": {
        "colab": {
          "referenced_widgets": [
            "c8bf250454644a51afdd53fdf53befda"
          ]
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T09:33:52.668225Z",
          "iopub.status.busy": "2022-01-07T09:33:52.665338Z",
          "iopub.status.idle": "2022-01-07T09:34:55.092477Z",
          "shell.execute_reply": "2022-01-07T09:34:55.092968Z",
          "shell.execute_reply.started": "2022-01-05T23:50:44.468215Z"
        },
        "id": "8f18867c",
        "outputId": "619dfbb3-44fa-4a1c-eefd-ce2e25889d1e",
        "papermill": {
          "duration": 62.460721,
          "end_time": "2022-01-07T09:34:55.093155",
          "exception": false,
          "start_time": "2022-01-07T09:33:52.632434",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c8bf250454644a51afdd53fdf53befda",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/159571 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "tqdm.pandas()\n",
        "df_train['text'] = df_train['text'].progress_apply(text_cleaning)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f794a464",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:34:55.160809Z",
          "iopub.status.busy": "2022-01-07T09:34:55.160104Z",
          "iopub.status.idle": "2022-01-07T09:34:55.175536Z",
          "shell.execute_reply": "2022-01-07T09:34:55.174827Z",
          "shell.execute_reply.started": "2022-01-05T23:51:46.020949Z"
        },
        "id": "f794a464",
        "papermill": {
          "duration": 0.051239,
          "end_time": "2022-01-07T09:34:55.175682",
          "exception": false,
          "start_time": "2022-01-07T09:34:55.124443",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "df = df_train.copy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ed79f101",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:34:55.248295Z",
          "iopub.status.busy": "2022-01-07T09:34:55.241299Z",
          "iopub.status.idle": "2022-01-07T09:34:55.254519Z",
          "shell.execute_reply": "2022-01-07T09:34:55.255082Z",
          "shell.execute_reply.started": "2022-01-05T23:51:46.049316Z"
        },
        "id": "ed79f101",
        "outputId": "d4577385-7c1a-4688-af52-07c13e419fa0",
        "papermill": {
          "duration": 0.048429,
          "end_time": "2022-01-07T09:34:55.255281",
          "exception": false,
          "start_time": "2022-01-07T09:34:55.206852",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.00    143346\n",
              "0.32      5666\n",
              "1.12      3800\n",
              "0.48      1758\n",
              "2.62      1738\n",
              "0.96      1215\n",
              "4.12       385\n",
              "0.16       317\n",
              "0.64       301\n",
              "1.82       290\n",
              "1.98       204\n",
              "0.80       181\n",
              "2.46       164\n",
              "1.50        76\n",
              "2.14        31\n",
              "5.62        31\n",
              "3.32        21\n",
              "2.30        20\n",
              "3.48        10\n",
              "3.96        10\n",
              "1.66         5\n",
              "3.96         1\n",
              "4.82         1\n",
              "Name: y, dtype: int64"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df['y'].value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "440f8ff9",
      "metadata": {
        "id": "440f8ff9",
        "papermill": {
          "duration": 0.031113,
          "end_time": "2022-01-07T09:34:55.317916",
          "exception": false,
          "start_time": "2022-01-07T09:34:55.286803",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Undersampling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "70582e81",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:34:55.385185Z",
          "iopub.status.busy": "2022-01-07T09:34:55.384483Z",
          "iopub.status.idle": "2022-01-07T09:34:55.393132Z",
          "shell.execute_reply": "2022-01-07T09:34:55.393649Z",
          "shell.execute_reply.started": "2022-01-05T23:51:46.064295Z"
        },
        "id": "70582e81",
        "outputId": "a412587a-31a6-4ce9-a5a2-4cee4f250666",
        "papermill": {
          "duration": 0.044453,
          "end_time": "2022-01-07T09:34:55.393820",
          "exception": false,
          "start_time": "2022-01-07T09:34:55.349367",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.00    0.898321\n",
              "0.32    0.035508\n",
              "1.12    0.023814\n",
              "0.48    0.011017\n",
              "2.62    0.010892\n",
              "0.96    0.007614\n",
              "4.12    0.002413\n",
              "0.16    0.001987\n",
              "0.64    0.001886\n",
              "1.82    0.001817\n",
              "1.98    0.001278\n",
              "0.80    0.001134\n",
              "2.46    0.001028\n",
              "1.50    0.000476\n",
              "2.14    0.000194\n",
              "5.62    0.000194\n",
              "3.32    0.000132\n",
              "2.30    0.000125\n",
              "3.48    0.000063\n",
              "3.96    0.000063\n",
              "1.66    0.000031\n",
              "3.96    0.000006\n",
              "4.82    0.000006\n",
              "Name: y, dtype: float64"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df['y'].value_counts(normalize=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "fca8b32d",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:34:55.464677Z",
          "iopub.status.busy": "2022-01-07T09:34:55.463935Z",
          "iopub.status.idle": "2022-01-07T09:34:55.516097Z",
          "shell.execute_reply": "2022-01-07T09:34:55.516644Z",
          "shell.execute_reply.started": "2022-01-05T23:51:46.086355Z"
        },
        "id": "fca8b32d",
        "outputId": "2f31d176-687f-4e28-f132-46cc3f228e74",
        "papermill": {
          "duration": 0.09103,
          "end_time": "2022-01-07T09:34:55.516842",
          "exception": false,
          "start_time": "2022-01-07T09:34:55.425812",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.00    16225\n",
              "0.32     5666\n",
              "1.12     3800\n",
              "0.48     1758\n",
              "2.62     1738\n",
              "0.96     1215\n",
              "4.12      385\n",
              "0.16      317\n",
              "0.64      301\n",
              "1.82      290\n",
              "1.98      204\n",
              "0.80      181\n",
              "2.46      164\n",
              "1.50       76\n",
              "5.62       31\n",
              "2.14       31\n",
              "3.32       21\n",
              "2.30       20\n",
              "3.48       10\n",
              "3.96       10\n",
              "1.66        5\n",
              "3.96        1\n",
              "4.82        1\n",
              "Name: y, dtype: int64"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "min_len = (df['y'] >= 0.1).sum()\n",
        "df_y0_undersample = df[df['y'] == 0].sample(n=min_len, random_state=201)\n",
        "df = pd.concat([df[df['y'] >= 0.1], df_y0_undersample])\n",
        "df['y'].value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "09b5cd77",
      "metadata": {
        "id": "09b5cd77",
        "papermill": {
          "duration": 0.031702,
          "end_time": "2022-01-07T09:34:55.580766",
          "exception": false,
          "start_time": "2022-01-07T09:34:55.549064",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# TF-IDF"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d0b45c97",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:34:55.674886Z",
          "iopub.status.busy": "2022-01-07T09:34:55.664216Z",
          "iopub.status.idle": "2022-01-07T09:35:28.967987Z",
          "shell.execute_reply": "2022-01-07T09:35:28.968523Z",
          "shell.execute_reply.started": "2022-01-05T23:51:46.159528Z"
        },
        "id": "d0b45c97",
        "outputId": "06e037c5-6583-4aa4-c484-4e31409c751a",
        "papermill": {
          "duration": 33.35525,
          "end_time": "2022-01-07T09:35:28.968703",
          "exception": false,
          "start_time": "2022-01-07T09:34:55.613453",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<32450x46000 sparse matrix of type '<class 'numpy.float64'>'\n",
              "\twith 13716685 stored elements in Compressed Sparse Row format>"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# vec = TfidfVectorizer(min_df= 3, max_df=0.5, analyzer = 'char_wb', ngram_range = (3,5))\n",
        "# vec = TfidfVectorizer(min_df= 3, max_df=0.5, analyzer = 'char_wb', ngram_range = (3,6))\n",
        "vec = TfidfVectorizer(min_df= 3, max_df=0.5, analyzer = 'char_wb', ngram_range = (2,5),max_features=46000)\n",
        "X = vec.fit_transform(df['text'])\n",
        "X"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "88e6dc04",
      "metadata": {
        "id": "88e6dc04",
        "papermill": {
          "duration": 0.032694,
          "end_time": "2022-01-07T09:35:29.033748",
          "exception": false,
          "start_time": "2022-01-07T09:35:29.001054",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "<h1>Fit Ridge</h1>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "20f71c18",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:29.103655Z",
          "iopub.status.busy": "2022-01-07T09:35:29.102946Z",
          "iopub.status.idle": "2022-01-07T09:35:34.420333Z",
          "shell.execute_reply": "2022-01-07T09:35:34.421795Z",
          "shell.execute_reply.started": "2022-01-05T23:52:20.741869Z"
        },
        "id": "20f71c18",
        "outputId": "5c7dbc7e-f1f9-4c9f-eff4-ba8495b4c8f7",
        "papermill": {
          "duration": 5.354433,
          "end_time": "2022-01-07T09:35:34.422085",
          "exception": false,
          "start_time": "2022-01-07T09:35:29.067652",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Ridge(alpha=0.5)"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model = Ridge(alpha=0.5)\n",
        "# model = Ridge(alpha=0.485)\n",
        "model.fit(X, df['y'])"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "e02ebe33",
      "metadata": {
        "id": "e02ebe33",
        "papermill": {
          "duration": 0.053118,
          "end_time": "2022-01-07T09:35:34.536621",
          "exception": false,
          "start_time": "2022-01-07T09:35:34.483503",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Prepare validation data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9c980439",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:34.606364Z",
          "iopub.status.busy": "2022-01-07T09:35:34.605686Z",
          "iopub.status.idle": "2022-01-07T09:35:35.181467Z",
          "shell.execute_reply": "2022-01-07T09:35:35.182029Z",
          "shell.execute_reply.started": "2022-01-05T23:52:26.447001Z"
        },
        "id": "9c980439",
        "papermill": {
          "duration": 0.612967,
          "end_time": "2022-01-07T09:35:35.182204",
          "exception": false,
          "start_time": "2022-01-07T09:35:34.569237",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "df_val = pd.read_csv(\"../input/jigsaw-toxic-severity-rating/validation_data.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "abbb4eb2",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:35.259679Z",
          "iopub.status.busy": "2022-01-07T09:35:35.258892Z",
          "iopub.status.idle": "2022-01-07T09:35:35.263004Z",
          "shell.execute_reply": "2022-01-07T09:35:35.262319Z",
          "shell.execute_reply.started": "2022-01-05T23:52:27.068549Z"
        },
        "id": "abbb4eb2",
        "outputId": "b96753a6-7e12-4447-f1ba-622d0d103ec1",
        "papermill": {
          "duration": 0.047619,
          "end_time": "2022-01-07T09:35:35.263147",
          "exception": false,
          "start_time": "2022-01-07T09:35:35.215528",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>worker</th>\n",
              "      <th>less_toxic</th>\n",
              "      <th>more_toxic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>313</td>\n",
              "      <td>This article sucks \\n\\nwoo woo wooooooo</td>\n",
              "      <td>WHAT!!!!!!!!?!?!!?!?!!?!?!?!?!!!!!!!!!!!!!!!!!!!!!!!!??????????????????????????????????????????????????????!!!!!!=\\nWHER IS YOUR SEXY PIC GONE FROM YOUR MAIN PAGE???????? put it back</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>188</td>\n",
              "      <td>\"And yes, people should recognize that but they usually don't. One of the first objections you hear directed at someone who says he favors free markets is invariably \"\"There's no such thing as a free market.\"\" Ridiculously trivial. \"</td>\n",
              "      <td>Daphne Guinness \\n\\nTop of the mornin' my favourite Alien Life Form. I'm sorry, I can't marry you because you are a poor commoner so, therefore, unimportant. Can I have my dresses back. I hope your fat Oirish arse hasn't stretched them.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>82</td>\n",
              "      <td>Western Media?\\n\\nYup, because every crime in the entire world is caused by the west.  Racists.</td>\n",
              "      <td>\"Atom you don't believe actual photos of masturbation is sexually explicit photos. I am sorry to say they are sexually explicit and so you said yourself that the disclaimer is used for that purpose. Also, Wikipedia itself is targeted for minors. In addition, you say that Wikipedia does not have ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>347</td>\n",
              "      <td>And you removed it! You numbskull! I don't care what you say anymore, this is my life! Go ahead with your own life, leave me alone! —</td>\n",
              "      <td>You seem to have sand in your vagina.\\n\\nMight want to wash that stinking hole out, bitch.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>539</td>\n",
              "      <td>smelly vagina \\n\\nBluerasberry why don't you be a model for a cheesy blue vagina syndrome. A lot are experiencing this vaginal disease.</td>\n",
              "      <td>hey \\n\\nway to support nazis, you racist</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   worker  \\\n",
              "0     313   \n",
              "1     188   \n",
              "2      82   \n",
              "3     347   \n",
              "4     539   \n",
              "\n",
              "                                                                                                                                                                                                                                  less_toxic  \\\n",
              "0                                                                                                                                                                                                    This article sucks \\n\\nwoo woo wooooooo   \n",
              "1  \"And yes, people should recognize that but they usually don't. One of the first objections you hear directed at someone who says he favors free markets is invariably \"\"There's no such thing as a free market.\"\" Ridiculously trivial. \"   \n",
              "2                                                                                                                                         Western Media?\\n\\nYup, because every crime in the entire world is caused by the west.  Racists.      \n",
              "3                                                                                                     And you removed it! You numbskull! I don't care what you say anymore, this is my life! Go ahead with your own life, leave me alone! —    \n",
              "4                                                                                                    smelly vagina \\n\\nBluerasberry why don't you be a model for a cheesy blue vagina syndrome. A lot are experiencing this vaginal disease.   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                    more_toxic  \n",
              "0                                                                                                                       WHAT!!!!!!!!?!?!!?!?!!?!?!?!?!!!!!!!!!!!!!!!!!!!!!!!!??????????????????????????????????????????????????????!!!!!!=\\nWHER IS YOUR SEXY PIC GONE FROM YOUR MAIN PAGE???????? put it back  \n",
              "1                                                                 Daphne Guinness \\n\\nTop of the mornin' my favourite Alien Life Form. I'm sorry, I can't marry you because you are a poor commoner so, therefore, unimportant. Can I have my dresses back. I hope your fat Oirish arse hasn't stretched them.  \n",
              "2  \"Atom you don't believe actual photos of masturbation is sexually explicit photos. I am sorry to say they are sexually explicit and so you said yourself that the disclaimer is used for that purpose. Also, Wikipedia itself is targeted for minors. In addition, you say that Wikipedia does not have ...  \n",
              "3                                                                                                                                                                                                                   You seem to have sand in your vagina.\\n\\nMight want to wash that stinking hole out, bitch.  \n",
              "4                                                                                                                                                                                                                                                                     hey \\n\\nway to support nazis, you racist  "
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_val.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "c4782a10",
      "metadata": {
        "id": "c4782a10",
        "papermill": {
          "duration": 0.033159,
          "end_time": "2022-01-07T09:35:35.331082",
          "exception": false,
          "start_time": "2022-01-07T09:35:35.297923",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "<h2>Text cleaning</h2>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f447ac23",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:35.466191Z",
          "iopub.status.busy": "2022-01-07T09:35:35.465537Z",
          "iopub.status.idle": "2022-01-07T09:35:35.467869Z",
          "shell.execute_reply": "2022-01-07T09:35:35.468385Z",
          "shell.execute_reply.started": "2022-01-05T23:52:27.082Z"
        },
        "id": "f447ac23",
        "papermill": {
          "duration": 0.040811,
          "end_time": "2022-01-07T09:35:35.468553",
          "exception": false,
          "start_time": "2022-01-07T09:35:35.427742",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "# tqdm.pandas()\n",
        "# df_val['less_toxic'] = df_val['less_toxic'].progress_apply(text_cleaning)\n",
        "# df_val['more_toxic'] = df_val['more_toxic'].progress_apply(text_cleaning)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1d073e50",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:35.540348Z",
          "iopub.status.busy": "2022-01-07T09:35:35.539663Z",
          "iopub.status.idle": "2022-01-07T09:35:35.543559Z",
          "shell.execute_reply": "2022-01-07T09:35:35.544015Z",
          "shell.execute_reply.started": "2022-01-05T23:52:27.090395Z"
        },
        "id": "1d073e50",
        "papermill": {
          "duration": 0.041647,
          "end_time": "2022-01-07T09:35:35.544189",
          "exception": false,
          "start_time": "2022-01-07T09:35:35.502542",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "# X_less_toxic = vec.transform(df_val['less_toxic'])\n",
        "# X_more_toxic = vec.transform(df_val['more_toxic'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c4ff32b2",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:35.614960Z",
          "iopub.status.busy": "2022-01-07T09:35:35.614284Z",
          "iopub.status.idle": "2022-01-07T09:35:35.616934Z",
          "shell.execute_reply": "2022-01-07T09:35:35.617571Z",
          "shell.execute_reply.started": "2022-01-05T23:52:27.104238Z"
        },
        "id": "c4ff32b2",
        "papermill": {
          "duration": 0.039691,
          "end_time": "2022-01-07T09:35:35.617741",
          "exception": false,
          "start_time": "2022-01-07T09:35:35.578050",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "# p1 = model.predict(X_less_toxic)\n",
        "# p2 = model.predict(X_more_toxic)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "026f6f47",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:35.689561Z",
          "iopub.status.busy": "2022-01-07T09:35:35.688817Z",
          "iopub.status.idle": "2022-01-07T09:35:35.692014Z",
          "shell.execute_reply": "2022-01-07T09:35:35.692599Z",
          "shell.execute_reply.started": "2022-01-05T23:52:27.11542Z"
        },
        "id": "026f6f47",
        "papermill": {
          "duration": 0.040899,
          "end_time": "2022-01-07T09:35:35.692784",
          "exception": false,
          "start_time": "2022-01-07T09:35:35.651885",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "# # Validation Accuracy\n",
        "# (p1 < p2).mean()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "fa572f5d",
      "metadata": {
        "id": "fa572f5d",
        "papermill": {
          "duration": 0.033267,
          "end_time": "2022-01-07T09:35:35.759950",
          "exception": false,
          "start_time": "2022-01-07T09:35:35.726683",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Prepare submission data "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "547516f3",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:35.831378Z",
          "iopub.status.busy": "2022-01-07T09:35:35.830627Z",
          "iopub.status.idle": "2022-01-07T09:35:35.878823Z",
          "shell.execute_reply": "2022-01-07T09:35:35.879425Z",
          "shell.execute_reply.started": "2022-01-05T23:52:27.126988Z"
        },
        "id": "547516f3",
        "papermill": {
          "duration": 0.086219,
          "end_time": "2022-01-07T09:35:35.879644",
          "exception": false,
          "start_time": "2022-01-07T09:35:35.793425",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "df_sub = pd.read_csv(\"../input/jigsaw-toxic-severity-rating/comments_to_score.csv\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "eb5fe957",
      "metadata": {
        "id": "eb5fe957",
        "papermill": {
          "duration": 0.034289,
          "end_time": "2022-01-07T09:35:35.947942",
          "exception": false,
          "start_time": "2022-01-07T09:35:35.913653",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "<h2>Text cleaning</h2>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "23e12555",
      "metadata": {
        "colab": {
          "referenced_widgets": [
            "7c234aebfec14ca192f0072e4bdfcdca"
          ]
        },
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:36.019891Z",
          "iopub.status.busy": "2022-01-07T09:35:36.019184Z",
          "iopub.status.idle": "2022-01-07T09:35:39.029403Z",
          "shell.execute_reply": "2022-01-07T09:35:39.028804Z",
          "shell.execute_reply.started": "2022-01-05T23:52:27.18487Z"
        },
        "id": "23e12555",
        "outputId": "f5355eae-913c-4078-f734-d9f2d672f84c",
        "papermill": {
          "duration": 3.047831,
          "end_time": "2022-01-07T09:35:39.029557",
          "exception": false,
          "start_time": "2022-01-07T09:35:35.981726",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7c234aebfec14ca192f0072e4bdfcdca",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/7537 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "tqdm.pandas()\n",
        "df_sub['text'] = df_sub['text'].progress_apply(text_cleaning)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a7928483",
      "metadata": {
        "id": "a7928483",
        "papermill": {
          "duration": 0.033954,
          "end_time": "2022-01-07T09:35:39.097726",
          "exception": false,
          "start_time": "2022-01-07T09:35:39.063772",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "<h2>Prediction</h2>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c060470a",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:39.185327Z",
          "iopub.status.busy": "2022-01-07T09:35:39.180203Z",
          "iopub.status.idle": "2022-01-07T09:35:49.382745Z",
          "shell.execute_reply": "2022-01-07T09:35:49.383289Z",
          "shell.execute_reply.started": "2022-01-05T23:52:30.18565Z"
        },
        "id": "c060470a",
        "papermill": {
          "duration": 10.251626,
          "end_time": "2022-01-07T09:35:49.383503",
          "exception": false,
          "start_time": "2022-01-07T09:35:39.131877",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "X_test = vec.transform(df_sub['text'])\n",
        "p3 = model.predict(X_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6b0b1394",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:49.457451Z",
          "iopub.status.busy": "2022-01-07T09:35:49.455215Z",
          "iopub.status.idle": "2022-01-07T09:35:49.459400Z",
          "shell.execute_reply": "2022-01-07T09:35:49.459948Z",
          "shell.execute_reply.started": "2022-01-05T23:52:40.537983Z"
        },
        "id": "6b0b1394",
        "papermill": {
          "duration": 0.041956,
          "end_time": "2022-01-07T09:35:49.460117",
          "exception": false,
          "start_time": "2022-01-07T09:35:49.418161",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "df_sub['score'] = p3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9ec0918c",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:49.535818Z",
          "iopub.status.busy": "2022-01-07T09:35:49.532437Z",
          "iopub.status.idle": "2022-01-07T09:35:49.538712Z",
          "shell.execute_reply": "2022-01-07T09:35:49.539223Z",
          "shell.execute_reply.started": "2022-01-05T23:52:40.545833Z"
        },
        "id": "9ec0918c",
        "outputId": "1504e603-a0a2-414a-cfb4-d29f2f853df6",
        "papermill": {
          "duration": 0.044742,
          "end_time": "2022-01-07T09:35:49.539413",
          "exception": false,
          "start_time": "2022-01-07T09:35:49.494671",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "7537"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_sub['score'].count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "71192260",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:49.613950Z",
          "iopub.status.busy": "2022-01-07T09:35:49.613221Z",
          "iopub.status.idle": "2022-01-07T09:35:49.615673Z",
          "shell.execute_reply": "2022-01-07T09:35:49.616153Z",
          "shell.execute_reply.started": "2022-01-05T23:52:40.564342Z"
        },
        "id": "71192260",
        "papermill": {
          "duration": 0.042322,
          "end_time": "2022-01-07T09:35:49.616344",
          "exception": false,
          "start_time": "2022-01-07T09:35:49.574022",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "df_sub['score'] = df_sub['score'] "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0d011fc5",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:49.690085Z",
          "iopub.status.busy": "2022-01-07T09:35:49.688526Z",
          "iopub.status.idle": "2022-01-07T09:35:49.694585Z",
          "shell.execute_reply": "2022-01-07T09:35:49.695094Z",
          "shell.execute_reply.started": "2022-01-05T23:52:40.574045Z"
        },
        "id": "0d011fc5",
        "outputId": "e5a016cb-4105-46d3-e96c-69964a32499e",
        "papermill": {
          "duration": 0.044602,
          "end_time": "2022-01-07T09:35:49.695291",
          "exception": false,
          "start_time": "2022-01-07T09:35:49.650689",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "7517"
            ]
          },
          "execution_count": 26,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# 9 comments will fail if compared one with the other\n",
        "df_sub['score'].nunique()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "31a17dd5",
      "metadata": {
        "id": "31a17dd5",
        "papermill": {
          "duration": 0.034382,
          "end_time": "2022-01-07T09:35:49.764419",
          "exception": false,
          "start_time": "2022-01-07T09:35:49.730037",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "<h2>Prepare submission file</h2>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cbd14f3d",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:49.840218Z",
          "iopub.status.busy": "2022-01-07T09:35:49.839158Z",
          "iopub.status.idle": "2022-01-07T09:35:49.879717Z",
          "shell.execute_reply": "2022-01-07T09:35:49.878994Z",
          "shell.execute_reply.started": "2022-01-05T23:52:40.588825Z"
        },
        "id": "cbd14f3d",
        "papermill": {
          "duration": 0.079643,
          "end_time": "2022-01-07T09:35:49.879875",
          "exception": false,
          "start_time": "2022-01-07T09:35:49.800232",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "df_sub[['comment_id', 'score']].to_csv(\"submission.csv\", index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8b688b90",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-07T09:35:49.957404Z",
          "iopub.status.busy": "2022-01-07T09:35:49.956711Z",
          "iopub.status.idle": "2022-01-07T09:35:49.967052Z",
          "shell.execute_reply": "2022-01-07T09:35:49.966461Z",
          "shell.execute_reply.started": "2022-01-05T23:52:40.636051Z"
        },
        "id": "8b688b90",
        "outputId": "74b32773-6027-4b4d-c756-166c128985a5",
        "papermill": {
          "duration": 0.052548,
          "end_time": "2022-01-07T09:35:49.967194",
          "exception": false,
          "start_time": "2022-01-07T09:35:49.914646",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>comment_id</th>\n",
              "      <th>text</th>\n",
              "      <th>score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>114890</td>\n",
              "      <td>Gjalexei you asked about whether there is an anti editorializing policy here There is and it s called wikipedia neutral point of view It discusses at some length the case of what we should do when writing about a subject which most of us find repugnant Whilst you re not likely to get too many de...</td>\n",
              "      <td>-0.034033</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>732895</td>\n",
              "      <td>Looks like be have an abuser can you please look into this thanks</td>\n",
              "      <td>0.258731</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1139051</td>\n",
              "      <td>I confess to having complete and apparently blissful ignorance of Jordan but I ve glanced at the article Is this a woman or a soap opera I don t think there was much to change in terms of the description of the various diseases It is mentioned that she is famous for the size of her breasts am I ...</td>\n",
              "      <td>0.016812</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1434512</td>\n",
              "      <td>Freud s ideas are certainly much discussed today and I would be the first to agree that they must be grappled with and dealt with seriously if only because of their currency So Freud deserves a long and thorough article in Wikipedia I believe that a balanced article would include A explanations ...</td>\n",
              "      <td>-0.105424</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2084821</td>\n",
              "      <td>It is not just you This is a laundry list of stupid allegations scooped up from god knows where Probably two thirds of it has little basis in fact</td>\n",
              "      <td>0.264088</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7532</th>\n",
              "      <td>504235362</td>\n",
              "      <td>Go away you annoying vandal</td>\n",
              "      <td>0.242439</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7533</th>\n",
              "      <td>504235566</td>\n",
              "      <td>This user is a vandal</td>\n",
              "      <td>0.274774</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7534</th>\n",
              "      <td>504308177</td>\n",
              "      <td>Sorry to sound like a pain but one by following me here is a tad stalking in all honesty but I m not too bothered about that But the main worry is that you have just assumed badly that I was referring about you when it is clear I wasn t One should never jump to conclusions without checking facts...</td>\n",
              "      <td>-0.351585</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7535</th>\n",
              "      <td>504570375</td>\n",
              "      <td>Well it s pretty fucking irrelevant now I m unblocked aint it 81 108 7 13</td>\n",
              "      <td>0.264866</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7536</th>\n",
              "      <td>504598250</td>\n",
              "      <td>The team name is Great Britain and Northern Ireland Your blind reverts are nonsensical</td>\n",
              "      <td>0.286673</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>7537 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      comment_id  \\\n",
              "0         114890   \n",
              "1         732895   \n",
              "2        1139051   \n",
              "3        1434512   \n",
              "4        2084821   \n",
              "...          ...   \n",
              "7532   504235362   \n",
              "7533   504235566   \n",
              "7534   504308177   \n",
              "7535   504570375   \n",
              "7536   504598250   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                             text  \\\n",
              "0     Gjalexei you asked about whether there is an anti editorializing policy here There is and it s called wikipedia neutral point of view It discusses at some length the case of what we should do when writing about a subject which most of us find repugnant Whilst you re not likely to get too many de...   \n",
              "1                                                                                                                                                                                                                                               Looks like be have an abuser can you please look into this thanks   \n",
              "2     I confess to having complete and apparently blissful ignorance of Jordan but I ve glanced at the article Is this a woman or a soap opera I don t think there was much to change in terms of the description of the various diseases It is mentioned that she is famous for the size of her breasts am I ...   \n",
              "3     Freud s ideas are certainly much discussed today and I would be the first to agree that they must be grappled with and dealt with seriously if only because of their currency So Freud deserves a long and thorough article in Wikipedia I believe that a balanced article would include A explanations ...   \n",
              "4                                                                                                                                                              It is not just you This is a laundry list of stupid allegations scooped up from god knows where Probably two thirds of it has little basis in fact   \n",
              "...                                                                                                                                                                                                                                                                                                           ...   \n",
              "7532                                                                                                                                                                                                                                                                                  Go away you annoying vandal   \n",
              "7533                                                                                                                                                                                                                                                                                        This user is a vandal   \n",
              "7534  Sorry to sound like a pain but one by following me here is a tad stalking in all honesty but I m not too bothered about that But the main worry is that you have just assumed badly that I was referring about you when it is clear I wasn t One should never jump to conclusions without checking facts...   \n",
              "7535                                                                                                                                                                                                                                    Well it s pretty fucking irrelevant now I m unblocked aint it 81 108 7 13   \n",
              "7536                                                                                                                                                                                                                       The team name is Great Britain and Northern Ireland Your blind reverts are nonsensical   \n",
              "\n",
              "         score  \n",
              "0    -0.034033  \n",
              "1     0.258731  \n",
              "2     0.016812  \n",
              "3    -0.105424  \n",
              "4     0.264088  \n",
              "...        ...  \n",
              "7532  0.242439  \n",
              "7533  0.274774  \n",
              "7534 -0.351585  \n",
              "7535  0.264866  \n",
              "7536  0.286673  \n",
              "\n",
              "[7537 rows x 3 columns]"
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_sub"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b61fc2f9",
      "metadata": {
        "id": "b61fc2f9",
        "papermill": {
          "duration": 0.036034,
          "end_time": "2022-01-07T09:35:50.038931",
          "exception": false,
          "start_time": "2022-01-07T09:35:50.002897",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.10"
    },
    "papermill": {
      "default_parameters": {},
      "duration": 134.133138,
      "end_time": "2022-01-07T09:35:50.987446",
      "environment_variables": {},
      "exception": null,
      "input_path": "__notebook__.ipynb",
      "output_path": "__notebook__.ipynb",
      "parameters": {},
      "start_time": "2022-01-07T09:33:36.854308",
      "version": "2.3.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}