HackerNewsV4.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HackerNewsV4.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "RmQM8bEqo6tk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from warnings import filterwarnings\n",
        "filterwarnings(\"ignore\")\n"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x4EnlcuvotZH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import json\n",
        "import urllib\n",
        "import requests"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EfumSQtyoTXt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KD2T0WNpo2mv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "e46ff953-9f95-4af5-dcd2-5531e5d52206"
      },
      "source": [
        "html = urllib.request.urlopen('https://hacker-news.firebaseio.com/v0/item/2169746.json?print=pretty')\n",
        "json.loads(html.read())\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'by': 'timrobinson',\n",
              " 'id': 2169746,\n",
              " 'parent': 2169669,\n",
              " 'text': \"After being down all day yesterday, the site's now working for me.<p>Not much crime in my small part of south-east London, except for an ominous 'other crime' at the end of the street...\",\n",
              " 'time': 1296642293,\n",
              " 'type': 'comment'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pdkw3f9tpMvE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "92725447-9657-4a10-8a00-c20ee496fe9d"
      },
      "source": [
        "x = urllib.request.urlopen('https://hacker-news.firebaseio.com/v0/maxitem.json?print=pretty')\n",
        "max_item = int(x.read())\n",
        "print(max_item)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "23990757\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v6fBur72pTjd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "fbf3a91a-4a20-4cea-d246-d61eacb423d0"
      },
      "source": [
        "data = []\n",
        "number_of_entries = 1000\n",
        "\n",
        "min_item = max_item - number_of_entries\n",
        "count = 0\n",
        "\n",
        "\n",
        "for i in range(min_item, max_item):\n",
        "    html = urllib.request.urlopen('https://hacker-news.firebaseio.com/v0/item/' + str(i) + '.json')\n",
        "    data.append(json.loads(html.read()))\n",
        "    count += 1\n",
        "    if count % 100 == 0:\n",
        "        print(f\"Loaded {count} rows\")"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded 100 rows\n",
            "Loaded 200 rows\n",
            "Loaded 300 rows\n",
            "Loaded 400 rows\n",
            "Loaded 500 rows\n",
            "Loaded 600 rows\n",
            "Loaded 700 rows\n",
            "Loaded 800 rows\n",
            "Loaded 900 rows\n",
            "Loaded 1000 rows\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nqH_FSIXq7u6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "01ead794-0753-4e80-e93e-0571382f8477"
      },
      "source": [
        "print (data[0])"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'by': 'nix23', 'id': 23989757, 'kids': [23989872, 23989888, 23989921], 'parent': 23989181, 'text': '&gt;There is a lot of implied ill-will on HN<p>So true, since Microsoft wants just the best for Humankind and is overall just a nice Company.', 'time': 1596046005, 'type': 'comment'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "edYTFyNmrIPf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = [i for i in data if i is not None]\n"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dxiozrLrqwZ8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 529
        },
        "outputId": "4c8043bd-6d0f-4cbf-d7fb-a41692bd2ca4"
      },
      "source": [
        "from pandas.io.json import json_normalize\n",
        "df = pd.DataFrame.from_dict(data)\n",
        "df.head(10)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>by</th>\n",
              "      <th>id</th>\n",
              "      <th>kids</th>\n",
              "      <th>parent</th>\n",
              "      <th>text</th>\n",
              "      <th>time</th>\n",
              "      <th>type</th>\n",
              "      <th>descendants</th>\n",
              "      <th>score</th>\n",
              "      <th>title</th>\n",
              "      <th>url</th>\n",
              "      <th>deleted</th>\n",
              "      <th>dead</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>nix23</td>\n",
              "      <td>23989757</td>\n",
              "      <td>[23989872, 23989888, 23989921]</td>\n",
              "      <td>23989181.0</td>\n",
              "      <td>&amp;gt;There is a lot of implied ill-will on HN&lt;p...</td>\n",
              "      <td>1596046005</td>\n",
              "      <td>comment</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>aliswe</td>\n",
              "      <td>23989758</td>\n",
              "      <td>NaN</td>\n",
              "      <td>23983330.0</td>\n",
              "      <td>Yes that being said the demo is an old version.</td>\n",
              "      <td>1596046007</td>\n",
              "      <td>comment</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>DeonPenny</td>\n",
              "      <td>23989759</td>\n",
              "      <td>NaN</td>\n",
              "      <td>23989527.0</td>\n",
              "      <td>I&amp;#x27;d do 27 months for millions of dollars....</td>\n",
              "      <td>1596046009</td>\n",
              "      <td>comment</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>blopeur</td>\n",
              "      <td>23989760</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1596046016</td>\n",
              "      <td>story</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>GPUs for Science 2020</td>\n",
              "      <td>https://www.nersc.gov/users/training/gpus-for-...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>msla</td>\n",
              "      <td>23989761</td>\n",
              "      <td>NaN</td>\n",
              "      <td>23989383.0</td>\n",
              "      <td>I just described how Minitel was completely di...</td>\n",
              "      <td>1596046021</td>\n",
              "      <td>comment</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Glench</td>\n",
              "      <td>23989762</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1596046021</td>\n",
              "      <td>story</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>To recharge America’s job market, build a gree...</td>\n",
              "      <td>https://fortune.com/2020/07/29/climate-change-...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>aspenmayer</td>\n",
              "      <td>23989763</td>\n",
              "      <td>[23989804]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1596046028</td>\n",
              "      <td>story</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>Phone backup and a new storage manager in the ...</td>\n",
              "      <td>https://blog.google/products/google-one/free-f...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>pc86</td>\n",
              "      <td>23989764</td>\n",
              "      <td>NaN</td>\n",
              "      <td>23987345.0</td>\n",
              "      <td>&amp;gt; See the problem now?&lt;p&gt;No, because I am a...</td>\n",
              "      <td>1596046032</td>\n",
              "      <td>comment</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>dheera</td>\n",
              "      <td>23989765</td>\n",
              "      <td>[23989830, 23989853, 23989813, 23989822]</td>\n",
              "      <td>23989527.0</td>\n",
              "      <td>Honest question here. Why are we, the taxpayer...</td>\n",
              "      <td>1596046041</td>\n",
              "      <td>comment</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>UncleMeat</td>\n",
              "      <td>23989766</td>\n",
              "      <td>[23990081, 23989988]</td>\n",
              "      <td>23989664.0</td>\n",
              "      <td>&amp;gt; You have to bake the security into your w...</td>\n",
              "      <td>1596046041</td>\n",
              "      <td>comment</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           by        id  ... deleted  dead\n",
              "0       nix23  23989757  ...     NaN   NaN\n",
              "1      aliswe  23989758  ...     NaN   NaN\n",
              "2   DeonPenny  23989759  ...     NaN   NaN\n",
              "3     blopeur  23989760  ...     NaN   NaN\n",
              "4        msla  23989761  ...     NaN   NaN\n",
              "5      Glench  23989762  ...     NaN   NaN\n",
              "6  aspenmayer  23989763  ...     NaN   NaN\n",
              "7        pc86  23989764  ...     NaN   NaN\n",
              "8      dheera  23989765  ...     NaN   NaN\n",
              "9   UncleMeat  23989766  ...     NaN   NaN\n",
              "\n",
              "[10 rows x 13 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H_45wcz7ror0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e432f5c2-1142-4706-a3b5-c06e5a0eb605"
      },
      "source": [
        "print(df.shape)\n"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1000, 13)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vzEBGrrz-5yn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "a6d5706f-9d2f-44a8-851b-8c6d83e9be0d"
      },
      "source": [
        "df.columns"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['by', 'id', 'kids', 'parent', 'text', 'time', 'type', 'descendants',\n",
              "       'score', 'title', 'url', 'deleted', 'dead'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9_fKq7kpr2Zj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "outputId": "8ce79afc-ca09-4580-ff5a-ad92e9f9b0ba"
      },
      "source": [
        "df_new = df.drop(columns = ['deleted', 'dead', 'descendants', 'score', 'kids', 'parent', 'title', 'url'])\n",
        "df_new.head(10)\n"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>by</th>\n",
              "      <th>id</th>\n",
              "      <th>text</th>\n",
              "      <th>time</th>\n",
              "      <th>type</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>nix23</td>\n",
              "      <td>23989757</td>\n",
              "      <td>&amp;gt;There is a lot of implied ill-will on HN&lt;p...</td>\n",
              "      <td>1596046005</td>\n",
              "      <td>comment</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>aliswe</td>\n",
              "      <td>23989758</td>\n",
              "      <td>Yes that being said the demo is an old version.</td>\n",
              "      <td>1596046007</td>\n",
              "      <td>comment</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>DeonPenny</td>\n",
              "      <td>23989759</td>\n",
              "      <td>I&amp;#x27;d do 27 months for millions of dollars....</td>\n",
              "      <td>1596046009</td>\n",
              "      <td>comment</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>blopeur</td>\n",
              "      <td>23989760</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1596046016</td>\n",
              "      <td>story</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>msla</td>\n",
              "      <td>23989761</td>\n",
              "      <td>I just described how Minitel was completely di...</td>\n",
              "      <td>1596046021</td>\n",
              "      <td>comment</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Glench</td>\n",
              "      <td>23989762</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1596046021</td>\n",
              "      <td>story</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>aspenmayer</td>\n",
              "      <td>23989763</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1596046028</td>\n",
              "      <td>story</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>pc86</td>\n",
              "      <td>23989764</td>\n",
              "      <td>&amp;gt; See the problem now?&lt;p&gt;No, because I am a...</td>\n",
              "      <td>1596046032</td>\n",
              "      <td>comment</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>dheera</td>\n",
              "      <td>23989765</td>\n",
              "      <td>Honest question here. Why are we, the taxpayer...</td>\n",
              "      <td>1596046041</td>\n",
              "      <td>comment</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>UncleMeat</td>\n",
              "      <td>23989766</td>\n",
              "      <td>&amp;gt; You have to bake the security into your w...</td>\n",
              "      <td>1596046041</td>\n",
              "      <td>comment</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           by        id  ...        time     type\n",
              "0       nix23  23989757  ...  1596046005  comment\n",
              "1      aliswe  23989758  ...  1596046007  comment\n",
              "2   DeonPenny  23989759  ...  1596046009  comment\n",
              "3     blopeur  23989760  ...  1596046016    story\n",
              "4        msla  23989761  ...  1596046021  comment\n",
              "5      Glench  23989762  ...  1596046021    story\n",
              "6  aspenmayer  23989763  ...  1596046028    story\n",
              "7        pc86  23989764  ...  1596046032  comment\n",
              "8      dheera  23989765  ...  1596046041  comment\n",
              "9   UncleMeat  23989766  ...  1596046041  comment\n",
              "\n",
              "[10 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_lsIA8wzsbYU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "9a2f7c44-9309-4b9f-8561-97b05f544560"
      },
      "source": [
        "df_new['text'][0]\n"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'&gt;There is a lot of implied ill-will on HN<p>So true, since Microsoft wants just the best for Humankind and is overall just a nice Company.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "htqSj1IXshyO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "57ae4b6f-e7ae-4621-f1a9-c1da8d370523"
      },
      "source": [
        "df_comments = df_new[df_new['type'] == 'comment']\n",
        "df_comments['text']\n"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0      &gt;There is a lot of implied ill-will on HN<p...\n",
              "1        Yes that being said the demo is an old version.\n",
              "2      I&#x27;d do 27 months for millions of dollars....\n",
              "4      I just described how Minitel was completely di...\n",
              "7      &gt; See the problem now?<p>No, because I am a...\n",
              "                             ...                        \n",
              "994    Can&#x27;t we just shoot a orbital space laser...\n",
              "995    Under figure 1 it says &quot;Comparisons are n...\n",
              "996    Would you please review the site guidelines an...\n",
              "997    CYCLADES was really influencial on that (<a hr...\n",
              "998    Forgive my ignorance on the subject but if it&...\n",
              "Name: text, Length: 872, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8EcuiY68qtb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "ee1ed270-377f-4a00-9b50-8d1e743c43fd"
      },
      "source": [
        "import nltk\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "from nltk.stem import WordNetLemmatizer,PorterStemmer\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "\n",
        "import re\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "stemmer = PorterStemmer() \n",
        "\n",
        "def preprocess(sentence):\n",
        "    sentence=str(sentence)\n",
        "    sentence = sentence.lower()\n",
        "    sentence=sentence.replace('{html}',\"\") \n",
        "    cleanr = re.compile('<.*?>')\n",
        "    cleantext = re.sub(cleanr, '', sentence)\n",
        "    rem_url=re.sub(r'http\\S+', '',cleantext)\n",
        "    rem_num = re.sub('[0-9]+', '', rem_url)\n",
        "    tokenizer = RegexpTokenizer(r'\\w+')\n",
        "    tokens = tokenizer.tokenize(rem_num)  \n",
        "    filtered_words = [w for w in tokens if len(w) > 2 if not w in stopwords.words('english')]\n",
        "    stem_words=[stemmer.stem(w) for w in filtered_words]\n",
        "    lemma_words=[lemmatizer.lemmatize(w) for w in stem_words]\n",
        "    return \" \".join(filtered_words)\n",
        "\n",
        "\n",
        "df_comments['clean_text']=df_comments['text'].map(lambda s:preprocess(s)) \n"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-9-e38cdae40475>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m \u001b[0mdf_comments\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'clean_text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdf_comments\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mpreprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'df_comments' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zd1mZtNKAxV3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "outputId": "dfaf381f-76e6-4537-9569-498f970d6294"
      },
      "source": [
        "df_comments.head(10)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>by</th>\n",
              "      <th>id</th>\n",
              "      <th>text</th>\n",
              "      <th>time</th>\n",
              "      <th>type</th>\n",
              "      <th>clean_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>nix23</td>\n",
              "      <td>23989757</td>\n",
              "      <td>&amp;gt;There is a lot of implied ill-will on HN&lt;p...</td>\n",
              "      <td>1596046005</td>\n",
              "      <td>comment</td>\n",
              "      <td>lot implied ill hnso true since microsoft want...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>aliswe</td>\n",
              "      <td>23989758</td>\n",
              "      <td>Yes that being said the demo is an old version.</td>\n",
              "      <td>1596046007</td>\n",
              "      <td>comment</td>\n",
              "      <td>yes said demo old version</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>DeonPenny</td>\n",
              "      <td>23989759</td>\n",
              "      <td>I&amp;#x27;d do 27 months for millions of dollars....</td>\n",
              "      <td>1596046009</td>\n",
              "      <td>comment</td>\n",
              "      <td>months millions dollars might even thousands d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>msla</td>\n",
              "      <td>23989761</td>\n",
              "      <td>I just described how Minitel was completely di...</td>\n",
              "      <td>1596046021</td>\n",
              "      <td>comment</td>\n",
              "      <td>described minitel completely different interne...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>pc86</td>\n",
              "      <td>23989764</td>\n",
              "      <td>&amp;gt; See the problem now?&lt;p&gt;No, because I am a...</td>\n",
              "      <td>1596046032</td>\n",
              "      <td>comment</td>\n",
              "      <td>see problem free speech advocate thing free sp...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>dheera</td>\n",
              "      <td>23989765</td>\n",
              "      <td>Honest question here. Why are we, the taxpayer...</td>\n",
              "      <td>1596046041</td>\n",
              "      <td>comment</td>\n",
              "      <td>honest question taxpayers paying months free h...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>UncleMeat</td>\n",
              "      <td>23989766</td>\n",
              "      <td>&amp;gt; You have to bake the security into your w...</td>\n",
              "      <td>1596046041</td>\n",
              "      <td>comment</td>\n",
              "      <td>bake security workflows processes code want ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>gmfawcett</td>\n",
              "      <td>23989767</td>\n",
              "      <td>Sorry to hear about the flashbacks. Having wri...</td>\n",
              "      <td>1596046044</td>\n",
              "      <td>comment</td>\n",
              "      <td>sorry hear flashbacks written couple dreadful ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>beervirus</td>\n",
              "      <td>23989768</td>\n",
              "      <td>Strange. Works beautifully on iOS.</td>\n",
              "      <td>1596046049</td>\n",
              "      <td>comment</td>\n",
              "      <td>strange works beautifully ios</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>awillen</td>\n",
              "      <td>23989769</td>\n",
              "      <td>You&amp;#x27;re obviously coming in with an anti-P...</td>\n",
              "      <td>1596046054</td>\n",
              "      <td>comment</td>\n",
              "      <td>obviously coming anti bent interested pull sta...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           by  ...                                         clean_text\n",
              "0       nix23  ...  lot implied ill hnso true since microsoft want...\n",
              "1      aliswe  ...                          yes said demo old version\n",
              "2   DeonPenny  ...  months millions dollars might even thousands d...\n",
              "4        msla  ...  described minitel completely different interne...\n",
              "7        pc86  ...  see problem free speech advocate thing free sp...\n",
              "8      dheera  ...  honest question taxpayers paying months free h...\n",
              "9   UncleMeat  ...  bake security workflows processes code want ma...\n",
              "10  gmfawcett  ...  sorry hear flashbacks written couple dreadful ...\n",
              "11  beervirus  ...                      strange works beautifully ios\n",
              "12    awillen  ...  obviously coming anti bent interested pull sta...\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "97IUEcCJFWRT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "40980dec-01b9-4b1b-d8d4-07a2ced4af92"
      },
      "source": [
        "df_comments['by'].value_counts()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "valuearb         8\n",
              "munk-a           8\n",
              "skbt             6\n",
              "oblio            6\n",
              "jandrese         6\n",
              "                ..\n",
              "Simulacra        1\n",
              "jaybeeayyy       1\n",
              "macspoofing      1\n",
              "searchableguy    1\n",
              "nightfly         1\n",
              "Name: by, Length: 589, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ur812Bwcs3Di",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4ba424dd-b23d-4123-9625-659dd6155eab"
      },
      "source": [
        "df_comments.shape\n"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(872, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2vnXeMXKs8p7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "ac950de2-f569-4936-97e7-2fbb9efc544b"
      },
      "source": [
        "df_comments['text'] = df_comments['text'].astype(str)\n",
        "df_comments['text'][:10]\n"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0     &gt;There is a lot of implied ill-will on HN<p...\n",
              "1       Yes that being said the demo is an old version.\n",
              "2     I&#x27;d do 27 months for millions of dollars....\n",
              "4     I just described how Minitel was completely di...\n",
              "7     &gt; See the problem now?<p>No, because I am a...\n",
              "8     Honest question here. Why are we, the taxpayer...\n",
              "9     &gt; You have to bake the security into your w...\n",
              "10    Sorry to hear about the flashbacks. Having wri...\n",
              "11                   Strange. Works beautifully on iOS.\n",
              "12    You&#x27;re obviously coming in with an anti-P...\n",
              "Name: text, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RHx6f8IbA6nu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "f94055d6-50f6-4e87-8371-094d5fa06af1"
      },
      "source": [
        "df_comments['clean_text'] = df_comments['clean_text'].astype(str)\n",
        "df_comments['clean_text'][:10]\n"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0     lot implied ill hnso true since microsoft want...\n",
              "1                             yes said demo old version\n",
              "2     months millions dollars might even thousands d...\n",
              "4     described minitel completely different interne...\n",
              "7     see problem free speech advocate thing free sp...\n",
              "8     honest question taxpayers paying months free h...\n",
              "9     bake security workflows processes code want ma...\n",
              "10    sorry hear flashbacks written couple dreadful ...\n",
              "11                        strange works beautifully ios\n",
              "12    obviously coming anti bent interested pull sta...\n",
              "Name: clean_text, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8przLjcy94j",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "64e951f7-aa80-4e0d-a236-e8fa930bd794"
      },
      "source": [
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "nltk.download('stopwords')\n",
        "\n"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NAmZGgfMzQDn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "091906ae-e777-4f7c-d762-e41ca111bc39"
      },
      "source": [
        "!pip install vaderSentiment"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: vaderSentiment in /usr/local/lib/python3.6/dist-packages (3.3.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from vaderSentiment) (2.23.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->vaderSentiment) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->vaderSentiment) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->vaderSentiment) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->vaderSentiment) (2.10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KX_7qEbwzKoF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
        "analyzer = SentimentIntensityAnalyzer()\n"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8v-69pWfzYO-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_compund_score(text):\n",
        "        score = analyzer.polarity_scores(text)\n",
        "        str(text)\n",
        "        return score['compound']\n"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5xkzdx2Kzs-O",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163
        },
        "outputId": "79738f2a-2045-4821-ec50-d47712fae9c0"
      },
      "source": [
        "get_compund_score(df_comments['text'][0])"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-20-1e260c8e4924>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_compund_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_comments\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'df_comments' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qvExbljFhaT8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FjdiIaUDBCIz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ddb18438-5ba4-4994-903f-216baf296f6f"
      },
      "source": [
        "get_compund_score(df_comments['clean_text'][0])"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7906"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GKCCuFPYz1vv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_comments['vader_score'] = df_comments['text'].apply(get_compund_score)\n"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lPLmFQJ1BGox",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_comments['clean_vader_score'] = df_comments['clean_text'].apply(get_compund_score)\n"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KnwRscgBz879",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "outputId": "c397de83-0299-4210-9268-8c481d21d735"
      },
      "source": [
        "df_comments.head(10)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>by</th>\n",
              "      <th>id</th>\n",
              "      <th>text</th>\n",
              "      <th>time</th>\n",
              "      <th>type</th>\n",
              "      <th>clean_text</th>\n",
              "      <th>vader_score</th>\n",
              "      <th>clean_vader_score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>nix23</td>\n",
              "      <td>23989757</td>\n",
              "      <td>&amp;gt;There is a lot of implied ill-will on HN&lt;p...</td>\n",
              "      <td>1596046005</td>\n",
              "      <td>comment</td>\n",
              "      <td>lot implied ill hnso true since microsoft want...</td>\n",
              "      <td>0.8689</td>\n",
              "      <td>0.7906</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>aliswe</td>\n",
              "      <td>23989758</td>\n",
              "      <td>Yes that being said the demo is an old version.</td>\n",
              "      <td>1596046007</td>\n",
              "      <td>comment</td>\n",
              "      <td>yes said demo old version</td>\n",
              "      <td>0.4019</td>\n",
              "      <td>0.4019</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>DeonPenny</td>\n",
              "      <td>23989759</td>\n",
              "      <td>I&amp;#x27;d do 27 months for millions of dollars....</td>\n",
              "      <td>1596046009</td>\n",
              "      <td>comment</td>\n",
              "      <td>months millions dollars might even thousands d...</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>msla</td>\n",
              "      <td>23989761</td>\n",
              "      <td>I just described how Minitel was completely di...</td>\n",
              "      <td>1596046021</td>\n",
              "      <td>comment</td>\n",
              "      <td>described minitel completely different interne...</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>pc86</td>\n",
              "      <td>23989764</td>\n",
              "      <td>&amp;gt; See the problem now?&lt;p&gt;No, because I am a...</td>\n",
              "      <td>1596046032</td>\n",
              "      <td>comment</td>\n",
              "      <td>see problem free speech advocate thing free sp...</td>\n",
              "      <td>0.7602</td>\n",
              "      <td>0.9040</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>dheera</td>\n",
              "      <td>23989765</td>\n",
              "      <td>Honest question here. Why are we, the taxpayer...</td>\n",
              "      <td>1596046041</td>\n",
              "      <td>comment</td>\n",
              "      <td>honest question taxpayers paying months free h...</td>\n",
              "      <td>0.8951</td>\n",
              "      <td>0.6808</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>UncleMeat</td>\n",
              "      <td>23989766</td>\n",
              "      <td>&amp;gt; You have to bake the security into your w...</td>\n",
              "      <td>1596046041</td>\n",
              "      <td>comment</td>\n",
              "      <td>bake security workflows processes code want ma...</td>\n",
              "      <td>0.4019</td>\n",
              "      <td>0.4215</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>gmfawcett</td>\n",
              "      <td>23989767</td>\n",
              "      <td>Sorry to hear about the flashbacks. Having wri...</td>\n",
              "      <td>1596046044</td>\n",
              "      <td>comment</td>\n",
              "      <td>sorry hear flashbacks written couple dreadful ...</td>\n",
              "      <td>0.8244</td>\n",
              "      <td>0.2263</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>beervirus</td>\n",
              "      <td>23989768</td>\n",
              "      <td>Strange. Works beautifully on iOS.</td>\n",
              "      <td>1596046049</td>\n",
              "      <td>comment</td>\n",
              "      <td>strange works beautifully ios</td>\n",
              "      <td>0.4404</td>\n",
              "      <td>0.4404</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>awillen</td>\n",
              "      <td>23989769</td>\n",
              "      <td>You&amp;#x27;re obviously coming in with an anti-P...</td>\n",
              "      <td>1596046054</td>\n",
              "      <td>comment</td>\n",
              "      <td>obviously coming anti bent interested pull sta...</td>\n",
              "      <td>0.7003</td>\n",
              "      <td>0.5423</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           by        id  ... vader_score  clean_vader_score\n",
              "0       nix23  23989757  ...      0.8689             0.7906\n",
              "1      aliswe  23989758  ...      0.4019             0.4019\n",
              "2   DeonPenny  23989759  ...      0.0000             0.0000\n",
              "4        msla  23989761  ...      0.0000             0.0000\n",
              "7        pc86  23989764  ...      0.7602             0.9040\n",
              "8      dheera  23989765  ...      0.8951             0.6808\n",
              "9   UncleMeat  23989766  ...      0.4019             0.4215\n",
              "10  gmfawcett  23989767  ...      0.8244             0.2263\n",
              "11  beervirus  23989768  ...      0.4404             0.4404\n",
              "12    awillen  23989769  ...      0.7003             0.5423\n",
              "\n",
              "[10 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UwTgYXBv0FrE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_comments['sentiment'] = df_comments['vader_score'].apply(lambda c: 'positive' if c >=0 else 'negative')\n"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qScSLIuSBPQj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_comments['clean_sentiment'] = df_comments['clean_vader_score'].apply(lambda c: 'positive' if c >=0 else 'negative')\n"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4aQrJ_TE0XZY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 824
        },
        "outputId": "2a000e18-85d0-4c15-9552-ed6af39d76e3"
      },
      "source": [
        "df_comments.head(25)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>by</th>\n",
              "      <th>id</th>\n",
              "      <th>text</th>\n",
              "      <th>time</th>\n",
              "      <th>type</th>\n",
              "      <th>clean_text</th>\n",
              "      <th>vader_score</th>\n",
              "      <th>clean_vader_score</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>clean_sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>nix23</td>\n",
              "      <td>23989757</td>\n",
              "      <td>&amp;gt;There is a lot of implied ill-will on HN&lt;p...</td>\n",
              "      <td>1596046005</td>\n",
              "      <td>comment</td>\n",
              "      <td>lot implied ill hnso true since microsoft want...</td>\n",
              "      <td>0.8689</td>\n",
              "      <td>0.7906</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>aliswe</td>\n",
              "      <td>23989758</td>\n",
              "      <td>Yes that being said the demo is an old version.</td>\n",
              "      <td>1596046007</td>\n",
              "      <td>comment</td>\n",
              "      <td>yes said demo old version</td>\n",
              "      <td>0.4019</td>\n",
              "      <td>0.4019</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>DeonPenny</td>\n",
              "      <td>23989759</td>\n",
              "      <td>I&amp;#x27;d do 27 months for millions of dollars....</td>\n",
              "      <td>1596046009</td>\n",
              "      <td>comment</td>\n",
              "      <td>months millions dollars might even thousands d...</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>msla</td>\n",
              "      <td>23989761</td>\n",
              "      <td>I just described how Minitel was completely di...</td>\n",
              "      <td>1596046021</td>\n",
              "      <td>comment</td>\n",
              "      <td>described minitel completely different interne...</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>pc86</td>\n",
              "      <td>23989764</td>\n",
              "      <td>&amp;gt; See the problem now?&lt;p&gt;No, because I am a...</td>\n",
              "      <td>1596046032</td>\n",
              "      <td>comment</td>\n",
              "      <td>see problem free speech advocate thing free sp...</td>\n",
              "      <td>0.7602</td>\n",
              "      <td>0.9040</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>dheera</td>\n",
              "      <td>23989765</td>\n",
              "      <td>Honest question here. Why are we, the taxpayer...</td>\n",
              "      <td>1596046041</td>\n",
              "      <td>comment</td>\n",
              "      <td>honest question taxpayers paying months free h...</td>\n",
              "      <td>0.8951</td>\n",
              "      <td>0.6808</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>UncleMeat</td>\n",
              "      <td>23989766</td>\n",
              "      <td>&amp;gt; You have to bake the security into your w...</td>\n",
              "      <td>1596046041</td>\n",
              "      <td>comment</td>\n",
              "      <td>bake security workflows processes code want ma...</td>\n",
              "      <td>0.4019</td>\n",
              "      <td>0.4215</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>gmfawcett</td>\n",
              "      <td>23989767</td>\n",
              "      <td>Sorry to hear about the flashbacks. Having wri...</td>\n",
              "      <td>1596046044</td>\n",
              "      <td>comment</td>\n",
              "      <td>sorry hear flashbacks written couple dreadful ...</td>\n",
              "      <td>0.8244</td>\n",
              "      <td>0.2263</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>beervirus</td>\n",
              "      <td>23989768</td>\n",
              "      <td>Strange. Works beautifully on iOS.</td>\n",
              "      <td>1596046049</td>\n",
              "      <td>comment</td>\n",
              "      <td>strange works beautifully ios</td>\n",
              "      <td>0.4404</td>\n",
              "      <td>0.4404</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>awillen</td>\n",
              "      <td>23989769</td>\n",
              "      <td>You&amp;#x27;re obviously coming in with an anti-P...</td>\n",
              "      <td>1596046054</td>\n",
              "      <td>comment</td>\n",
              "      <td>obviously coming anti bent interested pull sta...</td>\n",
              "      <td>0.7003</td>\n",
              "      <td>0.5423</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>vntok</td>\n",
              "      <td>23989770</td>\n",
              "      <td>Not necessarily. For it to be a type of intell...</td>\n",
              "      <td>1596046059</td>\n",
              "      <td>comment</td>\n",
              "      <td>necessarily type intellectual property would u...</td>\n",
              "      <td>0.0516</td>\n",
              "      <td>0.0516</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>syshum</td>\n",
              "      <td>23989771</td>\n",
              "      <td>PayPal is not a credit card processor, it is M...</td>\n",
              "      <td>1596046063</td>\n",
              "      <td>comment</td>\n",
              "      <td>paypal credit card processor money transmitter...</td>\n",
              "      <td>0.7283</td>\n",
              "      <td>0.8720</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>toomuchtodo</td>\n",
              "      <td>23989772</td>\n",
              "      <td>It’s okay to get caught (from a risk, not a mo...</td>\n",
              "      <td>1596046067</td>\n",
              "      <td>comment</td>\n",
              "      <td>okay get caught risk moral perspective whether...</td>\n",
              "      <td>-0.8020</td>\n",
              "      <td>-0.8020</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>minimaxir</td>\n",
              "      <td>23989773</td>\n",
              "      <td>A few comments:&lt;p&gt;1) &amp;quot;Teaching&amp;quot; GPT-...</td>\n",
              "      <td>1596046082</td>\n",
              "      <td>comment</td>\n",
              "      <td>comments quot teaching quot gpt misleading unl...</td>\n",
              "      <td>0.9286</td>\n",
              "      <td>0.9545</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>kelvin0</td>\n",
              "      <td>23989774</td>\n",
              "      <td>So I&amp;#x27;m not clear on what this is about? I...</td>\n",
              "      <td>1596046082</td>\n",
              "      <td>comment</td>\n",
              "      <td>clear seem missing piece puzzle seems related ...</td>\n",
              "      <td>-0.6134</td>\n",
              "      <td>0.7964</td>\n",
              "      <td>negative</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>Shared404</td>\n",
              "      <td>23989775</td>\n",
              "      <td>Out of curiosity, did you even read my respons...</td>\n",
              "      <td>1596046088</td>\n",
              "      <td>comment</td>\n",
              "      <td>curiosity even read response regurgitating tal...</td>\n",
              "      <td>0.7579</td>\n",
              "      <td>0.7579</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>jjcon</td>\n",
              "      <td>23989776</td>\n",
              "      <td>&amp;gt; not in keeping with the First Sale Doctri...</td>\n",
              "      <td>1596046091</td>\n",
              "      <td>comment</td>\n",
              "      <td>keeping first sale doctrine confused first sale</td>\n",
              "      <td>-0.3182</td>\n",
              "      <td>-0.3182</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>iaw</td>\n",
              "      <td>23989777</td>\n",
              "      <td>They had previously paid him $120M so it seems...</td>\n",
              "      <td>1596046097</td>\n",
              "      <td>comment</td>\n",
              "      <td>previously paid seems like actually ballpark t...</td>\n",
              "      <td>0.4144</td>\n",
              "      <td>0.3612</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>Impossible</td>\n",
              "      <td>23989778</td>\n",
              "      <td>I never said that Blender is not used in game ...</td>\n",
              "      <td>1596046097</td>\n",
              "      <td>comment</td>\n",
              "      <td>never said blender used game development speci...</td>\n",
              "      <td>0.2225</td>\n",
              "      <td>-0.2023</td>\n",
              "      <td>positive</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>sadfev</td>\n",
              "      <td>23989779</td>\n",
              "      <td>No, it was blatant theft of IP and some might ...</td>\n",
              "      <td>1596046099</td>\n",
              "      <td>comment</td>\n",
              "      <td>blatant theft might consider corporate espiona...</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>noisem4ker</td>\n",
              "      <td>23989781</td>\n",
              "      <td>I doubt it&amp;#x27;s news to you, but Android ope...</td>\n",
              "      <td>1596046103</td>\n",
              "      <td>comment</td>\n",
              "      <td>doubt news android openly allows app stores go...</td>\n",
              "      <td>-0.1901</td>\n",
              "      <td>-0.3612</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>rzzzt</td>\n",
              "      <td>23989782</td>\n",
              "      <td>The advantage is that you don&amp;#x27;t need to h...</td>\n",
              "      <td>1596046104</td>\n",
              "      <td>comment</td>\n",
              "      <td>advantage need another parser ready parse sche...</td>\n",
              "      <td>0.5423</td>\n",
              "      <td>0.5423</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>gav</td>\n",
              "      <td>23989783</td>\n",
              "      <td>I&amp;#x27;m not sure if it&amp;#x27;s just me, but wh...</td>\n",
              "      <td>1596046109</td>\n",
              "      <td>comment</td>\n",
              "      <td>sure see deletes use ctes like start get nervo...</td>\n",
              "      <td>0.9046</td>\n",
              "      <td>0.8779</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>starbugs</td>\n",
              "      <td>23989784</td>\n",
              "      <td>It can go on for a really long time unfortunat...</td>\n",
              "      <td>1596046124</td>\n",
              "      <td>comment</td>\n",
              "      <td>really long time unfortunately impossible pred...</td>\n",
              "      <td>-0.2168</td>\n",
              "      <td>-0.6394</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>scott_s</td>\n",
              "      <td>23989785</td>\n",
              "      <td>I proposed a scenario where it seems to me tha...</td>\n",
              "      <td>1596046124</td>\n",
              "      <td>comment</td>\n",
              "      <td>proposed scenario seems sms achieve something ...</td>\n",
              "      <td>0.2732</td>\n",
              "      <td>0.2732</td>\n",
              "      <td>positive</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             by        id  ... sentiment  clean_sentiment\n",
              "0         nix23  23989757  ...  positive         positive\n",
              "1        aliswe  23989758  ...  positive         positive\n",
              "2     DeonPenny  23989759  ...  positive         positive\n",
              "4          msla  23989761  ...  positive         positive\n",
              "7          pc86  23989764  ...  positive         positive\n",
              "8        dheera  23989765  ...  positive         positive\n",
              "9     UncleMeat  23989766  ...  positive         positive\n",
              "10    gmfawcett  23989767  ...  positive         positive\n",
              "11    beervirus  23989768  ...  positive         positive\n",
              "12      awillen  23989769  ...  positive         positive\n",
              "13        vntok  23989770  ...  positive         positive\n",
              "14       syshum  23989771  ...  positive         positive\n",
              "15  toomuchtodo  23989772  ...  negative         negative\n",
              "16    minimaxir  23989773  ...  positive         positive\n",
              "17      kelvin0  23989774  ...  negative         positive\n",
              "18    Shared404  23989775  ...  positive         positive\n",
              "19        jjcon  23989776  ...  negative         negative\n",
              "20          iaw  23989777  ...  positive         positive\n",
              "21   Impossible  23989778  ...  positive         negative\n",
              "22       sadfev  23989779  ...  positive         positive\n",
              "24   noisem4ker  23989781  ...  negative         negative\n",
              "25        rzzzt  23989782  ...  positive         positive\n",
              "26          gav  23989783  ...  positive         positive\n",
              "27     starbugs  23989784  ...  negative         negative\n",
              "28      scott_s  23989785  ...  positive         positive\n",
              "\n",
              "[25 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YNoiopCD2OZf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_comments.to_csv('hn_sentiments.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kRZj5eX9M5DV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import json\n",
        "import urllib\n",
        "import requests"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F7y3ynzmNEWw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install vaderSentiment"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tOTkNWNzGgmQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
        "analyzer = SentimentIntensityAnalyzer()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ulzaGBmgGpPA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_compund_score(text):\n",
        "        score = analyzer.polarity_scores(text)\n",
        "        str(text)\n",
        "        return score['compound']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n4V7bA5ZGuuj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def preprocess(sentence):\n",
        "    sentence=str(sentence)\n",
        "    sentence = sentence.lower()\n",
        "    sentence=sentence.replace('{html}',\"\") \n",
        "    cleanr = re.compile('<.*?>')\n",
        "    cleantext = re.sub(cleanr, '', sentence)\n",
        "    rem_url=re.sub(r'http\\S+', '',cleantext)\n",
        "    rem_num = re.sub('[0-9]+', '', rem_url)\n",
        "    tokenizer = RegexpTokenizer(r'\\w+')\n",
        "    tokens = tokenizer.tokenize(rem_num)  \n",
        "    filtered_words = [w for w in tokens if len(w) > 2 if not w in stopwords.words('english')]\n",
        "    stem_words=[stemmer.stem(w) for w in filtered_words]\n",
        "    lemma_words=[lemmatizer.lemmatize(w) for w in stem_words]\n",
        "    return \" \".join(filtered_words)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "42qz9BDeG5vt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "35bb51e3-a91f-42f1-d347-79967931672c"
      },
      "source": [
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "nltk.download('stopwords')\n"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ALu0ptT9BTXU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import json\n",
        "import urllib\n",
        "import requests\n",
        "\n",
        "def get_score(entries):\n",
        "  data = []\n",
        "  for id in entries:\n",
        "    html = urllib.request.urlopen('https://hacker-news.firebaseio.com/v0/item/' + str(id) + '.json')\n",
        "    data.append(json.loads(html.read()))\n",
        "\n",
        "  print(data[0])\n",
        "  data = [i for i in data if i is not None]\n",
        "\n",
        "  df = pd.DataFrame.from_dict(data)\n",
        "  df_comments = df[df['type'] == 'comment']\n",
        "\n",
        "  df_comments['clean_text']=df_comments['text'].map(lambda s:preprocess(s)) \n",
        "  df_comments['clean_vader_score'] = df_comments['clean_text'].apply(get_compund_score)\n",
        "\n",
        "  return df_comments['clean_vader_score'].sum() #we can use mean()\n",
        "\n",
        "  \n",
        " "
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MQnMHUcycg7E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wjCBP10yQmca",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import json\n",
        "import urllib\n",
        "import requests\n",
        "\n",
        "def get_cummulative_score(username):\n",
        "\n",
        "  data = []\n",
        "  html = urllib.request.urlopen('https://hacker-news.firebaseio.com/v0/user/' + str(username) + '.json?print=pretty')\n",
        "  data.append(json.loads(html.read()))\n",
        "  df2 = pd.DataFrame.from_dict(data)\n",
        "  entries =  (df2['submitted'][0])\n",
        "\n",
        "  score = get_score(entries)\n",
        "  return score\n",
        "\n",
        "\n",
        "    \n",
        "\n",
        "  \n",
        "\n"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eHct_Sx5RYQf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "outputId": "dcd3c519-13e6-4d98-b4fc-0b3dd6f7fdcb"
      },
      "source": [
        "\n",
        "score = get_cummulative_score('jl')\n",
        "print(\"Cummulative score for user \", score )"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'by': 'jl', 'descendants': 536, 'id': 19464269, 'kids': [19464832, 19465697, 19465135, 19464960, 19465786, 19465019, 19464652, 19466183, 19470050, 19465299, 19465037, 19467582, 19466713, 19466536, 19467580, 19465893, 19466505, 19464708, 19465389, 19467096, 19466016, 19465003, 19465088, 19469424, 19468661, 19468811, 19475637, 19471240, 19468924, 19465592, 19469386, 19466312, 19467740, 19466241, 19466112, 19465889, 19467249, 19475393, 19465474, 19465638, 19466066, 19466993, 19465036, 19467011, 19467899, 19465798, 19466036, 19465314, 19465127, 19465038, 19467292, 19464560], 'score': 268, 'time': 1553272932, 'title': 'Women: Learn to Program This Summer', 'type': 'story', 'url': 'http://foundersatwork.posthaven.com/women-learn-to-program-this-summer'}\n",
            "Cummulative score for user  39.2382\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:18: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:19: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ercETRGciY6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "3c727666-9075-4e51-85e1-5a1991e9d60f"
      },
      "source": [
        "test_message = \"This is really bad. Never going there again. Absolute worst\"\n",
        "print (get_compund_score(test_message))\n",
        "\n",
        "test_message = \"This is really good. Loved the place. Employees were super kind. Can't wait to go back again.\"\n",
        "print (get_compund_score(test_message))\n",
        "\n"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-0.8357\n",
            "0.94\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}