Sarcasm3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sarcasm3.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO7Cr4m1JFHUEbsDkQG3EtK"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qUdxUxrZfFd1",
        "outputId": "1484efe1-c52f-464d-919d-3782a6f929df"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "import re\n",
        "import tensorflow_datasets as tfds\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import backend as K\n",
        "import tensorflow_datasets as tfds\n",
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow as tf\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "import string\n",
        "import time\n",
        "\n",
        "\n",
        "nltk.download('wordnet')\n",
        "nltk.download('stopwords')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ClIMwjzofNAB"
      },
      "source": [
        "df1=pd.read_json(\"/content/Sarcasm_Headlines_Dataset_v2.json\",lines=True)\n",
        "df1=df1[['headline','is_sarcastic']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CNMHxqocGZRr",
        "outputId": "75cfd3d5-1041-45cd-c8a9-ef9de96438a3"
      },
      "source": [
        "df1.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 28619 entries, 0 to 28618\n",
            "Data columns (total 2 columns):\n",
            " #   Column        Non-Null Count  Dtype \n",
            "---  ------        --------------  ----- \n",
            " 0   headline      28619 non-null  object\n",
            " 1   is_sarcastic  28619 non-null  int64 \n",
            "dtypes: int64(1), object(1)\n",
            "memory usage: 447.3+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CZ5wgH1YfhYW"
      },
      "source": [
        "df=df1.drop(df1[df1['is_sarcastic']==1].sample(frac=0.10).index)\n",
        "df=df.reset_index()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "my5P86PtgKTi",
        "outputId": "c99e355f-c3f1-4d34-e490-ff61b1d676ce"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>headline</th>\n",
              "      <th>is_sarcastic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>thirtysomething scientists unveil doomsday clock of hair loss</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>dem rep. totally nails why congress is falling short on gender, racial equality</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>eat your veggies: 9 deliciously different recipes</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>inclement weather prevents liar from getting to work</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>my white inheritance</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   index  ... is_sarcastic\n",
              "0      0  ...            1\n",
              "1      1  ...            0\n",
              "2      2  ...            0\n",
              "3      3  ...            1\n",
              "4      5  ...            0\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g4kFd_b7Rxs2",
        "outputId": "8a0e79f0-5dda-4732-c012-359680871fbe"
      },
      "source": [
        "df.index"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RangeIndex(start=0, stop=27256, step=1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 280
        },
        "id": "QBZMuIc7gjLt",
        "outputId": "7760a5d2-24f2-4a67-fbbd-a599927d5a7f"
      },
      "source": [
        "sns.countplot(data=df,x=df.is_sarcastic);"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEHCAYAAABvHnsJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAVF0lEQVR4nO3df7BndX3f8ecLNqJEZflxS3AXXao7dhCTCjtA4ySTSAOLTbOMFQNjZCXUbRuMMTpRSFs3RelotaViIulGVsA6EsRQNi262SCGdirIosjyQ8MtouyWHxeWHxqKuOTdP76fxW/We+Hy2b3fL5f7fMx853vO+3zOOZ+zs7OvPed8vuekqpAkqcde4+6AJGn+MkQkSd0MEUlSN0NEktTNEJEkdVs07g6M2kEHHVTLli0bdzckaV658cYbH6iqiV3rCy5Eli1bxubNm8fdDUmaV5J8d7q6l7MkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3RbcL9Z311G/d8m4u6DnoBs/etq4uyCNhWcikqRuhogkqZshIknqZohIkroZIpKkboaIJKmbISJJ6maISJK6GSKSpG5zFiJJ1ie5P8kt0yx7b5JKclCbT5Lzk0wmuTnJkUNtVye5o31WD9WPSrKlrXN+kszVsUiSpjeXZyIXASt3LSY5FDge+N5Q+URgefusAS5obQ8A1gLHAEcDa5Ps39a5AHjH0Ho/sS9J0tyasxCpqmuB7dMsOg94H1BDtVXAJTVwHbA4ySHACcCmqtpeVQ8Bm4CVbdlLq+q6qirgEuCkuToWSdL0RnpPJMkqYFtVfXOXRUuAu4fmt7ba09W3TlOfab9rkmxOsnlqamo3jkCSNGxkIZJkX+D3gQ+Map87VdW6qlpRVSsmJiZGvXtJet4a5ZnIK4HDgG8muQtYCnw9yc8A24BDh9oubbWnqy+dpi5JGqGRhUhVbamqv1dVy6pqGYNLUEdW1b3ABuC0NkrrWOCRqroH2Agcn2T/dkP9eGBjW/ZokmPbqKzTgCtHdSySpIG5HOL7OeCrwKuTbE1yxtM0vwq4E5gE/gT4LYCq2g58ELihfc5pNVqbT7V1/g/wxbk4DknSzObszYZVdeozLF82NF3AmTO0Ww+sn6a+GThi93opSdod/mJdktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHWbs9fjShq9753z2nF3Qc9BL//Aljnb9pydiSRZn+T+JLcM1T6a5FtJbk5yRZLFQ8vOTjKZ5NtJThiqr2y1ySRnDdUPS3J9q/9pkhfM1bFIkqY3l5ezLgJW7lLbBBxRVT8L/DVwNkCSw4FTgNe0dT6ZZO8kewN/BJwIHA6c2toCfAQ4r6peBTwEnDGHxyJJmsachUhVXQts36X2F1W1o81eByxt06uAS6vqh1X1HWASOLp9Jqvqzqp6ArgUWJUkwBuAy9v6FwMnzdWxSJKmN84b678JfLFNLwHuHlq2tdVmqh8IPDwUSDvr00qyJsnmJJunpqb2UPclSWMJkST/GtgBfHYU+6uqdVW1oqpWTExMjGKXkrQgjHx0VpK3A78KHFdV1crbgEOHmi1tNWaoPwgsTrKonY0Mt5ckjchIz0SSrATeB/xaVT02tGgDcEqSfZIcBiwHvgbcACxvI7FewODm+4YWPtcAb27rrwauHNVxSJIG5nKI7+eArwKvTrI1yRnAHwIvATYluSnJHwNU1a3AZcBtwJeAM6vqyXaW8U5gI3A7cFlrC/B+4D1JJhncI7lwro5FkjS9ObucVVWnTlOe8R/6qjoXOHea+lXAVdPU72QwekuSNCY+9kSS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdZvLd6yvT3J/kluGagck2ZTkjva9f6snyflJJpPcnOTIoXVWt/Z3JFk9VD8qyZa2zvlJMlfHIkma3lyeiVwErNyldhZwdVUtB65u8wAnAsvbZw1wAQxCB1gLHMPgfeprdwZPa/OOofV23ZckaY7NWYhU1bXA9l3Kq4CL2/TFwElD9Utq4DpgcZJDgBOATVW1vaoeAjYBK9uyl1bVdVVVwCVD25Ikjcio74kcXFX3tOl7gYPb9BLg7qF2W1vt6epbp6lLkkZobDfW2xlEjWJfSdYk2Zxk89TU1Ch2KUkLwqhD5L52KYr2fX+rbwMOHWq3tNWerr50mvq0qmpdVa2oqhUTExO7fRCSpIFRh8gGYOcIq9XAlUP109oorWOBR9plr43A8Un2bzfUjwc2tmWPJjm2jco6bWhbkqQRWTRXG07yOeCXgIOSbGUwyurDwGVJzgC+C7ylNb8KeCMwCTwGnA5QVduTfBC4obU7p6p23qz/LQYjwF4EfLF9JEkjNGchUlWnzrDouGnaFnDmDNtZD6yfpr4ZOGJ3+ihJ2j3+Yl2S1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1G1WIZLk6tnUJEkLy9M+Cj7JC4F9GbwTZH8gbdFL8Z3mkrTgPdP7RP4F8G7gZcCN/DhEHgX+cA77JUmaB542RKrq48DHk/x2VX1iRH2SJM0Ts3qzYVV9IsnPA8uG16mqS+aoX5KkeWBWIZLkM8ArgZuAJ1u5AENEkhaw2b5jfQVweHsX+m5L8rvAP2cQRFuA04FDgEuBAxncf3lbVT2RZB8GYXUU8CDw61V1V9vO2cAZDILtXVW1cU/0T5I0O7P9ncgtwM/siR0mWQK8C1hRVUcAewOnAB8BzquqVwEPMQgH2vdDrX5ea0eSw9t6rwFWAp9Msvee6KMkaXZmGyIHAbcl2Zhkw87Pbux3EfCiJIsYDCG+B3gDcHlbfjFwUpte1eZpy49Lkla/tKp+WFXfASaBo3ejT5KkZ2m2l7P+YE/tsKq2JfkY8D3g/wF/weDy1cNVtaM128qPf4eyBLi7rbsjySMMLnktAa4b2vTwOn9HkjXAGoCXv/zle+pQJGnBm+3orL/aUztsP1pcBRwGPAx8nsHlqDlTVeuAdQArVqzYI/d1JEmzf+zJ95M82j6PJ3kyyaOd+/zHwHeqaqqqfgT8GfB6YHG7vAWwFNjWprcBh7Z+LAL2Y3CD/an6NOtIkkZgViFSVS+pqpdW1UuBFwH/DPhk5z6/BxybZN92b+M44DbgGuDNrc1q4Mo2vaHN05Z/uY0S2wCckmSfJIcBy4GvdfZJktThWT/Ftwb+G3BCzw6r6noGN8i/zmB4714MLjW9H3hPkkkG9zwubKtcCBzY6u8BzmrbuRW4jEEAfQk4s6qeRJI0MrP9seGbhmb3YvC7kcd7d1pVa4G1u5TvZJrRVVX1OHDyDNs5Fzi3tx+SpN0z29FZ/3RoegdwF4Ob45KkBWy2o7NOn+uOSJLmn9mOzlqa5Iok97fPF5IsnevOSZKe22Z7Y/3TDEZDvax9/rzVJEkL2GxDZKKqPl1VO9rnImBiDvslSZoHZhsiDyb5jSR7t89vMPjBnyRpAZttiPwm8BbgXgYPS3wz8PY56pMkaZ6Y7RDfc4DVVfUQQJIDgI8xCBdJ0gI12zORn90ZIABVtR143dx0SZI0X8w2RPZqT98FnjoTme1ZjCTpeWq2QfAfga8m+XybPxkfNyJJC95sf7F+SZLNDN4+CPCmqrpt7rolSZoPZn1JqoWGwSFJesqzfhS8JEk7GSKSpG6GiCSpmyEiSepmiEiSuhkikqRuYwmRJIuTXJ7kW0luT/KPkhyQZFOSO9r3/q1tkpyfZDLJzUmOHNrO6tb+jiSrx3EskrSQjetM5OPAl6rqHwA/B9wOnAVcXVXLgavbPMCJwPL2WQNcAE89emUtcAxwNLB2+NEskqS5N/IQSbIf8IvAhQBV9URVPQysAi5uzS4GTmrTq4BLauA6YHGSQ4ATgE1Vtb09HHITsHKEhyJJC944zkQOA6aATyf5RpJPJflp4OCquqe1uRc4uE0vAe4eWn9rq81U/wlJ1iTZnGTz1NTUHjwUSVrYxhEii4AjgQuq6nXA3/DjS1cAVFUBtad2WFXrqmpFVa2YmPCtvpK0p4wjRLYCW6vq+jZ/OYNQua9dpqJ939+WbwMOHVp/aavNVJckjcjIQ6Sq7gXuTvLqVjqOwYMdNwA7R1itBq5s0xuA09oorWOBR9plr43A8Un2bzfUj281SdKIjOvFUr8NfDbJC4A7gdMZBNplSc4Avsvgne4AVwFvBCaBx1pbqmp7kg8CN7R257Q3LkqSRmQsIVJVNwErpll03DRtCzhzhu2sB9bv2d5JkmbLX6xLkroZIpKkboaIJKmbISJJ6maISJK6GSKSpG6GiCSpmyEiSepmiEiSuhkikqRuhogkqZshIknqZohIkroZIpKkboaIJKmbISJJ6maISJK6GSKSpG5jC5Ekeyf5RpL/3uYPS3J9kskkf9rev06Sfdr8ZFu+bGgbZ7f6t5OcMJ4jkaSFa5xnIr8D3D40/xHgvKp6FfAQcEarnwE81OrntXYkORw4BXgNsBL4ZJK9R9R3SRJjCpEkS4F/AnyqzQd4A3B5a3IxcFKbXtXmacuPa+1XAZdW1Q+r6jvAJHD0aI5AkgTjOxP5z8D7gL9t8wcCD1fVjja/FVjSppcAdwO05Y+09k/Vp1lHkjQCIw+RJL8K3F9VN45wn2uSbE6yeWpqalS7laTnvXGcibwe+LUkdwGXMriM9XFgcZJFrc1SYFub3gYcCtCW7wc8OFyfZp2/o6rWVdWKqloxMTGxZ49GkhawkYdIVZ1dVUurahmDG+Nfrqq3AtcAb27NVgNXtukNbZ62/MtVVa1+Shu9dRiwHPjaiA5DkgQseuYmI/N+4NIkHwK+AVzY6hcCn0kyCWxnEDxU1a1JLgNuA3YAZ1bVk6PvtiQtXGMNkar6CvCVNn0n04yuqqrHgZNnWP9c4Ny566Ek6en4i3VJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1G3mIJDk0yTVJbktya5LfafUDkmxKckf73r/Vk+T8JJNJbk5y5NC2Vrf2dyRZPepjkaSFbhxnIjuA91bV4cCxwJlJDgfOAq6uquXA1W0e4ERgefusAS6AQegAa4FjgKOBtTuDR5I0GiMPkaq6p6q+3qa/D9wOLAFWARe3ZhcDJ7XpVcAlNXAdsDjJIcAJwKaq2l5VDwGbgJUjPBRJWvDGek8kyTLgdcD1wMFVdU9bdC9wcJteAtw9tNrWVpupPt1+1iTZnGTz1NTUHuu/JC10YwuRJC8GvgC8u6oeHV5WVQXUntpXVa2rqhVVtWJiYmJPbVaSFryxhEiSn2IQIJ+tqj9r5fvaZSra9/2tvg04dGj1pa02U12SNCLjGJ0V4ELg9qr6T0OLNgA7R1itBq4cqp/WRmkdCzzSLnttBI5Psn+7oX58q0mSRmTRGPb5euBtwJYkN7Xa7wMfBi5LcgbwXeAtbdlVwBuBSeAx4HSAqtqe5IPADa3dOVW1fTSHIEmCMYRIVf0vIDMsPm6a9gWcOcO21gPr91zvJEnPhr9YlyR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUrd5HyJJVib5dpLJJGeNuz+StJDM6xBJsjfwR8CJwOHAqUkOH2+vJGnhmNchAhwNTFbVnVX1BHApsGrMfZKkBWPRuDuwm5YAdw/NbwWO2bVRkjXAmjb7gyTfHkHfFoKDgAfG3Ynngnxs9bi7oJ/k38+d1mZPbOUV0xXne4jMSlWtA9aNux/PN0k2V9WKcfdDmo5/P0djvl/O2gYcOjS/tNUkSSMw30PkBmB5ksOSvAA4Bdgw5j5J0oIxry9nVdWOJO8ENgJ7A+ur6tYxd2sh8RKhnsv8+zkCqapx90GSNE/N98tZkqQxMkQkSd0MEXXxcTN6rkqyPsn9SW4Zd18WAkNEz5qPm9Fz3EXAynF3YqEwRNTDx83oOauqrgW2j7sfC4Uhoh7TPW5myZj6ImmMDBFJUjdDRD183IwkwBBRHx83IwkwRNShqnYAOx83cztwmY+b0XNFks8BXwVenWRrkjPG3afnMx97Iknq5pmIJKmbISJJ6maISJK6GSKSpG6GiCSpmyEiSepmiEgzSPK/x92HZyvJu5PsOzR/VZLF4+yTnt/8nYj0HJBkUfsR5+5u5y5gRVU9sPu9kp6ZZyLSDJL8oH0fkuTaJDcluSXJL8zQfu8kF7U2W5L8bqu/I8kNSb6Z5As7zxRa2z9Ocj3wH5K8KslftnZfT/LKJC9OcnWb35JkVVv3p5P8j9b2liS/nuRdwMuAa5Jc09rdleSgNn1akpvbOp+Z8z9ALQieiUgzSPKDqnpxkvcCL6yqc9sLufatqu9P0/4o4MNV9SttfnFVPZzkwKp6sNU+BNxXVZ9IchFwELCqqp5sYfLhqroiyQsZ/Cfviba/R1sYXAcsB94ErKyqd7Tt7ldVj+x6JrJzHjgYuAL4+ap6IMkBVeU7N7TbPBORntkNwOlJ/gB47XQB0twJ/P0kn0iyEni01Y9I8j+TbAHeCrxmaJ3PtwB5CbCkqq4AqKrHq+oxIMC/T3Iz8JcM3ttyMLAF+JUkH0nyC1X1yDMcwxvavh5o2zdAtEcYItIzaG/K+0UGj7u/KMlpM7R7CPg54CvAvwQ+1RZdBLyzql4L/DvghUOr/c0z7P6twARwVFX9Q+A+BmdFfw0cySBMPpTkA8/+yKTdZ4hIzyDJKxhcgvoTBsFw5AztDgL2qqovAP9mqN1LgHuS/BSDUPgJ7exma5KT2rb2afdO9gPur6ofJfll4BVt+cuAx6rqvwIfHdrX99v+dvVl4OQkB7b1D3g2fwbSTBaNuwPSPPBLwO8l+RHwA2DaMxEGl5o+nWTnf87Obt//FrgemGrf0/0jD/A24L8kOQf4EXAy8Fngz9ulsM3At1rb1wIfTfK3re2/avV1wJeS/N+q+uWdG66qW5OcC/xVkieBbwBvn93hSzPzxrokqZuXsyRJ3bycJXVow3H32aX8tqraMo7+SOPi5SxJUjcvZ0mSuhkikqRuhogkqZshIknq9v8BZg4dfwahMyoAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "id": "6_CvcBISgzN5",
        "outputId": "3a870b4e-dd51-4678-e3fb-28c878038550"
      },
      "source": [
        "pd.options.display.max_colwidth = 200\n",
        "pd.DataFrame(df[df['is_sarcastic']==1]['headline']).head(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>headline</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>thirtysomething scientists unveil doomsday clock of hair loss</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>inclement weather prevents liar from getting to work</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>richard branson's global-warming donation nearly as much as cost of failed balloon trips</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>shadow government getting too large to meet in marriott conference room b</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>ford develops new suv that runs purely on gasoline</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>area boy enters jumping-and-touching-tops-of-doorways phase</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>area man does most of his traveling by gurney</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>guard in video game under strict orders to repeatedly pace same stretch of hallway</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>secret service agent not so secret about being david alan grier fan</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>leading probability researchers confounded by three coworkers wearing same shirt color on same day</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                              headline\n",
              "0                                        thirtysomething scientists unveil doomsday clock of hair loss\n",
              "3                                                 inclement weather prevents liar from getting to work\n",
              "6             richard branson's global-warming donation nearly as much as cost of failed balloon trips\n",
              "7                            shadow government getting too large to meet in marriott conference room b\n",
              "13                                                  ford develops new suv that runs purely on gasoline\n",
              "15                                         area boy enters jumping-and-touching-tops-of-doorways phase\n",
              "16                                                       area man does most of his traveling by gurney\n",
              "20                  guard in video game under strict orders to repeatedly pace same stretch of hallway\n",
              "24                                 secret service agent not so secret about being david alan grier fan\n",
              "32  leading probability researchers confounded by three coworkers wearing same shirt color on same day"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rXkEji5-goO1"
      },
      "source": [
        "def decontract(text):\n",
        "    text = re.sub(r\"won\\'t\", \"will not\", text)\n",
        "    text = re.sub(r\"can\\'t\", \"can not\", text)\n",
        "    text = re.sub(r\"n\\'t\", \" not\", text)\n",
        "    text = re.sub(r\"\\'re\", \" are\", text)\n",
        "    text = re.sub(r\"\\'s$\", \" is\", text)\n",
        "    text = re.sub(r\"\\'d\", \" would\", text)\n",
        "    text = re.sub(r\"\\'ll\", \" will\", text)\n",
        "    text = re.sub(r\"\\'t\", \" not\", text)\n",
        "    text = re.sub(r\"\\'ve\", \" have\", text)\n",
        "    text = re.sub(r\"\\'m\", \" am\", text)\n",
        "    return text\n",
        "stopwords_english = set(stopwords.words('english'))-set(['No','no','not','Not'])\n",
        "def preprocess(text,stopwords=stopwords_english):\n",
        "    lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "\n",
        "    # remove stock market tickers like $GE\n",
        "    text = re.sub(r'\\$\\w*', '', text)\n",
        "    # remove old style retweet text \"RT\"\n",
        "    text = re.sub(r'^RT[\\s]+', '', text)\n",
        "    # remove hyperlinks\n",
        "    text = re.sub(r'https?:\\/\\/.*[\\r\\n]*', '', text)\n",
        "    #Decontract texts\n",
        "    text=decontract(text)\n",
        "    # tokenize texts\n",
        "\n",
        "\n",
        "    texts_clean = []\n",
        "    for word in text.split():\n",
        "        if (word not in stopwords_english and  # remove stopwords\n",
        "                word not in set(string.punctuation)-set(['!','?','.','@',':'])):  # remove punctuation\n",
        "            #Lemmatize word \n",
        "            lem_word = lemmatizer.lemmatize(word,\"v\")  # Lemmatizing word\n",
        "            texts_clean.append(lem_word)\n",
        "\n",
        "    return \" \".join(texts_clean)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qv7RuniYg4tq"
      },
      "source": [
        "inputs=list(df['headline'].apply(lambda x: preprocess(x)))\n",
        "labels=list(df['is_sarcastic'])\n",
        "X_train, X_test, y_train, y_test = train_test_split(inputs, labels, test_size=0.2)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "ccGOGVKiSX-h",
        "outputId": "062ac406-1a40-4150-8e2b-3cf11e8496a1"
      },
      "source": [
        "sns.countplot(x=y_train);"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAD4CAYAAAAtrdtxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQyUlEQVR4nO3df6xfdX3H8efLVvyxiYDcMG1hbWZjUp2LeANsJouBBQpzlhg0kG1UbNYlQ6fbMgWXrAvKgtGNAU6XZlTAGLBDHd2GsgZxZpn8KEqQHzJuYEgbsFeK6CTC6t774/u5+LXessunvd9vL/f5SL6557zP55zzPknDi/Pje76pKiRJ6vGCcTcgSVq4DBFJUjdDRJLUzRCRJHUzRCRJ3ZaOu4FRO/LII2vFihXjbkOSFpTbb7/9u1U1sXd90YXIihUr2L59+7jbkKQFJclDs9W9nCVJ6maISJK6GSKSpG6GiCSpmyEiSeo2byGSZHOSXUnuGqp9NMm3ktyZ5AtJDhtadn6SqST3JTllqL6m1aaSnDdUX5nkllb/bJJD5utYJEmzm88zkSuANXvVtgGvq6rXA/8JnA+QZDVwJvDats4nkixJsgT4W+BUYDVwVhsL8BHg4qp6NfA4sH4ej0WSNIt5C5Gq+iqwe6/av1bVnjZ7M7C8Ta8Frqmqp6rqQWAKOK59pqrqgap6GrgGWJskwInAtW39K4HT5+tYJEmzG+c9kXcBX2zTy4CHh5btaLV91V8BfG8okGbqs0qyIcn2JNunp6cPUPuSpLF8Yz3JnwF7gM+MYn9VtQnYBDA5Oblfv8L1xj+96oD0pOeX2z969rhbkMZi5CGS5J3AW4CT6ic/q7gTOHpo2PJWYx/1x4DDkixtZyPD4yVJIzLSy1lJ1gDvB95aVU8OLdoKnJnkRUlWAquAW4HbgFXtSaxDGNx839rC5ybgjLb+OuC6UR2HJGlgPh/xvRr4GvCaJDuSrAc+DrwM2JbkjiR/B1BVdwNbgHuALwHnVtWP21nGu4EbgHuBLW0swAeAP04yxeAeyeXzdSySpNnN2+WsqjprlvI+/0NfVRcCF85Svx64fpb6Awye3pIkjYnfWJckdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdZu3EEmyOcmuJHcN1Y5Isi3J/e3v4a2eJJcmmUpyZ5Jjh9ZZ18bfn2TdUP2NSb7Z1rk0SebrWCRJs5vPM5ErgDV71c4DbqyqVcCNbR7gVGBV+2wAPgmD0AE2AscDxwEbZ4Knjfm9ofX23pckaZ7NW4hU1VeB3XuV1wJXtukrgdOH6lfVwM3AYUleCZwCbKuq3VX1OLANWNOWHVpVN1dVAVcNbUuSNCKjvidyVFU90qYfBY5q08uAh4fG7Wi1Z6vvmKU+qyQbkmxPsn16enr/jkCS9Iyx3VhvZxA1on1tqqrJqpqcmJgYxS4laVEYdYh8p12Kov3d1eo7gaOHxi1vtWerL5+lLkkaoaUj3t9WYB1wUft73VD93UmuYXAT/YmqeiTJDcBfDt1MPxk4v6p2J/l+khOAW4CzgctGeSDSwejbF/zyuFvQQeiYP//mvG173kIkydXAm4Ejk+xg8JTVRcCWJOuBh4B3tOHXA6cBU8CTwDkALSw+BNzWxl1QVTM36/+AwRNgLwG+2D6SpBGatxCpqrP2seikWcYWcO4+trMZ2DxLfTvwuv3pUZK0f/zGuiSpmyEiSepmiEiSuhkikqRuhogkqZshIknqZohIkroZIpKkboaIJKmbISJJ6maISJK6GSKSpG6GiCSpmyEiSepmiEiSuhkikqRuhogkqZshIknqZohIkroZIpKkboaIJKmbISJJ6maISJK6GSKSpG6GiCSpmyEiSeo2lhBJ8kdJ7k5yV5Krk7w4ycoktySZSvLZJIe0sS9q81Nt+Yqh7Zzf6vclOWUcxyJJi9nIQyTJMuAPgcmqeh2wBDgT+AhwcVW9GngcWN9WWQ883uoXt3EkWd3Wey2wBvhEkiWjPBZJWuzGdTlrKfCSJEuBlwKPACcC17blVwKnt+m1bZ62/KQkafVrquqpqnoQmAKOG1H/kiTGECJVtRP4GPBtBuHxBHA78L2q2tOG7QCWtellwMNt3T1t/CuG67Os81OSbEiyPcn26enpA3tAkrSIjeNy1uEMziJWAq8Cfo7B5ah5U1WbqmqyqiYnJibmc1eStKiM43LWbwAPVtV0Vf0P8HngTcBh7fIWwHJgZ5veCRwN0Ja/HHhsuD7LOpKkERhHiHwbOCHJS9u9jZOAe4CbgDPamHXAdW16a5unLf9yVVWrn9me3loJrAJuHdExSJIY3OAeqaq6Jcm1wNeBPcA3gE3AvwDXJPlwq13eVrkc+HSSKWA3gyeyqKq7k2xhEEB7gHOr6scjPRhJWuRGHiIAVbUR2LhX+QFmebqqqn4EvH0f27kQuPCANyhJmhO/sS5J6maISJK6GSKSpG6GiCSpmyEiSepmiEiSuhkikqRuhogkqZshIknqZohIkroZIpKkboaIJKmbISJJ6janEEly41xqkqTF5VlfBZ/kxcBLgSPbz9qmLTqUffyeuSRp8fj/fk/k94H3Mfgt9Nv5SYh8H/j4PPYlSVoAnjVEquoS4JIk76mqy0bUkyRpgZjTLxtW1WVJfg1YMbxOVV01T31JkhaAOYVIkk8DvwTcAcz8jnkBhogkLWJz/Y31SWB1VdV8NiNJWljm+j2Ru4BfmM9GJEkLz1zPRI4E7klyK/DUTLGq3jovXUmSFoS5hshfzGcTkqSFaa5PZ/3bfDciSVp45vp01g8YPI0FcAjwQuCHVXXofDUmSTr4zfVM5GUz00kCrAVOmK+mJEkLw3N+i28N/CNwSu9OkxyW5Nok30pyb5JfTXJEkm1J7m9/D29jk+TSJFNJ7kxy7NB21rXx9ydZ19uPJKnPXC9nvW1o9gUMvjfyo/3Y7yXAl6rqjCSHMHjJ4weBG6vqoiTnAecBHwBOBVa1z/HAJ4HjkxwBbGy9FHB7kq1V9fh+9CVJeg7m+nTWbw1N7wH+i8ElrecsycuBXwfeCVBVTwNPJ1kLvLkNuxL4CoMQWQtc1b7oeHM7i3llG7utqna37W4D1gBX9/QlSXru5npP5JwDuM+VwDTwqSS/wuDtwO8FjqqqR9qYR4Gj2vQy4OGh9Xe02r7qPyPJBmADwDHHHHNgjkKSNOcfpVqe5AtJdrXP55Is79znUuBY4JNV9QbghwwuXT2jnXUcsFesVNWmqpqsqsmJiYkDtVlJWvTmemP9U8BWBr8r8irgn1qtxw5gR1Xd0uavZRAq32mXqWh/d7XlO4Gjh9Zf3mr7qkuSRmSuITJRVZ+qqj3tcwXQ9b/0VfUo8HCS17TSScA9DEJq5gmrdcB1bXorcHZ7SusE4Il22esG4OQkh7cnuU5uNUnSiMz1xvpjSX6Hn9y0Pgt4bD/2+x7gM+3JrAeAcxgE2pYk64GHgHe0sdcDpwFTwJNtLFW1O8mHgNvauAtmbrJLkkZjriHyLuAy4GIG9yr+g/Z0VY+quoPBo7l7O2mWsQWcu4/tbAY29/YhSdo/cw2RC4B1M9/BaN/R+BiDcJEkLVJzvSfy+uEv8bXLRm+Yn5YkSQvFXEPkBTOvIYFnzkTmehYjSXqemmsQ/BXwtST/0ObfDlw4Py1JkhaKuX5j/aok24ETW+ltVXXP/LUlSVoI5nxJqoWGwSFJesZzfhW8JEkzDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1G1sIZJkSZJvJPnnNr8yyS1JppJ8Nskhrf6iNj/Vlq8Y2sb5rX5fklPGcySStHiN80zkvcC9Q/MfAS6uqlcDjwPrW3098HirX9zGkWQ1cCbwWmAN8IkkS0bUuySJMYVIkuXAbwJ/3+YDnAhc24ZcCZzepte2edryk9r4tcA1VfVUVT0ITAHHjeYIJEkwvjORvwHeD/xvm38F8L2q2tPmdwDL2vQy4GGAtvyJNv6Z+izr/JQkG5JsT7J9enr6QB6HJC1qIw+RJG8BdlXV7aPaZ1VtqqrJqpqcmJgY1W4l6Xlv6Rj2+SbgrUlOA14MHApcAhyWZGk721gO7GzjdwJHAzuSLAVeDjw2VJ8xvI4kaQRGfiZSVedX1fKqWsHgxviXq+q3gZuAM9qwdcB1bXprm6ct/3JVVauf2Z7eWgmsAm4d0WFIkhjPmci+fAC4JsmHgW8Al7f65cCnk0wBuxkED1V1d5ItwD3AHuDcqvrx6NuWpMVrrCFSVV8BvtKmH2CWp6uq6kfA2/ex/oXAhfPXoSTp2fiNdUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktRt5CGS5OgkNyW5J8ndSd7b6kck2Zbk/vb38FZPkkuTTCW5M8mxQ9ta18bfn2TdqI9Fkha7cZyJ7AH+pKpWAycA5yZZDZwH3FhVq4Ab2zzAqcCq9tkAfBIGoQNsBI4HjgM2zgSPJGk0Rh4iVfVIVX29Tf8AuBdYBqwFrmzDrgROb9Nrgatq4GbgsCSvBE4BtlXV7qp6HNgGrBnhoUjSojfWeyJJVgBvAG4BjqqqR9qiR4Gj2vQy4OGh1Xa02r7qs+1nQ5LtSbZPT08fsP4labEbW4gk+Xngc8D7qur7w8uqqoA6UPuqqk1VNVlVkxMTEwdqs5K06I0lRJK8kEGAfKaqPt/K32mXqWh/d7X6TuDoodWXt9q+6pKkERnH01kBLgfuraq/Hlq0FZh5wmodcN1Q/ez2lNYJwBPtstcNwMlJDm831E9uNUnSiCwdwz7fBPwu8M0kd7TaB4GLgC1J1gMPAe9oy64HTgOmgCeBcwCqaneSDwG3tXEXVNXu0RyCJAnGECJV9e9A9rH4pFnGF3DuPra1Gdh84LqTJD0XfmNdktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdTNEJEndDBFJUjdDRJLUzRCRJHUzRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTNEJEkdVvwIZJkTZL7kkwlOW/c/UjSYrKgQyTJEuBvgVOB1cBZSVaPtytJWjwWdIgAxwFTVfVAVT0NXAOsHXNPkrRoLB13A/tpGfDw0PwO4Pi9ByXZAGxos/+d5L4R9LYYHAl8d9xNHAzysXXjbkE/y3+fMzbmQGzlF2crLvQQmZOq2gRsGncfzzdJtlfV5Lj7kGbjv8/RWOiXs3YCRw/NL281SdIILPQQuQ1YlWRlkkOAM4GtY+5JkhaNBX05q6r2JHk3cAOwBNhcVXePua3FxEuEOpj573MEUlXj7kGStEAt9MtZkqQxMkQkSd0MEXXxdTM6WCXZnGRXkrvG3ctiYIjoOfN1MzrIXQGsGXcTi4Uhoh6+bkYHrar6KrB73H0sFoaIesz2upllY+pF0hgZIpKkboaIevi6GUmAIaI+vm5GEmCIqENV7QFmXjdzL7DF183oYJHkauBrwGuS7Eiyftw9PZ/52hNJUjfPRCRJ3QwRSVI3Q0SS1M0QkSR1M0QkSd0MEUlSN0NEktTt/wDbb3VM9nSx3wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T8QV4WoIg-Jz"
      },
      "source": [
        "def subword_tokenize(train_corpus, vocab_size=2**14, max_length=50,tokenizer=None):\n",
        "  # Create the vocabulary using Subword tokenization\n",
        "  if(tokenizer==None):\n",
        "    tokenizer_corpus = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(train_corpus, target_vocab_size=2**14)\n",
        "  else:\n",
        "    tokenizer_corpus=tokenizer\n",
        "  # Get the final vocab size, adding the eos and sos tokens\n",
        "  vocab_size = tokenizer_corpus.vocab_size \n",
        "  \n",
        "  # Tokenize the corpus\n",
        "  sentences = [tokenizer_corpus.encode(sentence) for sentence in train_corpus]\n",
        "\n",
        "  #Pad the sentences\n",
        "  sentences = tf.keras.preprocessing.sequence.pad_sequences(sentences,value=0,padding='post',maxlen=50)\n",
        "  \n",
        "  return sentences, tokenizer_corpus, vocab_size"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TG68lIkzg-pR"
      },
      "source": [
        "tokenized_inputs,tokenizer,vocab_size=subword_tokenize(train_corpus=X_train)\n",
        "tokenized_test_inputs,_,vocab_size2=subword_tokenize(X_test,tokenizer=tokenizer)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q75qmBWchB0-"
      },
      "source": [
        "# Define a dataset \n",
        "dataset = tf.data.Dataset.from_tensor_slices((tokenized_inputs, y_train))\n",
        "dataset = dataset.shuffle(len(tokenized_inputs), reshuffle_each_iteration=True).batch(128, drop_remainder=True)\n",
        "\n",
        "dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_rO9uxMvhJKZ"
      },
      "source": [
        "def scaled_dot_product_attention(queries, keys, values, mask):\n",
        "    # Calculate the dot product, QK_transpose\n",
        "    product = tf.matmul(queries, keys, transpose_b=True)\n",
        "    # Get the scale factor\n",
        "    keys_dim = tf.cast(tf.shape(keys)[-1], tf.float32)\n",
        "    # Apply the scale factor to the dot product\n",
        "    scaled_product = product / tf.math.sqrt(keys_dim)\n",
        "    # Apply masking when it is requiered\n",
        "    if mask is not None:\n",
        "        scaled_product += (mask * -1e9)\n",
        "    # dot product with Values\n",
        "    attention = tf.matmul(tf.nn.softmax(scaled_product, axis=-1), values)\n",
        "    \n",
        "    return attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-zz6kUnBhKy2"
      },
      "source": [
        "class MultiHeadAttention(layers.Layer):\n",
        "    \n",
        "    def __init__(self, n_heads):\n",
        "        super(MultiHeadAttention, self).__init__()\n",
        "        self.n_heads = n_heads\n",
        "        \n",
        "    def build(self, input_shape):\n",
        "        self.d_model = input_shape[-1]\n",
        "        assert self.d_model % self.n_heads == 0\n",
        "        # Calculate the dimension of every head or projection\n",
        "        self.d_head = self.d_model // self.n_heads\n",
        "        # Set the weight matrices for Q, K and V\n",
        "        self.query_lin = layers.Dense(units=self.d_model)\n",
        "        self.key_lin = layers.Dense(units=self.d_model)\n",
        "        self.value_lin = layers.Dense(units=self.d_model)\n",
        "        # Set the weight matrix for the output of the multi-head attention W0\n",
        "        self.final_lin = layers.Dense(units=self.d_model)\n",
        "        \n",
        "    def split_proj(self, inputs, batch_size): # inputs: (batch_size, seq_length, d_model)\n",
        "        # Set the dimension of the projections\n",
        "        shape = (batch_size,\n",
        "                 -1,\n",
        "                 self.n_heads,\n",
        "                 self.d_head)\n",
        "        # Split the input vectors\n",
        "        splited_inputs = tf.reshape(inputs, shape=shape) # (batch_size, seq_length, nb_proj, d_proj)\n",
        "        return tf.transpose(splited_inputs, perm=[0, 2, 1, 3]) # (batch_size, nb_proj, seq_length, d_proj)\n",
        "    \n",
        "    def call(self, queries, keys, values, mask):\n",
        "        # Get the batch size\n",
        "        batch_size = tf.shape(queries)[0]\n",
        "        # Set the Query, Key and Value matrices\n",
        "        queries = self.query_lin(queries)\n",
        "        keys = self.key_lin(keys)\n",
        "        values = self.value_lin(values)\n",
        "        # Split Q, K y V between the heads or projections\n",
        "        queries = self.split_proj(queries, batch_size)\n",
        "        keys = self.split_proj(keys, batch_size)\n",
        "        values = self.split_proj(values, batch_size)\n",
        "        # Apply the scaled dot product\n",
        "        attention = scaled_dot_product_attention(queries, keys, values, mask)\n",
        "        # Get the attention scores\n",
        "        attention = tf.transpose(attention, perm=[0, 2, 1, 3])\n",
        "        # Concat the h heads or projections\n",
        "        concat_attention = tf.reshape(attention,\n",
        "                                      shape=(batch_size, -1, self.d_model))\n",
        "        # Apply W0 to get the output of the multi-head attention\n",
        "        outputs = self.final_lin(concat_attention)\n",
        "        \n",
        "        return outputs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OeauD4UhhLRQ"
      },
      "source": [
        "class PositionalEncoding(layers.Layer):\n",
        "\n",
        "    def __init__(self):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "    \n",
        "    def get_angles(self, pos, i, d_model): # pos: (seq_length, 1) i: (1, d_model)\n",
        "        angles = 1 / np.power(10000., (2*(i//2)) / np.float32(d_model))\n",
        "        return pos * angles # (seq_length, d_model)\n",
        "\n",
        "    def call(self, inputs):\n",
        "        # input shape batch_size, seq_length, d_model\n",
        "        seq_length = inputs.shape.as_list()[-2]\n",
        "        d_model = inputs.shape.as_list()[-1]\n",
        "        # Calculate the angles given the input\n",
        "        angles = self.get_angles(np.arange(seq_length)[:, np.newaxis],\n",
        "                                 np.arange(d_model)[np.newaxis, :],\n",
        "                                 d_model)\n",
        "        # Calculate the positional encodings\n",
        "        angles[:, 0::2] = np.sin(angles[:, 0::2])\n",
        "        angles[:, 1::2] = np.cos(angles[:, 1::2])\n",
        "        # Expand the encodings with a new dimension\n",
        "        pos_encoding = angles[np.newaxis, ...]\n",
        "        \n",
        "        return inputs + tf.cast(pos_encoding, tf.float32)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qMz_apUHhMzF"
      },
      "source": [
        "class EncoderLayer(layers.Layer):\n",
        "    \n",
        "    def __init__(self, FFN_units, n_heads, dropout_rate):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        # Hidden units of the feed forward component\n",
        "        self.FFN_units = FFN_units\n",
        "        # Set the number of projectios or heads\n",
        "        self.n_heads = n_heads\n",
        "        # Dropout rate\n",
        "        self.dropout_rate = dropout_rate\n",
        "    \n",
        "    def build(self, input_shape):\n",
        "        self.d_model = input_shape[-1]\n",
        "        # Build the multihead layer\n",
        "        self.multi_head_attention = MultiHeadAttention(self.n_heads)\n",
        "        self.dropout_1 = layers.Dropout(rate=self.dropout_rate)\n",
        "        # Layer Normalization\n",
        "        self.norm_1 = layers.LayerNormalization(epsilon=1e-6)\n",
        "        # Fully connected feed forward layer\n",
        "        self.ffn1_relu = layers.Dense(units=self.FFN_units, activation=\"relu\")\n",
        "        self.ffn2 = layers.Dense(units=self.d_model)\n",
        "        self.dropout_2 = layers.Dropout(rate=self.dropout_rate)\n",
        "        # Layer normalization\n",
        "        self.norm_2 = layers.LayerNormalization(epsilon=1e-6)\n",
        "        \n",
        "    def call(self, inputs, mask, training):\n",
        "        # Forward pass of the multi-head attention\n",
        "        attention = self.multi_head_attention(inputs,\n",
        "                                              inputs,\n",
        "                                              inputs,\n",
        "                                              mask)\n",
        "        attention = self.dropout_1(attention, training=training)\n",
        "        # Call to the residual connection and layer normalization\n",
        "        attention = self.norm_1(attention + inputs)\n",
        "        # Call to the FC layer\n",
        "        outputs = self.ffn1_relu(attention)\n",
        "        outputs = self.ffn2(outputs)\n",
        "        outputs = self.dropout_2(outputs, training=training)\n",
        "        # Call to residual connection and the layer normalization\n",
        "        outputs = self.norm_2(outputs + attention)\n",
        "        \n",
        "        return outputs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ANHZDbGchOxO"
      },
      "source": [
        "class Encoder(layers.Layer):\n",
        "    \n",
        "    def __init__(self,\n",
        "                 n_layers,\n",
        "                 FFN_units,\n",
        "                 n_heads,\n",
        "                 dropout_rate,\n",
        "                 vocab_size,\n",
        "                 d_model,\n",
        "                 name=\"encoder\"):\n",
        "        super(Encoder, self).__init__(name=name)\n",
        "        self.n_layers = n_layers\n",
        "        self.d_model = d_model\n",
        "        # The embedding layer\n",
        "        self.embedding = layers.Embedding(vocab_size, d_model)\n",
        "        # Positional encoding layer\n",
        "        self.pos_encoding = PositionalEncoding()\n",
        "        self.dropout = layers.Dropout(rate=dropout_rate)\n",
        "        # Stack of n layers of multi-head attention and FC\n",
        "        self.enc_layers = [EncoderLayer(FFN_units,\n",
        "                                        n_heads,\n",
        "                                        dropout_rate) \n",
        "                           for _ in range(n_layers)]\n",
        "        self.last_linear = layers.Dense(units=2, name=\"lin_ouput\")\n",
        "    \n",
        "    def call(self, inputs, mask, training):\n",
        "        # Get the embedding vectors\n",
        "        outputs = self.embedding(inputs)\n",
        "        # Scale the embeddings by sqrt of d_model\n",
        "        outputs *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "        # Positional encodding\n",
        "        outputs = self.pos_encoding(outputs)\n",
        "        outputs = self.dropout(outputs, training)\n",
        "        # Call the stacked layers\n",
        "        for i in range(self.n_layers):\n",
        "            outputs = self.enc_layers[i](outputs, mask, training)\n",
        "\n",
        "        logits=tf.math.reduce_mean(outputs,1)\n",
        "        #print(logits.shape)\n",
        "\n",
        "\n",
        "        logits=self.last_linear(logits)\n",
        "\n",
        "        return logits #outputs[:,0:1,:]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L0klrzE7hSe_"
      },
      "source": [
        "def create_padding_mask(seq): #seq: (batch_size, seq_length)\n",
        "# Create the mask for padding\n",
        "  mask = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
        "  return mask[:, tf.newaxis, tf.newaxis, :]\n",
        "def loss_function(target, pred):\n",
        "\n",
        "    \n",
        "    return tf.keras.losses.BinaryCrossentropy(from_logits=True)(target,pred)\n",
        "\n",
        "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "    \n",
        "    def __init__(self, d_model, warmup_steps=4000):\n",
        "        super(CustomSchedule, self).__init__()\n",
        "        \n",
        "        self.d_model = tf.cast(d_model, tf.float32)\n",
        "        self.warmup_steps = warmup_steps\n",
        "    \n",
        "    def __call__(self, step):\n",
        "        arg1 = tf.math.rsqrt(step)\n",
        "        arg2 = step * (self.warmup_steps**-1.5)\n",
        "        \n",
        "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qbe83tAAhTHL"
      },
      "source": [
        "def main_train(dataset, encoder, n_epochs, print_every=50):\n",
        "  ''' Train the transformer model for n_epochs using the data generator dataset'''\n",
        "  losses = []\n",
        "  accuracies = []\n",
        "  # In every epoch\n",
        "  for epoch in range(n_epochs):\n",
        "    print(\"Starting epoch {}\".format(epoch+1))\n",
        "    start = time.time()\n",
        "    # Reset the losss and accuracy calculations\n",
        "    train_loss.reset_states()\n",
        "    train_accuracy.reset_states()\n",
        "    # Get a batch of inputs and targets\n",
        "    for (batch, (inputs, targets)) in enumerate(dataset):\n",
        "        # Set the decoder inputs\n",
        "        #dec_inputs = targets[:, :-1]\n",
        "        # Set the target outputs, right shifted\n",
        "        targets = pd.get_dummies(targets).astype('float').values\n",
        "        with tf.GradientTape() as tape:\n",
        "            # Call the transformer and get the predicted output\n",
        "            predictions = encoder(inputs, create_padding_mask(inputs), True)\n",
        "            # Calculate the loss\n",
        "            loss = loss_function(targets, predictions)\n",
        "        # Update the weights and optimizer\n",
        "        gradients = tape.gradient(loss, encoder.trainable_variables)\n",
        "        optimizer.apply_gradients(zip(gradients, encoder.trainable_variables))\n",
        "        # Save and store the metrics\n",
        "        train_loss(loss)\n",
        "       \n",
        "        train_accuracy(targets, predictions)\n",
        "        \n",
        "        if batch % print_every == 0:\n",
        "            losses.append(train_loss.result())\n",
        "            accuracies.append(train_accuracy.result())\n",
        "            print(\"Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}\".format(\n",
        "                epoch+1, batch, train_loss.result(), train_accuracy.result()))\n",
        "            \n",
        "    # Checkpoint the model on every epoch        \n",
        "    #ckpt_save_path = ckpt_manager.save()\n",
        "    #print(\"Saving checkpoint for epoch {} in {}\".format(epoch+1,ckpt_save_path))\n",
        "    #print(\"Time for 1 epoch: {} secs\\n\".format(time.time() - start))\n",
        "\n",
        "  return losses, accuracies"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zpzNg_vVhUkU"
      },
      "source": [
        "tf.keras.backend.clear_session()\n",
        "# Create the Encoder model\n",
        "encoder = Encoder(6,512,8,0.1,vocab_size,256)\n",
        "\n",
        "\n",
        "# Define a metric to store the mean loss of every epoch\n",
        "train_loss = tf.keras.metrics.Mean(name=\"train_loss\")\n",
        "# Define a matric to save the accuracy in every epoch\n",
        "train_accuracy = tf.keras.metrics.BinaryAccuracy(name=\"train_accuracy\")\n",
        "# Create the scheduler for learning rate decay\n",
        "leaning_rate = CustomSchedule(256)\n",
        "# Create the Adam optimizer\n",
        "optimizer = tf.keras.optimizers.Adam(leaning_rate,\n",
        "                                     beta_1=0.9,\n",
        "                                     beta_2=0.98,\n",
        "                                     epsilon=1e-9)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SXP_yE2XhWHB",
        "outputId": "6869aee5-8fe5-4592-e1f5-c875f1daaaa6"
      },
      "source": [
        "losses, accuracies = main_train(dataset, encoder, 5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Starting epoch 1\n",
            "Epoch 1 Batch 0 Loss 0.2113 Accuracy 0.9023\n",
            "Epoch 1 Batch 50 Loss 0.2162 Accuracy 0.9082\n",
            "Epoch 1 Batch 100 Loss 0.2226 Accuracy 0.9069\n",
            "Epoch 1 Batch 150 Loss 0.2366 Accuracy 0.9001\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "437OfKxAh22Y",
        "outputId": "ab0e9e0a-30ff-415d-e5a5-5534f8c1cfbc"
      },
      "source": [
        "def predict(encoder,tokenized_sentences):\n",
        "  logits=encoder(tokenized_sentences,create_padding_mask(tokenized_sentences),False)\n",
        "  predictions=np.argmax(tf.keras.layers.Softmax()(logits),axis=1)\n",
        "  return predictions\n",
        "test_accuracy=sum((predict(encoder,tokenized_test_inputs)==y_test))\n",
        "test_accuracy/=len(y_test)\n",
        "print(test_accuracy)\n",
        "from sklearn.metrics import classification_report\n",
        "print(classification_report(predict(encoder,tokenized_test_inputs),y_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.8209831254585473\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.89      0.81      0.85      3355\n",
            "           1       0.73      0.84      0.78      2097\n",
            "\n",
            "    accuracy                           0.82      5452\n",
            "   macro avg       0.81      0.82      0.82      5452\n",
            "weighted avg       0.83      0.82      0.82      5452\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "05xL9XMEU_w2"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "llGjigt7iqMf",
        "outputId": "cf49ca0e-4726-4c08-d14a-b3e83f5ff1ad"
      },
      "source": [
        "class Encoder(layers.Layer):\n",
        "    \n",
        "    def __init__(self,\n",
        "                 n_layers,\n",
        "                 FFN_units,\n",
        "                 n_heads,\n",
        "                 dropout_rate,\n",
        "                 vocab_size,\n",
        "                 d_model,\n",
        "                 name=\"encoder_lstm\"):\n",
        "        super(Encoder, self).__init__(name=name)\n",
        "        self.n_layers = n_layers\n",
        "        self.d_model = d_model\n",
        "        # The embedding layer\n",
        "        self.embedding = layers.Embedding(vocab_size, d_model)\n",
        "        # Positional encoding layer\n",
        "        self.pos_encoding = PositionalEncoding()\n",
        "        self.dropout = layers.Dropout(rate=dropout_rate)\n",
        "        # Stack of n layers of multi-head attention and FC\n",
        "        self.enc_layers = [EncoderLayer(FFN_units,\n",
        "                                        n_heads,\n",
        "                                        dropout_rate) \n",
        "                           for _ in range(n_layers)]\n",
        "        self.lstm=layers.LSTM(128)\n",
        "        self.last_linear = layers.Dense(units=2, name=\"lin_ouput\")\n",
        "    \n",
        "    def call(self, inputs, mask, training):\n",
        "        # Get the embedding vectors\n",
        "        outputs = self.embedding(inputs)\n",
        "        # Scale the embeddings by sqrt of d_model\n",
        "        outputs *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "        # Positional encodding\n",
        "        outputs = self.pos_encoding(outputs)\n",
        "        outputs = self.dropout(outputs, training)\n",
        "        # Call the stacked layers\n",
        "        for i in range(self.n_layers):\n",
        "            outputs = self.enc_layers[i](outputs, mask, training)\n",
        "\n",
        "        #logits=tf.math.reduce_mean(outputs,1)\n",
        "        #print(logits.shape)\n",
        "        logits=self.lstm(outputs)\n",
        "\n",
        "\n",
        "        logits=self.last_linear(logits)\n",
        "\n",
        "        return logits #outputs[:,0:1,:]\n",
        "\n",
        "\n",
        "tf.keras.backend.clear_session()\n",
        "# Create the Encoder model\n",
        "encoder_lstm = Encoder(6,512,8,0.1,vocab_size,256)\n",
        "\n",
        "\n",
        "# Define a metric to store the mean loss of every epoch\n",
        "train_loss = tf.keras.metrics.Mean(name=\"train_loss\")\n",
        "# Define a matric to save the accuracy in every epoch\n",
        "train_accuracy = tf.keras.metrics.BinaryAccuracy(name=\"train_accuracy\")\n",
        "# Create the scheduler for learning rate decay\n",
        "leaning_rate = CustomSchedule(256)\n",
        "# Create the Adam optimizer\n",
        "optimizer = tf.keras.optimizers.Adam(leaning_rate,\n",
        "                                     beta_1=0.9,\n",
        "                                     beta_2=0.98,\n",
        "                                     epsilon=1e-9)\n",
        "\n",
        "losses, accuracies = main_train(dataset, encoder_lstm, 5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Starting epoch 1\n",
            "Epoch 1 Batch 0 Loss 0.7143 Accuracy 0.5352\n",
            "Epoch 1 Batch 50 Loss 0.7004 Accuracy 0.5061\n",
            "Epoch 1 Batch 100 Loss 0.6891 Accuracy 0.5120\n",
            "Epoch 1 Batch 150 Loss 0.6787 Accuracy 0.5324\n",
            "Starting epoch 2\n",
            "Epoch 2 Batch 0 Loss 0.6344 Accuracy 0.6328\n",
            "Epoch 2 Batch 50 Loss 0.6047 Accuracy 0.6505\n",
            "Epoch 2 Batch 100 Loss 0.5814 Accuracy 0.6748\n",
            "Epoch 2 Batch 150 Loss 0.5666 Accuracy 0.6892\n",
            "Starting epoch 3\n",
            "Epoch 3 Batch 0 Loss 0.4442 Accuracy 0.7930\n",
            "Epoch 3 Batch 50 Loss 0.4346 Accuracy 0.7864\n",
            "Epoch 3 Batch 100 Loss 0.4285 Accuracy 0.7910\n",
            "Epoch 3 Batch 150 Loss 0.4217 Accuracy 0.7956\n",
            "Starting epoch 4\n",
            "Epoch 4 Batch 0 Loss 0.2713 Accuracy 0.8867\n",
            "Epoch 4 Batch 50 Loss 0.2964 Accuracy 0.8670\n",
            "Epoch 4 Batch 100 Loss 0.3145 Accuracy 0.8581\n",
            "Epoch 4 Batch 150 Loss 0.3152 Accuracy 0.8581\n",
            "Starting epoch 5\n",
            "Epoch 5 Batch 0 Loss 0.2847 Accuracy 0.8789\n",
            "Epoch 5 Batch 50 Loss 0.2055 Accuracy 0.9154\n",
            "Epoch 5 Batch 100 Loss 0.2164 Accuracy 0.9115\n",
            "Epoch 5 Batch 150 Loss 0.2303 Accuracy 0.9048\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BHpkyrDOizPA",
        "outputId": "b3456349-9121-4f01-ca32-a485613f1c93"
      },
      "source": [
        "from sklearn.metrics import classification_report\n",
        "print(classification_report(predict(encoder_lstm,tokenized_test_inputs),y_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.79      0.86      0.83      2791\n",
            "           1       0.84      0.76      0.80      2661\n",
            "\n",
            "    accuracy                           0.81      5452\n",
            "   macro avg       0.82      0.81      0.81      5452\n",
            "weighted avg       0.82      0.81      0.81      5452\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u8ZM4cHPi1FN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f890ff4e-0b97-4160-8b38-aa5552578013"
      },
      "source": [
        "class Encoder(layers.Layer):\n",
        "    \n",
        "    def __init__(self,\n",
        "                 n_layers,\n",
        "                 FFN_units,\n",
        "                 n_heads,\n",
        "                 dropout_rate,\n",
        "                 vocab_size,\n",
        "                 d_model,\n",
        "                 name=\"encoder_cnn\"):\n",
        "        super(Encoder, self).__init__(name=name)\n",
        "        self.n_layers = n_layers\n",
        "        self.d_model = d_model\n",
        "        # The embedding layer\n",
        "        self.embedding = layers.Embedding(vocab_size, d_model)\n",
        "        # Positional encoding layer\n",
        "        self.pos_encoding = PositionalEncoding()\n",
        "        self.dropout = layers.Dropout(rate=dropout_rate)\n",
        "        # Stack of n layers of multi-head attention and FC\n",
        "        self.enc_layers = [EncoderLayer(FFN_units,\n",
        "                                        n_heads,\n",
        "                                        dropout_rate) \n",
        "                           for _ in range(n_layers)]\n",
        "        self.cnn=layers.Conv2D(1, (3,3), activation='relu', padding='same', input_shape=(50,256,1))\n",
        "        self.flatten=layers.Flatten()\n",
        "        self.last_linear = layers.Dense(units=2, name=\"lin_ouput\")\n",
        "    \n",
        "    def call(self, inputs, mask, training):\n",
        "        # Get the embedding vectors\n",
        "        outputs = self.embedding(inputs)\n",
        "        # Scale the embeddings by sqrt of d_model\n",
        "        outputs *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "        # Positional encodding\n",
        "        outputs = self.pos_encoding(outputs)\n",
        "        outputs = self.dropout(outputs, training)\n",
        "        # Call the stacked layers\n",
        "        for i in range(self.n_layers):\n",
        "            outputs = self.enc_layers[i](outputs, mask, training)\n",
        "\n",
        "        #logits=tf.math.reduce_mean(outputs,1)\n",
        "        #print(logits.shape)\n",
        "        logits=K.expand_dims(outputs,axis=-1)\n",
        "        logits=self.cnn(logits)\n",
        "        logits=self.flatten(logits)\n",
        "\n",
        "\n",
        "        logits=self.last_linear(logits)\n",
        "\n",
        "        return logits #outputs[:,0:1,:]\n",
        "tf.keras.backend.clear_session()\n",
        "# Create the Encoder model\n",
        "encoder_cnn = Encoder(6,512,8,0.1,vocab_size,256)\n",
        "\n",
        "\n",
        "# Define a metric to store the mean loss of every epoch\n",
        "train_loss = tf.keras.metrics.Mean(name=\"train_loss\")\n",
        "# Define a matric to save the accuracy in every epoch\n",
        "train_accuracy = tf.keras.metrics.BinaryAccuracy(name=\"train_accuracy\")\n",
        "# Create the scheduler for learning rate decay\n",
        "leaning_rate = CustomSchedule(256)\n",
        "# Create the Adam optimizer\n",
        "optimizer = tf.keras.optimizers.Adam(leaning_rate,\n",
        "                                     beta_1=0.9,\n",
        "                                     beta_2=0.98,\n",
        "                                     epsilon=1e-9)\n",
        "losses, accuracies = main_train(dataset, encoder_cnn, 4)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Starting epoch 1\n",
            "Epoch 1 Batch 0 Loss 0.7173 Accuracy 0.5391\n",
            "Epoch 1 Batch 50 Loss 0.7090 Accuracy 0.5152\n",
            "Epoch 1 Batch 100 Loss 0.7032 Accuracy 0.5186\n",
            "Epoch 1 Batch 150 Loss 0.6961 Accuracy 0.5265\n",
            "Starting epoch 2\n",
            "Epoch 2 Batch 0 Loss 0.7001 Accuracy 0.5430\n",
            "Epoch 2 Batch 50 Loss 0.6269 Accuracy 0.6208\n",
            "Epoch 2 Batch 100 Loss 0.6051 Accuracy 0.6480\n",
            "Epoch 2 Batch 150 Loss 0.5839 Accuracy 0.6701\n",
            "Starting epoch 3\n",
            "Epoch 3 Batch 0 Loss 0.5502 Accuracy 0.7109\n",
            "Epoch 3 Batch 50 Loss 0.4502 Accuracy 0.7803\n",
            "Epoch 3 Batch 100 Loss 0.4439 Accuracy 0.7836\n",
            "Epoch 3 Batch 150 Loss 0.4417 Accuracy 0.7830\n",
            "Starting epoch 4\n",
            "Epoch 4 Batch 0 Loss 0.2956 Accuracy 0.8594\n",
            "Epoch 4 Batch 50 Loss 0.3158 Accuracy 0.8582\n",
            "Epoch 4 Batch 100 Loss 0.3270 Accuracy 0.8554\n",
            "Epoch 4 Batch 150 Loss 0.3245 Accuracy 0.8566\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NlfAL00cXDaH",
        "outputId": "5acfc0e2-37a4-40ab-8899-9140cace4067"
      },
      "source": [
        "losses, accuracies = main_train(dataset, encoder_cnn, 1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Starting epoch 1\n",
            "Epoch 1 Batch 0 Loss 0.2595 Accuracy 0.8867\n",
            "Epoch 1 Batch 50 Loss 0.2479 Accuracy 0.8938\n",
            "Epoch 1 Batch 100 Loss 0.2385 Accuracy 0.8969\n",
            "Epoch 1 Batch 150 Loss 0.2489 Accuracy 0.8948\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Tnz7Liti81f",
        "outputId": "06ca6804-e2a2-4c76-da02-6e21ac39a844"
      },
      "source": [
        "from sklearn.metrics import classification_report\n",
        "print(classification_report(predict(encoder_cnn,tokenized_test_inputs),y_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.85      0.83      0.84      3117\n",
            "           1       0.78      0.80      0.79      2335\n",
            "\n",
            "    accuracy                           0.82      5452\n",
            "   macro avg       0.82      0.82      0.82      5452\n",
            "weighted avg       0.82      0.82      0.82      5452\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mBcVobkEibPd"
      },
      "source": [
        "examples=['Woman wins award for safe driving','I work forty hours a week for me to be this poor.','I would kill for a Nobel Peace Prize.','Depression is merely anger without enthusiasm.','Two wrongs don’t make a right take your parents as an example .','If I wanted to kill myself I’d climb your ego and jump to your IQ.','Always remember that you are absolutely unique just like everyone else .','Congratulations, If you press the elevator button three times it goes into hurry mode – really...','Hello my name is Raj and I study computer science!','Please get me some fruits from the store .','Want to hang out today ?','Presidency is probably the toughest job around','We should go out and play football','Discounted items end up being the most expensive']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VH02cwjhYysh",
        "outputId": "28a4da01-7cfc-4fae-8569-7ff722de46f1"
      },
      "source": [
        "example=\"Trump's tenure oversaw a massive economic decline , definitely going to vote for him again .\"\n",
        "sentence=tokenizer.encode(preprocess(example))\n",
        "sentence=tf.keras.preprocessing.sequence.pad_sequences([sentence],value=0,padding='post',maxlen=50)\n",
        "print(example)\n",
        "print(predict(encoder,sentence))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Trump's tenure oversaw a massive economic decline , definitely going to vote for him again .\n",
            "[1]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_18yuLw2Za4T",
        "outputId": "d11ff81c-603b-4ca8-c51b-9b9f2b1c06e8"
      },
      "source": [
        "example=\"Today is such a beautiful day , the weather is perfect to stay inside .\"\n",
        "sentence=tokenizer.encode(preprocess(example))\n",
        "sentence=tf.keras.preprocessing.sequence.pad_sequences([sentence],value=0,padding='post',maxlen=50)\n",
        "print(example)\n",
        "print(predict(encoder,sentence))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Today is such a beautiful day , the weather is perfect to stay inside .\n",
            "[1]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nU7Ho0bZigy-",
        "outputId": "4dbf3461-e071-4e47-f01d-c96222d50aa0"
      },
      "source": [
        "#example='Woman wins award for safe driving'\n",
        "for example in examples:\n",
        "  sentence=tokenizer.encode(preprocess(example))\n",
        "  sentence=tf.keras.preprocessing.sequence.pad_sequences([sentence],value=0,padding='post',maxlen=50)\n",
        "  print(example)\n",
        "  print(predict(encoder,sentence))\n",
        "  print(predict(encoder_lstm,sentence))\n",
        "  print(predict(encoder_cnn,sentence))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Woman wins award for safe driving\n",
            "[0]\n",
            "[0]\n",
            "[0]\n",
            "I work forty hours a week for me to be this poor.\n",
            "[1]\n",
            "[1]\n",
            "[1]\n",
            "I would kill for a Nobel Peace Prize.\n",
            "[0]\n",
            "[0]\n",
            "[0]\n",
            "Depression is merely anger without enthusiasm.\n",
            "[1]\n",
            "[1]\n",
            "[0]\n",
            "Two wrongs don’t make a right take your parents as an example .\n",
            "[0]\n",
            "[0]\n",
            "[0]\n",
            "If I wanted to kill myself I’d climb your ego and jump to your IQ.\n",
            "[1]\n",
            "[0]\n",
            "[0]\n",
            "Always remember that you are absolutely unique just like everyone else .\n",
            "[0]\n",
            "[1]\n",
            "[0]\n",
            "Congratulations, If you press the elevator button three times it goes into hurry mode – really...\n",
            "[1]\n",
            "[1]\n",
            "[1]\n",
            "Hello my name is Raj and I study computer science!\n",
            "[0]\n",
            "[0]\n",
            "[1]\n",
            "Please get me some fruits from the store .\n",
            "[1]\n",
            "[1]\n",
            "[1]\n",
            "Want to hang out today ?\n",
            "[0]\n",
            "[0]\n",
            "[0]\n",
            "Presidency is probably the toughest job around\n",
            "[1]\n",
            "[0]\n",
            "[1]\n",
            "We should go out and play football\n",
            "[0]\n",
            "[1]\n",
            "[0]\n",
            "Discounted items end up being the most expensive\n",
            "[1]\n",
            "[1]\n",
            "[1]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H_1Q5NzJmzlC"
      },
      "source": [
        "import os\n",
        "checkpoint_folder = \"ckpt/\"\n",
        "checkpoint_path = os.path.abspath(os.path.join('', checkpoint_folder))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "g5Is0T27op16",
        "outputId": "dc8f71f3-2dea-4f25-b69a-6431d66bf131"
      },
      "source": [
        "checkpoint_path"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/ckpt'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TzBgZdwspELU"
      },
      "source": [
        "ckpt = tf.train.Checkpoint(encoder=encoder,optimizer=optimizer,)\n",
        "\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
        "\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "    ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "    print(\"Last checkpoint restored.\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zi9jpLzhpINS"
      },
      "source": [
        "ckpt_save_path = ckpt_manager.save()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KxDWJtwjpT8I",
        "outputId": "065169a5-9348-4aca-d063-753574fd35bd"
      },
      "source": [
        "if ckpt_manager.latest_checkpoint:\n",
        "    ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "    print(\"Last checkpoint restored.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Last checkpoint restored.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0dOh2YbEamx0",
        "outputId": "21a538ba-0662-49fc-f0c2-7ab48defe47f"
      },
      "source": [
        "print(roc_auc_score(y_test, predict_proba2(encoder,tokenized_test_inputs)[:, 1]))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9047049028952252\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0I08LPJwaybQ"
      },
      "source": [
        "encoder=Encoder()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iZQ3bnyGqNkU"
      },
      "source": [
        "def predict_proba2(encoder,examples):\n",
        "    #sentences=[tokenizer.encode(example) for example in examples]\n",
        "    #sentences=tf.keras.preprocessing.sequence.pad_sequences(sentences,value=0,padding='post',maxlen=50)\n",
        "    logits=encoder(examples,create_padding_mask(examples),False)\n",
        "    return layers.Softmax()(logits).numpy()\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "89M760EAqGiu",
        "outputId": "18aed63f-4e7c-406c-971b-bc02ada07c6b"
      },
      "source": [
        "from sklearn.metrics import roc_auc_score\n",
        "print(roc_auc_score(y_test, predict_proba2(encoder,tokenized_test_inputs)[:, 1]))\n",
        "print(roc_auc_score(y_test, predict_proba2(encoder_lstm,tokenized_test_inputs)[:, 1]))\n",
        "print(roc_auc_score(y_test, predict_proba2(encoder_cnn,tokenized_test_inputs)[:, 1]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9047049028952252\n",
            "0.9028470707734592\n",
            "0.902234841493594\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EKyntIjrtJLO",
        "outputId": "568c3bf4-cfe5-4e09-ef7a-098ee1409f6a"
      },
      "source": [
        "print(roc_auc_score(y_test, predict_proba2(encoder_cnn,tokenized_test_inputs)[:, 1]))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.5286365388734204\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}