Text_summarization_using_transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## Import the dependencies"
      ],
      "metadata": {
        "id": "sNCDmCizqq6l"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install contractions"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "ZjsVoVCNexrG",
        "outputId": "303ebef7-2b06-4d13-e546-3c47a7ed1f02"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting contractions\n",
            "  Downloading contractions-0.1.73-py2.py3-none-any.whl (8.7 kB)\n",
            "Collecting textsearch>=0.0.21 (from contractions)\n",
            "  Downloading textsearch-0.0.24-py2.py3-none-any.whl (7.6 kB)\n",
            "Collecting anyascii (from textsearch>=0.0.21->contractions)\n",
            "  Downloading anyascii-0.3.2-py3-none-any.whl (289 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m289.9/289.9 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyahocorasick (from textsearch>=0.0.21->contractions)\n",
            "  Downloading pyahocorasick-2.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.7/110.7 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pyahocorasick, anyascii, textsearch, contractions\n",
            "Successfully installed anyascii-0.3.2 contractions-0.1.73 pyahocorasick-2.1.0 textsearch-0.0.24\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('all')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "jWmBlFJQh8K0",
        "outputId": "ee6dbfa4-559e-4535-96ce-6f3cf7cb96f8"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading collection 'all'\n",
            "[nltk_data]    | \n",
            "[nltk_data]    | Downloading package abc to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/abc.zip.\n",
            "[nltk_data]    | Downloading package alpino to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/alpino.zip.\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger_eng to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping\n",
            "[nltk_data]    |       taggers/averaged_perceptron_tagger_eng.zip.\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger_ru to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping\n",
            "[nltk_data]    |       taggers/averaged_perceptron_tagger_ru.zip.\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger_rus to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping\n",
            "[nltk_data]    |       taggers/averaged_perceptron_tagger_rus.zip.\n",
            "[nltk_data]    | Downloading package basque_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/basque_grammars.zip.\n",
            "[nltk_data]    | Downloading package bcp47 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package biocreative_ppi to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/biocreative_ppi.zip.\n",
            "[nltk_data]    | Downloading package bllip_wsj_no_aux to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/bllip_wsj_no_aux.zip.\n",
            "[nltk_data]    | Downloading package book_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/book_grammars.zip.\n",
            "[nltk_data]    | Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/brown.zip.\n",
            "[nltk_data]    | Downloading package brown_tei to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/brown_tei.zip.\n",
            "[nltk_data]    | Downloading package cess_cat to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cess_cat.zip.\n",
            "[nltk_data]    | Downloading package cess_esp to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cess_esp.zip.\n",
            "[nltk_data]    | Downloading package chat80 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/chat80.zip.\n",
            "[nltk_data]    | Downloading package city_database to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/city_database.zip.\n",
            "[nltk_data]    | Downloading package cmudict to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cmudict.zip.\n",
            "[nltk_data]    | Downloading package comparative_sentences to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/comparative_sentences.zip.\n",
            "[nltk_data]    | Downloading package comtrans to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package conll2000 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/conll2000.zip.\n",
            "[nltk_data]    | Downloading package conll2002 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/conll2002.zip.\n",
            "[nltk_data]    | Downloading package conll2007 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package crubadan to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/crubadan.zip.\n",
            "[nltk_data]    | Downloading package dependency_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/dependency_treebank.zip.\n",
            "[nltk_data]    | Downloading package dolch to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/dolch.zip.\n",
            "[nltk_data]    | Downloading package europarl_raw to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/europarl_raw.zip.\n",
            "[nltk_data]    | Downloading package extended_omw to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package floresta to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/floresta.zip.\n",
            "[nltk_data]    | Downloading package framenet_v15 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/framenet_v15.zip.\n",
            "[nltk_data]    | Downloading package framenet_v17 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/framenet_v17.zip.\n",
            "[nltk_data]    | Downloading package gazetteers to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/gazetteers.zip.\n",
            "[nltk_data]    | Downloading package genesis to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/genesis.zip.\n",
            "[nltk_data]    | Downloading package gutenberg to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/gutenberg.zip.\n",
            "[nltk_data]    | Downloading package ieer to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ieer.zip.\n",
            "[nltk_data]    | Downloading package inaugural to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/inaugural.zip.\n",
            "[nltk_data]    | Downloading package indian to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/indian.zip.\n",
            "[nltk_data]    | Downloading package jeita to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package kimmo to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/kimmo.zip.\n",
            "[nltk_data]    | Downloading package knbc to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package large_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/large_grammars.zip.\n",
            "[nltk_data]    | Downloading package lin_thesaurus to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/lin_thesaurus.zip.\n",
            "[nltk_data]    | Downloading package mac_morpho to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/mac_morpho.zip.\n",
            "[nltk_data]    | Downloading package machado to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package masc_tagged to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping chunkers/maxent_ne_chunker.zip.\n",
            "[nltk_data]    | Downloading package maxent_treebank_pos_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/maxent_treebank_pos_tagger.zip.\n",
            "[nltk_data]    | Downloading package moses_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/moses_sample.zip.\n",
            "[nltk_data]    | Downloading package movie_reviews to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/movie_reviews.zip.\n",
            "[nltk_data]    | Downloading package mte_teip5 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/mte_teip5.zip.\n",
            "[nltk_data]    | Downloading package mwa_ppdb to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping misc/mwa_ppdb.zip.\n",
            "[nltk_data]    | Downloading package names to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/names.zip.\n",
            "[nltk_data]    | Downloading package nombank.1.0 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package nonbreaking_prefixes to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/nonbreaking_prefixes.zip.\n",
            "[nltk_data]    | Downloading package nps_chat to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/nps_chat.zip.\n",
            "[nltk_data]    | Downloading package omw to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package omw-1.4 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package opinion_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/opinion_lexicon.zip.\n",
            "[nltk_data]    | Downloading package panlex_swadesh to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package paradigms to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/paradigms.zip.\n",
            "[nltk_data]    | Downloading package pe08 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pe08.zip.\n",
            "[nltk_data]    | Downloading package perluniprops to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping misc/perluniprops.zip.\n",
            "[nltk_data]    | Downloading package pil to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pil.zip.\n",
            "[nltk_data]    | Downloading package pl196x to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pl196x.zip.\n",
            "[nltk_data]    | Downloading package porter_test to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping stemmers/porter_test.zip.\n",
            "[nltk_data]    | Downloading package ppattach to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ppattach.zip.\n",
            "[nltk_data]    | Downloading package problem_reports to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/problem_reports.zip.\n",
            "[nltk_data]    | Downloading package product_reviews_1 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/product_reviews_1.zip.\n",
            "[nltk_data]    | Downloading package product_reviews_2 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/product_reviews_2.zip.\n",
            "[nltk_data]    | Downloading package propbank to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package pros_cons to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pros_cons.zip.\n",
            "[nltk_data]    | Downloading package ptb to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ptb.zip.\n",
            "[nltk_data]    | Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data]    | Downloading package qc to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/qc.zip.\n",
            "[nltk_data]    | Downloading package reuters to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package rslp to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping stemmers/rslp.zip.\n",
            "[nltk_data]    | Downloading package rte to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/rte.zip.\n",
            "[nltk_data]    | Downloading package sample_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/sample_grammars.zip.\n",
            "[nltk_data]    | Downloading package semcor to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package senseval to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/senseval.zip.\n",
            "[nltk_data]    | Downloading package sentence_polarity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sentence_polarity.zip.\n",
            "[nltk_data]    | Downloading package sentiwordnet to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sentiwordnet.zip.\n",
            "[nltk_data]    | Downloading package shakespeare to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/shakespeare.zip.\n",
            "[nltk_data]    | Downloading package sinica_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sinica_treebank.zip.\n",
            "[nltk_data]    | Downloading package smultron to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/smultron.zip.\n",
            "[nltk_data]    | Downloading package snowball_data to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package spanish_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/spanish_grammars.zip.\n",
            "[nltk_data]    | Downloading package state_union to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/state_union.zip.\n",
            "[nltk_data]    | Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data]    | Downloading package subjectivity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/subjectivity.zip.\n",
            "[nltk_data]    | Downloading package swadesh to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/swadesh.zip.\n",
            "[nltk_data]    | Downloading package switchboard to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/switchboard.zip.\n",
            "[nltk_data]    | Downloading package tagsets to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping help/tagsets.zip.\n",
            "[nltk_data]    | Downloading package tagsets_json to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping help/tagsets_json.zip.\n",
            "[nltk_data]    | Downloading package timit to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/timit.zip.\n",
            "[nltk_data]    | Downloading package toolbox to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/toolbox.zip.\n",
            "[nltk_data]    | Downloading package treebank to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/treebank.zip.\n",
            "[nltk_data]    | Downloading package twitter_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/twitter_samples.zip.\n",
            "[nltk_data]    | Downloading package udhr to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/udhr.zip.\n",
            "[nltk_data]    | Downloading package udhr2 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/udhr2.zip.\n",
            "[nltk_data]    | Downloading package unicode_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/unicode_samples.zip.\n",
            "[nltk_data]    | Downloading package universal_tagset to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/universal_tagset.zip.\n",
            "[nltk_data]    | Downloading package universal_treebanks_v20 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package vader_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package verbnet to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/verbnet.zip.\n",
            "[nltk_data]    | Downloading package verbnet3 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/verbnet3.zip.\n",
            "[nltk_data]    | Downloading package webtext to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/webtext.zip.\n",
            "[nltk_data]    | Downloading package wmt15_eval to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/wmt15_eval.zip.\n",
            "[nltk_data]    | Downloading package word2vec_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/word2vec_sample.zip.\n",
            "[nltk_data]    | Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package wordnet2021 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package wordnet2022 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/wordnet2022.zip.\n",
            "[nltk_data]    | Downloading package wordnet31 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package wordnet_ic to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/wordnet_ic.zip.\n",
            "[nltk_data]    | Downloading package words to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/words.zip.\n",
            "[nltk_data]    | Downloading package ycoe to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ycoe.zip.\n",
            "[nltk_data]    | \n",
            "[nltk_data]  Done downloading collection all\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "fb3b7110"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "from spacy.tokenizer import Tokenizer\n",
        "from sklearn.model_selection import train_test_split\n",
        "import pandas as pd\n",
        "import spacy\n",
        "import re\n",
        "import numpy as np\n",
        "import string\n",
        "import matplotlib.pyplot as plt\n",
        "import csv\n",
        "import contractions\n",
        "from nltk.corpus import stopwords\n",
        "from spacy.tokenizer import Tokenizer\n",
        "import random\n",
        "from tqdm import tqdm\n",
        "import math\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pd.set_option('display.max_colwidth', None)"
      ],
      "metadata": {
        "id": "Dj1O2ZJ0gDYQ"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(\"Using\",device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hq5GNaAkbngF",
        "outputId": "eccb537c-234b-4da0-e577-9e0083d93b46"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using cuda\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load the data"
      ],
      "metadata": {
        "id": "NjG29m8nqzit"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df1 = pd.read_csv(\"/content/news_summary.csv\",encoding='latin-1',usecols=['headlines','text'])\n",
        "df2 = pd.read_csv(\"/content/news_summary_more.csv\",encoding='iso-8859-1')"
      ],
      "metadata": {
        "id": "0vooAtvH8aRT"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df1.head(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 379
        },
        "id": "Cc7VXR3W-nHx",
        "outputId": "426a3c7f-e75d-4c98-b42e-e4fa471b1245"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                      headlines  \\\n",
              "0  Daman & Diu revokes mandatory Rakshabandhan in offices order   \n",
              "1  Malaika slams user who trolled her for 'divorcing rich man'    \n",
              "2          'Virgin' now corrected to 'Unmarried' in IGIMS' form   \n",
              "3      Aaj aapne pakad liya: LeT man Dujana before being killed   \n",
              "4  Hotel staff to get training to spot signs of sex trafficking   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                             text  \n",
              "0                                          The Administration of Union Territory Daman and Diu has revoked its order that made it compulsory for women to tie rakhis to their male colleagues on the occasion of Rakshabandhan on August 7. The administration was forced to withdraw the decision within 24 hours of issuing the circular after it received flak from employees and was slammed on social media.  \n",
              "1                                       Malaika Arora slammed an Instagram user who trolled her for \"divorcing a rich man\" and \"having fun with the alimony\". \"Her life now is all about wearing short clothes, going to gym or salon, enjoying vacation[s],\" the user commented. Malaika responded, \"You certainly got to get your damn facts right before spewing sh*t on me...when you know nothing about me.\"  \n",
              "2  The Indira Gandhi Institute of Medical Sciences (IGIMS) in Patna on Thursday made corrections in its Marital Declaration Form by changing 'Virgin' option to 'Unmarried'. Earlier, Bihar Health Minister defined virgin as being an unmarried woman and did not consider the term objectionable. The institute, however, faced strong backlash for asking new recruits to declare their virginity in the form.  \n",
              "3                                Lashkar-e-Taiba's Kashmir commander Abu Dujana, who was killed by security forces, said \"Kabhi hum aage, kabhi aap, aaj aapne pakad liya, mubarak ho aapko (Today you caught me. Congratulations)\" after being caught. He added that he won't surrender, and whatever is in his fate will happen to him. \"Hum nikley they shaheed hone (had left home for martyrdom),\" he added.  \n",
              "4                                  Hotels in Maharashtra will train their staff to spot signs of sex trafficking, including frequent requests for bed linen changes and 'Do not disturb' signs left on room doors for days. A mobile phone app called Rescue Me, which will allow staff to alert police of suspicious behaviour, will be developed. The initiative has been backed by the Maharashtra government.  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-87f3e3d0-6c4a-417d-9b74-3c917c501ccd\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>headlines</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Daman &amp; Diu revokes mandatory Rakshabandhan in offices order</td>\n",
              "      <td>The Administration of Union Territory Daman and Diu has revoked its order that made it compulsory for women to tie rakhis to their male colleagues on the occasion of Rakshabandhan on August 7. The administration was forced to withdraw the decision within 24 hours of issuing the circular after it received flak from employees and was slammed on social media.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Malaika slams user who trolled her for 'divorcing rich man'</td>\n",
              "      <td>Malaika Arora slammed an Instagram user who trolled her for \"divorcing a rich man\" and \"having fun with the alimony\". \"Her life now is all about wearing short clothes, going to gym or salon, enjoying vacation[s],\" the user commented. Malaika responded, \"You certainly got to get your damn facts right before spewing sh*t on me...when you know nothing about me.\"</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>'Virgin' now corrected to 'Unmarried' in IGIMS' form</td>\n",
              "      <td>The Indira Gandhi Institute of Medical Sciences (IGIMS) in Patna on Thursday made corrections in its Marital Declaration Form by changing 'Virgin' option to 'Unmarried'. Earlier, Bihar Health Minister defined virgin as being an unmarried woman and did not consider the term objectionable. The institute, however, faced strong backlash for asking new recruits to declare their virginity in the form.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Aaj aapne pakad liya: LeT man Dujana before being killed</td>\n",
              "      <td>Lashkar-e-Taiba's Kashmir commander Abu Dujana, who was killed by security forces, said \"Kabhi hum aage, kabhi aap, aaj aapne pakad liya, mubarak ho aapko (Today you caught me. Congratulations)\" after being caught. He added that he won't surrender, and whatever is in his fate will happen to him. \"Hum nikley they shaheed hone (had left home for martyrdom),\" he added.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Hotel staff to get training to spot signs of sex trafficking</td>\n",
              "      <td>Hotels in Maharashtra will train their staff to spot signs of sex trafficking, including frequent requests for bed linen changes and 'Do not disturb' signs left on room doors for days. A mobile phone app called Rescue Me, which will allow staff to alert police of suspicious behaviour, will be developed. The initiative has been backed by the Maharashtra government.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-87f3e3d0-6c4a-417d-9b74-3c917c501ccd')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-87f3e3d0-6c4a-417d-9b74-3c917c501ccd button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-87f3e3d0-6c4a-417d-9b74-3c917c501ccd');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-931263c4-3c88-402d-afe4-217354665f2a\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-931263c4-3c88-402d-afe4-217354665f2a')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-931263c4-3c88-402d-afe4-217354665f2a button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df1",
              "summary": "{\n  \"name\": \"df1\",\n  \"rows\": 4514,\n  \"fields\": [\n    {\n      \"column\": \"headlines\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4514,\n        \"samples\": [\n          \"South Delhi restaurants to allow public to use toilets at ?5\",\n          \"China bans Bieber from performing over bad behaviour\",\n          \"Pakistan accuses India of targeting CPEC\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4514,\n        \"samples\": [\n          \"All restaurants, including five-star hotels, in South Delhi have been directed by a Delhi civic body to make their toilets available to general public, who are not customers, at a fee of ?5 starting April 1. \\\"The move will be particularly beneficial for women who face problems due to lack of toilets in marketplaces,\\\" the South Delhi Municipal Corporation said.\",\n          \"The Chinese government has banned singer Justin Bieber from performing in the country as he has \\\"engaged in a series of bad behaviours, both in his social life and during a previous performance in China\\\". Bieber will perform in Asia later this year, with concerts scheduled in countries like Indonesia, Japan and Singapore, as part of his Purpose World Tour.\",\n          \"Pakistan on Saturday accused India of targeting the China-Pakistan Economic Corridor (CPEC) project through subversive activities. Pakistan further said that it is aware of \\\"enemy designs\\\" and is taking effective counter measures. Notably, India which has repeatedly dismissed all such allegations has raised sovereignty concerns over the project as it passes through Pakistan-occupied Kashmir.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df1.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_AwOKKrX-sci",
        "outputId": "07d94a21-4dbb-42d7-8922-0b81c8db0f9a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4514, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df2.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 379
        },
        "id": "BWijCfh1bGot",
        "outputId": "e47decf0-7ca7-4524-e5e9-774681e4fe14"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                             headlines  \\\n",
              "0    upGrad learner switches to career in ML & Al with 90% salary hike   \n",
              "1         Delhi techie wins free food from Swiggy for one year on CRED   \n",
              "2     New Zealand end Rohit Sharma-led India's 12-match winning streak   \n",
              "3             Aegon life iTerm insurance plan helps customers save tax   \n",
              "4  Have known Hirani for yrs, what if MeToo claims are not true: Sonam   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                      text  \n",
              "0  Saurav Kant, an alumnus of upGrad and IIIT-B's PG Program in Machine learning and Artificial Intelligence, was a Sr Systems Engineer at Infosys with almost 5 years of work experience. The program and upGrad's 360-degree career support helped him transition to a Data Scientist at Tech Mahindra with 90% salary hike. upGrad's Online Power Learning has powered 3 lakh+ careers.  \n",
              "1                              Kunal Shah's credit card bill payment platform, CRED, gave users a chance to win free food from Swiggy for one year. Pranav Kaushik, a Delhi techie, bagged this reward after spending 2000 CRED coins. Users get one CRED coin per rupee of bill paid, which can be used to avail rewards from brands like Ixigo, BookMyShow, UberEats, Cult.Fit and more.  \n",
              "2                 New Zealand defeated India by 8 wickets in the fourth ODI at Hamilton on Thursday to win their first match of the five-match ODI series. India lost an international match under Rohit Sharma's captaincy after 12 consecutive victories dating back to March 2018. The match witnessed India getting all out for 92, their seventh lowest total in ODI cricket history.  \n",
              "3                               With Aegon Life iTerm Insurance plan, customers can enjoy tax benefits on your premiums paid and save up to Ã¢ÂÂ¹46,800^ on taxes. The plan provides life cover up to the age of 100 years. Also, customers have options to insure against Critical Illnesses, Disability and Accidental Death Benefit Rider with a life cover up to the age of 80 years.  \n",
              "4       Speaking about the sexual harassment allegations against Rajkumar Hirani, Sonam Kapoor said, \"I've known Hirani for many years...What if it's not true, the [#MeToo] movement will get derailed.\" \"In the #MeToo movement, I always believe a woman. But in this case, we need to reserve our judgment,\" she added. Hirani has been accused by an assistant who worked in 'Sanju'.  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e07fe202-c401-45ec-9260-c20f03b36f22\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>headlines</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>upGrad learner switches to career in ML &amp; Al with 90% salary hike</td>\n",
              "      <td>Saurav Kant, an alumnus of upGrad and IIIT-B's PG Program in Machine learning and Artificial Intelligence, was a Sr Systems Engineer at Infosys with almost 5 years of work experience. The program and upGrad's 360-degree career support helped him transition to a Data Scientist at Tech Mahindra with 90% salary hike. upGrad's Online Power Learning has powered 3 lakh+ careers.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Delhi techie wins free food from Swiggy for one year on CRED</td>\n",
              "      <td>Kunal Shah's credit card bill payment platform, CRED, gave users a chance to win free food from Swiggy for one year. Pranav Kaushik, a Delhi techie, bagged this reward after spending 2000 CRED coins. Users get one CRED coin per rupee of bill paid, which can be used to avail rewards from brands like Ixigo, BookMyShow, UberEats, Cult.Fit and more.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>New Zealand end Rohit Sharma-led India's 12-match winning streak</td>\n",
              "      <td>New Zealand defeated India by 8 wickets in the fourth ODI at Hamilton on Thursday to win their first match of the five-match ODI series. India lost an international match under Rohit Sharma's captaincy after 12 consecutive victories dating back to March 2018. The match witnessed India getting all out for 92, their seventh lowest total in ODI cricket history.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Aegon life iTerm insurance plan helps customers save tax</td>\n",
              "      <td>With Aegon Life iTerm Insurance plan, customers can enjoy tax benefits on your premiums paid and save up to Ã¢ÂÂ¹46,800^ on taxes. The plan provides life cover up to the age of 100 years. Also, customers have options to insure against Critical Illnesses, Disability and Accidental Death Benefit Rider with a life cover up to the age of 80 years.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Have known Hirani for yrs, what if MeToo claims are not true: Sonam</td>\n",
              "      <td>Speaking about the sexual harassment allegations against Rajkumar Hirani, Sonam Kapoor said, \"I've known Hirani for many years...What if it's not true, the [#MeToo] movement will get derailed.\" \"In the #MeToo movement, I always believe a woman. But in this case, we need to reserve our judgment,\" she added. Hirani has been accused by an assistant who worked in 'Sanju'.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e07fe202-c401-45ec-9260-c20f03b36f22')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e07fe202-c401-45ec-9260-c20f03b36f22 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e07fe202-c401-45ec-9260-c20f03b36f22');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0a13e3a1-10ae-473b-ae75-591c823f0fbb\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0a13e3a1-10ae-473b-ae75-591c823f0fbb')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0a13e3a1-10ae-473b-ae75-591c823f0fbb button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df2",
              "summary": "{\n  \"name\": \"df2\",\n  \"rows\": 98401,\n  \"fields\": [\n    {\n      \"column\": \"headlines\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 98280,\n        \"samples\": [\n          \"Salman Khan asked to appear before court in Arms Act case\",\n          \"Man who beat Federer at Wimbledon once lost to AB de Villiers\",\n          \"Alwar lynching victim died of 'shock': Post-mortem report\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 98360,\n        \"samples\": [\n          \"Digital payments startup Paytm has raised $1.4 billion from SoftBank in India's largest funding round. This is also SoftBank's biggest investment in the Indian startup ecosystem till date. The latest investment by SoftBank will value Paytm at around $8 billion, up from its valuation of $4.8 billion in August 2016. \",\n          \"Oil companies on Thursday reduced the petrol price by \\u00c3\\u00a2\\u00c2\\u0082\\u00c2\\u00b91.12/litre and diesel by \\u00c3\\u00a2\\u00c2\\u0082\\u00c2\\u00b91.24/litre effective June 16. This is the first cut in prices following the decision to revise fuel prices daily and the revision shall be applicable from 6 AM to 6 AM the next day. Customers can know the revised prices through social media posts and 'Fuel@IOC' app.\",\n          \"The Indian Army has announced plans to deploy women officers for cyber warfare as they face cybersecurity threats from both state and non-state actors. In order to increase the number of female officers and provide them more opportunities, the Army is also planning to hire 850 of women in the military police branch and some in the language training branch.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df2.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zfVDFaFRIqnZ",
        "outputId": "766cd09d-c997-4a95-f25a-712b0bf8e705"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(98401, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_df = df2.copy()\n",
        "test_df = df1.copy()"
      ],
      "metadata": {
        "id": "oYjAu737bWEX"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_df.describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 209
        },
        "id": "_Rw6wyoibspG",
        "outputId": "81b8b80f-3544-44f4-b78e-5540fef1dee1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                          headlines  \\\n",
              "count                                                         98401   \n",
              "unique                                                        98280   \n",
              "top     Warne rightly predicted India-Eng 2011 WC match to be a tie   \n",
              "freq                                                              3   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                  text  \n",
              "count                                                                                                                                                                                                                                                                                                                                                                            98401  \n",
              "unique                                                                                                                                                                                                                                                                                                                                                                           98360  \n",
              "top     Former Pakistani cricketer Hanif Mohammad batted for 970 minutes in a Test against West Indies, during his knock of 337 runs, which ended on January 23, 1958. The innings, covering nine consecutive sessions and enough to encompass over 10 football matches, included 24 boundaries, 105 singles, 44 twos and 16 threes. The 337-run knock helped Pakistan draw the match.  \n",
              "freq                                                                                                                                                                                                                                                                                                                                                                                 2  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ffd4b3ff-5694-48b8-b5af-6a898fce9289\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>headlines</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>98401</td>\n",
              "      <td>98401</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>unique</th>\n",
              "      <td>98280</td>\n",
              "      <td>98360</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>top</th>\n",
              "      <td>Warne rightly predicted India-Eng 2011 WC match to be a tie</td>\n",
              "      <td>Former Pakistani cricketer Hanif Mohammad batted for 970 minutes in a Test against West Indies, during his knock of 337 runs, which ended on January 23, 1958. The innings, covering nine consecutive sessions and enough to encompass over 10 football matches, included 24 boundaries, 105 singles, 44 twos and 16 threes. The 337-run knock helped Pakistan draw the match.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>freq</th>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ffd4b3ff-5694-48b8-b5af-6a898fce9289')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ffd4b3ff-5694-48b8-b5af-6a898fce9289 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ffd4b3ff-5694-48b8-b5af-6a898fce9289');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-f46a22af-75ce-457a-9542-77eaf34bdca8\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f46a22af-75ce-457a-9542-77eaf34bdca8')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-f46a22af-75ce-457a-9542-77eaf34bdca8 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"train_df\",\n  \"rows\": 4,\n  \"fields\": [\n    {\n      \"column\": \"headlines\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          98280,\n          \"3\",\n          \"98401\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          98360,\n          \"2\",\n          \"98401\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df.describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 209
        },
        "id": "XkYFhQUSb32G",
        "outputId": "fd18e3d4-14fe-412b-b291-c896a5655ece"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                           headlines  \\\n",
              "count                                                           4514   \n",
              "unique                                                          4514   \n",
              "top     More than half of India's languages may die in 50yrs: Survey   \n",
              "freq                                                               1   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                  text  \n",
              "count                                                                                                                                                                                                                                                                                                                                                             4514  \n",
              "unique                                                                                                                                                                                                                                                                                                                                                            4514  \n",
              "top     At least 400 languages or more than half languages spoken by India's 1.3 billion people may die out in next 50 years, a study by the People's Linguistic Survey of India (PSLI) stated. Further claiming that India has already lost 250 languages in last five decades, the survey said, \"Each time a language is lost, the corresponding culture is killed.\"  \n",
              "freq                                                                                                                                                                                                                                                                                                                                                                 1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-38f4d5df-6b1f-4ea7-9390-80ca2948d987\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>headlines</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>4514</td>\n",
              "      <td>4514</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>unique</th>\n",
              "      <td>4514</td>\n",
              "      <td>4514</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>top</th>\n",
              "      <td>More than half of India's languages may die in 50yrs: Survey</td>\n",
              "      <td>At least 400 languages or more than half languages spoken by India's 1.3 billion people may die out in next 50 years, a study by the People's Linguistic Survey of India (PSLI) stated. Further claiming that India has already lost 250 languages in last five decades, the survey said, \"Each time a language is lost, the corresponding culture is killed.\"</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>freq</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-38f4d5df-6b1f-4ea7-9390-80ca2948d987')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-38f4d5df-6b1f-4ea7-9390-80ca2948d987 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-38f4d5df-6b1f-4ea7-9390-80ca2948d987');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c199af60-1734-474b-a3f1-1421014c3891\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c199af60-1734-474b-a3f1-1421014c3891')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c199af60-1734-474b-a3f1-1421014c3891 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"test_df\",\n  \"rows\": 4,\n  \"fields\": [\n    {\n      \"column\": \"headlines\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"4514\",\n          \"More than half of India's languages may die in 50yrs: Survey\",\n          \"1\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"4514\",\n          \"At least 400 languages or more than half languages spoken by India's 1.3 billion people may die out in next 50 years, a study by the People's Linguistic Survey of India (PSLI) stated. Further claiming that India has already lost 250 languages in last five decades, the survey said, \\\"Each time a language is lost, the corresponding culture is killed.\\\"\",\n          \"1\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Looks like training data has few duplicate rows. Let's drop these\n",
        "\n",
        "train_df.dropna(inplace=True)\n",
        "train_df.drop_duplicates(inplace=True,subset=['headlines'])\n",
        "train_df.drop_duplicates(inplace=True,subset=['text'])"
      ],
      "metadata": {
        "id": "b21nEbFh-0pa"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_df.describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 209
        },
        "id": "6Rdcazhmc9CY",
        "outputId": "28a8a9d3-9e72-4453-89ae-3f205b954fed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                      headlines  \\\n",
              "count                                                     98262   \n",
              "unique                                                    98262   \n",
              "top     Madhesi Morcha withdraws support to Nepalese government   \n",
              "freq                                                          1   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                          text  \n",
              "count                                                                                                                                                                                                                                                                                                                                                                                                    98262  \n",
              "unique                                                                                                                                                                                                                                                                                                                                                                                                   98262  \n",
              "top     Madhesi Morcha, an alliance of seven political parties, has withdrawn support to PM Pushpa Kamal Dahal-led Nepal government after it failed to meet a seven-day ultimatum to fulfil their demands including endorsement for the revised Constitution amendment bill. The Morcha has 36 seats in the Parliament, but despite the withdrawal of support, there is no immediate threat to the government.  \n",
              "freq                                                                                                                                                                                                                                                                                                                                                                                                         1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-46ab8681-1787-4436-9b08-7ca447cd09e0\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>headlines</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>98262</td>\n",
              "      <td>98262</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>unique</th>\n",
              "      <td>98262</td>\n",
              "      <td>98262</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>top</th>\n",
              "      <td>Madhesi Morcha withdraws support to Nepalese government</td>\n",
              "      <td>Madhesi Morcha, an alliance of seven political parties, has withdrawn support to PM Pushpa Kamal Dahal-led Nepal government after it failed to meet a seven-day ultimatum to fulfil their demands including endorsement for the revised Constitution amendment bill. The Morcha has 36 seats in the Parliament, but despite the withdrawal of support, there is no immediate threat to the government.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>freq</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-46ab8681-1787-4436-9b08-7ca447cd09e0')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-46ab8681-1787-4436-9b08-7ca447cd09e0 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-46ab8681-1787-4436-9b08-7ca447cd09e0');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-b45c74ba-4547-454c-9584-0166e47f82cc\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b45c74ba-4547-454c-9584-0166e47f82cc')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-b45c74ba-4547-454c-9584-0166e47f82cc button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"train_df\",\n  \"rows\": 4,\n  \"fields\": [\n    {\n      \"column\": \"headlines\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"98262\",\n          \"Madhesi Morcha withdraws support to Nepalese government\",\n          \"1\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"98262\",\n          \"Madhesi Morcha, an alliance of seven political parties, has withdrawn support to PM Pushpa Kamal Dahal-led Nepal government after it failed to meet a seven-day ultimatum to fulfil their demands including endorsement for the revised Constitution amendment bill. The Morcha has 36 seats in the Parliament, but despite the withdrawal of support, there is no immediate threat to the government.\",\n          \"1\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Clean/Preprocess Text"
      ],
      "metadata": {
        "id": "r70Qpb8Qqlt9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_df = train_df.rename(columns={'headlines':'target_text','text':'source_text'})\n",
        "test_df = test_df.rename(columns={'headlines':'target_text','text':'source_text'})"
      ],
      "metadata": {
        "id": "zWhbByFDdukm"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(train_df.columns)\n",
        "print(test_df.columns)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZLacIMWxd_q3",
        "outputId": "462608c0-7fd2-41ba-aae4-3628a293eb3b"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Index(['target_text', 'source_text'], dtype='object')\n",
            "Index(['target_text', 'source_text'], dtype='object')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "### Let's try to remove short forms like I'll -> I will etc.\n",
        "\n",
        "train_df['source_text'] = train_df['source_text'].apply(lambda x: contractions.fix(x))\n",
        "train_df['target_text'] = train_df['target_text'].apply(lambda x: contractions.fix(x))\n",
        "\n",
        "test_df['target_text'] = test_df['target_text'].apply(lambda x: contractions.fix(x))\n",
        "test_df['target_text'] = test_df['target_text'].apply(lambda x: contractions.fix(x))"
      ],
      "metadata": {
        "id": "wuyJouaBecZp"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stop_words = set(stopwords.words('english'))"
      ],
      "metadata": {
        "id": "KVSauothhIjL"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def cleantext(text):\n",
        "    new_text = str(text)\n",
        "    new_text = new_text.split()\n",
        "    words=[]\n",
        "    for t in new_text:\n",
        "        if t.isalpha():\n",
        "            words.append(t)\n",
        "    new_text =\" \".join(words)\n",
        "    new_text = new_text.lower()\n",
        "    new_text = re.sub(r\" e g \",\" eg \",new_text)\n",
        "    new_text = re.sub(r\"e-mail\",\"email\",new_text)\n",
        "    new_text = re.sub(r\"9\\\\/11\",\" 911 \",new_text)\n",
        "    new_text = re.sub(r\" u.s\",\" american \",new_text)\n",
        "    new_text = re.sub(r\" u.n\",\" united nations \",new_text)\n",
        "    new_text = re.sub(r\"\\n\",\" \",new_text)\n",
        "    new_text = re.sub(r\":\",\" \",new_text)\n",
        "    new_text = re.sub(r\"-\",\" \",new_text)\n",
        "    new_text = re.sub(r\"\\_\",\" \",new_text)\n",
        "    new_text = re.sub(r\"\\d+\",\" \",new_text)\n",
        "    new_text = re.sub(r\"[$#@%&*!~?%{}()]\",\" \",new_text)\n",
        "\n",
        "    tokens = [w for w in new_text.split() if not w in stop_words]\n",
        "\n",
        "\n",
        "    return \" \".join(tokens)"
      ],
      "metadata": {
        "id": "Cor89UCY-6zD"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## Apply the transformations on source and target texts\n",
        "for cols in train_df.columns:\n",
        "    train_df[cols] = train_df[cols].apply(cleantext)\n",
        "\n",
        "for cols in test_df.columns:\n",
        "    test_df[cols] = test_df[cols].apply(cleantext)"
      ],
      "metadata": {
        "id": "NAC6LlBhqa3u"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "id": "k4x9_GRyrGEM",
        "outputId": "58048ea0-48b5-4459-bab5-86c2fcbea3aa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                target_text  \\\n",
              "0          upgrad learner switches career ml al salary hike   \n",
              "1          delhi techie wins free food swiggy one year cred   \n",
              "2                      new zealand end rohit winning streak   \n",
              "3  aegon life iterm insurance plan helps customers save tax   \n",
              "4                           known hirani metoo claims sonam   \n",
              "\n",
              "                                                                                                                                                                                                                                 source_text  \n",
              "0                      saurav alumnus upgrad pg program machine learning artificial sr systems engineer infosys almost years work program career support helped transition data scientist tech mahindra salary online power learning powered  \n",
              "1                                                   kunal credit card bill payment gave users chance win free food swiggy one pranav delhi bagged reward spending cred users get one cred coin per rupee bill used avail rewards brands like  \n",
              "2  new zealand defeated india wickets fourth odi hamilton thursday win first match odi india lost international match rohit captaincy consecutive victories dating back march match witnessed india getting seventh lowest total odi cricket  \n",
              "3                                         aegon life iterm insurance customers enjoy tax benefits premiums paid save plan provides life cover age customers options insure critical disability accidental death benefit rider life cover age  \n",
              "4                                                                                    speaking sexual harassment allegations rajkumar sonam kapoor known hirani many movement get always believe need reserve hirani accused assistant worked  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a450519b-2edb-4502-aa88-6a1db0f9e019\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target_text</th>\n",
              "      <th>source_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>upgrad learner switches career ml al salary hike</td>\n",
              "      <td>saurav alumnus upgrad pg program machine learning artificial sr systems engineer infosys almost years work program career support helped transition data scientist tech mahindra salary online power learning powered</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>delhi techie wins free food swiggy one year cred</td>\n",
              "      <td>kunal credit card bill payment gave users chance win free food swiggy one pranav delhi bagged reward spending cred users get one cred coin per rupee bill used avail rewards brands like</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>new zealand end rohit winning streak</td>\n",
              "      <td>new zealand defeated india wickets fourth odi hamilton thursday win first match odi india lost international match rohit captaincy consecutive victories dating back march match witnessed india getting seventh lowest total odi cricket</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>aegon life iterm insurance plan helps customers save tax</td>\n",
              "      <td>aegon life iterm insurance customers enjoy tax benefits premiums paid save plan provides life cover age customers options insure critical disability accidental death benefit rider life cover age</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>known hirani metoo claims sonam</td>\n",
              "      <td>speaking sexual harassment allegations rajkumar sonam kapoor known hirani many movement get always believe need reserve hirani accused assistant worked</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a450519b-2edb-4502-aa88-6a1db0f9e019')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a450519b-2edb-4502-aa88-6a1db0f9e019 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a450519b-2edb-4502-aa88-6a1db0f9e019');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c8756563-c7df-4014-bbe6-ec86d025b879\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c8756563-c7df-4014-bbe6-ec86d025b879')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c8756563-c7df-4014-bbe6-ec86d025b879 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "train_df",
              "summary": "{\n  \"name\": \"train_df\",\n  \"rows\": 98262,\n  \"fields\": [\n    {\n      \"column\": \"target_text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 98038,\n        \"samples\": [\n          \"ditching friends love javadekar\",\n          \"rape girls revealing egypt lawyer\",\n          \"rajkot school mahatma gandhi studied shuts\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"source_text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 98232,\n        \"samples\": [\n          \"scientists discovered new species glass frog whose beating heart visible amazonian frog dark green spots back reproductive behaviour different glass species threatened due habitat destruction mining activities oil scientists\",\n          \"rejecting concerns release postal stamps portraying slain terrorist burhan wani pakistan said thousands people fighting pakistan post official reportedly said stamps issued express solidarity kashmiris indian\",\n          \"credit rating agency fitch wednesday said withdrawn ratings reliance communications firm longer provide analytical coverage ratings anil investors service withdrawn credit rating reliance communications defaulted interest payment\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_df.describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 192
        },
        "id": "J59rNsktoO54",
        "outputId": "fd033b2f-bdb8-46c5-aec0-2ae9ae1f9282"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       target_text  \\\n",
              "count        98262   \n",
              "unique       98038   \n",
              "top                  \n",
              "freq             9   \n",
              "\n",
              "                                                                                                                                                                                     source_text  \n",
              "count                                                                                                                                                                                      98262  \n",
              "unique                                                                                                                                                                                     98232  \n",
              "top     legend bradman smashed runs help australia post india test ended december india dismissed first innings second combined score could overhaul individual highest score india match chandu  \n",
              "freq                                                                                                                                                                                           2  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0f52eb8a-f047-424d-aa19-2a8ec8173cd2\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target_text</th>\n",
              "      <th>source_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>98262</td>\n",
              "      <td>98262</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>unique</th>\n",
              "      <td>98038</td>\n",
              "      <td>98232</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>top</th>\n",
              "      <td></td>\n",
              "      <td>legend bradman smashed runs help australia post india test ended december india dismissed first innings second combined score could overhaul individual highest score india match chandu</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>freq</th>\n",
              "      <td>9</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0f52eb8a-f047-424d-aa19-2a8ec8173cd2')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-0f52eb8a-f047-424d-aa19-2a8ec8173cd2 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-0f52eb8a-f047-424d-aa19-2a8ec8173cd2');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2340e8b2-4dd8-4429-ad58-1e1a35e0305f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2340e8b2-4dd8-4429-ad58-1e1a35e0305f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2340e8b2-4dd8-4429-ad58-1e1a35e0305f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"train_df\",\n  \"rows\": 4,\n  \"fields\": [\n    {\n      \"column\": \"target_text\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": \"1970-01-01 00:00:00.000000009\",\n        \"max\": \"1970-01-01 00:00:00.000098262\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          98038,\n          \"9\",\n          \"98262\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"source_text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          98232,\n          \"2\",\n          \"98262\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "id": "EcJyA2OFoG2g",
        "outputId": "afd8591e-a2b2-4499-e382-dfdb938b23bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                               target_text  \\\n",
              "0  daman diu revokes mandatory rakshabandhan offices order   \n",
              "1                          malaika slams user trolled rich   \n",
              "2                                           corrected form   \n",
              "3                    aaj aapne pakad let man dujana killed   \n",
              "4      hotel staff get training spot signs sex trafficking   \n",
              "\n",
              "                                                                                                                                                                                                                                          source_text  \n",
              "0  administration union territory daman diu revoked order made compulsory women tie rakhis male colleagues occasion rakshabandhan august administration forced withdraw decision within hours issuing circular received flak employees slammed social  \n",
              "1                                                                                    malaika arora slammed instagram user trolled rich fun life wearing short going gym enjoying user malaika certainly got get damn facts right spewing know nothing  \n",
              "2    indira gandhi institute medical sciences patna thursday made corrections marital declaration form changing option bihar health minister defined virgin unmarried woman consider term faced strong backlash asking new recruits declare virginity  \n",
              "3                                                                                               kashmir commander abu killed security said hum kabhi aaj aapne pakad mubarak ho aapko caught added whatever fate happen nikley shaheed hone left home  \n",
              "4                                  hotels maharashtra train staff spot signs sex including frequent requests bed linen changes signs left room doors mobile phone app called rescue allow staff alert police suspicious initiative backed maharashtra  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ff94d003-333c-42d8-9bf7-9da26cee14d9\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target_text</th>\n",
              "      <th>source_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>daman diu revokes mandatory rakshabandhan offices order</td>\n",
              "      <td>administration union territory daman diu revoked order made compulsory women tie rakhis male colleagues occasion rakshabandhan august administration forced withdraw decision within hours issuing circular received flak employees slammed social</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>malaika slams user trolled rich</td>\n",
              "      <td>malaika arora slammed instagram user trolled rich fun life wearing short going gym enjoying user malaika certainly got get damn facts right spewing know nothing</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>corrected form</td>\n",
              "      <td>indira gandhi institute medical sciences patna thursday made corrections marital declaration form changing option bihar health minister defined virgin unmarried woman consider term faced strong backlash asking new recruits declare virginity</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>aaj aapne pakad let man dujana killed</td>\n",
              "      <td>kashmir commander abu killed security said hum kabhi aaj aapne pakad mubarak ho aapko caught added whatever fate happen nikley shaheed hone left home</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>hotel staff get training spot signs sex trafficking</td>\n",
              "      <td>hotels maharashtra train staff spot signs sex including frequent requests bed linen changes signs left room doors mobile phone app called rescue allow staff alert police suspicious initiative backed maharashtra</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ff94d003-333c-42d8-9bf7-9da26cee14d9')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ff94d003-333c-42d8-9bf7-9da26cee14d9 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ff94d003-333c-42d8-9bf7-9da26cee14d9');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ce68190d-5e1e-4543-9dc4-7829b9f6b990\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ce68190d-5e1e-4543-9dc4-7829b9f6b990')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ce68190d-5e1e-4543-9dc4-7829b9f6b990 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "test_df",
              "summary": "{\n  \"name\": \"test_df\",\n  \"rows\": 4514,\n  \"fields\": [\n    {\n      \"column\": \"target_text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4512,\n        \"samples\": [\n          \"dept seizes crore cash karol bagh area\",\n          \"polish man killed court hears\",\n          \"wifi indian railway stations better google\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"source_text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4514,\n        \"samples\": [\n          \"including south delhi directed delhi civic body make toilets available general fee starting april move particularly beneficial women face problems due lack toilets south delhi municipal corporation\",\n          \"chinese government banned singer justin bieber performing country series bad social life previous performance bieber perform asia later concerts scheduled countries like japan part purpose world\",\n          \"pakistan saturday accused india targeting economic corridor project subversive pakistan said aware taking effective counter india repeatedly dismissed allegations raised sovereignty concerns project passes\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df.describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 192
        },
        "id": "EGGmqzp7oVEw",
        "outputId": "f4f98a9a-b421-4bea-aa6d-c3b1078480d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            target_text  \\\n",
              "count                                              4514   \n",
              "unique                                             4512   \n",
              "top     pakistan arrests indian fishermen gujarat coast   \n",
              "freq                                                  2   \n",
              "\n",
              "                                                                                                                                                                                       source_text  \n",
              "count                                                                                                                                                                                         4514  \n",
              "unique                                                                                                                                                                                        4514  \n",
              "top     least languages half languages spoken billion people may die next study linguistic survey india claiming india already lost languages last five survey time language corresponding culture  \n",
              "freq                                                                                                                                                                                             1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7653abe0-e398-43b9-8fc3-5d962a6fa357\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target_text</th>\n",
              "      <th>source_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>4514</td>\n",
              "      <td>4514</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>unique</th>\n",
              "      <td>4512</td>\n",
              "      <td>4514</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>top</th>\n",
              "      <td>pakistan arrests indian fishermen gujarat coast</td>\n",
              "      <td>least languages half languages spoken billion people may die next study linguistic survey india claiming india already lost languages last five survey time language corresponding culture</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>freq</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7653abe0-e398-43b9-8fc3-5d962a6fa357')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-7653abe0-e398-43b9-8fc3-5d962a6fa357 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-7653abe0-e398-43b9-8fc3-5d962a6fa357');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ec10be58-b92a-442e-b7ff-2f19f3ee6cab\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ec10be58-b92a-442e-b7ff-2f19f3ee6cab')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ec10be58-b92a-442e-b7ff-2f19f3ee6cab button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"test_df\",\n  \"rows\": 4,\n  \"fields\": [\n    {\n      \"column\": \"target_text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          4512,\n          \"2\",\n          \"4514\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"source_text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"4514\",\n          \"least languages half languages spoken billion people may die next study linguistic survey india claiming india already lost languages last five survey time language corresponding culture\",\n          \"1\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "tokenizer = Tokenizer(nlp.vocab)"
      ],
      "metadata": {
        "id": "9W1GA9qarBVK"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## Tokenize the data using NLTK\n",
        "\n",
        "for cols in train_df.columns:\n",
        "    train_df[cols] = train_df[cols].apply(lambda x: [token.text.lower() for token in tokenizer(x)])\n",
        "\n",
        "for cols in test_df.columns:\n",
        "    test_df[cols] = test_df[cols].apply(lambda x: [token.text.lower() for token in tokenizer(x)])"
      ],
      "metadata": {
        "id": "Kb-huDmvpYIA"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "id": "uUnIBDbArdjz",
        "outputId": "156b5c4f-7003-470f-e96e-920ac1cd12c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                          target_text  \\\n",
              "0           [upgrad, learner, switches, career, ml, al, salary, hike]   \n",
              "1          [delhi, techie, wins, free, food, swiggy, one, year, cred]   \n",
              "2                         [new, zealand, end, rohit, winning, streak]   \n",
              "3  [aegon, life, iterm, insurance, plan, helps, customers, save, tax]   \n",
              "4                               [known, hirani, metoo, claims, sonam]   \n",
              "\n",
              "                                                                                                                                                                                                                                                                   source_text  \n",
              "0                          [saurav, alumnus, upgrad, pg, program, machine, learning, artificial, sr, systems, engineer, infosys, almost, years, work, program, career, support, helped, transition, data, scientist, tech, mahindra, salary, online, power, learning, powered]  \n",
              "1                                                    [kunal, credit, card, bill, payment, gave, users, chance, win, free, food, swiggy, one, pranav, delhi, bagged, reward, spending, cred, users, get, one, cred, coin, per, rupee, bill, used, avail, rewards, brands, like]  \n",
              "2  [new, zealand, defeated, india, wickets, fourth, odi, hamilton, thursday, win, first, match, odi, india, lost, international, match, rohit, captaincy, consecutive, victories, dating, back, march, match, witnessed, india, getting, seventh, lowest, total, odi, cricket]  \n",
              "3                                              [aegon, life, iterm, insurance, customers, enjoy, tax, benefits, premiums, paid, save, plan, provides, life, cover, age, customers, options, insure, critical, disability, accidental, death, benefit, rider, life, cover, age]  \n",
              "4                                                                                                 [speaking, sexual, harassment, allegations, rajkumar, sonam, kapoor, known, hirani, many, movement, get, always, believe, need, reserve, hirani, accused, assistant, worked]  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b7f07e63-ab5a-4b07-9a3d-f08cc059ca47\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target_text</th>\n",
              "      <th>source_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[upgrad, learner, switches, career, ml, al, salary, hike]</td>\n",
              "      <td>[saurav, alumnus, upgrad, pg, program, machine, learning, artificial, sr, systems, engineer, infosys, almost, years, work, program, career, support, helped, transition, data, scientist, tech, mahindra, salary, online, power, learning, powered]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[delhi, techie, wins, free, food, swiggy, one, year, cred]</td>\n",
              "      <td>[kunal, credit, card, bill, payment, gave, users, chance, win, free, food, swiggy, one, pranav, delhi, bagged, reward, spending, cred, users, get, one, cred, coin, per, rupee, bill, used, avail, rewards, brands, like]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[new, zealand, end, rohit, winning, streak]</td>\n",
              "      <td>[new, zealand, defeated, india, wickets, fourth, odi, hamilton, thursday, win, first, match, odi, india, lost, international, match, rohit, captaincy, consecutive, victories, dating, back, march, match, witnessed, india, getting, seventh, lowest, total, odi, cricket]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[aegon, life, iterm, insurance, plan, helps, customers, save, tax]</td>\n",
              "      <td>[aegon, life, iterm, insurance, customers, enjoy, tax, benefits, premiums, paid, save, plan, provides, life, cover, age, customers, options, insure, critical, disability, accidental, death, benefit, rider, life, cover, age]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[known, hirani, metoo, claims, sonam]</td>\n",
              "      <td>[speaking, sexual, harassment, allegations, rajkumar, sonam, kapoor, known, hirani, many, movement, get, always, believe, need, reserve, hirani, accused, assistant, worked]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b7f07e63-ab5a-4b07-9a3d-f08cc059ca47')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b7f07e63-ab5a-4b07-9a3d-f08cc059ca47 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b7f07e63-ab5a-4b07-9a3d-f08cc059ca47');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-fa51db03-3bcf-48d4-9adf-f328a3ebf88d\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-fa51db03-3bcf-48d4-9adf-f328a3ebf88d')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-fa51db03-3bcf-48d4-9adf-f328a3ebf88d button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "train_df",
              "summary": "{\n  \"name\": \"train_df\",\n  \"rows\": 98262,\n  \"fields\": [\n    {\n      \"column\": \"target_text\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"source_text\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "id": "2cllRk13riDS",
        "outputId": "16cb7ede-28c8-42ec-f916-57ce91685d9b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                       target_text  \\\n",
              "0  [daman, diu, revokes, mandatory, rakshabandhan, offices, order]   \n",
              "1                            [malaika, slams, user, trolled, rich]   \n",
              "2                                                [corrected, form]   \n",
              "3                    [aaj, aapne, pakad, let, man, dujana, killed]   \n",
              "4     [hotel, staff, get, training, spot, signs, sex, trafficking]   \n",
              "\n",
              "                                                                                                                                                                                                                                                                         source_text  \n",
              "0  [administration, union, territory, daman, diu, revoked, order, made, compulsory, women, tie, rakhis, male, colleagues, occasion, rakshabandhan, august, administration, forced, withdraw, decision, within, hours, issuing, circular, received, flak, employees, slammed, social]  \n",
              "1                                                                                         [malaika, arora, slammed, instagram, user, trolled, rich, fun, life, wearing, short, going, gym, enjoying, user, malaika, certainly, got, get, damn, facts, right, spewing, know, nothing]  \n",
              "2   [indira, gandhi, institute, medical, sciences, patna, thursday, made, corrections, marital, declaration, form, changing, option, bihar, health, minister, defined, virgin, unmarried, woman, consider, term, faced, strong, backlash, asking, new, recruits, declare, virginity]  \n",
              "3                                                                                                     [kashmir, commander, abu, killed, security, said, hum, kabhi, aaj, aapne, pakad, mubarak, ho, aapko, caught, added, whatever, fate, happen, nikley, shaheed, hone, left, home]  \n",
              "4                                  [hotels, maharashtra, train, staff, spot, signs, sex, including, frequent, requests, bed, linen, changes, signs, left, room, doors, mobile, phone, app, called, rescue, allow, staff, alert, police, suspicious, initiative, backed, maharashtra]  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-be9de193-4907-482f-b08c-ee42a567633d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target_text</th>\n",
              "      <th>source_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[daman, diu, revokes, mandatory, rakshabandhan, offices, order]</td>\n",
              "      <td>[administration, union, territory, daman, diu, revoked, order, made, compulsory, women, tie, rakhis, male, colleagues, occasion, rakshabandhan, august, administration, forced, withdraw, decision, within, hours, issuing, circular, received, flak, employees, slammed, social]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[malaika, slams, user, trolled, rich]</td>\n",
              "      <td>[malaika, arora, slammed, instagram, user, trolled, rich, fun, life, wearing, short, going, gym, enjoying, user, malaika, certainly, got, get, damn, facts, right, spewing, know, nothing]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[corrected, form]</td>\n",
              "      <td>[indira, gandhi, institute, medical, sciences, patna, thursday, made, corrections, marital, declaration, form, changing, option, bihar, health, minister, defined, virgin, unmarried, woman, consider, term, faced, strong, backlash, asking, new, recruits, declare, virginity]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[aaj, aapne, pakad, let, man, dujana, killed]</td>\n",
              "      <td>[kashmir, commander, abu, killed, security, said, hum, kabhi, aaj, aapne, pakad, mubarak, ho, aapko, caught, added, whatever, fate, happen, nikley, shaheed, hone, left, home]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[hotel, staff, get, training, spot, signs, sex, trafficking]</td>\n",
              "      <td>[hotels, maharashtra, train, staff, spot, signs, sex, including, frequent, requests, bed, linen, changes, signs, left, room, doors, mobile, phone, app, called, rescue, allow, staff, alert, police, suspicious, initiative, backed, maharashtra]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-be9de193-4907-482f-b08c-ee42a567633d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-be9de193-4907-482f-b08c-ee42a567633d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-be9de193-4907-482f-b08c-ee42a567633d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-298cc886-c4d5-4329-9f39-d70f77ca1c12\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-298cc886-c4d5-4329-9f39-d70f77ca1c12')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-298cc886-c4d5-4329-9f39-d70f77ca1c12 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "test_df",
              "summary": "{\n  \"name\": \"test_df\",\n  \"rows\": 4514,\n  \"fields\": [\n    {\n      \"column\": \"target_text\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"source_text\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Let's try to collate the vocabulary for source and target texts\n",
        "\n",
        "all_text = list(train_df['source_text']) + list(test_df['source_text']) + list(train_df['target_text']) + list(test_df['target_text'])"
      ],
      "metadata": {
        "id": "GX7UbxCZrtm9"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_text[0:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "y9X4ono_rtg8",
        "outputId": "0a2b9034-0a3e-43ac-af5b-3e84f0511240"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['saurav',\n",
              "  'alumnus',\n",
              "  'upgrad',\n",
              "  'pg',\n",
              "  'program',\n",
              "  'machine',\n",
              "  'learning',\n",
              "  'artificial',\n",
              "  'sr',\n",
              "  'systems',\n",
              "  'engineer',\n",
              "  'infosys',\n",
              "  'almost',\n",
              "  'years',\n",
              "  'work',\n",
              "  'program',\n",
              "  'career',\n",
              "  'support',\n",
              "  'helped',\n",
              "  'transition',\n",
              "  'data',\n",
              "  'scientist',\n",
              "  'tech',\n",
              "  'mahindra',\n",
              "  'salary',\n",
              "  'online',\n",
              "  'power',\n",
              "  'learning',\n",
              "  'powered'],\n",
              " ['kunal',\n",
              "  'credit',\n",
              "  'card',\n",
              "  'bill',\n",
              "  'payment',\n",
              "  'gave',\n",
              "  'users',\n",
              "  'chance',\n",
              "  'win',\n",
              "  'free',\n",
              "  'food',\n",
              "  'swiggy',\n",
              "  'one',\n",
              "  'pranav',\n",
              "  'delhi',\n",
              "  'bagged',\n",
              "  'reward',\n",
              "  'spending',\n",
              "  'cred',\n",
              "  'users',\n",
              "  'get',\n",
              "  'one',\n",
              "  'cred',\n",
              "  'coin',\n",
              "  'per',\n",
              "  'rupee',\n",
              "  'bill',\n",
              "  'used',\n",
              "  'avail',\n",
              "  'rewards',\n",
              "  'brands',\n",
              "  'like'],\n",
              " ['new',\n",
              "  'zealand',\n",
              "  'defeated',\n",
              "  'india',\n",
              "  'wickets',\n",
              "  'fourth',\n",
              "  'odi',\n",
              "  'hamilton',\n",
              "  'thursday',\n",
              "  'win',\n",
              "  'first',\n",
              "  'match',\n",
              "  'odi',\n",
              "  'india',\n",
              "  'lost',\n",
              "  'international',\n",
              "  'match',\n",
              "  'rohit',\n",
              "  'captaincy',\n",
              "  'consecutive',\n",
              "  'victories',\n",
              "  'dating',\n",
              "  'back',\n",
              "  'march',\n",
              "  'match',\n",
              "  'witnessed',\n",
              "  'india',\n",
              "  'getting',\n",
              "  'seventh',\n",
              "  'lowest',\n",
              "  'total',\n",
              "  'odi',\n",
              "  'cricket'],\n",
              " ['aegon',\n",
              "  'life',\n",
              "  'iterm',\n",
              "  'insurance',\n",
              "  'customers',\n",
              "  'enjoy',\n",
              "  'tax',\n",
              "  'benefits',\n",
              "  'premiums',\n",
              "  'paid',\n",
              "  'save',\n",
              "  'plan',\n",
              "  'provides',\n",
              "  'life',\n",
              "  'cover',\n",
              "  'age',\n",
              "  'customers',\n",
              "  'options',\n",
              "  'insure',\n",
              "  'critical',\n",
              "  'disability',\n",
              "  'accidental',\n",
              "  'death',\n",
              "  'benefit',\n",
              "  'rider',\n",
              "  'life',\n",
              "  'cover',\n",
              "  'age'],\n",
              " ['speaking',\n",
              "  'sexual',\n",
              "  'harassment',\n",
              "  'allegations',\n",
              "  'rajkumar',\n",
              "  'sonam',\n",
              "  'kapoor',\n",
              "  'known',\n",
              "  'hirani',\n",
              "  'many',\n",
              "  'movement',\n",
              "  'get',\n",
              "  'always',\n",
              "  'believe',\n",
              "  'need',\n",
              "  'reserve',\n",
              "  'hirani',\n",
              "  'accused',\n",
              "  'assistant',\n",
              "  'worked']]"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## sort basis the word\n",
        "source_vocab = {actual_word: idx for idx, (word_num, actual_word) in enumerate(sorted(enumerate(set(token for tokens in all_text for token in tokens)), key=lambda x: x[1]))}\n",
        "target_vocab = {actual_word: idx for idx, (word_num, actual_word) in enumerate(sorted(enumerate(set(token for tokens in all_text for token in tokens)), key=lambda x: x[1]))}"
      ],
      "metadata": {
        "id": "B9cnSSz0rtZ8"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class CustomDataset(Dataset):\n",
        "    def __init__(self, source_texts, target_summaries, source_vocab, target_vocab):\n",
        "        self.source_texts = source_texts\n",
        "        self.target_summaries = target_summaries\n",
        "        self.source_vocab = source_vocab\n",
        "        self.target_vocab = target_vocab\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.source_texts)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        source_text = [self.source_vocab[word] for word in self.source_texts[idx]]\n",
        "        target_summary = [self.target_vocab[word] for word in self.target_summaries[idx]]\n",
        "        return torch.tensor(source_text), torch.tensor(target_summary)"
      ],
      "metadata": {
        "id": "F3n86JpxzaVp"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = CustomDataset(train_df['source_text'].tolist(), train_df['target_text'].tolist(),source_vocab, target_vocab)\n",
        "test_dataset = CustomDataset(test_df['source_text'].tolist(), test_df['target_text'].tolist(),source_vocab, target_vocab)"
      ],
      "metadata": {
        "id": "ZyJo2fs7UPNu"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_max_seqlen():\n",
        "    max_length = 0\n",
        "    for index, row in train_df.iterrows():\n",
        "        # Calculate the length of the current row\n",
        "        row_length = len(row['source_text'])\n",
        "        # Update the maximum length if the current row length is greater\n",
        "        max_length = max(max_length, row_length)\n",
        "    for index, row in test_df.iterrows():\n",
        "        # Calculate the length of the current row\n",
        "        row_length = len(row['source_text'])\n",
        "        # Update the maximum length if the current row length is greater\n",
        "        max_length = max(max_length, row_length)\n",
        "    print(\"Max length in dataset \",max_length)\n",
        "    return max_length"
      ],
      "metadata": {
        "id": "IT6LMCVXUll2"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def collate_fn(batch):\n",
        "    sources, targets = zip(*batch)\n",
        "    padded_sources = pad_sequence(sources, batch_first=True)\n",
        "    padded_targets = pad_sequence(targets, batch_first=True)\n",
        "    return padded_sources, padded_targets"
      ],
      "metadata": {
        "id": "1xoDeNNzU1xT"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class MultiHeadAttention(nn.Module):\n",
        "    def __init__(self, embedding_dim, num_heads):\n",
        "        super(MultiHeadAttention,self).__init__()\n",
        "        assert embedding_dim % num_heads == 0, \"embedding_dim must be divisible by num_heads\"\n",
        "\n",
        "        self.embedding_dim = embedding_dim\n",
        "        self.num_heads = num_heads\n",
        "        self.dim_perhead = embedding_dim // num_heads\n",
        "\n",
        "        self.W_q = nn.Linear(embedding_dim, embedding_dim)\n",
        "        self.W_k = nn.Linear(embedding_dim, embedding_dim)\n",
        "        self.W_v = nn.Linear(embedding_dim, embedding_dim)\n",
        "        self.W_o = nn.Linear(embedding_dim, embedding_dim)\n",
        "\n",
        "    def scaled_dot_product_attention(self,Q,K,V,mask=None):\n",
        "        # Q,K,V Shape : [Batch_Size X Num_Heads X Seq_len X Dim Per Head]\n",
        "\n",
        "        K = K.transpose(-2,-1) # K = K.permute(0,1,3,2) also works\n",
        "        # K Shape(after permute) : [Batch_Size X Num_Heads X Dim Per Head X Seq_len]\n",
        "        attn_scores = torch.matmul(Q,K) / math.sqrt(self.dim_perhead)\n",
        "        # attn_scores Shape : [Batch_Size X Num_Heads X Seq_len X Seq_len]\n",
        "        if mask is not None:\n",
        "            attn_scores = attn_scores.masked_fill(mask == 0, -1e9)\n",
        "        attn_probs = torch.softmax(attn_scores, dim=-1)\n",
        "        # attn_probs Shape : [Batch_Size X Num_Heads X Seq_len X Seq_len]\n",
        "        output = torch.matmul(attn_probs, V)\n",
        "        # output Shape : [Batch_Size X Num_Heads X Seq_len X Dim Per Head]\n",
        "        return output\n",
        "\n",
        "    def split_heads(self, x):\n",
        "        # X shape : [Batch_Size X Seq_len X Embedding Dim]\n",
        "        batch_size, seq_length, d_model = x.size()\n",
        "        x = x.view(batch_size, seq_length,self.num_heads,self.dim_perhead)\n",
        "        # X shape : [Batch_Size X Seq_len X Num_Heads X Dim Per Head]\n",
        "        x = x.transpose(1,2)\n",
        "        # X shape : [Batch_Size X Num_Heads X Seq_len X Dim Per Head]\n",
        "        return x\n",
        "\n",
        "    def combine_heads(self, x):\n",
        "        # x Shape : [Batch_Size X Num_Heads X Seq_len X Dim Per Head]\n",
        "        batch_size, _, seq_length, dim_perhead = x.size()\n",
        "        x = x.transpose(1,2).contiguous()\n",
        "        # x Shape : [Batch_Size X Seq_len X Num_Heads X Dim Per Head]\n",
        "        x = x.view(batch_size, seq_length,self.embedding_dim)\n",
        "        # x Shape : [Batch_Size X Seq_len X Embedding Dim]\n",
        "        return x\n",
        "\n",
        "    def forward(self, Q, K, V, mask=None):\n",
        "        # Q,K,V Shape : [Batch_Size X Seq_len X Embedding Dim]\n",
        "        Q = self.split_heads(self.W_q(Q))\n",
        "        K = self.split_heads(self.W_k(K))\n",
        "        V = self.split_heads(self.W_v(V))\n",
        "        # Q,K,V Shape : [Batch_Size X Num_Heads X Seq_len X Dim Per Head]\n",
        "        attn_output = self.scaled_dot_product_attention(Q, K, V, mask)\n",
        "        # attn_output Shape : [Batch_Size X Num_Heads X Seq_len X Dim Per Head]\n",
        "        output = self.W_o(self.combine_heads(attn_output))\n",
        "        # output Shape :  # x Shape : [Batch_Size X Seq_len X Embedding Dim]\n",
        "        return output"
      ],
      "metadata": {
        "id": "_y5ys_GaVQoW"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class PositionWiseFeedForward(nn.Module):\n",
        "    def __init__(self, d_model, d_ff):\n",
        "        super(PositionWiseFeedForward, self).__init__()\n",
        "        self.fc1 = nn.Linear(d_model, d_ff)\n",
        "        self.fc2 = nn.Linear(d_ff, d_model)\n",
        "\n",
        "    def forward(self, x):\n",
        "        # shape does not change here\n",
        "        return self.fc2(F.relu(self.fc1(x)))\n",
        "\n",
        "\n",
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self, d_model, max_seq_length):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "\n",
        "        pe = torch.zeros(max_seq_length, d_model)\n",
        "        position = torch.arange(0, max_seq_length,dtype=torch.float).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, d_model,2).float() * -(math.log(10000.0) / d_model))\n",
        "\n",
        "        pe[:, 0::2] = torch.sin(position * div_term)\n",
        "        pe[:, 1::2] = torch.cos(position * div_term)\n",
        "\n",
        "        self.register_buffer('pe',pe.unsqueeze(0))\n",
        "\n",
        "    def forward(self, x):\n",
        "        # shape does not change here, adding positional encoding information\n",
        "        return x + self.pe[:, :x.size(1)]"
      ],
      "metadata": {
        "id": "xnSVSll6VZd0"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class EncoderLayer(nn.Module):\n",
        "    def __init__(self, d_model, num_heads, d_ff, dropout):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        self.self_attn = MultiHeadAttention(d_model, num_heads)\n",
        "        self.feed_forward = PositionWiseFeedForward(d_model, d_ff)\n",
        "        self.norm1 = nn.LayerNorm(d_model)\n",
        "        self.norm2 = nn.LayerNorm(d_model)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "        # x shape [Batch_Size X Seq_len X Embedding Dim]\n",
        "        attn_output = self.self_attn(x, x, x,mask)\n",
        "        # attn_output shape [Batch_Size X Seq_len X Embedding Dim]\n",
        "        x = self.norm1(x + self.dropout(attn_output))\n",
        "        # x shape [Batch_Size X Seq_len X Embedding Dim]\n",
        "        ff_output = self.feed_forward(x)\n",
        "        # ff_output shape [Batch_Size X Seq_len X Embedding Dim]\n",
        "        x = self.norm2(x + self.dropout(ff_output))\n",
        "        # x shape [Batch_Size X Seq_len X Embedding Dim]\n",
        "        return x\n",
        "\n",
        "\n",
        "class DecoderLayer(nn.Module):\n",
        "    def __init__(self, d_model, num_heads, d_ff, dropout):\n",
        "        super(DecoderLayer, self).__init__()\n",
        "        self.self_attn = MultiHeadAttention(d_model, num_heads)\n",
        "        self.cross_attn = MultiHeadAttention(d_model, num_heads)\n",
        "        self.feed_forward = PositionWiseFeedForward(d_model, d_ff)\n",
        "        self.norm1 = nn.LayerNorm(d_model)\n",
        "        self.norm2 = nn.LayerNorm(d_model)\n",
        "        self.norm3 = nn.LayerNorm(d_model)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, x, enc_output, src_mask, tgt_mask):\n",
        "        attn_output = self.self_attn(x, x, x,tgt_mask)\n",
        "        x = self.norm1(x + self.dropout(attn_output))\n",
        "        attn_output = self.cross_attn(x,enc_output,enc_output,src_mask)\n",
        "        x = self.norm2(x + self.dropout(attn_output))\n",
        "        ff_output = self.feed_forward(x)\n",
        "        x = self.norm3(x + self.dropout(ff_output))\n",
        "        return x"
      ],
      "metadata": {
        "id": "3_yQ12J_WyoM"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Transformer(nn.Module):\n",
        "    def __init__(self, src_vocab_size, tgt_vocab_size, d_model, num_heads, num_layers, d_ff, max_seq_length, dropout):\n",
        "        super(Transformer, self).__init__()\n",
        "        self.encoder_embedding = nn.Embedding(src_vocab_size, d_model)\n",
        "        self.decoder_embedding = nn.Embedding(tgt_vocab_size, d_model)\n",
        "        self.positional_encoding = PositionalEncoding(d_model, max_seq_length)\n",
        "\n",
        "        self.encoder_layers = nn.ModuleList([EncoderLayer(d_model, num_heads, d_ff, dropout) for _ in range(num_layers)])\n",
        "        self.decoder_layers = nn.ModuleList([DecoderLayer(d_model, num_heads, d_ff, dropout) for _ in range(num_layers)])\n",
        "\n",
        "        self.fc = nn.Linear(d_model, tgt_vocab_size)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def generate_mask(self, src, tgt):\n",
        "        src_mask = (src != 0).unsqueeze(1).unsqueeze(2)\n",
        "        tgt_mask = (tgt != 0).unsqueeze(1).unsqueeze(3)\n",
        "        seq_length = tgt.size(1)\n",
        "        nopeak_mask = (1 - torch.triu(torch.ones(1, seq_length, seq_length,device=device), diagonal=1)).bool()\n",
        "        tgt_mask = tgt_mask & nopeak_mask\n",
        "        return src_mask, tgt_mask\n",
        "\n",
        "    def forward(self, src, tgt):\n",
        "        src_mask, tgt_mask = self.generate_mask(src, tgt)\n",
        "        src_embedded = self.dropout(self.positional_encoding(self.encoder_embedding(src)))\n",
        "        tgt_embedded = self.dropout(self.positional_encoding(self.decoder_embedding(tgt)))\n",
        "\n",
        "        enc_output = src_embedded\n",
        "        for enc_layer in self.encoder_layers:\n",
        "            enc_output = enc_layer(enc_output, src_mask)\n",
        "\n",
        "        dec_output = tgt_embedded\n",
        "        for dec_layer in self.decoder_layers:\n",
        "            dec_output = dec_layer(dec_output, enc_output, src_mask, tgt_mask)\n",
        "\n",
        "        output = self.fc(dec_output)\n",
        "        return output"
      ],
      "metadata": {
        "id": "p3b5lf-PW6qt"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "src_vocab_size = len(source_vocab)\n",
        "tgt_vocab_size = len(target_vocab)\n",
        "input_dim = 256\n",
        "num_heads = 4\n",
        "num_layers = 6\n",
        "d_ff = 2048\n",
        "max_seq_length = get_max_seqlen()\n",
        "dropout = 0.05\n",
        "num_workers = 2\n",
        "num_epochs = 15\n",
        "\n",
        "model = Transformer(src_vocab_size, tgt_vocab_size, input_dim, num_heads, num_layers, d_ff, max_seq_length, dropout)\n",
        "print(model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RX_zUplSXCcW",
        "outputId": "0822ea96-7f76-4717-dd0e-64caab44add5"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Max length in dataset  46\n",
            "Transformer(\n",
            "  (encoder_embedding): Embedding(65916, 256)\n",
            "  (decoder_embedding): Embedding(65916, 256)\n",
            "  (positional_encoding): PositionalEncoding()\n",
            "  (encoder_layers): ModuleList(\n",
            "    (0-5): 6 x EncoderLayer(\n",
            "      (self_attn): MultiHeadAttention(\n",
            "        (W_q): Linear(in_features=256, out_features=256, bias=True)\n",
            "        (W_k): Linear(in_features=256, out_features=256, bias=True)\n",
            "        (W_v): Linear(in_features=256, out_features=256, bias=True)\n",
            "        (W_o): Linear(in_features=256, out_features=256, bias=True)\n",
            "      )\n",
            "      (feed_forward): PositionWiseFeedForward(\n",
            "        (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
            "        (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
            "      )\n",
            "      (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      (dropout): Dropout(p=0.05, inplace=False)\n",
            "    )\n",
            "  )\n",
            "  (decoder_layers): ModuleList(\n",
            "    (0-5): 6 x DecoderLayer(\n",
            "      (self_attn): MultiHeadAttention(\n",
            "        (W_q): Linear(in_features=256, out_features=256, bias=True)\n",
            "        (W_k): Linear(in_features=256, out_features=256, bias=True)\n",
            "        (W_v): Linear(in_features=256, out_features=256, bias=True)\n",
            "        (W_o): Linear(in_features=256, out_features=256, bias=True)\n",
            "      )\n",
            "      (cross_attn): MultiHeadAttention(\n",
            "        (W_q): Linear(in_features=256, out_features=256, bias=True)\n",
            "        (W_k): Linear(in_features=256, out_features=256, bias=True)\n",
            "        (W_v): Linear(in_features=256, out_features=256, bias=True)\n",
            "        (W_o): Linear(in_features=256, out_features=256, bias=True)\n",
            "      )\n",
            "      (feed_forward): PositionWiseFeedForward(\n",
            "        (fc1): Linear(in_features=256, out_features=2048, bias=True)\n",
            "        (fc2): Linear(in_features=2048, out_features=256, bias=True)\n",
            "      )\n",
            "      (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      (norm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
            "      (dropout): Dropout(p=0.05, inplace=False)\n",
            "    )\n",
            "  )\n",
            "  (fc): Linear(in_features=256, out_features=65916, bias=True)\n",
            "  (dropout): Dropout(p=0.05, inplace=False)\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "print(trainable_params)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wEiKL1JaXQqA",
        "outputId": "bb794c9a-724f-41c7-8e71-51888e72a218"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "68052348\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "criterion = nn.CrossEntropyLoss(ignore_index=0)\n",
        "optimizer = optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.98), eps=1e-9)\n",
        "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0)"
      ],
      "metadata": {
        "id": "bO7cSkA1XkzJ"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, collate_fn=collate_fn, num_workers=num_workers)\n",
        "test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False, collate_fn=collate_fn, num_workers=num_workers)"
      ],
      "metadata": {
        "id": "87QXOg2QXvTr"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "source_dummy,target_dummy = next(iter(train_loader))"
      ],
      "metadata": {
        "id": "V3M0Y40KXypz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(source_dummy.shape,target_dummy.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8BPMW8L5X1mi",
        "outputId": "b25b0c4a-524d-4f0d-8003-24f780c5d475"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([4, 34]) torch.Size([4, 8])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(source_dummy[1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F0PigYk6X4OC",
        "outputId": "e507876b-aa3b-4617-e613-e821443a6f82"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([29829, 64754, 32864, 50366, 42869, 59298, 11704, 56936, 14787, 42869,\n",
            "        59437, 56020,  3643, 17811, 51874, 58054, 44280, 14256, 53025,  1894,\n",
            "         1775, 56020, 37102, 21582, 28600, 48104, 24424, 42943, 21132, 65257,\n",
            "        60649, 14845, 42869, 59437])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred = model(source_dummy,target_dummy)\n",
        "print(y_pred.shape,target_dummy.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I0-921PiYFnt",
        "outputId": "c6e13262-e426-4e3b-96a8-28e4af89273d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([4, 8, 65916]) torch.Size([4, 8])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred = y_pred.reshape(-1,len(target_vocab))\n",
        "target_dummy = target_dummy.reshape(-1)\n",
        "print(y_pred.shape,target_dummy.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sSIS5zz7YINN",
        "outputId": "2ed87acf-b870-4e61-a23e-ef80f0c1c071"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([32, 65916]) torch.Size([32])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check accuracy function\n",
        "def check_accuracy(output,labels):\n",
        "    _ , predpos = output.max(1)\n",
        "    num_samples=len(labels)\n",
        "    num_correct=(predpos==labels).sum()\n",
        "    return (num_correct/num_samples)*100\n",
        "\n",
        "# Save checkpoint\n",
        "def save_checkpoint(state,filename='weights.pth.tar'):\n",
        "    print('Saving weights-->')\n",
        "    torch.save(state,filename)\n",
        "\n",
        "# Load checkpoint\n",
        "def load_checkpoint(checkpoint,model,optim):\n",
        "    print('Loading weights-->')\n",
        "    model.load_state_dict(checkpoint['state_dict'])\n",
        "    optim.load_state_dict(checkpoint['optimizer'])"
      ],
      "metadata": {
        "id": "QxkgItm1Y0cC"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train_loop(model,dataloader,loss_fun,optimizer,device):\n",
        "    model.train()\n",
        "    model.to(device)\n",
        "    min_loss = None\n",
        "    for epoch in range(num_epochs):\n",
        "        losses = []\n",
        "        accuracies = []\n",
        "        loop = tqdm(enumerate(dataloader), total=len(dataloader), leave=True)\n",
        "        for batch,(x,y) in loop:\n",
        "            # put on cuda\n",
        "            x = torch.tensor(x).to(device).long()\n",
        "            y = torch.tensor(y).to(device).long()\n",
        "            # x = x.to(device)\n",
        "            # y = y.to(device)\n",
        "\n",
        "            # forward pass\n",
        "            y_pred = model(x,y)\n",
        "\n",
        "            # calculate loss & accuracy\n",
        "            loss = loss_fun(y_pred.reshape(-1,len(target_vocab)),y.reshape(-1))\n",
        "            losses.append(loss.detach().item())\n",
        "\n",
        "            accuracy = check_accuracy(y_pred.reshape(-1,len(target_vocab)),y.reshape(-1))\n",
        "            accuracies.append(accuracy.detach().item())\n",
        "\n",
        "            # zero out prior gradients\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            # backprop\n",
        "            loss.backward()\n",
        "\n",
        "            # update weights\n",
        "            optimizer.step()\n",
        "            scheduler.step()\n",
        "\n",
        "            # Update TQDM progress bar\n",
        "            loop.set_description(f\"Epoch [{epoch}/{num_epochs}] \")\n",
        "            loop.set_postfix(loss=loss.detach().item(), accuracy=accuracy.detach().item())\n",
        "\n",
        "        moving_loss = sum(losses) / len(losses)\n",
        "        moving_accuracy = sum(accuracies) / len(accuracies)\n",
        "        checkpoint = {'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict()}\n",
        "        # Save check point\n",
        "        if min_loss == None:\n",
        "            min_loss = moving_loss\n",
        "            save_checkpoint(checkpoint)\n",
        "        elif moving_loss < min_loss:\n",
        "            min_loss = moving_loss\n",
        "            save_checkpoint(checkpoint)\n",
        "        print('Epoch {0} : Loss = {1} , Training Accuracy={2}'.format(epoch, moving_loss, moving_accuracy))"
      ],
      "metadata": {
        "id": "uKQGjW4cYlf3"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_loop(model,train_loader,criterion,optimizer,device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WYhpRZYSY9bL",
        "outputId": "afbff6e0-6ee5-4083-a713-81b055a919d5"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch [0/15] : 100%|██████████| 1536/1536 [02:33<00:00,  9.99it/s, accuracy=65.3, loss=0.742]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving weights-->\n",
            "Epoch 0 : Loss = 2.283001027535647 , Training Accuracy=53.45129730185727\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch [1/15] : 100%|██████████| 1536/1536 [02:36<00:00,  9.83it/s, accuracy=73.4, loss=1.15]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving weights-->\n",
            "Epoch 1 : Loss = 0.5832049241871573 , Training Accuracy=63.929689303040504\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch [2/15] : 100%|██████████| 1536/1536 [02:36<00:00,  9.81it/s, accuracy=65.7, loss=0.31]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving weights-->\n",
            "Epoch 2 : Loss = 0.5271809907959929 , Training Accuracy=64.25877660264571\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch [3/15] : 100%|██████████| 1536/1536 [02:36<00:00,  9.81it/s, accuracy=63.6, loss=0.717]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3 : Loss = 0.5708986710233148 , Training Accuracy=64.09659028549989\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch [4/15] : 100%|██████████| 1536/1536 [02:37<00:00,  9.77it/s, accuracy=62.6, loss=0.864]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4 : Loss = 0.6011921786315119 , Training Accuracy=64.07614346841972\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Epoch [5/15] : 100%|██████████| 1536/1536 [02:38<00:00,  9.72it/s, accuracy=74.4, loss=0.474]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5 : Loss = 0.6046253215851417 , Training Accuracy=64.02116981893778\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch [6/15] : 100%|██████████| 1536/1536 [02:39<00:00,  9.66it/s, accuracy=72.7, loss=0.609]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6 : Loss = 0.6016832045182431 , Training Accuracy=64.0040827865402\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Epoch [7/15] : 100%|██████████| 1536/1536 [02:38<00:00,  9.70it/s, accuracy=61.1, loss=0.612]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7 : Loss = 0.5964318731275853 , Training Accuracy=64.13706595698993\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Epoch [8/15] : 100%|██████████| 1536/1536 [02:37<00:00,  9.77it/s, accuracy=76, loss=0.888]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8 : Loss = 0.5921243573150908 , Training Accuracy=64.09136859327555\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Epoch [9/15] : 100%|██████████| 1536/1536 [02:37<00:00,  9.77it/s, accuracy=68.2, loss=0.151]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9 : Loss = 0.5876103246603938 , Training Accuracy=64.05725449075301\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Epoch [10/15] : 100%|██████████| 1536/1536 [02:37<00:00,  9.78it/s, accuracy=72.7, loss=0.614]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10 : Loss = 0.5888285514277717 , Training Accuracy=64.05280249069135\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch [11/15] : 100%|██████████| 1536/1536 [02:37<00:00,  9.73it/s, accuracy=73.3, loss=0.614]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11 : Loss = 0.5859183813251244 , Training Accuracy=64.07891712834437\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Epoch [12/15] : 100%|██████████| 1536/1536 [02:37<00:00,  9.78it/s, accuracy=69.7, loss=0.391]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12 : Loss = 0.5899323651489491 , Training Accuracy=64.04612274716298\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch [13/15] : 100%|██████████| 1536/1536 [02:37<00:00,  9.78it/s, accuracy=77.3, loss=0.477]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13 : Loss = 0.5903360002848785 , Training Accuracy=64.10921336710453\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "Epoch [14/15] : 100%|██████████| 1536/1536 [02:36<00:00,  9.79it/s, accuracy=60.1, loss=0.631]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14 : Loss = 0.5937628798710648 , Training Accuracy=63.893871046602726\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Inference\n",
        "def test_loop(model,dataloader,loss_fun,device):\n",
        "    model.eval()\n",
        "    model.to(device)\n",
        "    losses = []\n",
        "    samples,correct = 0,0\n",
        "    loop = tqdm(enumerate(dataloader), total=len(dataloader), leave=True)\n",
        "    with torch.no_grad():\n",
        "        for batch,(x,y) in loop:\n",
        "            # put on cuda\n",
        "            x = torch.tensor(x).to(device).long()\n",
        "            y = torch.tensor(y).to(device).long()\n",
        "\n",
        "            # forward pass\n",
        "            y_pred = model(x,y)\n",
        "\n",
        "            # caclulate test loss\n",
        "            loss = loss_fun(y_pred.reshape(-1,len(target_vocab)),y.reshape(-1))\n",
        "            losses.append(loss.detach().item())\n",
        "\n",
        "            # accuracy over entire dataset\n",
        "            _,predpos=y_pred.reshape(-1,len(target_vocab)).max(1)\n",
        "            samples+=len(y.reshape(-1))\n",
        "            correct+=(predpos==y.reshape(-1)).sum().item()\n",
        "\n",
        "            # Update TQDM progress bar\n",
        "            loop.set_postfix(loss=loss.item())\n",
        "\n",
        "    print(\"Final Test Accuracy = \",100 * (correct/samples))\n"
      ],
      "metadata": {
        "id": "8_LOfKyDa-wg"
      },
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_loop(model,test_loader,criterion,device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WhvX7-qimmbZ",
        "outputId": "b70b1874-f317-4c67-aec8-38d4a733a086"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 71/71 [00:02<00:00, 32.91it/s, loss=0.696]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Final Test Accuracy =  65.29104022754979\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "10O4tKE3m8gi"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}