check_scores.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-12T10:25:28.329838Z",
     "start_time": "2022-11-12T10:25:27.482040Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score,f1_score,roc_auc_score,recall_score,precision_score\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "from nltk import word_tokenize\n",
    "import nltk\n",
    "from nltk.translate import meteor\n",
    "from nltk.translate.bleu_score import SmoothingFunction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-12T10:25:28.360518Z",
     "start_time": "2022-11-12T10:25:28.330961Z"
    }
   },
   "outputs": [],
   "source": [
    "from ekphrasis.classes.preprocessor import TextPreProcessor\n",
    "from ekphrasis.classes.tokenizer import SocialTokenizer\n",
    "from ekphrasis.dicts.emoticons import emoticons\n",
    "import numpy as np\n",
    "\n",
    "import re\n",
    "import io\n",
    "\n",
    "def processText(text):\n",
    "    text = re.sub(r\"\\S*https?:\\S*\", \"\", text)\n",
    "    #text = re.sub('<user>','',text)\n",
    "    #text = re.sub('<url>','',text)\n",
    "    text = re.sub('<.*?>','',text)\n",
    "    text = re.sub(r'[.!\"\\/<\\*>!@#$%^&*]', r'', text)\n",
    "    text = re.sub(\"^\\d+\\s|\\s\\d+\\s|\\s\\d+$\", '', text)\n",
    "    text = re.sub(' +', ' ', text)\n",
    "    _RE_COMBINE_WHITESPACE = re.compile(r\"(?a:\\s+)\")\n",
    "    _RE_STRIP_WHITESPACE = re.compile(r\"(?a:^\\s+|\\s+$)\")\n",
    "    text = _RE_COMBINE_WHITESPACE.sub(\" \", text)\n",
    "    text = _RE_STRIP_WHITESPACE.sub(\"\", text)\n",
    "    text = text.strip()\n",
    "    return text\n",
    "\n",
    "\n",
    "def remove_emojis(text):\n",
    "    emoj = re.compile(\"[\"\n",
    "        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "        u\"\\U00002500-\\U00002BEF\"  # chinese char\n",
    "        u\"\\U00002702-\\U000027B0\"\n",
    "        u\"\\U00002702-\\U000027B0\"\n",
    "        u\"\\U000024C2-\\U0001F251\"\n",
    "        u\"\\U0001f926-\\U0001f937\"\n",
    "        u\"\\U00010000-\\U0010ffff\"\n",
    "        u\"\\u2640-\\u2642\" \n",
    "        u\"\\u2600-\\u2B55\"\n",
    "        u\"\\u200d\"\n",
    "        u\"\\u23cf\"\n",
    "        u\"\\u23e9\"\n",
    "        u\"\\u231a\"\n",
    "        u\"\\ufe0f\" \n",
    "        u\"\\u3030\"\n",
    "                      \"]+\", re.UNICODE)\n",
    "    return re.sub(emoj, '', text)\n",
    "\n",
    "def preprocess(text):\n",
    "    text = remove_emojis(text)\n",
    "    text = processText(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-12T10:25:28.374423Z",
     "start_time": "2022-11-12T10:25:28.361672Z"
    }
   },
   "outputs": [],
   "source": [
    "def hate_refrences(data,test_set):          ###############returns pair of <hate,refrences>  \n",
    "    hate  = []\n",
    "    reply = []\n",
    "    refrences = []\n",
    "    for sample in data:\n",
    "        ht , rep = sample[0] , sample[1]\n",
    "        hate.append(ht)\n",
    "        reply.append(rep)\n",
    "    hate = list(set(hate))\n",
    "    mp={}\n",
    "    for ht_i in hate:\n",
    "        refs = []\n",
    "        for sample in data:\n",
    "            ht_j , rep =  sample[0] , sample[1]\n",
    "            if ht_j == ht_i:\n",
    "                refs.append(rep)\n",
    "        mp[ht_i] = refs\n",
    "        refrences.append(refs)\n",
    "    hate = list(set([x[0] for x in test_set]))\n",
    "    refs = [mp[ht_i] for ht_i in hate]\n",
    "    return hate,refs             # a given hate instance and refrences(replies) for metrics evaluation\n",
    "\n",
    "\n",
    "# In[7]:\n",
    "\n",
    "\n",
    "def training_corpus(train_set):    # returns training corpus\n",
    "    replies = []\n",
    "    for sample in train_set:\n",
    "        rep = sample[1]\n",
    "        replies.append(rep)\n",
    "    replies = list(set(replies))\n",
    "    return replies                # returns the sentences used while training \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def evaluate(params, model, test_dataloader, device):\n",
    "    # Loop to handle MNLI double evaluation (matched, mis-matched)\n",
    "    eval_loss = 0.0\n",
    "    nb_eval_steps = 0\n",
    "    model.eval()\n",
    "    for step, batch in tqdm(enumerate(test_dataloader), total=len(test_dataloader), desc=\"Evaluating\"):\n",
    "        inputs, labels = (batch[0], batch[0])\n",
    "        inputs = inputs.to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            outputs = model(inputs, labels=labels)\n",
    "            lm_loss = outputs[0]\n",
    "            eval_loss += lm_loss.mean().item()\n",
    "        nb_eval_steps += 1\n",
    "        \n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    perplexity = torch.exp(torch.tensor(eval_loss))\n",
    "    return perplexity\n",
    "\n",
    "\n",
    "###################################### BLEU_SCORE , METEOR #######################################\n",
    "def hate_refrences(data, test_set):          ###############returns pair of <hate,refrences>  \n",
    "    hate  = []\n",
    "    reply = []\n",
    "    refrences = []\n",
    "    for ind in data.index:\n",
    "        ht , rep = data['input_text'][ind] , data['target_text'][ind]\n",
    "        hate.append(ht)\n",
    "        reply.append(rep)\n",
    "    hate = list(set(hate))\n",
    "    mp={}\n",
    "    for ht_i in hate:\n",
    "        refs = []\n",
    "        for ind in data.index:\n",
    "            ht_j , rep =  data['input_text'][ind] , data['target_text'][ind]\n",
    "            if ht_j == ht_i:\n",
    "                refs.append(rep)\n",
    "        mp[ht_i] = refs\n",
    "        refrences.append(refs)\n",
    "    #hate = list(set([x[0] for x in test_set]))\n",
    "    #refs = [mp[ht_i] for ht_i in hate]\n",
    "    return hate, refrences   \n",
    "\n",
    "\n",
    "\n",
    "############################################ JACCARD SIMILARITY #################################\n",
    "def get_jaccard_sim(str1, str2):   \n",
    "    if isinstance(str1, float) or isinstance(str2, float):\n",
    "        return (-1)\n",
    "    try:\n",
    "        a = set(str1.split()) \n",
    "        b = set(str2.split())\n",
    "        c = a.intersection(b)\n",
    "        return float(len(c)) / (len(a) + len(b) - len(c))\n",
    "    except:\n",
    "        print((str1))\n",
    "        print(type(str2))\n",
    "        return 0\n",
    "\n",
    "\n",
    "############################################### NOVELTY #########################################\n",
    "def get_novelty(sent, training_corpus):\n",
    "    max_overlap = 0\n",
    "    for instance in training_corpus:\n",
    "        max_overlap = max(max_overlap,get_jaccard_sim(instance,sent))\n",
    "    return 1-max_overlap\n",
    "\n",
    "def avg_novelty(sentences,training_corpus):\n",
    "    avg = 0\n",
    "    for sent in sentences:\n",
    "        avg += get_novelty(sent,training_corpus)\n",
    "    avg = (avg/float(len(sentences)))\n",
    "    return avg\n",
    "\n",
    "\n",
    "\n",
    "############################################### DIVERSITY ########################################\n",
    "def get_diversity(sentences):\n",
    "    avg = 0.0\n",
    "    for i in range(len(sentences)):\n",
    "        max_overlap = 0\n",
    "        for j in range(len(sentences)):\n",
    "            if i!=j:\n",
    "                max_overlap = max(max_overlap,get_jaccard_sim(sentences[i],sentences[j]))\n",
    "        avg = avg + (1-max_overlap)\n",
    "    avg = (avg/len(sentences))\n",
    "    return avg, len(sentences)\n",
    "    \n",
    "def diversity_and_novelty(training_corpus, gen_replies):\n",
    "    diversity = get_diversity(gen_replies)\n",
    "    novelty   = 0#avg_novelty(gen_replies,training_corpus)\n",
    "    return diversity,novelty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-12T10:36:09.734823Z",
     "start_time": "2022-11-12T10:36:09.324262Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# data_train=data_final/Exp3A/Hindi/Bengali/bengali2hindi_train_pairs.csv\n",
    "# data_val=data_final/Exp3A/Hindi/Bengali/bengali2hindi_val_pairs.csv\n",
    "# data_test=data_final/Exp3A/Hindi/Bengali/hindi_test_pairs.csv\n",
    "# data_pred=outputs/Exp3A/Hindi/Bengali/counter_mbart_bengali2hindi.csv\n",
    "#HateAlert_Folder/JointDir/Saurabh/outputs/Exp3B/Bengali/English/bloom/counter_bloom_english2bengali_0.txt\n",
    "\n",
    "df_train = pd.read_csv('/home/mithundas/HateAlert_Folder/JointDir/Saurabh/data_final/Exp4/joint_train_pairs.csv',\n",
    "                       lineterminator='\\n')\n",
    "df_test = pd.read_csv('/home/mithundas/HateAlert_Folder/JointDir/Saurabh/data_final/Exp4/hindi_test_pairs.csv',\n",
    "                      lineterminator='\\n')\n",
    "df_pred = pd.read_csv('/home/mithundas/HateAlert_Folder/JointDir/Saurabh/outputs/Exp4/counter_bloom_hindi_joint.csv',\n",
    "                      lineterminator='\\n')   \n",
    "df_pred = df_pred.fillna('')\n",
    "\n",
    "\n",
    "for ind in df_pred.index:\n",
    "        df_pred['input_text'][ind] =  preprocess(df_pred['input_text'][ind])\n",
    "        df_pred['predicted_text'][ind] =  preprocess(df_pred['predicted_text'][ind])\n",
    "        \n",
    "for ind in df_train.index:\n",
    "        df_train['input_text'][ind] =  preprocess(df_train['input_text'][ind])\n",
    "        df_train['target_text'][ind] =  preprocess(df_train['target_text'][ind])\n",
    "\n",
    "for ind in df_test.index:\n",
    "        df_test['input_text'][ind] =  preprocess(df_test['input_text'][ind])\n",
    "        df_test['target_text'][ind] =  preprocess(df_test['target_text'][ind])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-12T10:38:19.006753Z",
     "start_time": "2022-11-12T10:36:09.736068Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Diversity Scores\n",
      "Input pred:  (0.0, 524)\n",
      "Predicted pred:  (0.007134345666386647, 524)\n",
      "Novelty Score between predicted and test counters :  0.593115636672502\n"
     ]
    }
   ],
   "source": [
    "## Diversity Scores\n",
    "print(\"Diversity Scores\")\n",
    "# print(\"Input train: \", get_diversity(df_train['input_text']))\n",
    "# print(\"Target train: \", get_diversity(df_train['target_text']))\n",
    "\n",
    "# print(\"Input test: \", get_diversity(df_test['input_text']))\n",
    "# print(\"Target test: \", get_diversity(df_test['target_text']))\n",
    "\n",
    "print(\"Input pred: \", get_diversity(df_pred['input_text']))\n",
    "print(\"Predicted pred: \", get_diversity(df_pred['predicted_text']))\n",
    "\n",
    "\n",
    "## Novelty Scores\n",
    "# print(\"Novelty Scores\")\n",
    "# print(avg_novelty(df_train['input_text'], df_train['input_text']), avg_novelty(df_train['input_text'], df_train['target_text']))\n",
    "# print(avg_novelty(df_train['input_text'], df_test['input_text']), avg_novelty(df_train['input_text'], df_test['target_text']))\n",
    "# print(avg_novelty(df_train['input_text'], df_pred['input_text']), avg_novelty(df_train['input_text'], df_pred['predicted_text']))\n",
    "\n",
    "# print(avg_novelty(df_train['target_text'], df_train['input_text']), avg_novelty(df_train['target_text'], df_train['target_text']))\n",
    "# print(avg_novelty(df_train['target_text'], df_test['input_text']), avg_novelty(df_train['target_text'], df_test['target_text']))\n",
    "# print(avg_novelty(df_train['target_text'], df_pred['input_text']), avg_novelty(df_train['target_text'], df_pred['predicted_text']))\n",
    "\n",
    "# print(avg_novelty(df_test['input_text'], df_train['input_text']), avg_novelty(df_test['input_text'], df_train['target_text']))\n",
    "# print(avg_novelty(df_test['input_text'], df_test['input_text']), avg_novelty(df_test['input_text'], df_test['target_text']))\n",
    "# print(avg_novelty(df_test['input_text'], df_pred['input_text']), avg_novelty(df_test['input_text'], df_pred['predicted_text']))\n",
    "\n",
    "# print(avg_novelty(df_test['target_text'], df_train['input_text']), avg_novelty(df_test['target_text'], df_train['target_text']))\n",
    "# print(avg_novelty(df_test['target_text'], df_test['input_text']), avg_novelty(df_test['target_text'], df_test['target_text']))\n",
    "# print(avg_novelty(df_test['target_text'], df_pred['input_text']), avg_novelty(df_test['target_text'], df_pred['predicted_text']))\n",
    "\n",
    "# print(avg_novelty(df_pred['predicted_text'], df_train['input_text']), avg_novelty(df_pred['predicted_text'], df_train['target_text']))\n",
    "# print(avg_novelty(df_pred['predicted_text'], df_test['input_text']), avg_novelty(df_pred['predicted_text'], df_test['target_text']))\n",
    "# print(avg_novelty(df_pred['predicted_text'], df_pred['input_text']), avg_novelty(df_pred['predicted_text'], df_pred['predicted_text']))\n",
    "\n",
    "\n",
    "print(\"Novelty Score between predicted and test counters : \",avg_novelty(df_pred['predicted_text'], df_test['target_text']))\n",
    "#print(, avg_novelty(df_pred['predicted_text'], df_train['target_text']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-12T10:38:30.089256Z",
     "start_time": "2022-11-12T10:38:19.007938Z"
    }
   },
   "outputs": [],
   "source": [
    "## bleu and meteor scores\n",
    "hate  = []\n",
    "reply = []\n",
    "refrences = []\n",
    "for ind in df_train.index:\n",
    "    ht , rep = df_train['input_text'][ind] , df_train['target_text'][ind]\n",
    "    hate.append(ht)\n",
    "    reply.append(rep)\n",
    "\n",
    "for ind in df_test.index:\n",
    "    ht , rep = df_test['input_text'][ind] , df_test['target_text'][ind]\n",
    "    hate.append(ht)\n",
    "    reply.append(rep)\n",
    "\n",
    "hate = list(set(hate))\n",
    "mp={}\n",
    "\n",
    "for ht_i in hate:\n",
    "    refs = []\n",
    "    for ind in df_train.index:\n",
    "        ht_j , rep =  df_train['input_text'][ind] , df_train['target_text'][ind]\n",
    "        if ht_j == ht_i:\n",
    "            refs.append(rep)\n",
    "    for ind in df_test.index:\n",
    "        ht_j , rep =  df_test['input_text'][ind] , df_test['target_text'][ind]\n",
    "        if ht_j == ht_i:\n",
    "            refs.append(rep)\n",
    "    mp[ht_i] = refs\n",
    "    refrences.append(refs)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-12T10:38:30.093159Z",
     "start_time": "2022-11-12T10:38:30.090371Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/mithundas/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /home/mithundas/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     /home/mithundas/nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('omw-1.4')\n",
    "\n",
    "from nltk.translate.meteor_score import meteor_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-12T10:38:31.661748Z",
     "start_time": "2022-11-12T10:38:30.094164Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bleu 1 :  0.12589624856901427\n",
      "Bleu 2 :  0.08969732664188472\n",
      "Bleu 3 :  0.07599577814823956\n",
      "Bleu 4 :  0.06905791954426893\n",
      "Meteor Score 0.07370179672461374\n"
     ]
    }
   ],
   "source": [
    "bleu = bleu_2 = bleu_1 = bleu_3 = bleu_4 = meteor_ = 0.0\n",
    "\n",
    "for ind in df_pred.index:\n",
    "    hates = df_pred['input_text'][ind]\n",
    "    counters = df_pred['predicted_text'][ind]\n",
    "    ref = mp[hates]\n",
    "\n",
    "    ref_list = []\n",
    "    for i in range(len(ref)):\n",
    "        ref_list.append(word_tokenize(ref[i]))\n",
    "    bleu += nltk.translate.bleu_score.sentence_bleu(ref_list, word_tokenize(counters))\n",
    "    bleu_1  += nltk.translate.bleu_score.sentence_bleu(ref_list, word_tokenize(counters), smoothing_function=SmoothingFunction().method2, weights=(1.0, 0, 0, 0))\n",
    "    bleu_2  += nltk.translate.bleu_score.sentence_bleu(ref_list, word_tokenize(counters), smoothing_function=SmoothingFunction().method2, weights=(0.5, 0.5, 0, 0))\n",
    "    bleu_3  += nltk.translate.bleu_score.sentence_bleu(ref_list, word_tokenize(counters), smoothing_function=SmoothingFunction().method2, weights=(0.33, 0.33, 0.33, 0))\n",
    "    bleu_4  += nltk.translate.bleu_score.sentence_bleu(ref_list, word_tokenize(counters), smoothing_function=SmoothingFunction().method2, weights=(0.25, 0.25, 0.25, 0.25))\n",
    "    meteor_ += meteor_score(ref_list, word_tokenize(counters))\n",
    "\n",
    "bleu    /= len(df_pred)\n",
    "bleu_2  /= len(df_pred)\n",
    "bleu_1  /= len(df_pred)\n",
    "bleu_3  /= len(df_pred)\n",
    "bleu_4  /= len(df_pred)\n",
    "meteor_ /= len(df_pred)\n",
    "\n",
    "#print(\"Bleu Score \", bleu)\n",
    "\n",
    "print(\"Bleu 1 : \", bleu_1)\n",
    "print(\"Bleu 2 : \", bleu_2)\n",
    "print(\"Bleu 3 : \", bleu_3)\n",
    "print(\"Bleu 4 : \", bleu_4)\n",
    "print(\"Meteor Score\", meteor_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-12T10:38:32.158987Z",
     "start_time": "2022-11-12T10:38:31.662654Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  0.11107068199723155\n",
      "Recall:  0.2975581526972649\n",
      "F-score:  0.1617604248541627\n"
     ]
    }
   ],
   "source": [
    "def rec(str1, str2):\n",
    "    match = 0.0\n",
    "    tok1 = word_tokenize(str1)\n",
    "    tok2 = word_tokenize(str2)\n",
    "    if(len(tok1)==0 or len(tok2)==0):\n",
    "        return -999\n",
    "    for i in tok1:\n",
    "        for j in tok2:\n",
    "            if i == j:\n",
    "                match += 1.0\n",
    "                break;\n",
    "    return match/len(tok1)\n",
    "\n",
    "def rec2(str1, str2):\n",
    "    match = 0.0\n",
    "    tok1 = word_tokenize(str1)\n",
    "    tok2 = word_tokenize(str2)\n",
    "    for i in tok2:\n",
    "        for j in tok1:\n",
    "            if i == j:\n",
    "                match += 1.0\n",
    "                break;\n",
    "    return match/len(tok2)\n",
    "\n",
    "recall = 0.0\n",
    "\n",
    "for ind in df_pred.index:\n",
    "    recall2 = 0.0\n",
    "    hates = df_pred['input_text'][ind]\n",
    "    counters = df_pred['predicted_text'][ind]\n",
    "    ref = mp[hates]\n",
    "\n",
    "    for i in range(len(ref)):\n",
    "        recall2 = max(recall2, rec(counters, ref[i]))\n",
    "        #print(recall2)\n",
    "    \n",
    "    recall += recall2\n",
    "\n",
    "recall    /= len(df_pred)\n",
    "\n",
    "\n",
    "\n",
    "precision = 0.0\n",
    "\n",
    "for ind in df_pred.index:\n",
    "    recall2 = 0.0\n",
    "    hates = df_pred['input_text'][ind]\n",
    "    counters = df_pred['predicted_text'][ind]\n",
    "    ref = mp[hates]\n",
    "\n",
    "    for i in range(len(ref)):\n",
    "        recall2 = max(recall2, rec2(counters, ref[i]))\n",
    "        #print(recall2)\n",
    "    \n",
    "    precision += recall2\n",
    "\n",
    "precision    /= len(df_pred)\n",
    "\n",
    "print(\"Precision: \", precision)\n",
    "print(\"Recall: \", recall)\n",
    "print(\"F-score: \", 2*precision*recall/(precision+recall))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_diversity(sentences):\n",
    "    avg = 0.0\n",
    "    for i in range(len(sentences)):\n",
    "        max_overlap = 0\n",
    "        for j in range(len(sentences)):\n",
    "            if i!=j and sentences[i]!=sentences[j]:\n",
    "                #print(sentences[i])\n",
    "                #print()\n",
    "                #print(sentences[j])\n",
    "                max_overlap = max(max_overlap,get_jaccard_sim(sentences[i],sentences[j]))\n",
    "                #print(max_overlap)\n",
    "        avg = avg + (1-max_overlap)\n",
    "    print(\"Avg : \",avg)\n",
    "    print(\"Len : \",len(sentences))\n",
    "    avg = (avg/len(sentences))\n",
    "    return avg, len(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Diversity Scores\n",
      "Avg :  1429.3393995128574\n",
      "Len :  1818\n",
      "Input train:  (0.7862152912611977, 1818)\n",
      "Avg :  1240.9902994597046\n",
      "Len :  1818\n",
      "Target train:  (0.6826129259954371, 1818)\n",
      "Avg :  440.8900923786468\n",
      "Len :  524\n",
      "Input test:  (0.8413933060661198, 524)\n",
      "Avg :  386.6260382663821\n",
      "Len :  524\n",
      "Target test:  (0.7378359508900422, 524)\n",
      "Avg :  440.8900923786468\n",
      "Len :  524\n",
      "Input pred:  (0.8413933060661198, 524)\n",
      "Avg :  88.9443055546406\n",
      "Len :  524\n",
      "Predicted pred:  (0.16974104113481028, 524)\n"
     ]
    }
   ],
   "source": [
    "## Diversity Scores\n",
    "print(\"Diversity Scores\")\n",
    "print(\"Input train: \", get_diversity(df_train['input_text']))\n",
    "print(\"Target train: \", get_diversity(df_train['target_text']))\n",
    "\n",
    "print(\"Input test: \", get_diversity(df_test['input_text']))\n",
    "print(\"Target test: \", get_diversity(df_test['target_text']))\n",
    "\n",
    "print(\"Input pred: \", get_diversity(df_pred['input_text']))\n",
    "print(\"Predicted pred: \", get_diversity(df_pred['predicted_text']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Input train:  (0.7862152912611977, 1818)\n",
    "Target train:  (0.6826129259954371, 1818)\n",
    "Input test:  (0.8413933060661198, 524)\n",
    "Target test:  (0.7378359508900422, 524)\n",
    "Input pred:  (0.8413933060661198, 524)\n",
    "Predicted pred:  (0.16974104113481028, 524)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_path='/home/mithun-binny/HateAlert_Folder/JointDir/Saurabh/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.load(base_path + 'saved_models/Exp2/mt5/mt5_hindi_large.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}