Main_Baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import csv\n",
    "import nltk\n",
    "from matplotlib import pyplot as plt\n",
    "import os\n",
    "import re\n",
    "import warnings\n",
    "import random\n",
    "warnings.filterwarnings('ignore')\n",
    "from collections import Counter,defaultdict\n",
    "from nltk import word_tokenize\n",
    "from math import log\n",
    "import string\n",
    "import pickle\n",
    "from math import log\n",
    "import gensim\n",
    "from gensim import corpora\n",
    "from gensim import models\n",
    "from string import digits\n",
    "from collections import defaultdict\n",
    "from scipy.spatial.distance import cosine\n",
    "import ast\n",
    "cwd = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "osname = os.name\n",
    "if osname =='Windows':\n",
    "    sym = \"//\"\n",
    "else:\n",
    "    sym = \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_path=cwd+sym+'nips_reviewer_data'+sym+'reviewers.txt'\n",
    "review_df=pd.read_csv(r_path,sep='\\t',header=None)\n",
    "review_df.columns=['sno','name']\n",
    "review_lt=list(review_df.name)\n",
    "review_sno={val:ind for ind,val in enumerate(review_lt)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Necessary files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [],
   "source": [
    "path1=cwd+sym+'Saved_Items'+sym\n",
    "with open(path1+'dictionary.pkl','rb') as f:\n",
    "    dictionary=pickle.load(f)\n",
    "with open(path1+'tfidf.pkl','rb') as f:\n",
    "    tfidf=pickle.load(f)\n",
    "with open(path1+'ldaModel250','rb') as f:\n",
    "    lda_model=pickle.load(f)\n",
    "with open(path1+'cocit.pkl','rb') as f:\n",
    "    cocit_mat=pickle.load(f)\n",
    "with open(path1+'bibcoupling.pkl','rb') as f:\n",
    "    bc_mat=pickle.load(f)\n",
    "with open(path1+'distance.pkl','rb') as f:\n",
    "    dist=pickle.load(f)\n",
    "with open(path1+'paper_vec2.pkl','rb') as f:\n",
    "    paper_vec2=pickle.load(f)\n",
    "with open(path1+'paper_id.pkl','rb') as f:\n",
    "    paper_id=pickle.load(f)\n",
    "with open(path1+'reviewer_vec.pkl','rb') as f:\n",
    "    reviewer_vec=pickle.load(f)\n",
    "with open(path1+'topic_community.pkl','rb') as f:\n",
    "    topic_comm=pickle.load(f)\n",
    "with open(path1+'paper_auth.pkl','rb') as f:\n",
    "    paper_auth=pickle.load(f)\n",
    "with open(path1+'reviewer_matrix.pkl','rb') as f:\n",
    "    reviewer_matrix=pickle.load(f)\n",
    "with open(path1+'reviewer_vec_bow.pkl','rb') as f:\n",
    "    reviewer_vec_bow=pickle.load(f)\n",
    "reviewer_detail=pd.read_pickle(path1+'auth_detail.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path1+'reviewer_matrix_temp.pkl','rb') as f:\n",
    "    reviewer_matrix=pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## New Paper's Title and Abstract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "title='Learning from Multiple Sources'\n",
    "abstract='''\n",
    "Categorization is a central activity of human cognition. When an individual is asked to categorize a sequence of items, context effects arise: categorization of one item influences category decisions for subsequent items. Specifically, when experimental subjects are shown an exemplar of some target category, the category prototype appears to be pulled toward the exemplar, and the prototypes of all nontarget categories appear to be pushed away. These push and pull effects diminish with experience, and likely reflect long-term learning of category boundaries. We propose and evaluate four principled probabilistic (Bayesian) accounts of context effects in categorization. In all four accounts, the probability of an exemplar given a category is encoded as a Gaussian density in feature space, and categorization involves computing category posteriors given an exemplar. The models differ in how the uncertainty distribution of category prototypes is represented (localist or distributed), and how it is updated following each experience (using a maximum likelihood gradient ascent, or a Kalman filter update). We find that the distributed maximum-likelihood model can explain the key experimental phenomena. Further, the model predicts other phenomena that were confirmed via reanalysis of the experimental data. Categorization is a central activity of human cognition. We continually make decisions about characteristics of objects and individuals: Is the fruit ripe? Does your friend seem unhappy? Is your car tire flat? Should this manuscript be accepted for publication in NIPS? When an individual is asked to categorize a sequence of items, context effects arise: categorization of one item influences category decisions for subsequent items. Intuitive naturalistic scenarios in which context effects occur are easy to imagine. For example, if one lifts a medium-weight object after lifting a light-weight or heavy-weight object, the medium weight feels heavier following the light weight than following the heavy weight. Although the object-contrast effect might be due to fatigue of sensory-motor systems, many context effects in categorization are purely cognitive and cannot easily be attributed to neural habituation. For example, if you are reviewing a set of conference papers, and the first three in the set are dreadful, then even a mediocre paper seems like it might be above threshold for acceptance. Another example of a category boundary shift due to context is the following. Suppose you move from San Diego to Pittsburgh and notice that your neighbors repeatedly describe muggy, somewhat overcast days as Ólovely.Ó Eventually, your notion of what constitutes a lovely day accommodates to your new surroundings. As we describe shortly, experimental studies have shown a fundamental link between context effects in categorization and long-term learning of category boundaries. We believe that context effects can be viewed as a reflection of a trial-to-trial learning, and the cumulative effect of these trial-to-trial modulations corresponds to what we classically consider to be category learning. Consequently, any compelling model of category learning should also be capable of explaining context effects. stimulus dimension ABCD example Figure 1: Schematic depiction of sequential effects in categorization \n",
    "'''\n",
    "text=title+abstract"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PreProcessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'groot'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords \n",
    "from nltk.stem import PorterStemmer\n",
    "st = set(stopwords.words('english'))\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "import re\n",
    "\n",
    "def cleantext(text):\n",
    "    \n",
    "    remove_digits = str.maketrans('', '', digits)\n",
    "    text = text.translate(remove_digits)\n",
    "    \n",
    "    # remove punctuation\n",
    "    table = str.maketrans(string.punctuation, ' '*len(string.punctuation))\n",
    "    strip = text.translate(table)\n",
    "    \n",
    "    \n",
    "    # Tokenizer\n",
    "    tokens = word_tokenize(strip)\n",
    "    \n",
    "    # Convert into lower case \n",
    "    proc_text = [w.lower() for w in tokens]\n",
    "    \n",
    "    # Remove stopwords\n",
    "    proc_text = [word for word in proc_text if word not in st]\n",
    "    \n",
    "    \n",
    "    #Storing only Lemmmatized words\n",
    "    lemmatizer=WordNetLemmatizer()\n",
    "    lemma_text=[lemmatizer.lemmatize(word) for word in proc_text]\n",
    "\n",
    "    return \" \".join(lemma_text)\n",
    "\n",
    "\n",
    "t = \"i am groot2'7\"    \n",
    "cleantext(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_column_slice(array_or_dataframe,coloumn):\n",
    "    return np.asarray(array_or_dataframe)[:, coloumn]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_topics=250"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Consolidated Functions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def topic_level(text):\n",
    "    # Loading saved Lda model,dictionary,tfidf vectorizer\n",
    "    \n",
    "    clean_text=list(word_tokenize(cleantext(text))) \n",
    "    bow_text=dictionary.doc2bow(clean_text)\n",
    "    tfidf_text=tfidf[bow_text]\n",
    "    topic=lda_model[tfidf_text]\n",
    "\n",
    "    topic_vec=np.zeros(num_topics)\n",
    "    for ind,wt in topic:\n",
    "        topic_vec[ind]=wt\n",
    "    \n",
    "    # Topic of interest and community\n",
    "    \n",
    "    no_top_topic=5\n",
    "    toi=[]#topic of interest\n",
    "    if len(topic)<no_top_topic:\n",
    "        for ind,wt in topic:\n",
    "            toi.append(ind)\n",
    "    else:\n",
    "        topic.sort(key=lambda x:x[1],reverse=True)\n",
    "        for i in range(no_top_topic):\n",
    "            toi.append(topic[i][0])\n",
    "\n",
    "    \n",
    "    toi_comm=[topic_comm[i] for i in toi]\n",
    "    toi_comm=set(toi_comm)\n",
    "    topic_rest=[]\n",
    "    for index,comm_no in enumerate(topic_comm):\n",
    "        if comm_no in toi_comm and not index in toi:\n",
    "            topic_rest.append(index)\n",
    "    \n",
    "    return topic_vec,toi,topic_rest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def paper_level(toi,topic_rest):\n",
    "    poi=[]#papers having maximuim content of toi\n",
    "    paper_vec_lt=list(paper_vec2.values())\n",
    "    for i in toi:\n",
    "        samp=get_column_slice(paper_vec_lt,i)\n",
    "        ind=np.argmax(np.array(samp))\n",
    "        poi.append(ind)\n",
    "    puc=[]#papers under consideration\n",
    "    for i in topic_rest:\n",
    "        topic_paper=get_column_slice(paper_vec_lt,i)\n",
    "        n1=len(np.where(topic_paper>0)[0])\n",
    "        thres=sum(topic_paper)/n1\n",
    "        ind_list=np.where(topic_paper>thres)[0]\n",
    "        for ele in ind_list:\n",
    "            puc.append(ele)\n",
    "    puc=list(set(puc))\n",
    "    \n",
    "    paper_scr=[]\n",
    "    for i in puc:\n",
    "        num=bc_mat[i]+cocit_mat[i]\n",
    "        den=min([dist[i][j] for j in poi])\n",
    "        if den==0:\n",
    "            scr=float('inf')\n",
    "        else:\n",
    "            scr=num/den\n",
    "        paper_scr.append((scr,i))\n",
    "    \n",
    "    scr=[scr for scr,ind in paper_scr]\n",
    "    paper_scr.sort(reverse=True)\n",
    "    for i in range(300):\n",
    "        poi.append(paper_scr[i][1])\n",
    "    val=np.percentile(scr,75)# taking into acccount only top 25% entries\n",
    "    potential_paper=[]\n",
    "    for score,ind in paper_scr:\n",
    "        if score>val:\n",
    "            potential_paper.append(ind)\n",
    "    \n",
    "    \n",
    "    return poi,potential_paper\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reviewer_pred(poi,potential_paper,topic_vec,text):\n",
    "    roi=[]\n",
    "    ct_roi=[]\n",
    "    for i in poi:\n",
    "        roi=roi+paper_auth[i]\n",
    "    #print(len(set(roi)))\n",
    "    rest_reviewer=[]\n",
    "    for i in potential_paper:\n",
    "        if not paper_auth[i] in roi:\n",
    "            rest_reviewer=rest_reviewer+paper_auth[i]\n",
    "    \n",
    "    \n",
    "    \n",
    "    npaper=list(reviewer_detail['pc'])\n",
    "    citn=list(reviewer_detail['cn'])\n",
    "    \n",
    "    clean_text=list(word_tokenize(cleantext(text))) \n",
    "    bow_text=dictionary.doc2bow(clean_text)\n",
    "    tfidf_text=tfidf[bow_text]\n",
    "    n=len(dictionary)\n",
    "\n",
    "    bow_vec=np.zeros(n, dtype = float)\n",
    "    for ind,wt in tfidf_text:\n",
    "        bow_vec[ind]=wt    \n",
    "    \n",
    "    dec_scr=[]\n",
    "    for i in roi:\n",
    "        sim=1-cosine(reviewer_vec[i],topic_vec)\n",
    "        bow_scr=1-cosine(reviewer_vec_bow[i],bow_vec)\n",
    "        if np.isnan(sim):\n",
    "            sim=0\n",
    "        if np.isnan(bow_scr):\n",
    "            bow_scr=0\n",
    "        #val=0.1*npaper[i]/max(npaper)+0.1*citn[i]/max(citn)+0.8*sim\n",
    "        val=0.6*sim+0.4*bow_scr\n",
    "        #val2=npaper[i]/max(npaper)+citn[i]/max(citn)\n",
    "        dec_scr.append((val,i))\n",
    "    dec_scr.sort(reverse=True)\n",
    "    start_node=dec_scr[0][1]\n",
    "    \n",
    "    n=len(review_lt)\n",
    "    total_node=set([i for i in range(n)])\n",
    "    invalid_node=total_node-set(roi)-set(rest_reviewer)\n",
    "    \n",
    "    custom_matrix=[]\n",
    "    for i in range(n):\n",
    "        temp=[]\n",
    "        for j in range(n):\n",
    "            temp.append(reviewer_matrix[i][j])\n",
    "        custom_matrix.append(np.array(temp))\n",
    "    for node in invalid_node:\n",
    "        for i in range(n):\n",
    "            custom_matrix[node][i]=0\n",
    "            custom_matrix[i][node]=0\n",
    "    col_sum=[sum(x) for x in zip(*custom_matrix)]\n",
    "\n",
    "    transition_prob=[]\n",
    "    for i in range(n):\n",
    "        op=[]\n",
    "        for j in range(n):\n",
    "            if col_sum[j]:\n",
    "                val=custom_matrix[i][j]/col_sum[j]\n",
    "            else:\n",
    "                val=0\n",
    "            op.append(val)\n",
    "        transition_prob.append(op)\n",
    "        \n",
    "    ini_state=[0]*n\n",
    "    ini_state[start_node]=1\n",
    "    len_op=10#No of items to be shown as output\n",
    "    new_state=ini_state\n",
    "    for i in range(len_op):\n",
    "        new_state=np.dot(0.85,np.dot(transition_prob,new_state))+np.dot(0.15,ini_state)\n",
    "        \n",
    "    op=new_state.argsort()[-10:][::-1]\n",
    "    #op=new_state.argsort()[::-1]\n",
    "    res=[ele+1 for ele in op]\n",
    "    \n",
    "    return res\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(text):\n",
    "    topic_vec,toi,topic_rest=topic_level(text)\n",
    "    poi,potential_paper=paper_level(toi,topic_rest)\n",
    "    pred=reviewer_pred(poi,potential_paper,topic_vec,text)\n",
    "    \n",
    "    return pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "path=path1+'Testing'+sym+'trec_judgments.txt'\n",
    "ordinal=[]\n",
    "value=[]\n",
    "judge={}\n",
    "with open(path,'r',encoding=\"utf8\")as f:\n",
    "    for line in f:\n",
    "        ordinal=list(line.split('\\t'))\n",
    "        value=[int(ordinal[2]),int(ordinal[3][:-1])]\n",
    "        if int(ordinal[0][4:]) in judge:\n",
    "            judge[int(ordinal[0][4:])].append(value)\n",
    "        else :\n",
    "            judge[int(ordinal[0][4:])]=[value]\n",
    "            \n",
    "path=path1+'Testing'+sym+'trec_judgments_additional.txt'\n",
    "with open(path,'r',encoding=\"utf8\")as f:\n",
    "    for line in f:\n",
    "        ordinal=list(line.split('\\t'))\n",
    "        value=[int(ordinal[2]),int(ordinal[3][:-1])]\n",
    "        if int(ordinal[0]) in judge:\n",
    "            judge[int(ordinal[0])].append(value)\n",
    "        else :\n",
    "            judge[int(ordinal[0])]=[value]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def precision_k(res,gnd_tht,k,paper_no):\n",
    "    actual=gnd_tht[paper_no]\n",
    "    actual_auth=[i[0] for i in actual]\n",
    "    rating=[i[1] for i in actual]\n",
    "    n=len(actual)\n",
    "    soft_p=0\n",
    "    for i in range(k):\n",
    "        pred=res[i]\n",
    "        for j in range(n):\n",
    "            if actual_auth[j]==pred:\n",
    "                if rating[j]==3:\n",
    "                    soft_p+=1\n",
    "                elif rating[j]==2:\n",
    "                    soft_p+=0.67\n",
    "                elif rating[j]==1:\n",
    "                    soft_p+=0.33\n",
    "                else:\n",
    "                    break\n",
    "    soft_p=soft_p/k\n",
    "    return soft_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_abs=pd.read_pickle('test_abs')\n",
    "n=len(test_abs)\n",
    "ctr=0\n",
    "avg_precision=[0]*11\n",
    "valid_file=list(judge.keys())\n",
    "for i in range(n):\n",
    "    if i in valid_file:        \n",
    "        res=predict(test_abs['abstract'].iloc[i])\n",
    "        #print(res)\n",
    "        for k in range(1,11):\n",
    "            avg_precision[k]+=precision_k(res,judge,k,i)\n",
    "        ctr+=1\n",
    "avg_precision=[ele/ctr for ele in avg_precision]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.3332352941176471,\n",
       " 0.396764705882353,\n",
       " 0.316764705882353,\n",
       " 0.2669852941176471,\n",
       " 0.27041176470588235,\n",
       " 0.2547549019607844,\n",
       " 0.24357142857142863,\n",
       " 0.23025735294117644,\n",
       " 0.22098039215686271,\n",
       " 0.2194705882352941]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_precision[1:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handling Reviewers not present in Ground Thruth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reviewer_404(res,ground_tht,paper_no):\n",
    "    actual=ground_tht[paper_no]\n",
    "    actual_auth=[i[0] for i in actual]\n",
    "    rating=[i[1] for i in actual]\n",
    "    n=len(actual)\n",
    "    output=[]\n",
    "    for auth in res:\n",
    "        if auth not in actual_auth:\n",
    "            sim_scr=[0]*4\n",
    "            count=[0]*4\n",
    "            vec1=reviewer_vec_bow[auth-1]\n",
    "            for i in range(n):\n",
    "                val=1-cosine(vec1,reviewer_vec_bow[actual_auth[i]-1])\n",
    "                if np.isnan(val):\n",
    "                    val=0\n",
    "                sim_scr[rating[i]]+=val\n",
    "                count[rating[i]]+=1\n",
    "            sim_scr=[sim_scr[i]/count[i] for i in range(4) if count[i]]\n",
    "            max_scr=max(sim_scr)\n",
    "            ind=sim_scr.index(max_scr)\n",
    "\n",
    "            output.append([auth,ind])\n",
    "            #print(sim_scr)\n",
    "    return output\n",
    "                \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "res1=predict(test_abs['abstract'].iloc[0])\n",
    "op=reviewer_404(res1,judge,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[162, 3], [234, 2], [23, 3], [12, 0], [164, 0]]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 536,
   "metadata": {},
   "outputs": [],
   "source": [
    "def precision_k_404(res,gnd_tht,k,paper_no):\n",
    "    actual=gnd_tht[paper_no]\n",
    "    actual_auth=[i[0] for i in actual]\n",
    "    rating=[i[1] for i in actual]\n",
    "    n=len(actual)\n",
    "    soft_p=0\n",
    "    for i in range(k):\n",
    "        pred=res[i]\n",
    "        for j in range(n):\n",
    "            if actual_auth[j]==pred:\n",
    "                if rating[j]==3:\n",
    "                    soft_p+=1\n",
    "                elif rating[j]==2:\n",
    "                    soft_p+=1\n",
    "                elif rating[j]==1:\n",
    "                    soft_p+=0\n",
    "                else:\n",
    "                    break\n",
    "    rating_gen=reviewer_404(res,gnd_tht,paper_no)\n",
    "    auth_gen=[i[0] for i in rating_gen]\n",
    "    rating_gen=[i[1] for i in rating_gen]\n",
    "    n1=len(auth_gen)\n",
    "    for i in range(k):\n",
    "        pred=res[i]\n",
    "        for j in range(n1):\n",
    "            if auth_gen[j]==pred:\n",
    "                if rating_gen[j]==3:\n",
    "                    soft_p+=1\n",
    "                elif rating_gen[j]==2:\n",
    "                    soft_p+=1\n",
    "                elif rating_gen[j]==1:\n",
    "                    soft_p+=0\n",
    "                else:\n",
    "                    break\n",
    "    soft_p=soft_p/k\n",
    "    return soft_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_abs=pd.read_pickle('test_abs')\n",
    "n=len(test_abs)\n",
    "ctr=0\n",
    "avg_precision=[0]*11\n",
    "valid_file=list(judge.keys())\n",
    "for i in range(n):\n",
    "    if i in valid_file:        \n",
    "        res=predict(test_abs['abstract'].iloc[i])\n",
    "        #print(res)\n",
    "        for k in range(1,11):\n",
    "            avg_precision[k]+=precision_k_404(res,judge,k,i)\n",
    "        ctr+=1\n",
    "avg_precision=[ele/ctr for ele in avg_precision]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.5588235294117647,\n",
       " 0.5,\n",
       " 0.5490196078431372,\n",
       " 0.5514705882352942,\n",
       " 0.5470588235294118,\n",
       " 0.5441176470588236,\n",
       " 0.5504201680672269,\n",
       " 0.5551470588235294,\n",
       " 0.5522875816993464,\n",
       " 0.5588235294117647]"
      ]
     },
     "execution_count": 545,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_precision[1:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Insight about our results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gndtht_stats(res,gnd_tht,paper_no):\n",
    "    actual=gnd_tht[paper_no]\n",
    "    actual_auth=[i[0] for i in actual]\n",
    "    rating=[i[1] for i in actual]\n",
    "    ctr_3=0\n",
    "    ctr_2=0\n",
    "    ctr_1=0\n",
    "    ctr_0=0\n",
    "    ctr_not=0    \n",
    "    for ele in res:\n",
    "        if ele in actual_auth:\n",
    "            ind=actual_auth.index(ele)\n",
    "            if rating[ind]==3:\n",
    "                ctr_3+=1\n",
    "            elif rating[ind]==2:\n",
    "                ctr_2+=1\n",
    "            elif rating[ind]==1:\n",
    "                ctr_1+=1\n",
    "            else:\n",
    "                ctr_0+=1\n",
    "        else:\n",
    "            ctr_not+=1\n",
    "    return ctr_3,ctr_2,ctr_1,ctr_0,ctr_not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_abs=pd.read_pickle('test_abs')\n",
    "n=len(test_abs)\n",
    "ctr=0\n",
    "avg_precision=[0]*11\n",
    "valid_file=list(judge.keys())\n",
    "op=[]\n",
    "for i in range(n):\n",
    "    if i in valid_file:        \n",
    "        res=predict(test_abs['abstract'].iloc[i])\n",
    "        #print(res)\n",
    "        res1=gndtht_stats(res,judge,i)\n",
    "        op.append([i,res1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File Id: 0\n",
      "No of judgement with rel 3: 2\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 3\n",
      "No of judgement with rel 0: 0\n",
      "No of judgement not in ground thruth: 5\n",
      "######################################\n",
      "File Id: 1\n",
      "No of judgement with rel 3: 0\n",
      "No of judgement with rel 2: 4\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 4\n",
      "######################################\n",
      "File Id: 2\n",
      "No of judgement with rel 3: 0\n",
      "No of judgement with rel 2: 1\n",
      "No of judgement with rel 1: 2\n",
      "No of judgement with rel 0: 2\n",
      "No of judgement not in ground thruth: 5\n",
      "######################################\n",
      "File Id: 3\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 0\n",
      "No of judgement with rel 0: 3\n",
      "No of judgement not in ground thruth: 6\n",
      "######################################\n",
      "File Id: 4\n",
      "No of judgement with rel 3: 0\n",
      "No of judgement with rel 2: 2\n",
      "No of judgement with rel 1: 0\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 7\n",
      "######################################\n",
      "File Id: 5\n",
      "No of judgement with rel 3: 2\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 6\n",
      "######################################\n",
      "File Id: 10\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 2\n",
      "No of judgement with rel 0: 4\n",
      "No of judgement not in ground thruth: 3\n",
      "######################################\n",
      "File Id: 11\n",
      "No of judgement with rel 3: 2\n",
      "No of judgement with rel 2: 1\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 5\n",
      "######################################\n",
      "File Id: 13\n",
      "No of judgement with rel 3: 3\n",
      "No of judgement with rel 2: 2\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 0\n",
      "No of judgement not in ground thruth: 4\n",
      "######################################\n",
      "File Id: 14\n",
      "No of judgement with rel 3: 0\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 2\n",
      "No of judgement not in ground thruth: 7\n",
      "######################################\n",
      "File Id: 16\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 1\n",
      "No of judgement with rel 1: 3\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 4\n",
      "######################################\n",
      "File Id: 17\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 3\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 4\n",
      "######################################\n",
      "File Id: 18\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 1\n",
      "No of judgement with rel 1: 2\n",
      "No of judgement with rel 0: 2\n",
      "No of judgement not in ground thruth: 4\n",
      "######################################\n",
      "File Id: 22\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 2\n",
      "No of judgement with rel 1: 3\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 3\n",
      "######################################\n",
      "File Id: 26\n",
      "No of judgement with rel 3: 0\n",
      "No of judgement with rel 2: 1\n",
      "No of judgement with rel 1: 5\n",
      "No of judgement with rel 0: 0\n",
      "No of judgement not in ground thruth: 4\n",
      "######################################\n",
      "File Id: 29\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 2\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 5\n",
      "######################################\n",
      "File Id: 34\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 0\n",
      "No of judgement with rel 0: 4\n",
      "No of judgement not in ground thruth: 5\n",
      "######################################\n",
      "File Id: 35\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 3\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 5\n",
      "######################################\n",
      "File Id: 36\n",
      "No of judgement with rel 3: 0\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 0\n",
      "No of judgement with rel 0: 0\n",
      "No of judgement not in ground thruth: 10\n",
      "######################################\n",
      "File Id: 40\n",
      "No of judgement with rel 3: 3\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 2\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 4\n",
      "######################################\n",
      "File Id: 45\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 2\n",
      "No of judgement with rel 1: 0\n",
      "No of judgement with rel 0: 2\n",
      "No of judgement not in ground thruth: 5\n",
      "######################################\n",
      "File Id: 52\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 2\n",
      "No of judgement not in ground thruth: 6\n",
      "######################################\n",
      "File Id: 56\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 1\n",
      "No of judgement with rel 1: 3\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 4\n",
      "######################################\n",
      "File Id: 63\n",
      "No of judgement with rel 3: 0\n",
      "No of judgement with rel 2: 3\n",
      "No of judgement with rel 1: 2\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 4\n",
      "######################################\n",
      "File Id: 79\n",
      "No of judgement with rel 3: 0\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 1\n",
      "No of judgement not in ground thruth: 8\n",
      "######################################\n",
      "File Id: 83\n",
      "No of judgement with rel 3: 0\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 2\n",
      "No of judgement with rel 0: 0\n",
      "No of judgement not in ground thruth: 8\n",
      "######################################\n",
      "File Id: 88\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 2\n",
      "No of judgement with rel 1: 0\n",
      "No of judgement with rel 0: 3\n",
      "No of judgement not in ground thruth: 4\n",
      "######################################\n",
      "File Id: 93\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 1\n",
      "No of judgement with rel 1: 0\n",
      "No of judgement with rel 0: 3\n",
      "No of judgement not in ground thruth: 5\n",
      "######################################\n",
      "File Id: 103\n",
      "No of judgement with rel 3: 2\n",
      "No of judgement with rel 2: 2\n",
      "No of judgement with rel 1: 3\n",
      "No of judgement with rel 0: 0\n",
      "No of judgement not in ground thruth: 3\n",
      "######################################\n",
      "File Id: 128\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 2\n",
      "No of judgement not in ground thruth: 6\n",
      "######################################\n",
      "File Id: 139\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 1\n",
      "No of judgement with rel 1: 2\n",
      "No of judgement with rel 0: 3\n",
      "No of judgement not in ground thruth: 3\n",
      "######################################\n",
      "File Id: 144\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 2\n",
      "No of judgement not in ground thruth: 6\n",
      "######################################\n",
      "File Id: 145\n",
      "No of judgement with rel 3: 1\n",
      "No of judgement with rel 2: 0\n",
      "No of judgement with rel 1: 0\n",
      "No of judgement with rel 0: 0\n",
      "No of judgement not in ground thruth: 9\n",
      "######################################\n",
      "File Id: 147\n",
      "No of judgement with rel 3: 2\n",
      "No of judgement with rel 2: 2\n",
      "No of judgement with rel 1: 1\n",
      "No of judgement with rel 0: 0\n",
      "No of judgement not in ground thruth: 5\n",
      "######################################\n"
     ]
    }
   ],
   "source": [
    "for entry in op:\n",
    "    file_id=entry[0]\n",
    "    stats=entry[1]\n",
    "    label=['No of judgement with rel 3:','No of judgement with rel 2:','No of judgement with rel 1:','No of judgement with rel 0:','No of judgement not in ground thruth:']\n",
    "    print('File Id:',file_id)\n",
    "    for i in range(5):\n",
    "        print(label[i],stats[i])\n",
    "    print('######################################')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Expertise Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expertise(res,k,query):\n",
    "    clean_text=list(word_tokenize(cleantext(query))) \n",
    "    bow_text=dictionary.doc2bow(clean_text)\n",
    "    tfidf_text=tfidf[bow_text]\n",
    "    topic=lda_model[tfidf_text]\n",
    "\n",
    "    topic_vec=np.zeros(num_topics)\n",
    "    for ind,wt in topic:\n",
    "        topic_vec[ind]=wt\n",
    "    \n",
    "    expert_scr=0\n",
    "    for i in range(k):\n",
    "        temp_scr=1-cosine(topic_vec,reviewer_vec[res[i]-1])\n",
    "        if np.isnan(temp_scr):\n",
    "            temp_scr=0\n",
    "        expert_scr+=temp_scr\n",
    "    \n",
    "    expert_scr=expert_scr/k\n",
    "    \n",
    "    return expert_scr\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_abs=pd.read_pickle('test_abs')\n",
    "n=len(test_abs)\n",
    "ctr=0\n",
    "expertise_k=[0]*11\n",
    "valid_file=list(judge.keys())\n",
    "for i in range(n):\n",
    "    if i in valid_file:        \n",
    "        res=predict(test_abs['abstract'].iloc[i])\n",
    "        #print(res)\n",
    "        for k in range(1,11):\n",
    "            expertise_k[k]+=expertise(res,k,test_abs['abstract'].iloc[i])\n",
    "        ctr+=1\n",
    "avg_expertise_k=[ele/ctr for ele in expertise_k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.7668277469595643,\n",
       " 0.7212900274873323,\n",
       " 0.7128708883252811,\n",
       " 0.6981370907646861,\n",
       " 0.6881152635070239,\n",
       " 0.6897492248827924,\n",
       " 0.6846105824308703,\n",
       " 0.6793597162338412,\n",
       " 0.6770969799301807,\n",
       " 0.6828991189643149]"
      ]
     },
     "execution_count": 338,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_expertise_k[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75"
      ]
     },
     "execution_count": 549,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import jaccard_similarity_score\n",
    "a=[0,0,0,1]\n",
    "b=[1,0,0,1]\n",
    "jaccard_similarity_score(a,b)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Authority Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [],
   "source": [
    "path1=cwd+sym+'Saved_Items'+sym\n",
    "hind=pd.read_csv(path1+'hIndex.txt',header=None)\n",
    "hind=list(hind[0])\n",
    "\n",
    "def authority_k(res,k):\n",
    "    authority=0\n",
    "    for i in range(k):\n",
    "        authority+=hind[res[i]-1]\n",
    "    authority=authority/k\n",
    "    return authority\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_abs=pd.read_pickle('test_abs')\n",
    "n=len(test_abs)\n",
    "ctr=0\n",
    "authority_k_net=[0]*11\n",
    "valid_file=list(judge.keys())\n",
    "for i in range(n):\n",
    "    if i in valid_file:        \n",
    "        res=predict(test_abs['abstract'].iloc[i])\n",
    "        #print(res)\n",
    "        for k in range(1,11):\n",
    "            authority_k_net[k]+=authority_k(res,k)\n",
    "        ctr+=1\n",
    "authority_k_res=[ele/ctr for ele in authority_k_net]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[21.88235294117647,\n",
       " 38.661764705882355,\n",
       " 37.2156862745098,\n",
       " 37.080882352941174,\n",
       " 37.23529411764706,\n",
       " 37.50490196078431,\n",
       " 37.7142857142857,\n",
       " 37.23529411764706,\n",
       " 36.98366013071895,\n",
       " 37.247058823529414]"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authority_k_res[1:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Diversity Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 473,
   "metadata": {},
   "outputs": [],
   "source": [
    "def KL(p,q):\n",
    "    \"\"\" Epsilon is used here to avoid conditional code for\n",
    "    checking that neither P nor Q is equal to 0. \"\"\"\n",
    "    epsilon = 0.00001\n",
    "    \n",
    "    # You may want to instead make copies to avoid changing the np arrays.\n",
    "    vec1=reviewer_vec[p-1]\n",
    "    vec2=reviewer_vec[q-1]\n",
    "    \n",
    "    vec1=np.array(vec1)\n",
    "    v1sum=np.sum(vec1)\n",
    "    vec2=np.array(vec2)\n",
    "    v2sum=np.sum(vec2)\n",
    "\n",
    "    vec1=np.divide(vec1,v1sum)\n",
    "    vec2=np.divide(vec2,v2sum)\n",
    "        \n",
    "    vec1 = vec1+epsilon\n",
    "    vec2 = vec2+epsilon\n",
    "    \n",
    "    if v1sum==0 or v2sum==0:\n",
    "        return 0\n",
    "\n",
    "    divergence = np.sum(vec1*np.log10(vec1/vec2))\n",
    "    return divergence\n",
    "\n",
    "def diversity(res):\n",
    "    div=0\n",
    "    n=len(res)\n",
    "    for i in range(n):\n",
    "        for j in range(n):\n",
    "            div+=KL(res[i],res[j])\n",
    "    div=div/(n*(n-1))\n",
    "    return div\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_abs=pd.read_pickle('test_abs')\n",
    "n=len(test_abs)\n",
    "ctr=0\n",
    "diverse=0\n",
    "valid_file=list(judge.keys())\n",
    "for i in range(n):\n",
    "    if i in valid_file:        \n",
    "        res=predict(test_abs['abstract'].iloc[i])\n",
    "        diverse+=diversity(res)\n",
    "        ctr+=1\n",
    "diverse=diverse/ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4415717476301809"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diverse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Alternative Diversity Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "def diversity(res):\n",
    "    div=0\n",
    "    n=len(res)\n",
    "    for i in range(n):\n",
    "        for j in range(n):\n",
    "            sim=cosine_similarity([reviewer_vec[res[i]-1],reviewer_vec[res[j]-1]])\n",
    "            sim=sim[0][1]\n",
    "            div+=1-sim\n",
    "    div=div/(n*(n-1))\n",
    "    return div\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_abs=pd.read_pickle('test_abs')\n",
    "n=len(test_abs)\n",
    "ctr=0\n",
    "diverse=0\n",
    "valid_file=list(judge.keys())\n",
    "for i in range(n):\n",
    "    if i in valid_file:        \n",
    "        res=predict(test_abs['abstract'].iloc[i])\n",
    "        diverse+=diversity(res)\n",
    "        ctr+=1\n",
    "diverse=diverse/ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2998115047827517"
      ]
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diverse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path1+'reviewer_text.pkl','rb') as f:\n",
    "    reviewer_text=pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_tokenize = lambda x: re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_set=list(reviewer_text.values())\n",
    "docs=list(map(cleantext,text_set))\n",
    "tokenize_doc=[list(word_tokenize(doc)) for doc in docs]\n",
    "corpus=' '.join(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.util import ngrams\n",
    "bigram_doc=[]\n",
    "for doc in tokenize_doc:\n",
    "    c=Counter(ngrams(doc,2))\n",
    "    bigram_doc.append(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenize_corpus=list(word_tokenize(corpus))\n",
    "bigram_corpus=Counter(ngrams(tokenize_corpus,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_size=len(tokenize_corpus)\n",
    "def predict_lm(bigram_corpus,bigram_doc,query,tokenize_doc):\n",
    "    prob_score=[]\n",
    "    token_list=list(word_tokenize(query))\n",
    "    bigram_query=Counter(ngrams(token_list,2))\n",
    "    mu=2000 #Dirchlet smoothing Parameter\n",
    "    for ind,doc in enumerate(bigram_doc):\n",
    "        nd=len(tokenize_doc[ind])\n",
    "        scr=1\n",
    "        for bigram in bigram_query.keys():\n",
    "            if bigram in doc:\n",
    "                prob_doc=doc[bigram]/(nd-1)\n",
    "                prob_net=bigram_corpus[bigram]/(corpus_size-1)\n",
    "            else:\n",
    "                prob_doc=0\n",
    "                if bigram in bigram_corpus:\n",
    "                    prob_net=bigram_corpus[bigram]/(corpus_size-1)\n",
    "                else:\n",
    "                    prob_net=1/(2*corpus_size-1)               \n",
    "            scr=scr*(((nd*prob_doc)/(nd+mu))+(mu*prob_net)/(nd+mu))\n",
    "        prob_score.append(scr)\n",
    "    \n",
    "    prob_score=np.array(prob_score)\n",
    "    val=prob_score.argsort()[-10:][::-1]\n",
    "    #op=new_state.argsort()[::-1]\n",
    "    res=[ele+1 for ele in val]\n",
    "    \n",
    "    return res\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res=predict_lm(bigram_corpus,bigram_doc,test_abs['abstract'].iloc[i],tokenize_doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_abs=pd.read_pickle('test_abs')\n",
    "n=len(test_abs)\n",
    "ctr=0\n",
    "avg_precision=[0]*11\n",
    "valid_file=list(judge.keys())\n",
    "for i in range(n):\n",
    "    if i in valid_file:        \n",
    "        res=res=predict_lm(bigram_corpus,bigram_doc,test_abs['abstract'].iloc[i],tokenize_doc)\n",
    "        #print(res)\n",
    "        for k in range(1,11):\n",
    "            avg_precision[k]+=precision_k_404(res,judge,k,i)\n",
    "        ctr+=1\n",
    "avg_precision=[ele/ctr for ele in avg_precision]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Diversity vs Precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reviewer_label(res,gnd_tht,paper_no):\n",
    "    actual=gnd_tht[paper_no]\n",
    "    actual_auth=[i[0] for i in actual]\n",
    "    rating=[i[1] for i in actual]\n",
    "    n=len(actual)\n",
    "    soft_p=0\n",
    "    n1=len(res)\n",
    "    output=[]\n",
    "    \n",
    "    for i in range(n1):\n",
    "        pred=res[i]\n",
    "        for j in range(n):\n",
    "            if actual_auth[j]==pred:\n",
    "                if rating[j]==3:\n",
    "                    output.append((pred,3))\n",
    "                elif rating[j]==2:\n",
    "                    output.append((pred,2))\n",
    "                elif rating[j]==1:\n",
    "                    output.append((pred,1))\n",
    "                else:\n",
    "                    output.append((pred,0))\n",
    "                    break\n",
    "    rating_gen=reviewer_404(res,gnd_tht,paper_no)\n",
    "    auth_gen=[i[0] for i in rating_gen]\n",
    "    rating_gen=[i[1] for i in rating_gen]\n",
    "    n1=len(auth_gen)\n",
    "    for i in range(k):\n",
    "        pred=res[i]\n",
    "        for j in range(n1):\n",
    "            if auth_gen[j]==pred:\n",
    "                if rating_gen[j]==3:\n",
    "                    output.append((pred,3))\n",
    "                elif rating_gen[j]==2:\n",
    "                    output.append((pred,2))\n",
    "                elif rating_gen[j]==1:\n",
    "                    output.append((pred,1))\n",
    "                else:\n",
    "                    output.append((pred,0))\n",
    "                    break\n",
    "    \n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_abs=pd.read_pickle('test_abs')\n",
    "n=len(test_abs)\n",
    "ctr=0\n",
    "avg_precision=[0]*11\n",
    "valid_file=list(judge.keys())\n",
    "reviewer_rel=[]\n",
    "for i in range(n):\n",
    "    if i in valid_file:        \n",
    "        res=predict(test_abs['abstract'].iloc[i])\n",
    "        paper_reviewer_rel=reviewer_label(res,judge,i)\n",
    "        reviewer_rel=reviewer_rel+paper_reviewer_rel\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 428,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 429,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(164, 2), (234, 3), (295, 1), (164, 2), (214, 0)]"
      ]
     },
     "execution_count": 429,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.sample(reviewer_rel,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 430,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(183, 3),\n",
       " (9, 3),\n",
       " (53, 1),\n",
       " (108, 1),\n",
       " (34, 1),\n",
       " (162, 3),\n",
       " (234, 2),\n",
       " (23, 3),\n",
       " (12, 0),\n",
       " (164, 0),\n",
       " (9, 1),\n",
       " (162, 2),\n",
       " (53, 0),\n",
       " (34, 2),\n",
       " (108, 2),\n",
       " (211, 2),\n",
       " (174, 2),\n",
       " (23, 2),\n",
       " (234, 2),\n",
       " (12, 3),\n",
       " (225, 1),\n",
       " (9, 0),\n",
       " (34, 0),\n",
       " (108, 1),\n",
       " (295, 2),\n",
       " (12, 2),\n",
       " (162, 2),\n",
       " (234, 2),\n",
       " (53, 2),\n",
       " (164, 2),\n",
       " (270, 0),\n",
       " (9, 0),\n",
       " (295, 3),\n",
       " (34, 0),\n",
       " (162, 1),\n",
       " (234, 1),\n",
       " (108, 1),\n",
       " (211, 1),\n",
       " (164, 1),\n",
       " (12, 1),\n",
       " (213, 0),\n",
       " (9, 2),\n",
       " (10, 2),\n",
       " (162, 2),\n",
       " (234, 2),\n",
       " (108, 2),\n",
       " (106, 1),\n",
       " (145, 2),\n",
       " (164, 2),\n",
       " (53, 2),\n",
       " (218, 3),\n",
       " (9, 1),\n",
       " (108, 3),\n",
       " (34, 0),\n",
       " (234, 1),\n",
       " (162, 1),\n",
       " (164, 1),\n",
       " (53, 1),\n",
       " (106, 1),\n",
       " (12, 1),\n",
       " (214, 0),\n",
       " (9, 3),\n",
       " (108, 1),\n",
       " (211, 1),\n",
       " (228, 0),\n",
       " (34, 0),\n",
       " (234, 3),\n",
       " (162, 1),\n",
       " (164, 1),\n",
       " (23, 3),\n",
       " (9, 2),\n",
       " (162, 3),\n",
       " (34, 1),\n",
       " (53, 3),\n",
       " (197, 3),\n",
       " (234, 3),\n",
       " (108, 3),\n",
       " (211, 3),\n",
       " (23, 3),\n",
       " (164, 3),\n",
       " (78, 1),\n",
       " (9, 3),\n",
       " (108, 3),\n",
       " (12, 2),\n",
       " (34, 3),\n",
       " (295, 2),\n",
       " (162, 3),\n",
       " (234, 3),\n",
       " (53, 3),\n",
       " (164, 3),\n",
       " (295, 1),\n",
       " (162, 0),\n",
       " (34, 0),\n",
       " (203, 0),\n",
       " (309, 0),\n",
       " (166, 3),\n",
       " (12, 0),\n",
       " (164, 3),\n",
       " (211, 3),\n",
       " (86, 3),\n",
       " (235, 0),\n",
       " (9, 1),\n",
       " (108, 3),\n",
       " (34, 2),\n",
       " (295, 1),\n",
       " (10, 1),\n",
       " (234, 3),\n",
       " (162, 3),\n",
       " (164, 3),\n",
       " (53, 3),\n",
       " (224, 3),\n",
       " (9, 2),\n",
       " (12, 2),\n",
       " (108, 2),\n",
       " (34, 1),\n",
       " (295, 0),\n",
       " (234, 2),\n",
       " (162, 2),\n",
       " (164, 2),\n",
       " (53, 2),\n",
       " (9, 2),\n",
       " (162, 0),\n",
       " (108, 3),\n",
       " (34, 1),\n",
       " (295, 1),\n",
       " (53, 0),\n",
       " (209, 3),\n",
       " (234, 3),\n",
       " (164, 3),\n",
       " (211, 3),\n",
       " (132, 3),\n",
       " (106, 1),\n",
       " (34, 1),\n",
       " (295, 0),\n",
       " (9, 2),\n",
       " (99, 2),\n",
       " (10, 1),\n",
       " (162, 2),\n",
       " (234, 2),\n",
       " (164, 2),\n",
       " (132, 1),\n",
       " (106, 1),\n",
       " (34, 1),\n",
       " (295, 2),\n",
       " (9, 1),\n",
       " (10, 1),\n",
       " (162, 1),\n",
       " (234, 1),\n",
       " (99, 1),\n",
       " (164, 1),\n",
       " (234, 3),\n",
       " (9, 0),\n",
       " (108, 2),\n",
       " (34, 1),\n",
       " (295, 2),\n",
       " (162, 3),\n",
       " (164, 3),\n",
       " (211, 3),\n",
       " (53, 3),\n",
       " (23, 3),\n",
       " (9, 0),\n",
       " (108, 3),\n",
       " (34, 0),\n",
       " (211, 0),\n",
       " (295, 0),\n",
       " (209, 1),\n",
       " (162, 1),\n",
       " (234, 1),\n",
       " (164, 1),\n",
       " (53, 1),\n",
       " (197, 1),\n",
       " (9, 1),\n",
       " (162, 0),\n",
       " (53, 1),\n",
       " (34, 3),\n",
       " (234, 3),\n",
       " (108, 3),\n",
       " (211, 3),\n",
       " (23, 3),\n",
       " (164, 3),\n",
       " (36, 2),\n",
       " (68, 2),\n",
       " (325, 2),\n",
       " (75, 2),\n",
       " (198, 0),\n",
       " (232, 2),\n",
       " (217, 0),\n",
       " (248, 2),\n",
       " (148, 2),\n",
       " (58, 2),\n",
       " (286, 3),\n",
       " (10, 3),\n",
       " (9, 1),\n",
       " (34, 1),\n",
       " (295, 0),\n",
       " (176, 3),\n",
       " (162, 1),\n",
       " (234, 1),\n",
       " (108, 1),\n",
       " (211, 1),\n",
       " (12, 2),\n",
       " (9, 3),\n",
       " (108, 0),\n",
       " (34, 2),\n",
       " (295, 0),\n",
       " (162, 2),\n",
       " (234, 2),\n",
       " (53, 2),\n",
       " (164, 2),\n",
       " (211, 2),\n",
       " (309, 0),\n",
       " (295, 3),\n",
       " (34, 0),\n",
       " (9, 1),\n",
       " (162, 1),\n",
       " (106, 3),\n",
       " (234, 1),\n",
       " (164, 3),\n",
       " (108, 1),\n",
       " (53, 3),\n",
       " (119, 1),\n",
       " (9, 1),\n",
       " (34, 3),\n",
       " (53, 1),\n",
       " (10, 0),\n",
       " (295, 2),\n",
       " (162, 3),\n",
       " (234, 3),\n",
       " (108, 1),\n",
       " (164, 3),\n",
       " (197, 1),\n",
       " (9, 2),\n",
       " (108, 2),\n",
       " (23, 0),\n",
       " (34, 1),\n",
       " (53, 2),\n",
       " (234, 2),\n",
       " (162, 2),\n",
       " (211, 2),\n",
       " (164, 2),\n",
       " (118, 1),\n",
       " (122, 0),\n",
       " (334, 1),\n",
       " (364, 2),\n",
       " (124, 2),\n",
       " (116, 0),\n",
       " (117, 1),\n",
       " (119, 1),\n",
       " (120, 2),\n",
       " (121, 2),\n",
       " (9, 1),\n",
       " (34, 1),\n",
       " (85, 3),\n",
       " (162, 3),\n",
       " (23, 3),\n",
       " (234, 3),\n",
       " (12, 3),\n",
       " (53, 3),\n",
       " (108, 3),\n",
       " (164, 3),\n",
       " (119, 2),\n",
       " (9, 0),\n",
       " (34, 2),\n",
       " (53, 0),\n",
       " (10, 3),\n",
       " (295, 0),\n",
       " (162, 0),\n",
       " (234, 0),\n",
       " (108, 0),\n",
       " (164, 0),\n",
       " (251, 3),\n",
       " (9, 0),\n",
       " (295, 0),\n",
       " (12, 2),\n",
       " (34, 0),\n",
       " (250, 3),\n",
       " (234, 3),\n",
       " (162, 2),\n",
       " (106, 3),\n",
       " (211, 3),\n",
       " (9, 3),\n",
       " (108, 3),\n",
       " (34, 1),\n",
       " (211, 2),\n",
       " (295, 1),\n",
       " (53, 1),\n",
       " (23, 2),\n",
       " (234, 2),\n",
       " (162, 2),\n",
       " (164, 2),\n",
       " (295, 0),\n",
       " (34, 3),\n",
       " (106, 0),\n",
       " (9, 1),\n",
       " (309, 3),\n",
       " (162, 3),\n",
       " (234, 3),\n",
       " (164, 3),\n",
       " (108, 3),\n",
       " (53, 3),\n",
       " (226, 2),\n",
       " (9, 0),\n",
       " (162, 0),\n",
       " (108, 1),\n",
       " (53, 0),\n",
       " (12, 3),\n",
       " (34, 1),\n",
       " (234, 0),\n",
       " (164, 3),\n",
       " (211, 3),\n",
       " (9, 3),\n",
       " (34, 1),\n",
       " (108, 0),\n",
       " (295, 0),\n",
       " (225, 1),\n",
       " (162, 1),\n",
       " (234, 1),\n",
       " (53, 1),\n",
       " (164, 1),\n",
       " (12, 1),\n",
       " (9, 3),\n",
       " (108, 1),\n",
       " (34, 0),\n",
       " (53, 0),\n",
       " (295, 0),\n",
       " (91, 0),\n",
       " (162, 1),\n",
       " (234, 1),\n",
       " (164, 1),\n",
       " (6, 0),\n",
       " (145, 2),\n",
       " (9, 3),\n",
       " (108, 3),\n",
       " (34, 2),\n",
       " (10, 1),\n",
       " (162, 3),\n",
       " (234, 3),\n",
       " (53, 3),\n",
       " (12, 3),\n",
       " (164, 3)]"
      ]
     },
     "execution_count": 430,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviewer_rel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 516,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins=[[] for i in range(5)]#Range 0-0.2,0.2-0.4,so on\n",
    "min_entry=15\n",
    "#Upper Bound inclusive "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 517,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(5):\n",
    "    min_lim=0.2*i\n",
    "    max_lim=0.2*(i+1)\n",
    "    entry=0\n",
    "    while(entry<min_entry):\n",
    "        small_set=random.sample(reviewer_rel,5)\n",
    "        n=len(small_set)\n",
    "        prec_set=[tupule[1] for tupule in small_set]\n",
    "        review_set=[tupule[0] for tupule in small_set]\n",
    "        prec=sum(prec_set)/(len(prec_set)*3)\n",
    "        div=diversity(review_set)\n",
    "        if div>min_lim and div<=max_lim:\n",
    "            entry+=1\n",
    "            bins[i].append((prec,div))\n",
    "            \n",
    "   \n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 518,
   "metadata": {},
   "outputs": [],
   "source": [
    "prec_set=[]\n",
    "div_set=[]\n",
    "for i in range(5):\n",
    "    samp=[entry[0] for entry in bins[i]]\n",
    "    prec=sum(samp)/min_entry\n",
    "    prec_set.append(prec)\n",
    "    div=0.1+0.2*i\n",
    "    div_set.append(div)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 519,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Precision')"
      ]
     },
     "execution_count": 519,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4VOXZx/HvnZ0lCQHCThKWhCQgCokoqIiobFqs+65UqxWrVqu2Lm1FfbWutW7VKlqXulBpbUGQVVwBJeyEEAiBsAmEJSwhe+73jxl0jJGZQGbOZHJ/rmsu5pw5M+eXIZl7znme5zyiqhhjjDFHEuZ0AGOMMcHPioUxxhivrFgYY4zxyoqFMcYYr6xYGGOM8cqKhTHGGK+sWBhjjPHKioUxxhivrFgYY4zxKsLpAI2lffv2mpKS4nQMY4xpUhYvXrxLVRO9bRcyxSIlJYWcnBynYxhjTJMiIkW+bGenoYwxxnhlxcIYY4xXViyMMcZ4ZcXCGGOMV1YsjDHGeGXFwhhjjFdWLIwxxngVMuMsjAlWM1ZtZ39ZFQOTE+iV2AoRcTqSMQ1mxcIYP6mtVR6dnsfELzd8t65Ny0gGJiWQlZzAwKQEju8eT8so+zM0wc9+S43xg8rqWn43eTn/XbaNcUNSuPKkJJZs2svior0s2VTCJ2t2AhAeJmR0jiUrKYGBya4i0rVNCzv6MEFHVNXpDI0iOztb7XIfJhgcrKhm/D8X88W6Xdw9sg83D+v1ow//kkOVLN1UwuIiVwFZtrmEsqoaADrGRX9/9JGcQN8ucURHhDvxo5hmQEQWq2q2t+3syMKYRrTrYAXXvbGI3G37eeKi/lyS3b3e7dq0jOKM9A6ckd4BgOqaWtZsP/Dd0cfior18vGo7AFERYfTvGk9WcgID3EUkMTY6YD+TMWBHFsY0mk27D3HN61+zfX85L14xkDMzOh7T6+3YX86Sor3fFZBVW/dTWVMLQFLblt8deWQlJdCnUyzhYXbqyjScr0cWViyMaQS52/Zx7euLqKqp5fVxJ5KVnNDo+yivqiF32z5Xu0dRCTlFe9l1sAKAVlHhnJDU5ru2jwFJCcS3iGz0DCb02GkoYwJk/vpd3PjWYuJiInj/xsH07hDrl/3ERIaTldyWrOS2AKgqW/aWfXfaanHRXl6YV0Ct+/tfaofW3x99JCfQs7112zVHz44sjDkG01Z8yx2TlpHSviVvXjeIzvEtHM1zsKKaFZvdDeeb9rKkaC/7y6sBSHB32x1o3XaNBzuyMMbP3py/kQlTc8lKSuC1a08kvqXzp31aR0cwpHd7hvRuD7jGehTuOviDo4+5Ht12MzvHfXf0MTCpjXXbNT/JjiyMaSBV5elZa3lhXgFnZXTkhSsGEBPZdLq27i2tZOlmV7tHfd12Dw8YzEpOoG+XeKIi7KpAocyOLIzxg+qaWu7/cBWTcjZz+aDuPHxePyLCm9aHaUKrKIand2R4uqu31uFuu4ePPJZs2sv0lT/utnv49JV1222e7MjCGB+VVdZw63tLmZO3g9uG9+aOs9NC9pTN4W67h9s+cj267Sa3a0lWUgIDrNtuSLCus8Y0opJDlVz/Zg5LNu3lobF9uXpwitORAsqz267rVvKDbrsDkr5v97Buu02LnYYyppFsKynj2te/oWj3IV68YiBjjuvsdKSA86nb7ifrqFUQ8ei262776GHddps8O7Iw5gjW7TjANa9/w8Hyal65JpvBvdo5HSloHayoZrm72+6SI3TbzUpO4PhubWgR1XQ6BYQyO7Iw5hgtLtrDdW/kEBURxqRfDSazS5zTkYJa6+gITundnlM8uu2uLz74g4bzw912I8KEzC5xPyggXeJj7OgjiNmRhTH1mLN6B79+dwld2rTgresG0b1tS6cjhYTD3XYPF5Dlm/d91223U1yM+2KJbRh7Qhc6xMY4nLZ5sAZuY47SpEWbuO/DVfTrEsfr406kXWvrKuovdbvtLi7ay9aSMvp0jGX6b06zXlYBYKehjGkgVeXFeQU8NWstQ9MSeenKgbSKtj8Rf4oID6Nf13j6dY3n2iEpAExZvo3b3lvK5MWbufTEJGcDmu80rdFExvhJTa0yYUouT81ay/kDujLxmmwrFA75Wf/ODExqw9Oz1lJaUe10HONmxcI0exXVNdz23lLeXFDEjUN78vTFx9slLhwkItx/TiY7D1Tw6heFTscxbvYXYZq1/eVVjHt9EdNWfsv9YzK4b0wGYXae3HFZyQmMOa4Tr3xeyM795U7HMfi5WIjIKBHJF5ECEbnnJ7a5RERWi0iuiLzrsf5aEVnnvl3rz5ymedp5oJzL/r6QRRv38Mylx3PD0J5ORzIefjcynaqaWp6Zs9bpKAY/FgsRCQdeBEYDmcDlIpJZZ5tU4F7gFFXtC9zuXt8WeAA4CRgEPCAijT/1mGm2Nuwq5cKX5rNxdykTr83m/AHdnI5k6khp34qrT05h0qLNrN1xwOk4zZ4/jywGAQWqWqiqlcD7wHl1trkBeFFV9wKo6k73+pHAbFXd435sNjDKj1lNM7JiSwkXvTSf0ooa3rvhZIb16eB0JPMTbh3em1bREfx5ep7TUZo9fxaLrsBmj+Ut7nWe0oA0EflKRBaKyKgGPNeYBvt8bTGXvbKQFlHhTL5pMMd3b+N0JHMECa2iuHV4b+blF/NVwS6n4zRr/iwW9bUS1h0BGAGkAsOAy4GJItLGx+ciIjeKSI6I5BQXFx9jXBPq/rt0K9e9sYjkdq34z/gh9Exs7XQk44NrBqfQLaEFj0zLo7Y2NAYRN0X+LBZbgO4ey92AbfVs8z9VrVLVDUA+ruLhy3NR1VdUNVtVsxMTExs1vAktE78o5PZJy8hOSWDSr06mQ5xdSqKpiIkM5+6RfVj97X4+XLrV6TjNlj+LxSIgVUR6iEgUcBkwpc42/wXOABCR9rhOSxUCM4ERIpLgbtge4V5nTIPU1ip/np7H/03LY8xxnXjjF4OIi7G5Fpqan/XvwvHd4nlqVj7l7mtJmcDyW7FQ1WrgFlwf8nnAv1Q1V0QeEpGx7s1mArtFZDUwD7hbVXer6h7gYVwFZxHwkHudMT6rqqnlrg+W8/fPC7n65GSev3xgk5or23wvLEy4b0wG3+4r57UvNzgdp1myCwmakFRaUc3N7yzhs7XF3Hl2GrcM722Xvw4BN7yVw4L1u/n07mG0tws8NgpfLyRoI7hNyNlTWskVE7/mi3XFPHbBcdx6ZqoVihBxz+h0yqpqeHbOOqejNDtWLExI2bznEBe9NJ813+7n5auyuGyQXbU0lPRKbM2VJyXx7jebWF980Ok4zYoVCxMy8r7dz4UvzWfXwQr++cuTGNG3k9ORjB/85sxUWkSG89jHa5yO0qxYsTAhYWHhbi55eQFhIkweP4QTU9o6Hcn4SbvW0Ywf1ovZq3fwdeFup+M0G1YsTJM3Y9W3XPP6N3SMj+HfNw8hrWOs05GMn11/ag86x8fw6HQbqBcoVixMk/b2wiLGv7OEfl3i+OBXg+napoXTkUwAxESGc9eIPizfso+pK340Xtf4gRUL0ySpKn+ZvZY//ncVw/t04J1fnkxCqyinY5kAOn9AVzI7x/HEDBuoFwhWLEyTU11Ty30fruK5ueu4OKsbf786ixZRNtiuuQkLE+4/J4OtJWW8tWCj03FCnhUL06SUV9Vw8ztLeO+bTfz6jF48cVF/IsLt17i5OqV3e87ok8jznxSwt7TS6Tghzf7KTJOx71AVV7/2NbPzdjDhZ5ncPTLdBtsZ7h2TQWlFNc99YgP1/MmKhWkStu8r55K/L2DZ5hKev3wA407p4XQkEyTSOsZy6Ynd+efCIjbuKnU6TsiyYmGCXsHOA1zwt6/YWlLGm78YxLn9uzgdyQSZO85KIzI8jCdm2kA9f7FiYYLa4qK9XPTyAiprlPdvPJkhvds7HckEoQ5xMfxqaC+mr9zO4iK7QLU/WLEwQeuTNTu4cuJC4ltE8p/xQ+jXNd7pSCaI3TC0Bx1io3lkWh6hcjXtYGLFwgSlf+Vs5oa3FtO7Q2sm3zSEpHYtnY5kglzLqAjuHJHGkk0lfLxqu9NxQo4VCxNUVJW/fVrA7yavYHDPdrx/42ASY23eAuObi7K606djLI/PWENlda3TcUKKFQsTNGprlQenruaJGfmMPb4Lr487kdbREU7HMk1IeJhw75h0inYf4p8Li5yOE1KsWJigUFFdw28mLeON+Ru57pQe/PXSE4iKsF9P03CnpyVyWmp7nvtkHfvKqpyOEzLsr9E47kB5Fde9sYipy7dxz+h0/nhuBmFhNtjOHB0R4d7RGewrq+Jv8wqcjhMyrFgYRxUfqOCyVxaysHAPT198PDed3stGZZtjltkljgsHduMfX21k855DTscJCVYsjGOKdpdy0cvzKSwuZeI12VyY1c3pSCaE3DkijbAweHJmvtNRQoIVC+OIVVv3ceFL89lfVsW7N5zEGekdnI5kQkzn+BbccFpPpizfxvLNJU7HafKsWJiA+3LdLi79+wKiI8KZPH4IA5ISnI5kQtSvTu9F+9ZRPDLdBuodKysWJqCmLN/GL974hm4JLfn3+CH0SmztdCQTwlpHR3D7WWl8s2EPs1fvcDpOk2bFwgTM619u4Lb3ljKgewL/umkwneJjnI5kmoHLTuxOr8RWPDZjDVU1NlDvaFmxMH6nqjw+Yw0PfbSakX078tb1g4hvEel0LNNMRISHce/oDAqLS3n/m01Ox2myrFgYv6qqqeXuySt46dP1XHFSEn+7MouYSJsC1QTWmRkdOLlnW/46Zx0Hym2g3tGwYmH85lBlNTe+lcPkxVu4/axUHvl5P8JtsJ1xgIhw/5hMdpdW8vJn652O0yRZsTB+sbe0kite/ZrP1hbzfz/vx+1npdlgO+Oo47rF8/MTujDxiw1sKylzOk6TY8XCNLqtJWVc9PJ8Vn+7n79dmcVVJyc7HckYAO4a2QcFnpplA/Uayq/FQkRGiUi+iBSIyD31PD5ORIpFZJn79kuPx54QkVwRyROR58S+ljYJa7bv54K/fcXOAxW8fd0gRvXr5HQkY77TLaElvzglhQ+XbmXV1n1Ox2lS/FYsRCQceBEYDWQCl4tIZj2bTlLVE9y3ie7nDgFOAfoD/YATgdP9ldU0jm827OHilxcA8MFNgzmpZzuHExnzYzcP602bFpE8agP1GsSfRxaDgAJVLVTVSuB94Dwfn6tADBAFRAORgI2oCWIzc7dz1Wtfkxgbzb/HDyG9U5zTkYypV3yLSH5zZirz1+/m0/xip+M0Gf4sFl2BzR7LW9zr6rpQRFaIyGQR6Q6gqguAecC37ttMVc3zY1ZzDN79ehPj/7mYzM5xTL5pCN0SbApUE9yuOCmZlHYteXR6HtU2UM8n/iwW9bUx1D3mmwqkqGp/YA7wJoCI9AYygG64CsxwERn6ox2I3CgiOSKSU1xs3xACTVV5ds467vtwJUPTEnn3hpNo2yrK6VjGeBUVEcY9o9NZt/MgHyze4nScJsGfxWIL0N1juRuwzXMDVd2tqhXuxVeBLPf984GFqnpQVQ8CHwMn192Bqr6iqtmqmp2YmNjoP4D5aTW1yh/+u4pn5qzlwoHdePWabFpG2RSopukY2bcT2ckJPD1rLaUV1U7HCXr+LBaLgFQR6SEiUcBlwBTPDUSks8fiWODwqaZNwOkiEiEikbgat+00VJAor6rh1+8s4Z2vN3HT6b146uL+RIZbL2zTtIgI952Twa6DFbzyeaHTcYKe3/7CVbUauAWYieuD/l+qmisiD4nIWPdmt7m7xy4HbgPGuddPBtYDK4HlwHJVneqvrMZ3NbXKdW8sYkbudv54bib3jE63wXamyRqYlMA5/TvzyueF7Nhf7nScoCah0nUsOztbc3JynI4R8t5eWMQf/7uKR88/jitOSnI6jjHHbNPuQ5z5l0+5YEA3Hr+ov9NxAk5EFqtqtrft7NyB8dne0kqenpXP4J7tuHxQd+9PMKYJSGrXkmsGp/DB4s2s2b7f6ThBy4qF8dmTs/I5UF7NhLF97dSTCSm3Du9N6+gI/jx9jdNRgpYVC+OTVVv38d43m7j65GT6dIp1Oo4xjapNyyhuOzOVz9YW88U664ZfHysWxitV5YEpubRtGcUdZ6c5HccYv7h6cDLd27bgkWl51NSGRltuY7JiYbz6cOlWFhft5Xej+tgMdyZkRUeE87uR6azZfoD/LLGBenVZsTBHdKC8ij9/vIbju8VzcZY1apvQdm7/zhzfvQ1Pz1pLWWWN03GCihULc0TPf1JA8YEKHjyvH2E2y50Jca4Z9TLYvr+c1760gXqerFiYn1Sw8yCvf7mBS7K7cUL3Nk7HMSYgBvVoy8i+HXnp0/UUH6jw/oRmwoqFqZeq8uDUXFpEhfO7UelOxzEmoH4/Kp2K6lqenbvW6ShBw+diISJdRWSIiAw9fPNnMOOsWat38MW6XdxxVhrtW0c7HceYgOqZ2JorT0rivW82U7DzgNNxgoJPxUJEHge+Av4A3O2+3eXHXMZB5VU1PPzRatI6tubqwTZ/tmmebjszlZaR4Tz2sQ3UA/D1mtI/B/p4XE7chLCXP1vPlr1lvHvDSXY1WdNstWsdzfgzevHEjHwWrN/N4F7Ne5pgXz8JCnFNbWpC3OY9h3jp0/Wcc1xnhvRq73QcYxx13Sk96BIfw6PT86ht5gP1fC0Wh4BlIvJ3EXnu8M2fwYwzHpmWhwjcd06G01GMcVxMZDh3j+rDyq37mLJ8m/cnhDBfi8UU4GFgPrDY42ZCyJfrdjEjdzu/Htabrm1aOB3HmKBw3vFd6dc1jidn5lNe1XwH6vlULFT1TeA9vi8S77rXmRBRVVPLhKm5JLVtyQ1Dezodx5igERYm3Dc6g60lZbwxf6PTcRzja2+oYcA64EXgb8Ba6zobWt6cv5GCnQf507mZxESGOx3HmKAypHd7zkzvwIufFLCntNLpOI7w9TTU08AIVT1dVYcCI4Fn/BfLBNLOA+X8dc46hvVJ5MyMDk7HMSYo3TM6ndLKap6bu87pKI7wtVhEqmr+4QVVXYv1jgoZj3+cT0V1DX86N9MmNTLmJ6R2jOWyQUn8c2ERG3aVOh0n4HwtFjki8pqIDHPfXsUauEPC4qK9/HvJFq4/tSc9E1s7HceYoHb7WalER4TxeDMcqOdrsRgP5AK3Ab8BVgM3+SuUCYyaWmXClFw6xkVz6/DeTscxJuh1iI3hV6f3YkbudnI27nE6TkD52huqQlX/oqoXqOr5qvqMjeZu+iYt2szKrfu4b0wGraJ9HcxvTPP2y9N60DEumv+blodq8xmod8RiISL/cv+7UkRW1L0FJqLxh5JDlTw5cw2DUtoy9vguTscxpsloGRXBnSP6sGxzCdNWfut0nIDx9nXyN+5/z/V3EBNYf5m9ln1lVUwY29catY1poAsHduP1LzfwxIx8zs7sSHRE6Hc3P+KRhaoeLpu7gM2qWgREA8cDzXvsexO2ett+/rmwiCtPSiazS5zTcYxpcsLDhPvGZLBpzyHeXlDkdJyA8LWB+3MgRkS6AnOBXwBv+CuU8R9VV6N2fItI7hyR5nQcY5qsoWmJnJbanuc/KWDfoSqn4/idr8VCVPUQcAHwvKqeD2T6L5bxlynLt/HNxj3cPTKdNi2jnI5jTJN235gM9pdX8cK80B+o53OxEJHBwJXANPc66z7TxJRWVPPo9Dz6dY3j0hO7Ox3HmCYvo3McF2d14835RWzec8jpOH7la7G4HbgX+FBVc0WkJzDPf7GMP7wwr4Ad+yt4cGw/wsOsUduYxvDbs/sQFgZPzMz3vnET5us4i89UdayqPu5eLlTV2/wbzTSmDbtKmfhFIRcM7EpWcoLTcYwJGZ3iY7jxtJ5MXb6NpZv2Oh3Hb7yNs/ir+9+pIjKl7s3bi4vIKBHJF5ECEbmnnsfHiUixiCxz337p8ViSiMwSkTwRWS0iKQ3/8Qy4GrUfnJpLdEQ494xOdzqOMSHnxtN70b51NI9OD92Bet7aHd52//tUQ19YRMJxXdL8bGALsEhEpqjq6jqbTlLVW+p5ibeAR1R1toi0BmobmsG4zM3byaf5xdw/JoMOsTFOxzEm5LSOjuCOs1O5/8NVzFq9g5F9OzkdqdEdsVio6uGLBeYAZapaC98Vgmgvrz0IKFDVQvdz3gfOw3VdqSMSkUwgQlVnu3Mc9PYcU7/yqhoe+mg1vRJbce2QFKfjGBOyLs3uzj++2shjH69heHoHIsN9bRJuGnz9aeYCLT2WWwBzvDynK7DZY3mLe11dF7ovHzJZRA530UkDSkTkPyKyVESedBeoHxCRG0UkR0RyiouLffxRmpeJXxSyac8hJoztS1REaP3yGhNMIsLDuG9MOht2lfLu15ucjtPofP30iPH8du++3/II2wPU192m7sm8qUCKqvbHVXwOT9UaAZwG3AWcCPQExv3oxVRfUdVsVc1OTEz05edoVraVlPHivPWM7NuR01Lt/THG387o04HBPdvx7Nx17C8PrYF6vhaLUhEZeHhBRLKAMi/P2QJ4dubvRp1LhKjqbo+r174KZHk8d6m711U18F9gIKZBHpmeR60qfzjHxk8aEwgiwv3nZLCntJKXPl3vdJxG1ZBxFh+IyBci8gUwCaivUdrTIiBVRHqISBRwGfCDHlQi0tljcSyQ5/HcBBE5/HV4OD60dZjvzV+/i2krvmX8sF50b+vtINAY01j6dY3nggFdee3LDWwt8faduunwdZzFIiAd1yRINwMZHo3fP/WcalwFZSauIvAv94C+h0RkrHuz20QkV0SW45pYaZz7uTW4TkHNFZGVuE5pvdrQH665qq6p5cEpq+mW0IKbTu/ldBxjmp07R/YB4OkQGqjn0yU7RKQl8FsgWVVvEJFUEemjqh8d6XmqOh2YXmfdnzzu34trZHh9z50N9Pcln/mhtxcWkb/jAC9flUVMZOhfOtmYYNO1TQuuP7UHL326nutO7UG/rvFORzpmvp6G+gdQCQx2L28B/s8vicwx2XWwgr/MXstpqe0Z2bej03GMabbGD+tF21ZRPBIiM+r5Wix6qeoTQBWAqpZRf28n47AnZ+RTVlnDAz+zSY2McVJcTCS/OTOVBYW7mZe/0+k4x8zXYlEpIi1wd30VkV6AzcEdZJZtLmFSzmZ+cUoKvTu0djqOMc3eFScl0aN9Kx6dvobqmqZ9EQpfi8UDwAygu4i8g2uQ3u/8lso0WG2t8sD/VpEYG81tZ6Y6HccYA0SGh3HP6HQKdh5kUs5m708IYl6LhbjOZazBNfHROOA9IFtVP/VrMtMgkxdvYfmWfdwzKp3YmEin4xhj3EZkduTElASemb2OgxXVTsc5al6LhbpaZv7rHkA3TVU/UtVdAchmfLSvrIrHZ6xhYFIbzh9Q3xVVjDFOEXHN173rYAWvfNZ0B+r5ehpqoYic6Nck5qj9dc5a9hyq5KHz+hFmkxoZE3QGJCVwbv/OvPJFIdv3lTsd56j4WizOwFUw1rsv+rdSRFb4M5jxTf72A7y1oIjLByWFRF9uY0LV70elU1sLf5ndNAfq+TqP9mi/pjBHRVWZMCWX1tER3D2ij9NxjDFH0L1tS64dkszELzfwi1N6kNE5zulIDeJtprwYEbkduBsYBWxV1aLDt4AkND9p+srtLCjczV0j0khoFeV0HGOMF7eckUpcTCSPTs/zvnGQ8XYa6k0gG1iJ6+jiab8nMj45VFnNI9NWk9E5jitOSnY6jjHGB/EtI7l1eG++WLeLz9Y2rTl4vBWLTFW9SlX/DlyEa44JEwRe+nQ92/aV8+DYvoRbo7YxTcbVg5NJatuSP0/Po6a26VwGxFux+G72DvdVZE0QKNpdyt8/K+S8E7owqEdbp+MYYxogOiKc343qw5rtB/j34i1Ox/GZt2JxvIjsd98OAP0P3xeR/YEIaH7s4Y9WExEu3Ds6w+koxpijcM5xnTmhexuenp3Pocqm8T38iMVCVcNVNc59i1XVCI/7TaspP0TMy9/JnLyd3Do8lU7xMU7HMcYcBRHhD+dksGN/BRO/2OB0HJ/4Os7CBIGK6hoemrqaHu1bcd2pKU7HMcYcg+yUtozq24mXP1vPzgPBP1DPikUT8vqXG9mwq5QHfpZJdIRNamRMU/f70elUVtfy1znrnI7ilRWLJmL7vnKe/2QdZ2V0ZFifDk7HMcY0gh7tW3HVycm8/80m1u044HScI7Ji0UT8+eM8qmuVP52b6XQUY0wjuu3MVFpFRfDYx2ucjnJEViyagG827OF/y7bxq6E9SWrX0uk4xphG1LZVFL8e3pu5a3YyvyB4L+htxSLI1dQqD0zJpUt8DDcP6+10HGOMH4wbkkLXNi14ZHoetUE6UM+KRZB79+si8r7dz/3nZNIiyhq1jQlFMZHh3D2yD7nb9vO/5VudjlMvKxZBbE9pJU/NWsvgnu0Yc1wnp+MYY/xo7PFdOK5rPE/OyKe8qsbpOD9ixSKIPTkzn4MV1Tx4Xl9cs9saY0JVWJhrRr1t+8p5/avgG6hnxSJIrdyyj/cXbeKawcmkdYx1Oo4xJgAG92rHWRkdeGneenYfrHA6zg9YsQhCtbXKA1NW0bZlFLefleZ0HGNMAN0zOp1DVTU8Nze4BupZsQhCHy7dypJNJfx+dDrxLSKdjmOMCaDeHWK57MTuvPP1JgqLDzod5ztWLILMgfIq/vzxGk7o3oaLBnZzOo4xxgG3n5VGdEQYj88InoF6ViyCzHNz17G7tIIHx/YlzCY1MqZZSoyNZvywXszM3cE3G/Y4HQfwc7EQkVEiki8iBSJyTz2PjxORYhFZ5r79ss7jcSKyVURe8GfOYFGw8wD/+Gojl2R15/jubZyOY4xx0PWn9qRTXAyPTM9D1fmBen4rFiISDryIa+7uTOByEanvwkaTVPUE921incceBj7zV8ZgoqpMmLKaFlHh3D2qj9NxjDEOaxEVzp0j0li+uYSPVnzrdBy/HlkMAgpUtVBVK4H3gfN8fbKIZAEdgVl+yhdUZuZu58uCXfz27DTat452Oo4xJghcMLAbGZ3jeHzGGiqqnR2o589i0RXY7LG8xb2urgtFZIWITBaR7gAiEga3YJq1AAAPMklEQVQ8Ddztx3xBo6yyhoc/yqNPx1iuPjnZ6TjGmCARHibcNyadLXvLeGt+kaNZ/Fks6mudrXvibSqQoqr9gTnAm+71NwPTVXUzRyAiN4pIjojkFBcXH3Ngp7z82Xq2lpQxYWxfIsKtz4Ex5nunpSZyeloiz3+yjpJDlY7l8Ocn0xagu8dyN2Cb5waqultVDw9TfBXIct8fDNwiIhuBp4BrROSxujtQ1VdUNVtVsxMTExs7f0Bs3nOIlz9bzzn9OzO4Vzun4xhjgtB9YzI4WFHN858UOJbBn8ViEZAqIj1EJAq4DJjiuYGIdPZYHAvkAajqlaqapKopwF3AW6r6o95UoeD/pq0mTIT7x2Q4HcUYE6T6dIrl4qzuvLVgI0W7Sx3J4LdioarVwC3ATFxF4F+qmisiD4nIWPdmt4lIrogsB24DxvkrTzD6Yl0xM3N3cMvw3nRp08LpOMaYIPbbEWlEhIXxxMx8R/YvwdB/tzFkZ2drTk6O0zF8Vlldy+hnP6e6Vpl1x1CiI2yuCmPMkT0zey3Pzl3Hv8cPISs5oVFeU0QWq2q2t+2sNdUhb87fyPriUv50bqYVCmOMT24c2pPE2GgedWCgnhULB+zcX86zc9dxRp9Ezszo6HQcY0wT0So6gt+encbior3MzN0e0H1bsXDAYzPWUFldy59+1tfpKMaYJubirG6kdWzNYx+7PkcCxYpFgC0u2sN/lmzl+tN60KN9K6fjGGOamIjwMO4dncHG3Yd49+vADdSzYhFANbXKn/6XS6e4GG45o7fTcYwxTdSwPomc0rsdz85dx76yqoDs04pFAL2/aBO52/Zz75h0WkVHOB3HGNNEibjm6y4pq+JvnwZmoJ4ViwApOVTJUzPzGdSjLWOP7+J0HGNME9e3SzznD+jKP77ayJa9h/y+PysWAfL0rLXsK6tiws/6ImKTGhljjt1dI/ogwFMBGKhn50ICYPW2/bzzdRFXn5xMZpc4p+MYY0JElzYtuHlYbw5VVaOqfv0iasXCz1yTGuXSpmUUvz3bJjUyxjSu35yVGpD92GkoP5uyfBvfbNzD3SP7EN8y0uk4xhhzVKxY+FFpRTWPTs/juK7xXJLd3fsTjDEmSNlpKD96/pMCduyv4KWrsggPs0ZtY0zTZUcWfrK++CCvfVnIhQO7MTCpca4OaYwxTrFi4QeqykNTVxMdEc7vR1ujtjGm6bNi4Qdz8nby2dpibj8rlQ6xMU7HMcaYY2bFopGVV9Xw8Eer6d2hNdcOSXE6jjHGNApr4G5kr35eyKY9h/jn9ScRGW612BgTGuzTrBFtLSnjxU8LGNW3E6emtnc6jjHGNBorFo3o0Wl5qMIfzs1wOooxxjQqKxaNZH7BLqat/Jabh/WmW0JLp+MYY0yjsmLRCKpqapkwNZduCS341ek9nY5jjDGNzopFI3h7QRFrdxzkj+dmEhMZ7nQcY4xpdFYsjlHxgQqemb2W01LbMyKzo9NxjDHGL6xYHKMnZqyhrKqGB2xSI2NMCLNicQyWbtrLB4u3cN2pPejdobXTcYwxxm+sWByl2lrXpEaJsdHcOry303GMMcavrFgcpQ8Wb2b5ln3cOzqd2Bib1MgYE9qsWByFfWVVPDEjn6zkBM4f0NXpOMYY43d2baij8Mzstew5VMmbYwdZo7Yxplnw65GFiIwSkXwRKRCRe+p5fJyIFIvIMvftl+71J4jIAhHJFZEVInKpP3M2RP72A7y9sIgrBiXRr2u803GMMSYg/HZkISLhwIvA2cAWYJGITFHV1XU2naSqt9RZdwi4RlXXiUgXYLGIzFTVEn/l9YWq8sCUVcTGRHDXCJvUyBjTfPjzyGIQUKCqhapaCbwPnOfLE1V1raquc9/fBuwEEv2W1EfTVn7LwsI93DmiDwmtopyOY4wxAePPYtEV2OyxvMW9rq4L3aeaJotI97oPisggIApY75+YvjlUWc0j0/LI7BzHFYOSnIxijDEB589iUV/Lr9ZZngqkqGp/YA7w5g9eQKQz8DbwC1Wt/dEORG4UkRwRySkuLm6k2PV7cV4B3+4r58Hz+hIeZo3axpjmxZ/FYgvgeaTQDdjmuYGq7lbVCvfiq0DW4cdEJA6YBvxBVRfWtwNVfUVVs1U1OzHRf2epNu4q5dXPN/DzE7pwYkpbv+3HGGOClT+LxSIgVUR6iEgUcBkwxXMD95HDYWOBPPf6KOBD4C1V/cCPGX3y8EeriQwX7h1jkxoZY5onv/WGUtVqEbkFmAmEA6+raq6IPATkqOoU4DYRGQtUA3uAce6nXwIMBdqJyOF141R1mb/y/pR5a3Yyd81O7hmdTse4mEDv3hhjgoKo1m1GaJqys7M1JyenUV+zorqGkc98TpgIM24fSlSEDXg3xoQWEVmsqtnetrMR3Efw2pcb2Lj7EG9eN8gKhTGmWbNPwJ+wfV85L3xSwNmZHTk9zfEhHsYY4ygrFj/h0el5VNcqfzwn0+koxhjjOCsW9fi6cDdTlm/jpqE9SWrX0uk4xhjjOCsWdVTX1PLAlFy6tmnB+GE2qZExxoAVix955+tNrNl+gPvPyaBFVLjTcYwxJihYsfCw+2AFT8/KZ0ivdozu18npOMYYEzSsWHh4alY+pZU1TBjb1yY1MsYYD1Ys3FZsKeH9RZu5dnAKaR1jnY5jjDFBxYoFUFurPDAll3atorj97FSn4xhjTNCxYgH8Z+lWlm4q4fej0omLiXQ6jjHGBJ1mXyz2l1fx2MdrGJDUhgsHdnM6jjHGBKVmf22o8qoaBia14ddn9CbMJjUyxph6Nfti0SE2hleu8XrBRWOMadaa/WkoY4wx3lmxMMYY45UVC2OMMV5ZsTDGGOOVFQtjjDFeWbEwxhjjlRULY4wxXlmxMMYY45WoqtMZGoWIFANFx/AS7YFdjRSnMVmuhrFcDWO5GiYUcyWraqK3jUKmWBwrEclR1aAbym25GsZyNYzlapjmnMtOQxljjPHKioUxxhivrFh87xWnA/wEy9UwlqthLFfDNNtc1mZhjDHGKzuyMMYY41WzKhYiMkpE8kWkQETuqefxoSKyRESqReSiIMr1WxFZLSIrRGSuiCQHUbabRGSliCwTkS9FJDMYcnlsd5GIqIgEpAeLD+/XOBEpdr9fy0Tkl8GQy73NJe7fs1wReTcYconIMx7v1VoRKQmSXEkiMk9Elrr/LscESa5k92fEChH5VEQab/pPVW0WNyAcWA/0BKKA5UBmnW1SgP7AW8BFQZTrDKCl+/54YFIQZYvzuD8WmBEMudzbxQKfAwuB7GDIBYwDXgjE/18Dc6UCS4EE93KHYMhVZ/tbgdeDIReuNoLx7vuZwMYgyfUBcK37/nDg7cbaf3M6shgEFKhqoapWAu8D53luoKobVXUFUBtkueap6iH34kIgUJOF+5Jtv8diKyAQjWBec7k9DDwBlAcgU0NyBZovuW4AXlTVvQCqujNIcnm6HHgvSHIpEOe+Hw9sC5JcmcBc9/159Tx+1JpTsegKbPZY3uJe57SG5roe+Nivib7nUzYR+bWIrMf1wXxbMOQSkQFAd1X9KAB5fM7ldqH7NMFkEekeJLnSgDQR+UpEForIqCDJBbhOrwA9gE+CJNcE4CoR2QJMx3XUEwy5lgMXuu+fD8SKSLvG2HlzKhZSz7pg6Armcy4RuQrIBp70ayKPXdaz7kfZVPVFVe0F/B74g99TecklImHAM8CdAcjiyZf3ayqQoqr9gTnAm35P5VuuCFynoobh+gY/UUTaBEGuwy4DJqtqjR/zHOZLrsuBN1S1GzAGeNv9e+d0rruA00VkKXA6sBWoboydN6disQXw/BbXjcAcOnrjUy4ROQu4HxirqhXBlM3D+8DP/ZrIxVuuWKAf8KmIbAROBqYEoJHb6/ulqrs9/v9eBbL8nMmnXO5t/qeqVaq6AcjHVTycznXYZQTmFBT4lut64F8AqroAiMF1fSZHc6nqNlW9QFUH4Pq8QFX3Ncre/d0oEyw3XN+cCnEdyh5uHOr7E9u+QeAauL3mAgbgathKDbb3zDMT8DMgJxhy1dn+UwLTwO3L+9XZ4/75wMIgyTUKeNN9vz2u0x3tnM7l3q4PsBH3uLAgeb8+Bsa572fg+tD2az4fc7UHwtz3HwEearT9B+LND5YbrsPFte4P3vvd6x7C9W0d4ERc1bsU2A3kBkmuOcAOYJn7NiWI3rNngVx3rnlH+tAOZK462wakWPj4fv3Z/X4td79f6UGSS4C/AKuBlcBlwZDLvTwBeCwQeRrwfmUCX7n/H5cBI4Ik10XAOvc2E4Hoxtq3jeA2xhjjVXNqszDGGHOUrFgYY4zxyoqFMcYYr6xYGGOM8cqKhTHGGK+sWBhTh4jUuK9ymisiy91X/Q1zP5YtIs/5ef/z3f+miMgV/tyXMb6yrrPG1CEiB1W1tft+B+Bd4CtVfaAR9xGhqke8DIOIDAPuUtVzG2u/xhwtO7Iw5gjUdfXVG4FbxGWYiHwkImEistHz+knuOQY6ikiiiPxbRBa5b6e4H58gIq+IyCzgLRHpKyLfuI9iVohIqnu7g+6XfAw4zf34HSLyhYic4LG/r0Skf8DeDNOsRTgdwJhgp6qF7tNQHTzW1YrI/3BdsuMfInISrjkNdrgnDnpGVb8UkSRgJq5LQoDrWlCnqmqZiDwPPKuq74hIFK75Cjzdg8eRhYjswTUfxu0ikoZrdO4Kv/3gxniwIwtjfFPfFT8nAZe671/mXgY4C3hBRJYBU4A4EYl1PzZFVcvc9xcA94nI74Fkj/U/5QPgXBGJBK7DdQ0zYwLCioUxXohIT6AGqDsh0AKgt4gk4rra7n/c68OAwap6gvvWVVUPuB8rPfxkVX0X1+yCZcBMERl+pBzqmgBrNq4JbS7B1ZZiTEBYsTDmCNyF4GVcU6H+oDeIe/lDXBfgy1PV3e6HZgG3eLzGCdTDXYQKVfU5XEcgddsfDuC63LqnicBzwCJV3XNUP5QxR8GKhTE/1uJw11lcV/ydBTz4E9tOAq7i+1NQ4JotMNvdaL0auOknnnspsMp9uiod19zvnlYA1e7uu3cAqOpiYD/wj6P4uYw5atZ11pgmRES64LrkerqqBnKueNPM2ZGFMU2EiFwDfI1rHgMrFCag7MjCGGOMV3ZkYYwxxisrFsYYY7yyYmGMMcYrKxbGGGO8smJhjDHGKysWxhhjvPp/QbQpP7V/ccIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(div_set,prec_set) \n",
    "plt.xlabel('Diversity') \n",
    "plt.ylabel('Precision') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 521,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.5111111111111112,\n",
       " 0.5777777777777778,\n",
       " 0.6133333333333333,\n",
       " 0.6044444444444443,\n",
       " 0.5333333333333333]"
      ]
     },
     "execution_count": 521,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prec_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}