multiSEQ_to_csv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 322,
   "id": "valued-boutique",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import jsonlines\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "id": "exotic-price",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(875, 15)\n"
     ]
    }
   ],
   "source": [
    "optionlim = 0.1\n",
    "predfolder = 'multitask4mixbidspec2verbose2_prin_ml_1024_50_bs50_seq3_4bs_2DAYS' ## folder containing pred file\n",
    "\n",
    "goldfolder = 'multitask4mix_bi' ## folder containing gold file\n",
    "\n",
    "dataset = 'ml' ## dataset code (or part of it), is a column in gold file\n",
    "\n",
    "gfile = goldfolder+'/testml.csv' ## gold file\n",
    "#gfile = goldfolder+'/test.csv'\n",
    "\n",
    "#opfile = goldfolder+'/testmavenevex.csv'\n",
    "\n",
    "task = 'EV-EX'\n",
    "\n",
    "\n",
    "\n",
    "add_to_main_file = False\n",
    "\n",
    "\n",
    "golddf1 = pd.read_csv(gfile)\n",
    "\n",
    "if('subtask' in golddf1.columns):\n",
    "    golddf = golddf1[golddf1.subtask==task]\n",
    "    golddf = golddf[golddf.dataset.str.contains(dataset)]\n",
    "    print(golddf.shape)\n",
    "else:\n",
    "    golddf = golddf1\n",
    "\n",
    "return_seqs = 3\n",
    "\n",
    "textcol = 'text'\n",
    "relcol = 'src_label'\n",
    "\n",
    "#if('src21' in predfolder):\n",
    "#    relcol = 'src_label21'\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "modified-cleveland",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "id": "finished-suspect",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def reverse_21(pred):\n",
    "    ## reversed outputs/ convert to src_label1 format\n",
    "    predz=[]\n",
    "    for p in pred:\n",
    "        fin=[]\n",
    "        events = [x.strip() for x in p.split('|')]\n",
    "        for e in events:\n",
    "            #print(e)\n",
    "            if('->' not in e):\n",
    "                typ,trigs=['','']\n",
    "            else:\n",
    "                typ,trigs=e.split('->')[0:2]\n",
    "            \n",
    "            for tg in trigs.split('@'):\n",
    "                fin.append(tg+'->'+typ)\n",
    "        if(p.strip()=='->'):\n",
    "            predz.append('NONE')\n",
    "        else:\n",
    "            predz.append(' | '.join(fin))\n",
    "        \n",
    "    pred = predz.copy()\n",
    "    return pred\n",
    "\n",
    "    \n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "def reverse_12(pred):\n",
    "    ## reversed outputs/ convert to src_label1 format\n",
    "    predz=[]\n",
    "    for idx,p in enumerate(pred):\n",
    "        nm=[]\n",
    "        if('NONE' in p):\n",
    "            predz.append('NONE')\n",
    "            continue\n",
    "        \n",
    "        events = [x.strip() for x in p.split('|')]\n",
    "        #print(events,sb.iloc[idx])\n",
    "        for e in events:\n",
    "            if('->' not in e):\n",
    "                continue\n",
    "            tg = e.split('->')[0]\n",
    "            tps = e.split('->')[1].split('@')\n",
    "            nm += [tg+'->'+p for p in tps]\n",
    "        if(len(nm)==0):\n",
    "            nm = ['NONE']\n",
    "        predz.append(' | '.join(nm))\n",
    "        \n",
    "    pred = predz.copy()\n",
    "    return pred\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "informational-corps",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "id": "split-communication",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "875\n"
     ]
    }
   ],
   "source": [
    "gold = golddf[relcol]\n",
    "print(len(gold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "id": "extra-payday",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10499\n",
      "10500\n"
     ]
    }
   ],
   "source": [
    "pred = open('results/'+predfolder+'/generated_predictions.txt').readlines()\n",
    "print(len(pred))\n",
    "pred.append('\\n')\n",
    "print(len(pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "front-nightmare",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "public-weight",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "id": "furnished-annual",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([\"b'The types of events in this text are negative_regulation, localization, gene_expression, growth, positive_regulation, planned_process'\\n\",\n",
       "  \"b'The types of events in this text are localization, negative_regulation, gene_expression, growth, positive_regulation, planned_process'\\n\",\n",
       "  \"b'The types of events in this text are negative_regulation, growth, gene_expression, localization, positive_regulation, planned_process'\\n\",\n",
       "  '\\n',\n",
       "  \"b'The event triggers in this text are mediated, inhibition, growth, metastases, transfer, expression'\\n\"],\n",
       " 2     inhibition->negative_regulation | transfer->pl...\n",
       " 5     Inhibition->negative_regulation | angiogenesis...\n",
       " 8                  angiogenic->blood_vessel_development\n",
       " 11    angiogenic->blood_vessel_development | express...\n",
       " 14    suppression->negative_regulation | induced->po...\n",
       " Name: src_label, dtype: object)"
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred[0:5],gold[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "id": "rough-assignment",
   "metadata": {},
   "outputs": [],
   "source": [
    "if('verb' in predfolder):\n",
    "    p1 = pd.Series(pred).str.replace('The types of events in this text are ','').str.replace('The event triggers in this text are ','').str.replace('Events in this text: ','').str.replace('The event triggers and their types in this text are: ','')\n",
    "    p2 = p1.str.replace(' is a trigger of type ','->').str.replace(',',' |')\n",
    "    pred = p2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "id": "orange-hundred",
   "metadata": {},
   "outputs": [],
   "source": [
    "pls=[]\n",
    "\n",
    "plsi=[]\n",
    "for p in pred:\n",
    "    \n",
    "    if(p=='\\n'):\n",
    "        pls.append(plsi)\n",
    "        plsi=[]\n",
    "    else:\n",
    "        plsi.append(p[2:-2])\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "id": "careful-treaty",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pls[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "id": "bottom-detector",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2625, 3, 875)"
      ]
     },
     "execution_count": 331,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pls),len(pls[0]),len(gold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caring-mount",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "reserved-trance",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "id": "natural-transparency",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['negative_regulation | localization | gene_expression | growth | positive_regulation | planned_process',\n",
       "  'localization | negative_regulation | gene_expression | growth | positive_regulation | planned_process',\n",
       "  'negative_regulation | growth | gene_expression | localization | positive_regulation | planned_process'],\n",
       " ['mediated | inhibition | growth | metastases | transfer | expression',\n",
       "  'mediated | loss | growth | inhibition | transfer | expression',\n",
       "  'mediated | inhibition | growth | transfer | expression'],\n",
       " ['growth->growth | mediated->regulation | inhibition->negative_regulation | expression->gene_expression | metastases->localization | transfer->planned_process | results->positive_regulation',\n",
       "  'growth->growth | inhibition->negative_regulation | expression->gene_expression | mediated->positive_regulation | metastases->localization | transfer->planned_process | results->positive_regulation',\n",
       "  'growth->growth | inhibition->negative_regulation | expression->gene_expression | transfer->planned_process | results->positive_regulation | mediated->positive_regulation | metastases->localization'],\n",
       " ['negative_regulation | blood_vessel_development | planned_process',\n",
       "  'negative_regulation | blood_vessel_development | positive_regulation | planned_process',\n",
       "  'negative_regulation | blood_vessel_development | planned_process | negative_regulation | blood_vessel_development | planned_process']]"
      ]
     },
     "execution_count": 332,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pls[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "id": "august-december",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(875, 2625)"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(golddf),len(golddf1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "engaged-collectible",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "id": "scheduled-immigration",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Int64Index([   2,    5,    8,   11,   14,   17,   20,   23,   26,   29,\n",
       "             ...\n",
       "             2597, 2600, 2603, 2606, 2609, 2612, 2615, 2618, 2621, 2624],\n",
       "            dtype='int64', length=875),\n",
       " 2625)"
      ]
     },
     "execution_count": 334,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "golddf.index, len(pls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "id": "caring-vector",
   "metadata": {},
   "outputs": [],
   "source": [
    "if(len(golddf)<len(golddf1)):\n",
    "    pls = np.asarray(pls)\n",
    "    pls = pls[golddf.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "id": "direct-jimmy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(875, 3)"
      ]
     },
     "execution_count": 336,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pls = np.asarray(pls)\n",
    "pls.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "royal-shoot",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "id": "varied-throw",
   "metadata": {},
   "outputs": [],
   "source": [
    "op = pd.DataFrame(data={'text':golddf[textcol],'gold':gold})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "id": "hungarian-promise",
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in range(0,return_seqs):\n",
    "    op['top_'+str(k+1)] = pls[:,k]\n",
    "    if('src12' in predfolder):\n",
    "        op['top_'+str(k+1)] = reverse_12(op['top_'+str(k+1)])\n",
    "    if('src21' in predfolder):\n",
    "        op['top_'+str(k+1)] = reverse_21(op['top_'+str(k+1)])\n",
    "        \n",
    "        \n",
    "    \n",
    "    if('verb' in predfolder):\n",
    "        p1 = op['top_'+str(k+1)].str.replace('The types of events in this text are ','').str.replace('The event triggers in this text are ','').str.replace('Events in this text: ','').str.replace('The event triggers and their types in this text are: ','')\n",
    "        p2 = p1.str.replace(' is a trigger of type ','->').str.replace(',',' |')\n",
    "        op['top_'+str(k+1)] = p2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "id": "expensive-testing",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>gold</th>\n",
       "      <th>top_1</th>\n",
       "      <th>top_2</th>\n",
       "      <th>top_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>EV-EX&lt;/s&gt;mlee_processed_sent&lt;/s&gt;Adenovirus-med...</td>\n",
       "      <td>inhibition-&gt;negative_regulation | transfer-&gt;pl...</td>\n",
       "      <td>growth-&gt;growth | mediated-&gt;regulation | inhibi...</td>\n",
       "      <td>growth-&gt;growth | inhibition-&gt;negative_regulati...</td>\n",
       "      <td>growth-&gt;growth | inhibition-&gt;negative_regulati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>EV-EX&lt;/s&gt;mlee_processed_sent&lt;/s&gt;Inhibition of ...</td>\n",
       "      <td>Inhibition-&gt;negative_regulation | angiogenesis...</td>\n",
       "      <td>Inhibition-&gt;negative_regulation | therapy-&gt;pla...</td>\n",
       "      <td>therapy-&gt;planned_process | Inhibition-&gt;negativ...</td>\n",
       "      <td>Inhibition-&gt;negative_regulation | angiogenesis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>EV-EX&lt;/s&gt;mlee_processed_sent&lt;/s&gt;However, its w...</td>\n",
       "      <td>angiogenic-&gt;blood_vessel_development</td>\n",
       "      <td>angiogenic-&gt;blood_vessel_development</td>\n",
       "      <td>production-&gt;gene_expression | angiogenic-&gt;bloo...</td>\n",
       "      <td>produce-&gt;gene_expression | angiogenic-&gt;blood_v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>EV-EX&lt;/s&gt;mlee_processed_sent&lt;/s&gt;This limitatio...</td>\n",
       "      <td>angiogenic-&gt;blood_vessel_development | express...</td>\n",
       "      <td>angiogenic-&gt;blood_vessel_development | express...</td>\n",
       "      <td>expression-&gt;gene_expression | angiogenic-&gt;bloo...</td>\n",
       "      <td>express-&gt;gene_expression | angiogenic-&gt;blood_v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>EV-EX&lt;/s&gt;mlee_processed_sent&lt;/s&gt;We have constr...</td>\n",
       "      <td>suppression-&gt;negative_regulation | induced-&gt;po...</td>\n",
       "      <td>expresses-&gt;gene_expression | induced-&gt;positive...</td>\n",
       "      <td>induced-&gt;positive_regulation | expresses-&gt;gene...</td>\n",
       "      <td>expresses-&gt;gene_expression | induce-&gt;positive_...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text  \\\n",
       "2   EV-EX</s>mlee_processed_sent</s>Adenovirus-med...   \n",
       "5   EV-EX</s>mlee_processed_sent</s>Inhibition of ...   \n",
       "8   EV-EX</s>mlee_processed_sent</s>However, its w...   \n",
       "11  EV-EX</s>mlee_processed_sent</s>This limitatio...   \n",
       "14  EV-EX</s>mlee_processed_sent</s>We have constr...   \n",
       "\n",
       "                                                 gold  \\\n",
       "2   inhibition->negative_regulation | transfer->pl...   \n",
       "5   Inhibition->negative_regulation | angiogenesis...   \n",
       "8                angiogenic->blood_vessel_development   \n",
       "11  angiogenic->blood_vessel_development | express...   \n",
       "14  suppression->negative_regulation | induced->po...   \n",
       "\n",
       "                                                top_1  \\\n",
       "2   growth->growth | mediated->regulation | inhibi...   \n",
       "5   Inhibition->negative_regulation | therapy->pla...   \n",
       "8                angiogenic->blood_vessel_development   \n",
       "11  angiogenic->blood_vessel_development | express...   \n",
       "14  expresses->gene_expression | induced->positive...   \n",
       "\n",
       "                                                top_2  \\\n",
       "2   growth->growth | inhibition->negative_regulati...   \n",
       "5   therapy->planned_process | Inhibition->negativ...   \n",
       "8   production->gene_expression | angiogenic->bloo...   \n",
       "11  expression->gene_expression | angiogenic->bloo...   \n",
       "14  induced->positive_regulation | expresses->gene...   \n",
       "\n",
       "                                                top_3  \n",
       "2   growth->growth | inhibition->negative_regulati...  \n",
       "5   Inhibition->negative_regulation | angiogenesis...  \n",
       "8   produce->gene_expression | angiogenic->blood_v...  \n",
       "11  express->gene_expression | angiogenic->blood_v...  \n",
       "14  expresses->gene_expression | induce->positive_...  "
      ]
     },
     "execution_count": 339,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "op.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "irish-regulation",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "id": "recognized-astrology",
   "metadata": {},
   "outputs": [],
   "source": [
    "def div(a,b):\n",
    "    if b==0:\n",
    "        return 0;\n",
    "    else:\n",
    "        return a/b\n",
    "\n",
    "def scoring(gold,pred,task,indices=None,verbose=True,skipneg=True):\n",
    "    tp = 0\n",
    "    predn=0\n",
    "    goldn=0\n",
    "    skipped=0\n",
    "    \n",
    "    x=0\n",
    "    cor=[]\n",
    "    posind=[]\n",
    "\n",
    "    for ind,row in enumerate(pred):\n",
    "        #row = row.replace(': ','->')\n",
    "        \n",
    "        if(row=='->' or 'NONE' in row):\n",
    "            row='NONE'\n",
    "        if(indices is not None):\n",
    "            gold2 = gold[indices[x]]\n",
    "        x+=1\n",
    "        \n",
    "        if(str(gold2)=='nan'):\n",
    "            gold2=''\n",
    "        \n",
    "        '''\n",
    "        if(gold2=='NONE'):\n",
    "            continue\n",
    "        '''\n",
    "        \n",
    "        posind.append(ind)    \n",
    "        row = row.split('|')\n",
    "        gold2 = gold2.split('|')\n",
    "        \n",
    "        \n",
    "        gold2 = [g.strip() for g in gold2]\n",
    "        row = [g.strip() for g in row]\n",
    "        \n",
    "        r2 = row.copy()\n",
    "        g2 = gold2.copy()\n",
    "        \n",
    "        #print(gold2,'\\n\\n',row)\n",
    "        #break\n",
    "        r2 = [r for r in row if len(r)>1]\n",
    "        \n",
    "        \n",
    "        #gold[i] = str(gold[i]).split(' |')\n",
    "        \n",
    "        if(task=='EV-CLF'):\n",
    "            row = list(set(row))\n",
    "            gold2 = list(set(gold2))\n",
    "            \n",
    "        row.sort()\n",
    "        gold2.sort()\n",
    "        cor.append(row==gold2)\n",
    "        \n",
    "        if(skipneg and gold2[0]=='NONE'):\n",
    "            skipped+=1\n",
    "            continue\n",
    "\n",
    "        \n",
    "        for elem in row:\n",
    "            if(len(elem)<1):\n",
    "                continue\n",
    "            \n",
    "                \n",
    "            \n",
    "            if(elem in gold2): # exact phrase present\n",
    "                tp+=1\n",
    "\n",
    "            \n",
    "                \n",
    "                # NEW\n",
    "                gold2.remove(elem)\n",
    "            \n",
    "        gold2 = g2.copy()\n",
    "        for g in gold2: # part of pred phrase present in gold\n",
    "            if(len(g)<1):\n",
    "                continue\n",
    "            \n",
    "        \n",
    "        predn+=len(row)-len([p for p in row if len(p)<1])\n",
    "        goldn+=len(gold2)-len([p for p in gold2 if len(p)<1])\n",
    "        #print('tp: ',tp,' predn: ',predn,' goldn: ',goldn)\n",
    "        #print('tp_rel: ',tp_relaxed)\n",
    "        #print('tp_rel2: ',tp_relaxed2)\n",
    "\n",
    "    p = div(tp,predn)\n",
    "    r = div(tp,goldn)\n",
    "    f1 = div(2*p*r,(p+r))\n",
    "\n",
    "    if(verbose):\n",
    "        print('skipped: ',skipped)\n",
    "        print('\\nPrecision: ',p,'\\nRecall: ',r,'\\nF1-score: ',f1)\n",
    "\n",
    "    \n",
    "    '''\n",
    "    prel = div(tp_relaxed,predn)\n",
    "    rrel = div(tp_relaxed,goldn)\n",
    "    f1rel = div(2*prel*rrel,(prel+rrel))\n",
    "    \n",
    "    print('\\nRELAXED SCORING 1\\nPrecision: ',prel,'\\nRecall: ',rrel,'\\nF1-score: ',f1rel)\n",
    "\n",
    "    prel2 = div(tp_relaxed2,predn)\n",
    "    rrel2 = div(tp_relaxed2,goldn)\n",
    "    f1rel2 = div(2*prel2*rrel2,(prel2+rrel2))\n",
    "\n",
    "    print('\\nRELAXED SCORING 2\\nPrecision: ',prel2,'\\nRecall: ',rrel2,'\\nF1-score: ',f1rel2)\n",
    "    '''\n",
    "    return posind,cor,p,r,f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "id": "dense-cradle",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "**************\n",
      "K is  1\n",
      "skipped:  172\n",
      "\n",
      "Precision:  0.7433284813197477 \n",
      "Recall:  0.7580405739732805 \n",
      "F1-score:  0.7506124448799608\n",
      "\n",
      "WITH NEGATIVE\n",
      "skipped:  0\n",
      "\n",
      "Precision:  0.7357651245551602 \n",
      "Recall:  0.7542179662562699 \n",
      "F1-score:  0.7448772798919163\n",
      "\n",
      "**************\n",
      "K is  2\n",
      "skipped:  172\n",
      "\n",
      "Precision:  0.6773465703971119 \n",
      "Recall:  0.7427016328550222 \n",
      "F1-score:  0.7085201793721971\n",
      "\n",
      "WITH NEGATIVE\n",
      "skipped:  0\n",
      "\n",
      "Precision:  0.6623913942904427 \n",
      "Recall:  0.7300501595987232 \n",
      "F1-score:  0.6945770065075921\n",
      "\n",
      "**************\n",
      "K is  3\n",
      "skipped:  172\n",
      "\n",
      "Precision:  0.6360895779500431 \n",
      "Recall:  0.7308263236021771 \n",
      "F1-score:  0.6801749942436104\n",
      "\n",
      "WITH NEGATIVE\n",
      "skipped:  0\n",
      "\n",
      "Precision:  0.622626582278481 \n",
      "Recall:  0.7177382580939352 \n",
      "F1-score:  0.6668078796865071\n"
     ]
    }
   ],
   "source": [
    "corlist=[]\n",
    "for k in range(0,return_seqs):\n",
    "    print('\\n**************\\nK is ',str(k+1))\n",
    "    posind,cor,_,_,_ = scoring(op['gold'],op['top_'+str(k+1)],indices=op.index,task=task,skipneg=True)\n",
    "    print('\\nWITH NEGATIVE')\n",
    "    posind,cor,_,_,_ = scoring(op['gold'],op['top_'+str(k+1)],indices=op.index,task=task,skipneg=False)\n",
    "    corlist.append(cor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "id": "fatal-external",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "875"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## combining top1 and top2\n",
    "a2 = [p+q for p,q in zip(op['top_1'].str.split('|'),op['top_2'].str.split('|'))]# + [p for p in op['top_2'].str.split('|')]\n",
    "az=[]\n",
    "for a in a2:\n",
    "    az.append([p.strip() for p in a])\n",
    "a2 = az\n",
    "a2 = [list(set(a)) for a in a2]\n",
    "a2 = pd.Series(a2).str.join(' | ')\n",
    "op['top_2_combined'] = a2\n",
    "len(a2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "id": "trying-participation",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "skipped:  172\n",
      "\n",
      "Precision:  0.6701461377870563 \n",
      "Recall:  0.7941613062840178 \n",
      "F1-score:  0.7269021739130435\n",
      "WITH NEGATIVE\n",
      "skipped:  0\n",
      "\n",
      "Precision:  0.6622086358425678 \n",
      "Recall:  0.7902416780665754 \n",
      "F1-score:  0.7205821205821205\n"
     ]
    }
   ],
   "source": [
    "## combining top1 and top2\n",
    "posind,cor,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=True)\n",
    "print('WITH NEGATIVE')\n",
    "posind,cor,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "id": "distant-therapy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"## combining top1 and top2 and top3\\na3 = [p+q for p,q in zip(op['top_2_combined'].str.split('|'),op['top_3'].str.split('|'))]# + [p for p in op['top_2'].str.split('|')]\\naz=[]\\nfor a in a3:\\n    az.append([p.strip() for p in a])\\na3 = az\\na3 = [list(set(a)) for a in a3]\\na3 = pd.Series(a3).str.join(' | ')\\nop['top_3_combined'] = a3\\nlen(a3)\""
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''## combining top1 and top2 and top3\n",
    "a3 = [p+q for p,q in zip(op['top_2_combined'].str.split('|'),op['top_3'].str.split('|'))]# + [p for p in op['top_2'].str.split('|')]\n",
    "az=[]\n",
    "for a in a3:\n",
    "    az.append([p.strip() for p in a])\n",
    "a3 = az\n",
    "a3 = [list(set(a)) for a in a3]\n",
    "a3 = pd.Series(a3).str.join(' | ')\n",
    "op['top_3_combined'] = a3\n",
    "len(a3)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "id": "official-representative",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"## combining top1 and top2 and top3\\nposind,cor,_,_,_ = scoring(op['gold'],a3,indices=op.index,task=task,skipneg=True)\\nprint('WITH NEGATIVE')\\nposind,cor,_,_,_ = scoring(op['gold'],a3,indices=op.index,task=task,skipneg=False)\""
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''## combining top1 and top2 and top3\n",
    "posind,cor,_,_,_ = scoring(op['gold'],a3,indices=op.index,task=task,skipneg=True)\n",
    "print('WITH NEGATIVE')\n",
    "posind,cor,_,_,_ = scoring(op['gold'],a3,indices=op.index,task=task,skipneg=False)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "id": "accredited-edinburgh",
   "metadata": {},
   "outputs": [],
   "source": [
    "op['cor1'] = corlist[0]\n",
    "op['cor2'] = corlist[1]\n",
    "op['cor3'] = corlist[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "american-sculpture",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "id": "armed-composition",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "skipped:  172\n",
      "\n",
      "Precision:  0.7539370078740157 \n",
      "Recall:  0.7580405739732805 \n",
      "F1-score:  0.7559832223044658\n",
      "skipped:  0\n",
      "\n",
      "Precision:  0.7453808021631365 \n",
      "Recall:  0.7542179662562699 \n",
      "F1-score:  0.7497733454215775\n"
     ]
    }
   ],
   "source": [
    "optionlim=0.1\n",
    "\n",
    "## option 1. top 1 + lower(.5*top2)\n",
    "a2=[]\n",
    "for i,r in op.iterrows():\n",
    "    p = r['top_1'].split('|')\n",
    "    q = r['top_2'].split('|')\n",
    "    len2 = int(np.floor(len(q)*optionlim))\n",
    "    a = p+q[0:len2]   \n",
    "    \n",
    "    az=[]\n",
    "    for k in a:\n",
    "        az.append(k.strip())\n",
    "    \n",
    "    az = list(set(az))\n",
    "    a2.append(' | '.join(az))\n",
    "\n",
    "posind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=True)\n",
    "posind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=False)\n",
    "op['option_1'] = a2\n",
    "op['cor_option_1'] = corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "id": "moderate-convention",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "skipped:  172\n",
      "\n",
      "Precision:  0.7012929112795363 \n",
      "Recall:  0.7783275606135577 \n",
      "F1-score:  0.7378048780487805\n",
      "skipped:  0\n",
      "\n",
      "Precision:  0.6933115823817292 \n",
      "Recall:  0.7751937984496124 \n",
      "F1-score:  0.7319698600645856\n"
     ]
    }
   ],
   "source": [
    "optionlim = 0.4\n",
    "\n",
    "## option 2. top 2 + upper(.5*top2)\n",
    "a2=[]\n",
    "for i,r in op.iterrows():\n",
    "    p = r['top_1'].split('|')\n",
    "    q = r['top_2'].split('|')\n",
    "    len2 = int(np.ceil(len(q)*optionlim))\n",
    "    a = p+q[0:len2]   \n",
    "    \n",
    "    az=[]\n",
    "    for k in a:\n",
    "        az.append(k.strip())\n",
    "    \n",
    "    az = list(set(az))\n",
    "    a2.append(' | '.join(az))\n",
    "\n",
    "posind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=True)    \n",
    "posind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=False)\n",
    "op['option_2'] = a2\n",
    "op['cor_option_2'] = corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "id": "missing-summit",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "skipped:  172\n",
      "\n",
      "Precision:  0.7015873015873015 \n",
      "Recall:  0.7654626422563088 \n",
      "F1-score:  0.7321344060577378\n",
      "skipped:  0\n",
      "\n",
      "Precision:  0.6949646275488972 \n",
      "Recall:  0.7615139078887369 \n",
      "F1-score:  0.7267188859878155\n"
     ]
    }
   ],
   "source": [
    "## option 3. if top 1 in top 2, take top 2. else take top 1\n",
    "a2=[]\n",
    "for i,r in op.iterrows():\n",
    "    if(r['top_1'] in r['top_2']):\n",
    "        a2.append(r['top_2'])\n",
    "    else:\n",
    "        a2.append(r['top_1'])\n",
    "        \n",
    "posind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=True)\n",
    "posind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=False)\n",
    "op['option_3'] = a2\n",
    "op['cor_option_3'] = corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "synthetic-rescue",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "id": "incident-excuse",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"optionlim=0.2\\n    \\n## option 4. top 1 + lower(.5*top2)\\na2=[]\\nfor i,r in op.iterrows():\\n    p = r['top_1'].split('|')\\n    q = r['top_2'].split('|')\\n    len2 = int(np.floor(len(q)*optionlim))\\n    a = p+q[-len2:]   \\n    \\n    az=[]\\n    for k in a:\\n        az.append(k.strip())\\n    \\n    az = list(set(az))\\n    a2.append(' | '.join(az))\\n\\nposind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=True)\\nposind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=False)\\nop['option_4'] = a2\\nop['cor_option_4'] = corr\""
      ]
     },
     "execution_count": 350,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''optionlim=0.2\n",
    "    \n",
    "## option 4. top 1 + lower(.5*top2)\n",
    "a2=[]\n",
    "for i,r in op.iterrows():\n",
    "    p = r['top_1'].split('|')\n",
    "    q = r['top_2'].split('|')\n",
    "    len2 = int(np.floor(len(q)*optionlim))\n",
    "    a = p+q[-len2:]   \n",
    "    \n",
    "    az=[]\n",
    "    for k in a:\n",
    "        az.append(k.strip())\n",
    "    \n",
    "    az = list(set(az))\n",
    "    a2.append(' | '.join(az))\n",
    "\n",
    "posind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=True)\n",
    "posind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=False)\n",
    "op['option_4'] = a2\n",
    "op['cor_option_4'] = corr'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "id": "arbitrary-wound",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"optionlim=0.4\\n\\n## option 5. top 2 + upper(.5*top2)\\na2=[]\\nfor i,r in op.iterrows():\\n    p = r['top_1'].split('|')\\n    q = r['top_2'].split('|')\\n    len2 = int(np.ceil(len(q)*optionlim))\\n    a = p+q[-len2:]   \\n    \\n    az=[]\\n    for k in a:\\n        az.append(k.strip())\\n    \\n    az = list(set(az))\\n    a2.append(' | '.join(az))\\n\\nposind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=True)    \\nposind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=False)\\nop['option_5'] = a2\\nop['cor_option_5'] = corr\""
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''optionlim=0.4\n",
    "\n",
    "## option 5. top 2 + upper(.5*top2)\n",
    "a2=[]\n",
    "for i,r in op.iterrows():\n",
    "    p = r['top_1'].split('|')\n",
    "    q = r['top_2'].split('|')\n",
    "    len2 = int(np.ceil(len(q)*optionlim))\n",
    "    a = p+q[-len2:]   \n",
    "    \n",
    "    az=[]\n",
    "    for k in a:\n",
    "        az.append(k.strip())\n",
    "    \n",
    "    az = list(set(az))\n",
    "    a2.append(' | '.join(az))\n",
    "\n",
    "posind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=True)    \n",
    "posind,corr,_,_,_ = scoring(op['gold'],a2,indices=op.index,task=task,skipneg=False)\n",
    "op['option_5'] = a2\n",
    "op['cor_option_5'] = corr'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "presidential-implementation",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "intellectual-above",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "every-partition",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "id": "pacific-spice",
   "metadata": {},
   "outputs": [],
   "source": [
    "op.to_csv('results/'+predfolder+'/compare_top_'+str(return_seqs)+'.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "id": "informed-bottom",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'multitask4mixbidspec2verbose2_prin_ml_1024_50_bs50_seq3_4bs_2DAYS'"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predfolder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caroline-pillow",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "id": "subtle-entry",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"if(add_to_main_file):\\n    \\n    odf = pd.read_csv(opfile)\\n    \\n    if('evclf' in predfolder):\\n        odf['text'] = odf['text'] + ['</s>EVENT_TYPES</s>']*len(odf) + op['top_2_combined']\\n        odf['text_in'] = odf['text_in'] + ['</s>EVENT_TYPES</s>']*len(odf) + op['top_2_combined']\\n    elif('evid' in predfolder):\\n        odf['text'] = odf['text'] + ['</s>EVENT_TRIGGERS</s>']*len(odf) + op['top_2_combined']\\n        odf['text_in'] = odf['text_in'] + ['</s>EVENT_TRIGGERS</s>']*len(odf) + op['top_2_combined']\\n    elif('evex' in predfolder):\\n        odf['text'] = odf['text'] + ['</s>POSSIBLE EVENTS</s>']*len(odf) + op['top_2_combined']\\n        odf['text_in'] = odf['text_in'] + ['</s>POSSIBLE EVENTS</s>']*len(odf) + op['top_2_combined']\\n        \\n    \\n    odf.to_csv(opfile2,index=False)\\n    num = int(0.9*len(odf))\\n    \\n    af = opfile2.split('.')[0]+'_90p.csv'\\n    odf[0:num].to_csv(af,index=False)\\n    bf = opfile2.split('.')[0]+'_10p.csv'\\n    odf[num:].to_csv(bf,index=False)\""
      ]
     },
     "execution_count": 354,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''if(add_to_main_file):\n",
    "    \n",
    "    odf = pd.read_csv(opfile)\n",
    "    \n",
    "    if('evclf' in predfolder):\n",
    "        odf['text'] = odf['text'] + ['</s>EVENT_TYPES</s>']*len(odf) + op['top_2_combined']\n",
    "        odf['text_in'] = odf['text_in'] + ['</s>EVENT_TYPES</s>']*len(odf) + op['top_2_combined']\n",
    "    elif('evid' in predfolder):\n",
    "        odf['text'] = odf['text'] + ['</s>EVENT_TRIGGERS</s>']*len(odf) + op['top_2_combined']\n",
    "        odf['text_in'] = odf['text_in'] + ['</s>EVENT_TRIGGERS</s>']*len(odf) + op['top_2_combined']\n",
    "    elif('evex' in predfolder):\n",
    "        odf['text'] = odf['text'] + ['</s>POSSIBLE EVENTS</s>']*len(odf) + op['top_2_combined']\n",
    "        odf['text_in'] = odf['text_in'] + ['</s>POSSIBLE EVENTS</s>']*len(odf) + op['top_2_combined']\n",
    "        \n",
    "    \n",
    "    odf.to_csv(opfile2,index=False)\n",
    "    num = int(0.9*len(odf))\n",
    "    \n",
    "    af = opfile2.split('.')[0]+'_90p.csv'\n",
    "    odf[0:num].to_csv(af,index=False)\n",
    "    bf = opfile2.split('.')[0]+'_10p.csv'\n",
    "    odf[num:].to_csv(bf,index=False)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "terminal-banking",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "id": "directed-feature",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'odf' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-355-1d92ea295b9c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0modf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'odf' is not defined"
     ]
    }
   ],
   "source": [
    "odf.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aggressive-corps",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}