2. Hybrid extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# For training BioBERT NER on nala\r\n",
    "Run this in CLI -  \r\n",
    "\r\n",
    "python train_ner.py --model_name_or_path dmis-lab/biobert-base-cased-v1.1 --train_file data/nala/train_dev.json --validation_file data/nala/devel.json --text_column_name tokens --label_column_name tags --pad_to_max_length --max_length 192 --per_device_train_batch_size 8 --learning_rate 2e-5 --num_train_epochs 10 --output_dir models/nala --seed 1\r\n",
    "  \r\n",
    "Note:  Training took <30 mins on 1660 ti. You can decrease the num_train_epochs count to 5 without any substantial difference in accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 Regex \n",
    "### Has high precision and low recall.    \n",
    "### Consists of 3 parts - MutationFinder, tmVar and some custom patterns from WB papers \n",
    "### 1.1 Mutation Finder [Link](https://github.com/divyashan/MutationFinder), Modified regex from SETH [Link](https://github.com/rockt/SETH/blob/master/resources/mutations.txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import time\n",
    " \n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import glob\n",
    "from pathlib import Path\n",
    "\n",
    "from sklearn.metrics import precision_recall_fscore_support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.misc.regex_block import MutationFinder, TmVar, BOWdictionary, CustomWBregex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mf_mut_extract = MutationFinder('data/regexs/mutationfinder_regex/seth_modified.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mf_mut_extract('wa7 ) is a null allele, because a 1-bp deletion')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 tmVar [Link](https://www.ncbi.nlm.nih.gov/research/bionlp/Tools/tmvar/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/Users/Rishab/Documents/GitHub/genomic-info-from-papers/genomic-info-from-papers/utils/misc/regex_block.py:102: FutureWarning: Possible nested set at position 15\n",
      "  self._regular_expressions.append(re.compile(reg))\n"
     ]
    }
   ],
   "source": [
    "tmvar_mut_extract = TmVar('data/regexs/tmvar_regex/final_regex_path')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmvar_mut_extract('wa7 ) is a null allele, because a 1-bp deletion')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3 Extra custom regexs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import configparser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['utils/all_config.cfg']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db_config = configparser.ConfigParser()\r\n",
    "db_config.read('utils/all_config.cfg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_mut_extract = CustomWBregex(db_config, extra_regex=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- in the statement ced-3(n2888) \r\n",
    "n2888 bp AAAAAAAAAAAAAAAAA TTTTTTTTTTTTTTTTTT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['n2888 bp AAAAAAAAAAAAAAAAA TTTTTTTTTTTTTTTTTT',\n",
       "  'n2888 bp AAAAAAAAAAAAAAAAA TTTTTTTTTTTTTTTTTT']]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "custom_mut_extract('n2888 bp AAAAAAAAAAAAAAAAA TTTTTTTTTTTTTTTTTT')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['MET-1', 'Just gene']]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "custom_mut_extract.get_genes('Our results suggest that in C. elegans ,the trimethylation of histone H3 lysine 36 by MET-1/Set2 promotes a transcriptional repression cascade mediated by a NuRD-like complex and by the trimethylation of histone H3K9 by a SETDB1-like HMT.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['ce4', 'Genome Version']]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "custom_mut_extract.get_genome_vers('The genome version used in this paper is ce4.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['WS128', 'Annotation Version']]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "custom_mut_extract.get_annotation_vers('When needing to refer to a specific version of the reference genome, it was therefore sufficient (and convenient) to use the WormBase release number that the genome was taken from (e.g. WS128).')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.4 Bag of words "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "bow_mut_extract = BOWdictionary()\r\n",
    "# bow_mut_extract('This mutation deletes 471bp of the promoter region, the transcriptional start and 56 amino acids of the second exon.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.5 MF + tmVar + Custom regex + BOW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unique_rows(a):\n",
    "    a = np.ascontiguousarray(a)\n",
    "    unique_a = np.unique(a.view([('', a.dtype)]*a.shape[1]))\n",
    "    return unique_a.view(a.dtype).reshape((unique_a.shape[0], a.shape[1]))\n",
    "\n",
    "\n",
    "def regex_block(sentence, span_size=150):\n",
    "    mut_and_snippets = []\n",
    "    \n",
    "    # MutationFinder\n",
    "    mut_and_snippets = mut_and_snippets + mf_mut_extract(sentence, span_size=span_size)\n",
    "    # tmVar\n",
    "    mut_and_snippets = mut_and_snippets + tmvar_mut_extract(sentence, span_size=span_size)\n",
    "    # Custom patterns\n",
    "    mut_and_snippets = mut_and_snippets + custom_mut_extract(sentence, span_size=span_size)\n",
    "    # Bag of words\n",
    "    mut_and_snippets = mut_and_snippets + bow_mut_extract(sentence)\n",
    "    \n",
    "    if mut_and_snippets:\n",
    "        mut_and_snippets = unique_rows(mut_and_snippets).tolist()\n",
    "    return mut_and_snippets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex_block(' asdf gpa-2 ::Tc1 asdf as')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.6 * Additional details  \n",
    "These will get extracted regardless of whether the sentence has genomic information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extra_info_block(sentence, span_size=150):\r\n",
    "    info_and_snippets = []\r\n",
    "\r\n",
    "    # look for gene and variant combo\r\n",
    "    info_and_snippets = info_and_snippets + custom_mut_extract.var_and_gene_close(sentence, span_size=span_size)\r\n",
    "    \r\n",
    "    if info_and_snippets:\r\n",
    "        info_and_snippets = unique_rows(info_and_snippets).tolist()\r\n",
    "    return info_and_snippets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['ced-3(n2888', 'Gene & Variant']]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extra_info_block('in the statement ced-3(n2888)')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 BioBERT NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/risubu/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /home/risubu/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoConfig, AutoModelForTokenClassification\n",
    "from transformers import TokenClassificationPipeline\n",
    "\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sklearn as sk\n",
    "import math \n",
    "import string\n",
    "import time\n",
    "import json\n",
    "import csv\n",
    "import shutil\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name_or_path = 'models/nala'\r\n",
    "config = AutoConfig.from_pretrained(model_name_or_path)\r\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)\r\n",
    "model = AutoModelForTokenClassification.from_pretrained(\r\n",
    "    model_name_or_path,\r\n",
    "    from_tf=bool(\".ckpt\" in model_name_or_path),\r\n",
    "    config=config,\r\n",
    ")\r\n",
    "# LABEL_0 - B-mut, LABEL_1 - I-mut, LABEL_2 - O\r\n",
    "nala_ner  = TokenClassificationPipeline(model=model, tokenizer=tokenizer, task='ner', aggregation_strategy='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = set(nltk.corpus.stopwords.words('english'))\n",
    "stop_words = [w for w in stop_words if len(w) > 1]\n",
    "\n",
    "def ner_mutations(sentence):\n",
    "    mutations = []\n",
    "    try:\n",
    "        ner_output = nala_ner(sentence)\n",
    "        for i, grp in enumerate(ner_output):\n",
    "            if grp['entity_group'] == 'LABEL_0':\n",
    "                mut = grp['word']\n",
    "                for j in range(i+1, len(ner_output)):\n",
    "                    if ner_output[j]['entity_group'] == 'LABEL_1':\n",
    "                        mut  = mut + ' ' + ner_output[j]['word']\n",
    "                    else:\n",
    "                        # NER would be handling only data in NL form\n",
    "                        if len(mut.split()) > 3 and any(word in mut.split() for word in stop_words):\n",
    "                            mutations.append([mut, sentence])\n",
    "                        break\n",
    "    except:\n",
    "        pass\n",
    "    return mutations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3 Testing\n",
    "Using the IDP4+ dataset downloaded and setup in notebook 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "nala_db = pd.read_csv('data/nala/nala_binary.csv').to_numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 RegEx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total sentences to process:  19235\n",
      "500 1000 1500 2000 2500 3000 3500 4000 4500 5000 5500 6000 6500 7000 7500 8000 8500 9000 9500 10000 10500 11000 11500 12000 12500 13000 13500 14000 14500 15000 15500 16000 16500 17000 17500 18000 18500 19000 "
     ]
    }
   ],
   "source": [
    "y_true = []\n",
    "y_pred = []\n",
    "print('Total sentences to process: ', len(nala_db))\n",
    "for i, row in enumerate(nala_db):\n",
    "    if (i+1) % 500 == 0: print(f\"{i+1}\", end = \" \")\n",
    "    sentence = row[0]\n",
    "    true = row[1]\n",
    "    if regex_block(sentence):\n",
    "        pred = 1\n",
    "    else:\n",
    "        pred = 0\n",
    "    y_true.append(true)\n",
    "    y_pred.append(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9758194519075766, 0.6515966989594546, 0.7814113597246128, None)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_recall_fscore_support(y_true, y_pred, average='binary')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 BioBERT NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total sentences to process:  19235\n",
      "500 1000 1500 2000 2500 3000 3500 4000 4500 5000 5500 6000 6500 7000 7500 8000 8500 9000 9500 10000 10500 11000 11500 12000 12500 13000 13500 14000 14500 15000 15500 16000 16500 17000 17500 18000 18500 19000 "
     ]
    }
   ],
   "source": [
    "y_true = []\n",
    "y_pred = []\n",
    "print('Total sentences to process: ', len(nala_db))\n",
    "for i, row in enumerate(nala_db):\n",
    "    if (i+1) % 500 == 0: print(f\"{i+1}\", end = \" \")\n",
    "    sentence = row[0]\n",
    "    true = row[1]\n",
    "    if ner_mutations(sentence):\n",
    "        pred = 1\n",
    "    else:\n",
    "        pred = 0\n",
    "    y_true.append(true)\n",
    "    y_pred.append(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6675148430873622, 0.28238249013275923, 0.39687342410489157, None)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_recall_fscore_support(y_true, y_pred, average='binary')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 RegEx (custom) + NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total sentences to process:  19235\n",
      "100 200 300 400 500 600 700 800 900 1000 Time for 1000 lines:  3\n",
      "1100 1200 1300 1400 1500 1600 1700 1800 1900 2000 Time for 1000 lines:  4\n",
      "2100 2200 2300 2400 2500 2600 2700 2800 2900 3000 Time for 1000 lines:  3\n",
      "3100 3200 3300 3400 3500 3600 3700 3800 3900 4000 Time for 1000 lines:  2\n",
      "4100 4200 4300 4400 4500 4600 4700 4800 4900 5000 Time for 1000 lines:  3\n",
      "5100 5200 5300 5400 5500 5600 5700 5800 5900 6000 Time for 1000 lines:  3\n",
      "6100 6200 6300 6400 6500 6600 6700 6800 6900 7000 Time for 1000 lines:  3\n",
      "7100 7200 7300 7400 7500 7600 7700 7800 7900 8000 Time for 1000 lines:  3\n",
      "8100 8200 8300 8400 8500 8600 8700 8800 8900 9000 Time for 1000 lines:  3\n",
      "9100 9200 9300 9400 9500 9600 9700 9800 9900 10000 Time for 1000 lines:  4\n",
      "10100 10200 10300 10400 10500 10600 10700 10800 10900 11000 Time for 1000 lines:  3\n",
      "11100 11200 11300 11400 11500 11600 11700 11800 11900 12000 Time for 1000 lines:  3\n",
      "12100 12200 12300 12400 12500 12600 12700 12800 12900 13000 Time for 1000 lines:  4\n",
      "13100 13200 13300 13400 13500 13600 13700 13800 13900 14000 Time for 1000 lines:  4\n",
      "14100 14200 14300 14400 14500 14600 14700 14800 14900 15000 Time for 1000 lines:  3\n",
      "15100 15200 15300 15400 15500 15600 15700 15800 15900 16000 Time for 1000 lines:  3\n",
      "16100 16200 16300 16400 16500 16600 16700 16800 16900 17000 Time for 1000 lines:  3\n",
      "17100 17200 17300 17400 17500 17600 17700 17800 17900 18000 Time for 1000 lines:  3\n",
      "18100 18200 18300 18400 18500 18600 18700 18800 18900 19000 Time for 1000 lines:  3\n",
      "19100 19200 "
     ]
    }
   ],
   "source": [
    "y_true = []\n",
    "y_pred = []\n",
    "start_time = time.time()\n",
    "print('Total sentences to process: ', len(nala_db))\n",
    "for i, row in enumerate(nala_db):\n",
    "    if (i+1) % 500 == 0: print(f\"{i+1}\", end = \" \")\n",
    "    if (i+1) % 1000 == 0:\n",
    "        print('Time for 1000 lines:', int((time.time() - start_time)/60))\n",
    "        start_time = time.time()\n",
    "    sentence = row[0]\n",
    "    true = row[1]\n",
    "    if regex_block(sentence) or ner_mutations(sentence):\n",
    "        pred = 1\n",
    "    else:\n",
    "        pred = 0\n",
    "    y_true.append(true)\n",
    "    y_pred.append(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8377769432970334, 0.800502332256907, 0.8187155963302752, None)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_recall_fscore_support(y_true, y_pred, average='binary')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 Run on WB papers\n",
    "### Get the paper texts from textpresso API and wbtools"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How it works - First paper ID is searched through textpresso API. If the recieved output is blank, then wbtools is used.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os.path\n",
    "import argparse\n",
    "import subprocess\n",
    "import requests\n",
    "import csv\n",
    "import json\n",
    "import nltk.data\n",
    "import os\n",
    "from xml.dom import minidom\n",
    "from bs4 import BeautifulSoup\n",
    "from wbtools.literature.corpus import CorpusManager\n",
    "import platform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Uncomment and run the cell below only once to save the paper sentences in your local computer to avoid pulling paper text everytime during dev  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nCreate a separate temporary numpy file to store the paper sentences  \\nto avoid spending time on pulling sentences while in development \\nFormat - [paper_id, sentence]\\n'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Create a temporary cvs file to store the paper sentences  \n",
    "to avoid spending time on pulling sentences while in development \n",
    "Format - [paper_id, sentence]\n",
    "'''\n",
    "# final_data = []\n",
    "# # remove_sections = [PaperSections.ACKNOWLEDGEMENTS, PaperSections.REFERENCES, PaperSections.RELATED_WORK, PaperSections.INTRODUCTION]\n",
    "# remove_sections = []\n",
    "# # random 100 papers mentioned the remarks ace file in data/gsoc\n",
    "# paper_ids = np.load('data/top100.npy').tolist()\n",
    "# cm = CorpusManager()\n",
    "# for i, paper_id in enumerate(paper_ids):\n",
    "#     paper_id = paper_id[7:]\n",
    "#     cm.load_from_wb_database(db_name=db_config['wb_database']['db_name'], db_user=db_config['wb_database']['db_user'], db_password=db_config['wb_database']['db_password'],\n",
    "#         db_host=db_config['wb_database']['db_host'], paper_ids=[paper_id],\n",
    "#         ssh_host=db_config['wb_database']['ssh_host'], ssh_user=db_config['wb_database']['ssh_user'], ssh_passwd=db_config['wb_database']['ssh_passwd'],\n",
    "#         load_bib_info=False, load_afp_info=False, load_curation_info=False)\n",
    "#     sentences = cm.get_paper(paper_id).get_text_docs(remove_sections=remove_sections,split_sentences=True)\n",
    "#     for sent in sentences:\n",
    "#         final_data.append([paper_id, sent])\n",
    "#     print(i, end = \" \")\n",
    "# final_data = pd.DataFrame(final_data[:], columns=['WBPaper ID', 'Sentence'])\n",
    "# final_data.to_csv(\"data/id_and_sentence.csv\", index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def textpresso_paper_text(wbpid, path, token):\n",
    "    \"\"\"This sub takes a wbpid eg WBPaper00056731 and returns the fulltext paper in sentences\"\"\"\n",
    "    ft=[0];\n",
    "    # Check that wbpid is a valid WBPaper\n",
    "    if not re.match( 'WBPaper', wbpid):\n",
    "        print (wbpid, \"is not a valid WBPaper ID\")\n",
    "        return ft\n",
    "    # Download paper if it doesn't exist\n",
    "    fn = path + '/temp/' + wbpid + '.json'\n",
    "\n",
    "    if os.path.exists(fn) and os.path.getsize(fn) > 16:\n",
    "        pass\n",
    "    else:\n",
    "        com1 = '-o '+fn +'\\n-k '+ '\\n'+'-d \"{\\\\\"token\\\\\":\\\\\"'+ token + '\\\\\", \\\\\"query\\\\\": {\\\\\"accession\\\\\": \\\\\"' + wbpid +'\\\\\", \\\\\"type\\\\\": \\\\\"document\\\\\", \\\\\"corpora\\\\\": [\\\\\"C. elegans\\\\\"]}, \\\\\"include_fulltext\\\\\": true}\"'\n",
    "        configf= path +'/temp/' + wbpid + '.tmp.config'\n",
    "        curlf = open(configf,'w')\n",
    "        print (com1, file=curlf)\n",
    "        curlf.close()\n",
    "        command = 'curl -o '+ fn +' -K '+ configf+' https://textpressocentral.org:18080/v1/textpresso/api/search_documents' \n",
    "        comlist = command.split()\n",
    "        os.system(command)\n",
    "\n",
    "    # Read the paper, and split into sentences\n",
    "    if os.path.exists(fn) and os.path.getsize(fn) > 20:\n",
    "        # Open our JSON file and load it into python\n",
    "        input_file = open (fn)\n",
    "        json_array = json.load(input_file)\n",
    "        for item in json_array:\n",
    "            abs = item[\"abstract\"]\n",
    "            fullt =  item[\"fulltext\"]\n",
    "            tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "            ft = tokenizer.tokenize(abs)\n",
    "            ftt=tokenizer.tokenize(fullt)\n",
    "            ft = ft +ftt\n",
    "    else:\n",
    "        # some paper texts are blank for some reason\n",
    "        # pipeline uses wbtools to get sentences in such case\n",
    "        pass\n",
    "\n",
    "    outfilen = os.path.join(path, 'text_flatfiles', wbpid+'.txt')\n",
    "    outf = open(outfilen, 'w')\n",
    "    for sen in ft:\n",
    "        sen =str(sen)\n",
    "        print(sen, file=outf)\n",
    "    outf.close()\n",
    "    return outfilen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Sentences out of wbtools are sometimes weird, especially for the old papers\n",
    "This increases the false neg rate so this is used only when the textpresso api provides bad text\n",
    "Most of the issues have band aid fixes in the text preprocessing cell below but there are some \n",
    "with no easy fix or need to be worked on - \n",
    "1. Some sentences have white space between every character and are also somehow inverted (???)\n",
    "    e.g. Check out the sentences of WBPaper00002018 (1994)\n",
    "    Line 154 -  '0 7 1 u ( 7 - c e m    ) F (   .'\n",
    "    inverted and without the white space is (mec-7)u170 which is extremely useful and will get missed \n",
    "    by the pipeline unless processed correctly.\n",
    "    Only a problem on very old papers.\n",
    "'''\n",
    "cm = CorpusManager()\n",
    "def wbtools_paper_text(wbpid, db_name, db_user, db_password, db_host, ssh_host,\\\n",
    "    ssh_user, ssh_passwd):\n",
    "    # sectioning might not be always correct, text processing is done separately in the pipeline\n",
    "    # remove_sections = [PaperSections.ACKNOWLEDGEMENTS, PaperSections.REFERENCES, PaperSections.RELATED_WORK, PaperSections.INTRODUCTION]\n",
    "    remove_sections = []\n",
    "    paper_id = wbpid[7:]\n",
    "    cm.load_from_wb_database(db_name=db_name, db_user=db_user, db_password=db_password,\n",
    "        db_host=db_host, paper_ids=[paper_id],\n",
    "        ssh_host=ssh_host, ssh_user=ssh_user, ssh_passwd=ssh_passwd,\n",
    "        load_bib_info=False, load_afp_info=False, load_curation_info=False)\n",
    "    sentences = cm.get_paper(paper_id).get_text_docs(remove_sections=remove_sections,split_sentences=True)\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_paper_sentences(wbpids, config, store_ppr_path):\n",
    "    '''\n",
    "    Takes WB Paper IDs and returns a list of sentences from those papers after filtering\n",
    "    Arg:\n",
    "    wbpids - List of wb papers ids \n",
    "        e.g. ['WBPaper00002379']\n",
    "    config_path - Config file path\n",
    "    store_ppr_path - Folder path to store the paper flatfiles retrieved from TextPresso for future use\n",
    "    Returns:\n",
    "    paperid_sentence_list: List of paper ID and sentence\n",
    "        e.g. [['WBPaper00002379', 'First sentence'], ['WBPaper00002379', 'Second sentence'], ....]\n",
    "    '''\n",
    "    stop_words = set(nltk.corpus.stopwords.words('english'))\n",
    "    stop_words = [w for w in stop_words if len(w) > 1]\n",
    "\n",
    "    all_special_chars = []\n",
    "    with open('data/nala/train_dev.json') as f:\n",
    "        for jsonObj in f:\n",
    "            nala_json = json.loads(jsonObj)['tokens']\n",
    "            for word in nala_json:\n",
    "                if not word.isalnum():\n",
    "                    all_special_chars.append(word)\n",
    "    # list of special characters to keep during inference\n",
    "    # helps with clearing out the bad characters from old papers\n",
    "    all_special_chars = list(set(all_special_chars))\n",
    "    \n",
    "    token = config['textpresso']['token']\n",
    "    db_name=config['wb_database']['db_name']\n",
    "    db_user=config['wb_database']['db_user']\n",
    "    db_password=config['wb_database']['db_password']\n",
    "    db_host=config['wb_database']['db_host']\n",
    "    ssh_host=config['wb_database']['ssh_host']\n",
    "    ssh_user=config['wb_database']['ssh_user']\n",
    "    ssh_passwd=config['wb_database']['ssh_passwd']\n",
    "\n",
    "    temp_paperid_sentence = np.array([])\n",
    "    if os.path.isfile('data/id_and_sentence.csv'):\n",
    "        temp_paperid_sentence = pd.read_csv(\"data/id_and_sentence.csv\", lineterminator='\\n', dtype = str).to_numpy() # WBPaper ID, Sentence\n",
    "    paperid_sentence_list = []\n",
    "    \n",
    "    for curr_ppr_i, id in enumerate(wbpids):\n",
    "        print(f\"{curr_ppr_i+1}\", end = \" \")\n",
    "        # textpresso_paper_text() also saves the text in flatfiles for future use \n",
    "        start_time = time.time()\n",
    "        paper_path = textpresso_paper_text(id, store_ppr_path, token)\n",
    "        txt = Path(paper_path).read_text().split('\\n')\n",
    "        # deals with empty text files with only \"0\"\n",
    "        if len(txt) == 2:\n",
    "            if temp_paperid_sentence.size != 0:\n",
    "                txt = temp_paperid_sentence[temp_paperid_sentence[:, 0] == id[7:]][:, 1]\n",
    "                # incase the loaded numpy file didn't have the required paper \n",
    "                if len(txt) == 0 and platform.system() != 'Windows':\n",
    "                    txt = wbtools_paper_text(id, db_name, db_user, db_password, db_host, ssh_host,\\\n",
    "                        ssh_user, ssh_passwd)\n",
    "            elif platform.system() != 'Windows' :\n",
    "                txt = wbtools_paper_text(id, db_name, db_user, db_password, db_host, ssh_host,\\\n",
    "                    ssh_user, ssh_passwd)\n",
    "            \n",
    "        for row in txt: \n",
    "            if row.find('fifi') != -1:\n",
    "                if temp_paperid_sentence.size != 0:\n",
    "                    txt = temp_paperid_sentence[temp_paperid_sentence[:, 0] == id[7:]][:, 1]\n",
    "                    # incase the loaded numpy file didn't have the required paper \n",
    "                    if len(txt) == 0 and platform.system() != 'Windows':\n",
    "                        txt = wbtools_paper_text(id, db_name, db_user, db_password, db_host, ssh_host,\\\n",
    "                            ssh_user, ssh_passwd)\n",
    "                elif platform.system() != 'Windows':\n",
    "                    txt = wbtools_paper_text(id, db_name, db_user, db_password, db_host, ssh_host,\\\n",
    "                        ssh_user, ssh_passwd)\n",
    "                break\n",
    "            \n",
    "        count_total_rows = len(txt)\n",
    "        for current_i, row in enumerate(txt):\n",
    "            if row.lower().find(\"we thank\") == 0 or row.lower().find(\"this work was supported\") == 0 \\\n",
    "                or row.lower().find(\"references\") == 0 or row.lower().find(\"we also thank\") == 0 \\\n",
    "                or row.lower().find(\"this research was supported\") == 0 or row.lower().find(\"we acknowledge\") == 0 \\\n",
    "                or row.lower().find(\"acknowledgments\") == 0 or row.lower().find('literature cited') != -1:\n",
    "                if current_i > count_total_rows/3:\n",
    "                    break\n",
    "\n",
    "            # usually is bad sentence\n",
    "            if len(row) < 40 or not any(word in row.lower().split() for word in stop_words):\n",
    "                continue\n",
    "            # remove sentences with links and email ids\n",
    "            if re.search('\\S+@\\S+\\.', row) or re.search('www\\.\\S+\\.', row) or re.search('http.?://', row):\n",
    "                continue\n",
    "            # filters one word sentences\n",
    "            if len(row.split()) == 1:\n",
    "                continue\n",
    "            # sentences comprised of only single characters \n",
    "            # ^ seems to be issue with wbtools extraction pipeline \n",
    "            if all(len(word) < 5 for word in row.split()):\n",
    "                continue\n",
    "            row = re.sub(\"\\( *cid *: *\\d+ *\\)\", \" \", row)\n",
    "            # TODO: replace this block with a regex sub \n",
    "            temp_row = row\n",
    "            for c in temp_row:\n",
    "                if (not c.isalnum() and not c == ' ') and c not in all_special_chars:\n",
    "                        row = row.replace(c, \"\")\n",
    "                        \n",
    "            # fixes bad space between each character of flanking sequence from old papers\n",
    "            # Switching this off as it increases the processing time\n",
    "            # also affects very small subset of old papers so not worth the extra time\n",
    "            flanking_regex = re.compile('([ACTG]( +)){4,}')\n",
    "            for m in flanking_regex.finditer(row):\n",
    "                span = (m.start(0), m.end(0))   \n",
    "                span = row[span[0]:span[1]-1]\n",
    "                correct_flank = re.sub('([ACTG])( +)', r'\\1', row)\n",
    "                row = row.replace(span, correct_flank)\n",
    "            row = 'Line '+ str(current_i) + ': ' + row.strip()\n",
    "            paperid_sentence_list.append((id, row))\n",
    "    return paperid_sentence_list[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 "
     ]
    }
   ],
   "source": [
    "# papers mentioned the remarks ace file in data/gsoc\n",
    "ids_to_extract = np.load('data/top100.npy').tolist()[70:]\n",
    "paperid_sentence_list = get_paper_sentences(ids_to_extract, db_config, store_ppr_path='data/wbpapers')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove duplicates keeping the order\n",
    "seen = set()\n",
    "paperid_sentence_list = np.array([x for x in paperid_sentence_list if x not in seen and not seen.add(x)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sentences and characters:  8539 1444637\n"
     ]
    }
   ],
   "source": [
    "print('Number of sentences and characters: ', end=' ')\n",
    "print(len(paperid_sentence_list), sum([len(sent[1]) for sent in paperid_sentence_list]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This cell takes a while to run - mainly due to the huge regex blocks   \n",
    "~1 hour per 10 papers   \n",
    "Can't really switch them off though. There might be a smarter way to work with the regex block but this pipeline would be probably running every month on 50 or so papers so low priority.   \n",
    "That being said, if you want to reduce the processing time NOW, remove the travesing of lines below  \n",
    "by changing the limit value to 'min(ppr_sen_idx+1, total_sentences)'  \n",
    "Traversing was originally implemented due to bad sentence extraction from super old papers. If paper text extraction works well (which it usually does on new papers), traversing can be removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total sentences to process  8539\n",
      "{total sentences processed}>{snippets with useful info}\n",
      "50>21 100>62 150>95 200>139 250>180 300>205 350>227 400>249 450>283 500>316 550>341 600>361 650>382 700>409 750>451 800>495 850>536 900>572 950>605 1000>652 1050>721 1100>781 1150>835 1200>888 1250>916 1300>940 1350>971 1400>1000 1450>1030 1500>1053 1550>1091 1600>1108 1650>1126 1700>1161 1750>1197 1800>1234 1850>1256 1900>1272 1950>1299 2000>1320 2050>1346 2100>1394 2150>1423 2200>1455 2250>1479 2300>1508 2350>1549 2400>1587 2450>1620 2500>1670 2550>1723 2600>1769 2650>1797 2700>1799 2750>1799 2800>1799 2850>1800 2900>1803 2950>1821 3000>1839 3050>1880 3100>1929 3150>1973 3200>2015 3250>2034 3300>2082 3350>2119 3400>2157 3450>2179 3500>2217 3550>2263 3600>2287 3650>2319 3700>2357 3750>2403 3800>2449 3850>2495 3900>2535 3950>2568 4000>2576 4050>2588 4100>2617 4150>2644 4200>2663 4250>2686 4300>2712 4350>2739 4400>2779 4450>2801 4500>2817 4550>2833 4600>2852 4650>2877 4700>2901 4750>2927 4800>2950 4850>2984 4900>3002 4950>3043 5000>3091 5050>3115 5100>3144 5150>3168 5200>3198 5250>3221 5300>3250 5350>3275 5400>3321 5450>3367 5500>3405 5550>3430 5600>3476 5650>3502 5700>3526 5750>3557 5800>3602 5850>3650 5900>3674 5950>3708 6000>3730 6050>3751 6100>3761 6150>3794 6200>3827 6250>3842 6300>3874 6350>3909 6400>3926 6450>3929 6500>3939 6550>3979 6600>4015 6650>4027 6700>4039 6750>4077 6800>4121 6850>4156 6900>4197 6950>4235 7000>4275 7050>4309 7100>4351 7150>4396 7200>4438 7250>4465 7300>4507 7350>4551 7400>4587 7450>4626 7500>4652 7550>4670 7600>4683 7650>4707 7700>4734 7750>4758 7800>4784 7850>4808 7900>4829 7950>4857 8000>4884 8050>4911 8100>4941 8150>4966 8200>4990 8250>5026 8300>5062 8350>5096 8400>5155 8450>5199 8500>5226 Total time for processing in minutes:  148\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "final = [\n",
    "    ['temporary', 'temporary', 'temporary', 'temporary', 'temporary', 'temporary'],\\\n",
    "    ['WBPaper ID', 'Method', '*Genes', '*Gene-Variant combo', 'Mutation', 'Sentence']]\n",
    "total_sentences = len(paperid_sentence_list)\n",
    "print_snips = False\n",
    "print('Total sentences to process ', len(paperid_sentence_list))\n",
    "# there would be lots of duplicates in the gene col which might cause high count of\n",
    "# snippets with useful info. they'll get filtered out in the third notebook\n",
    "print('{total sentences processed}>{snippets with useful info}')\n",
    "for ppr_sen_idx, row in enumerate(paperid_sentence_list):\n",
    "    if (ppr_sen_idx+1) % 50 == 0: print(f\"{ppr_sen_idx+1}>{len(final)-1}\", end = \" \")\n",
    "    paper_id = row[0]\n",
    "    sentence = str()\n",
    "    # traverse upto 2 sentences at a time (if they aren't super long)\n",
    "    # this should be removed if table content thing from wbtools is ever fixed\n",
    "    # or else, it might group lines of table which will result in higher false positive count\n",
    "    limit = min(ppr_sen_idx+2, total_sentences)\n",
    "    # some sentences - mostly table content are all in a single sentence\n",
    "    # temp fix, need to have a nice sentence splitter to minimize manual verification time\n",
    "    not_single_sentence = False\n",
    "    for i in range(ppr_sen_idx, limit):\n",
    "\n",
    "        sentence = sentence + paperid_sentence_list[i][1] + ' '\n",
    "        raw_sentence = sentence\n",
    "\n",
    "        if (len(sentence) > 250 and not_single_sentence):\n",
    "            break\n",
    "        if paper_id != paperid_sentence_list[i][0]:\n",
    "            break\n",
    "        \n",
    "        var_plus_genes = ''\n",
    "        # Look for the special data e.g. gene-variant combo (e.g 'ced-3(n2888)') only on single sentences \n",
    "        if not not_single_sentence:\n",
    "            var_plus_genes = []\n",
    "            all_genes = []\n",
    "            genome_versions = []\n",
    "            annotation_versions = []\n",
    "            \n",
    "            for data_and_cat in custom_mut_extract.var_and_gene_close(sentence.strip()):\n",
    "                var_plus_genes.append(data_and_cat[0])\n",
    "            if var_plus_genes:\n",
    "                var_plus_genes  = list(set(var_plus_genes))\n",
    "                var_plus_genes = \"'\" + \"', '\".join(var_plus_genes) + \"'\"\n",
    "            else:\n",
    "                var_plus_genes = ''\n",
    "                \n",
    "            for data_and_cat in custom_mut_extract.get_genes(sentence.strip()):\n",
    "                all_genes.append(data_and_cat[0])\n",
    "            if all_genes:\n",
    "                all_genes  = list(set(all_genes))\n",
    "                # Removing gene mentions from sentence\n",
    "                # e.g. \"lysine 36 by MET-1/Set2\" regex will classify this as protein mutation\n",
    "                # but MET-1 is a gene name so the mutation isn't valid\n",
    "                for gene in all_genes:\n",
    "                    sentence = sentence.replace(gene, \"\")\n",
    "                all_genes = \"'\" + \"', '\".join(all_genes) + \"'\"\n",
    "            else:\n",
    "                all_genes = ''   \n",
    "                \n",
    "        output = regex_block(sentence.strip())\n",
    "        if output:\n",
    "            mutations = []\n",
    "            for mut_and_snip in output:\n",
    "                # temp fix to deal with same mutation getting detected due to stiching multiple sentences\n",
    "                if (mut_and_snip[0] not in final[-1][4][1:-1].split(\", \") and mut_and_snip[0] not in final[-2][4][1:-1].split(\", \")) \\\n",
    "                            and mut_and_snip[0] not in mutations:\n",
    "                    mutations.append(mut_and_snip[0])\n",
    "            if mutations:\n",
    "                mutations = \"'\" + \"', '\".join(mutations) + \"'\"\n",
    "                if print_snips: print(1, mutations)\n",
    "                final.append([paper_id, 'Regex', all_genes, var_plus_genes, mutations, raw_sentence.strip()])\n",
    "            break\n",
    "\n",
    "        output = ner_mutations(sentence.strip())\n",
    "        if output:\n",
    "            mutations = []\n",
    "            for mut_and_snip in output:\n",
    "                # temp fix to deal with same mutation getting detected due to stiching multiple sentences\n",
    "                if (mut_and_snip[0] not in final[-1][4][1:-1].split(\", \") and mut_and_snip[0] not in final[-2][4][1:-1].split(\", \")) \\\n",
    "                        and not all(len(word) < 4 for word in mut_and_snip[0].split())\\\n",
    "                    and mut_and_snip[0] not in mutations:\n",
    "                    mutations.append(mut_and_snip[0])\n",
    "            if mutations:\n",
    "                mutations = \"'\" + \"', '\".join(mutations) + \"'\"\n",
    "                if print_snips: print(2, mutations)\n",
    "                final.append([paper_id, 'NER', all_genes, var_plus_genes, mutations, raw_sentence.strip()])\n",
    "            break\n",
    "        \n",
    "        # these data, if found, are going to be important if no mutations are in that sentence\n",
    "        if var_plus_genes or all_genes:\n",
    "            final.append([paper_id, '', all_genes, var_plus_genes, '', raw_sentence.strip()])\n",
    "        \n",
    "        not_single_sentence = True\n",
    "print('Total time for processing in minutes: ', int((time.time() - start_time)/60))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = final[2:] # removing the temporary first row and header\n",
    "\n",
    "# this sheet will contain high number of duplicates - which will get filtered in 3rd notebook\n",
    "# columns with asterisk contain data which are useful regardless of whether the sentence has  mutation info\n",
    "temp = pd.DataFrame(temp[:], columns=['WBPaper ID', 'Method', '* Genes', '* Gene-Variant combo', 'Mutation', 'Sentence'])\n",
    "temp.to_csv(\"data/model_output/extracted_snippets_20_50.csv\", index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "90bac3f7a4bb879b9d06605bdeda624e0779c88b1a5b8631d7aaa6d430fa2aec"
  },
  "kernelspec": {
   "display_name": "wb_env",
   "language": "python",
   "name": "wb_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}