1. Preparing data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download IDP4+ corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-08-03 15:44:47--  https://s3.amazonaws.com/net.tagtog.public/resources/corpora/tagtog_IDP4%2B_anndoc.zip\n",
      "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.217.201.16\n",
      "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.217.201.16|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 3083160 (2.9M) [application/zip]\n",
      "Saving to: ‘tagtog_IDP4+_anndoc.zip’\n",
      "\n",
      "tagtog_IDP4+_anndoc 100%[===================>]   2.94M   151KB/s    in 16s     \n",
      "\n",
      "2021-08-03 15:45:04 (186 KB/s) - ‘tagtog_IDP4+_anndoc.zip’ saved [3083160/3083160]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://s3.amazonaws.com/net.tagtog.public/resources/corpora/tagtog_IDP4%2B_anndoc.zip\n",
    "!unzip -qq tagtog_IDP4+_anndoc.zip -d data/nala\n",
    "!rm tagtog_IDP4+_anndoc.zip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare IDP4+ corpus to BIO2 format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/risubu/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from utils.nala.readers import HTMLReader\n",
    "from utils.nala.annotation_readers import AnnJsonAnnotationReader, AnnJsonMergerAnnotationReader\n",
    "from utils.nala.definers import ExclusiveNLDefiner\n",
    "from utils.nala.tokenizers import TmVarTokenizer\n",
    "from utils.nala.spliters import NLTK_SPLITTER\n",
    "from utils.nala.labelers import BIOLabeler\n",
    "\n",
    "import math\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/Users/Rishab/Documents/GitHub/genomic-info-from-papers/genomic-info-from-papers/utils/nala/data.py:908: UserWarning: \"annotations\" (and \"predicted_annotations\") are meant to be \"entities\". This and related attributes will soon be renamed\n",
      "  warnings.warn('\"annotations\" (and \"predicted_annotations\") are meant to be \"entities\". This and related attributes will soon be renamed')\n",
      "/mnt/c/Users/Rishab/Documents/GitHub/genomic-info-from-papers/genomic-info-from-papers/utils/nala/annotation_readers.py:201: UserWarning: AnnJsonMergerAnnotationReader has not been thouroughly tested. The logic should be reduced to only merge given read annotations (i.e. not a reader)\n",
      "  warnings.warn('AnnJsonMergerAnnotationReader has not been thouroughly tested. The logic should be reduced to only merge given read annotations (i.e. not a reader)')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset(157 documents and 3632 entities (['e_2: Counter({False: 3632})']))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MUT_CLASS_ID = 'e_2'\n",
    "base_folder = 'data/nala/tagtog_IDP4+_anndoc/tagtog_IDP4'\n",
    "html_folder = os.path.join(base_folder, 'html')\n",
    "annjson_folder = os.path.join(base_folder, 'annjson')\n",
    "dataset = HTMLReader(html_folder).read()\n",
    "AnnJsonMergerAnnotationReader(\n",
    "    os.path.join(annjson_folder, 'members'),\n",
    "    read_only_class_id=MUT_CLASS_ID,\n",
    "    strategy='union',\n",
    "    entity_strategy='priority',\n",
    "    priority=['Ectelion', 'abojchevski', 'sanjeevkrn', 'Shpendi'],\n",
    "    delete_incomplete_docs=True).annotate(dataset)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset(503 documents and 4897 entities (['e_2: Counter({False: 4897})']))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_folder = 'data/nala/tagtog_IDP4+_anndoc/tagtog_nala_anndoc'\n",
    "html_folder = os.path.join(base_folder, 'nala_plain_html', 'pool')\n",
    "annjson_folder = os.path.join(base_folder, 'nala_members_json')\n",
    "nala_anndoc_dataset = HTMLReader(html_folder).read()\n",
    "AnnJsonMergerAnnotationReader(\n",
    "    os.path.join(annjson_folder, 'pool'),\n",
    "    read_only_class_id=MUT_CLASS_ID,\n",
    "    strategy='union',\n",
    "    entity_strategy='priority',\n",
    "    priority=['abojchevski', 'cuhlig', 'jmcejuela'],\n",
    "    delete_incomplete_docs=True).annotate(nala_anndoc_dataset)\n",
    "dataset.extend_dataset(nala_anndoc_dataset)\n",
    "nala_anndoc_dataset = None\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/Users/Rishab/Documents/GitHub/genomic-info-from-papers/genomic-info-from-papers/utils/nala/data.py:308: UserWarning: the offsets (TP73Δex2/3 != TP73Î”ex2/) do not match in: 26168399/s1p2/Entity(class_id: e_2, offset: 594, text: TP73Δex2/3)\n",
      "  warnings.warn('the offsets ({} != {}) do not match in: {}/{}/{}'.format(e.text, part.text[e.offset:e.offset + len(e.text)], docid, partid, e))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset(552 documents and 5045 entities (['e_2: Counter({False: 5045})']))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_folder = 'data/nala/tagtog_IDP4+_anndoc/tagtog_nala_discoveries'\n",
    "html_folder = os.path.join(base_folder, 'html')\n",
    "annjson_folder = os.path.join(base_folder, 'annjson')\n",
    "nala_dis_dataset = HTMLReader(html_folder).read()\n",
    "AnnJsonAnnotationReader(\n",
    "    annjson_folder,\n",
    "    read_only_class_id=MUT_CLASS_ID,\n",
    "    delete_incomplete_docs=True).annotate(nala_dis_dataset)\n",
    "dataset.extend_dataset(nala_dis_dataset)\n",
    "nala_dis_dataset = None\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset(552 documents and 5045 entities (['e_2: Counter({0: 4071, 1: 808, 2: 166})']))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "definer = ExclusiveNLDefiner()\n",
    "definer.define(dataset)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset(552 documents and 5045 entities (['e_2: Counter({0: 4071, 1: 808, 2: 166})']))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NLTK_SPLITTER.split(dataset) \n",
    "tokenizer = TmVarTokenizer()\n",
    "tokenizer.tokenize(dataset=dataset)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/c/Users/Rishab/Documents/GitHub/genomic-info-from-papers/genomic-info-from-papers/utils/nala/data.py:439: UserWarning: annotations actually means entities. This method and related attributes will soon be renamed\n",
      "  warnings.warn('annotations actually means entities. This method and related attributes will soon be renamed')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset(552 documents and 974 entities (['e_2: Counter({1: 808, 2: 166})']))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 0 (standard), 1(natural language) or 2 (semi standard)\n",
    "remove_subclasses = [0] \n",
    "dataset.delete_subclass_annotations(subclasses=remove_subclasses)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset(552 documents and 5045 entities (['e_2: Counter({0: 4071, 1: 808, 2: 166})']))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labeler = BIOLabeler() \n",
    "labeler.label(dataset)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentences count :  19235\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Run this cell to create file with sentence and label of true if it contains mutation, false if it doesn't.\n",
    "Used in notebook 2 during testing.\n",
    "NOTE: Do not run delete_subclass_annotations() above to store mutations of all three kinds - ST, SST and NL\n",
    "'''\n",
    "\n",
    "final_binary = []\n",
    "for i, part in enumerate(dataset.parts()):\n",
    "    for tokenized_sent, raw_sent in zip(part.sentences, part.sentences_):\n",
    "        mark_postitive = 0\n",
    "        # sanity check, checking if first character same, very dumb way \n",
    "        assert str(tokenized_sent[0])[0] == raw_sent[0], f'{str(tokenized_sent[0])[0], raw_sent[0]}'\n",
    "        for token in tokenized_sent:\n",
    "            if mark_postitive:\n",
    "                break\n",
    "            for ann in part.annotations:\n",
    "                start = ann.offset\n",
    "                end = ann.offset + len(ann.text)\n",
    "                if start <= token.start < end:\n",
    "                    mark_postitive = 1\n",
    "                    break\n",
    "        final_binary.append([raw_sent, mark_postitive])\n",
    "\n",
    "print('Sentences count : ', len(final_binary))\n",
    "\n",
    "data = pd.DataFrame(final_binary[:], columns=[\"Sentence\", \"Contains mutation?\"])\n",
    "data.to_csv(\"data/nala/nala_binary.csv\", index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.prune_sentences(0.1) # tried with 0.8, no change in NER acc "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43524"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final = []\n",
    "for doc_id, doc in dataset.documents.items(): \n",
    "    for part_id, part in doc.parts.items():\n",
    "\n",
    "        for sentence in part.sentences:\n",
    "            for token in sentence:\n",
    "                final.append([token.word, token.original_labels[0].value.split('-')[0]])\n",
    "            final.append([])\n",
    "len(final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000\n",
      "10000\n",
      "10000\n",
      "10000\n",
      "3524\n"
     ]
    }
   ],
   "source": [
    "chunk_size = 10000\n",
    "total = len(final)\n",
    "devel_thres = math.ceil(total*0.8)\n",
    " \n",
    "for i in range(math.ceil(len(final)/chunk_size)):\n",
    "    temp = final[i*chunk_size:(i+1)*chunk_size]\n",
    "    print(len(temp))\n",
    "\n",
    "    if i > math.ceil(len(final)//10000)*0.8:\n",
    "        file = open(\"data/nala/devel.txt\", \"a\", encoding=\"utf-8\")\n",
    "\n",
    "    file = open(\"data/nala/train_dev.txt\", \"a\", encoding=\"utf-8\")\n",
    "\n",
    "    for index in range(len(temp)):\n",
    "        if i*chunk_size + index > devel_thres and ok_to_switch:\n",
    "            file.close()\n",
    "            file = open(\"data/nala/devel.txt\", \"a\", encoding=\"utf-8\")\n",
    "        if temp[index]:\n",
    "            file.write(str(temp[index][0]) + \" \" + str(temp[index][1]) + \"\\n\")\n",
    "            ok_to_switch = False\n",
    "        else:\n",
    "            file.write(\"\\n\")\n",
    "            ok_to_switch = True\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert BIO2 text files to JSON "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deals with [Issue #8698](https://github.com/huggingface/transformers/issues/8698)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "with open('data/nala/devel.txt', 'r', encoding=\"utf-8\") as f_in:\n",
    "    for line in f_in:\n",
    "        line = line.split()   \n",
    "        data.append(line) \n",
    "\n",
    "token = []\n",
    "label = []\n",
    "for row in data:\n",
    "    if row:\n",
    "        token.append(row[0])\n",
    "        label.append(row[1])\n",
    "    else:\n",
    "        assert len(token) == len(label)\n",
    "        # for l in label:\n",
    "        #     if l not in ['B', 'O', 'I']:\n",
    "        #         print('Error')\n",
    "        #         break\n",
    "        dictionary = { \n",
    "            \"tokens\" : token, \n",
    "            \"tags\" : label, \n",
    "        }  \n",
    "        with open(\"data/nala/devel.json\", \"a\", encoding=\"utf-8\") as outfile: \n",
    "            json.dump(dictionary, outfile)\n",
    "            outfile.write('\\n')\n",
    "        token = []\n",
    "        label = []\n",
    "\n",
    "data = []\n",
    "with open('data/nala/train_dev.txt', 'r', encoding=\"utf-8\") as f_in:\n",
    "    for line in f_in:\n",
    "        line = line.split()   \n",
    "        data.append(line) \n",
    "\n",
    "token = []\n",
    "label = []\n",
    "for row in data:\n",
    "    if row:\n",
    "        token.append(row[0])\n",
    "        label.append(row[1])\n",
    "    else:\n",
    "        assert len(token) == len(label)\n",
    "        # for l in label:\n",
    "        #     if l not in ['B', 'O', 'I']:\n",
    "        #         print('Error')\n",
    "        #         break\n",
    "        dictionary = { \n",
    "            \"tokens\" : token, \n",
    "            \"tags\" : label, \n",
    "        }  \n",
    "        with open(\"data/nala/train_dev.json\", \"a\", encoding=\"utf-8\") as outfile: \n",
    "            json.dump(dictionary, outfile)\n",
    "            outfile.write('\\n')\n",
    "        token = []\n",
    "        label = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "90bac3f7a4bb879b9d06605bdeda624e0779c88b1a5b8631d7aaa6d430fa2aec"
  },
  "kernelspec": {
   "display_name": "wb_env",
   "language": "python",
   "name": "wb_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}