diseases.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute disease-disease-cooccurrence for Hetionet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import io\n",
    "import gzip\n",
    "\n",
    "import pandas\n",
    "import requests\n",
    "import networkx\n",
    "\n",
    "import eutility\n",
    "import cooccurrence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_code</th>\n",
       "      <th>doid_name</th>\n",
       "      <th>mesh_id</th>\n",
       "      <th>mesh_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:2531</td>\n",
       "      <td>hematologic cancer</td>\n",
       "      <td>D019337</td>\n",
       "      <td>Hematologic Neoplasms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:1319</td>\n",
       "      <td>brain cancer</td>\n",
       "      <td>D001932</td>\n",
       "      <td>Brain Neoplasms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DOID:1324</td>\n",
       "      <td>lung cancer</td>\n",
       "      <td>D008175</td>\n",
       "      <td>Lung Neoplasms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DOID:263</td>\n",
       "      <td>kidney cancer</td>\n",
       "      <td>D007680</td>\n",
       "      <td>Kidney Neoplasms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DOID:1793</td>\n",
       "      <td>pancreatic cancer</td>\n",
       "      <td>D010190</td>\n",
       "      <td>Pancreatic Neoplasms</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   doid_code           doid_name  mesh_id              mesh_name\n",
       "0  DOID:2531  hematologic cancer  D019337  Hematologic Neoplasms\n",
       "1  DOID:1319        brain cancer  D001932        Brain Neoplasms\n",
       "2  DOID:1324         lung cancer  D008175         Lung Neoplasms\n",
       "3   DOID:263       kidney cancer  D007680       Kidney Neoplasms\n",
       "4  DOID:1793   pancreatic cancer  D010190   Pancreatic Neoplasms"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read mappings for DO Slim terms\n",
    "url = 'https://raw.githubusercontent.com/dhimmel/disease-ontology/72614ade9f1cc5a5317b8f6836e1e464b31d5587/data/xrefs-slim.tsv'\n",
    "disease_df = pandas.read_table(url)\n",
    "disease_df = disease_df.query('resource == \"MSH\"').drop('resource', 1)\n",
    "disease_df = disease_df.rename(columns={'resource_id': 'mesh_id'})\n",
    "\n",
    "# Read MeSH terms to MeSH names\n",
    "url = 'https://raw.githubusercontent.com/dhimmel/mesh/e561301360e6de2140dedeaa7c7e17ce4714eb7f/data/terms.tsv'\n",
    "mesh_df = pandas.read_table(url)\n",
    "disease_df = disease_df.merge(mesh_df)\n",
    "\n",
    "# Manually remove problematic xrefs\n",
    "# https://github.com/obophenotype/human-disease-ontology/issues/45\n",
    "disease_df = disease_df.query(\"mesh_id != 'D003327' and mesh_id != 'D017202'\")\n",
    "disease_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Query PubMed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10320 articles for Hematologic Neoplasms\n",
      "122727 articles for Brain Neoplasms\n",
      "180844 articles for Lung Neoplasms\n",
      "60494 articles for Kidney Neoplasms\n",
      "57863 articles for Pancreatic Neoplasms\n",
      "100038 articles for Skin Neoplasms\n",
      "104535 articles for Bone Neoplasms\n",
      "27302 articles for Pharyngeal Neoplasms\n",
      "65991 articles for Ovarian Neoplasms\n",
      "226835 articles for Breast Neoplasms\n",
      "63189 articles for Glioma\n",
      "107447 articles for Uterine Neoplasms\n",
      "24447 articles for Adrenal Gland Neoplasms\n",
      "40010 articles for Esophageal Neoplasms\n",
      "14552 articles for Salivary Gland Neoplasms\n",
      "97203 articles for Prostatic Neoplasms\n",
      "77286 articles for Stomach Neoplasms\n",
      "45208 articles for Urinary Bladder Neoplasms\n",
      "18495 articles for Peripheral Nervous System Neoplasms\n",
      "40519 articles for Thyroid Neoplasms\n",
      "130963 articles for Liver Neoplasms\n",
      "60840 articles for Uterine Cervical Neoplasms\n",
      "4780 articles for Vaginal Neoplasms\n",
      "249626 articles for Head and Neck Neoplasms\n",
      "38987 articles for Rectal Neoplasms\n",
      "34076 articles for Eye Neoplasms\n",
      "68917 articles for Colonic Neoplasms\n",
      "24448 articles for Laryngeal Neoplasms\n",
      "283101 articles for Neoplasms, Germ Cell and Embryonal\n",
      "9735 articles for Thymus Neoplasms\n",
      "11737 articles for Myosarcoma\n",
      "5565 articles for Duodenal Neoplasms\n",
      "2617 articles for Ileal Neoplasms\n",
      "117808 articles for Sarcoma\n",
      "2355 articles for Appendiceal Neoplasms\n",
      "4612 articles for Penile Neoplasms\n",
      "4139 articles for Ureteral Neoplasms\n",
      "3249 articles for Tracheal Neoplasms\n",
      "7161 articles for Vulvar Neoplasms\n",
      "1940 articles for Jejunal Neoplasms\n",
      "12425 articles for Peritoneal Neoplasms\n",
      "2738 articles for Vascular Neoplasms\n",
      "11841 articles for Mesothelioma\n",
      "76390 articles for Melanoma\n",
      "2380 articles for Fallopian Tube Neoplasms\n",
      "22613 articles for Testicular Neoplasms\n",
      "7358 articles for Gallbladder Neoplasms\n",
      "20327 articles for Meningeal Neoplasms\n",
      "14052 articles for Bile Duct Neoplasms\n",
      "12274 articles for Mediastinal Neoplasms\n",
      "9418 articles for Spinal Cord Neoplasms\n",
      "7944 articles for Retroperitoneal Neoplasms\n",
      "31533 articles for Crohn Disease\n",
      "46287 articles for Multiple Sclerosis\n",
      "91140 articles for Diabetes Mellitus, Type 2\n",
      "28289 articles for Colitis, Ulcerative\n",
      "62862 articles for Diabetes Mellitus, Type 1\n",
      "95295 articles for Arthritis, Rheumatoid\n",
      "40786 articles for Coronary Artery Disease\n",
      "148894 articles for Obesity\n",
      "16725 articles for Celiac Disease\n",
      "49965 articles for Lupus Erythematosus, Systemic\n",
      "26855 articles for Refractive Errors\n",
      "7065 articles for Liver Cirrhosis, Biliary\n",
      "4078 articles for Vitiligo\n",
      "16971 articles for Macular Degeneration\n",
      "21070 articles for Metabolic Syndrome X\n",
      "108236 articles for Asthma\n",
      "30896 articles for Psoriasis\n",
      "87056 articles for Schizophrenia\n",
      "22222 articles for Migraine Disorders\n",
      "69752 articles for Alzheimer Disease\n",
      "14577 articles for Graves Disease\n",
      "49349 articles for Parkinson Disease\n",
      "14898 articles for Dermatitis, Atopic\n",
      "32534 articles for Bipolar Disorder\n",
      "12161 articles for Spondylitis, Ankylosing\n",
      "10757 articles for Polycystic Ovary Syndrome\n",
      "214731 articles for Hypertension\n",
      "17041 articles for Scleroderma, Systemic\n",
      "7686 articles for Behcet Syndrome\n",
      "4802 articles for Osteitis Deformans\n",
      "20395 articles for Leprosy\n",
      "22378 articles for Intracranial Aneurysm\n",
      "43355 articles for Glaucoma\n",
      "13589 articles for Amyotrophic Lateral Sclerosis\n",
      "2744 articles for Restless Legs Syndrome\n",
      "4773 articles for Mucocutaneous Lymph Node Syndrome\n",
      "24584 articles for Atherosclerosis\n",
      "2509 articles for Alopecia Areata\n",
      "45971 articles for Osteoporosis\n",
      "28909 articles for Hypothyroidism\n",
      "4960 articles for Glomerulonephritis, IGA\n",
      "67451 articles for Alcoholism\n",
      "5771 articles for Creutzfeldt-Jakob Syndrome\n",
      "1206 articles for Azoospermia\n",
      "132583 articles for Epilepsy\n",
      "47571 articles for Hepatitis B\n",
      "38605 articles for Pulmonary Disease, Chronic Obstructive\n",
      "14411 articles for Aortic Aneurysm, Abdominal\n",
      "79638 articles for Kidney Failure, Chronic\n",
      "45631 articles for Osteoarthritis\n",
      "3974 articles for Arthritis, Psoriatic\n",
      "8353 articles for Tobacco Use Disorder\n",
      "2448 articles for Glomerulonephritis, Membranous\n",
      "7669 articles for Diabetes, Gestational\n",
      "52704 articles for Malaria\n",
      "16500 articles for Autistic Disorder\n",
      "13355 articles for Cardiomyopathy, Dilated\n",
      "920 articles for Arthritis, Gouty\n",
      "17621 articles for Leiomyoma\n",
      "3033 articles for Cholangitis, Sclerosing\n",
      "3065 articles for Narcolepsy\n",
      "1884 articles for Intervertebral Disc Degeneration\n",
      "12123 articles for Cleft Lip\n",
      "1442 articles for Idiopathic Pulmonary Fibrosis\n",
      "21145 articles for Attention Deficit Disorder with Hyperactivity\n",
      "3636 articles for Tourette Syndrome\n",
      "8889 articles for Aortic Aneurysm, Thoracic\n",
      "83521 articles for Depressive Disorder\n",
      "17875 articles for Rhinitis, Allergic\n",
      "44312 articles for Pancreatitis\n",
      "16146 articles for Nephrolithiasis\n",
      "24223 articles for Periodontitis\n",
      "6418 articles for Barrett Esophagus\n",
      "782 articles for Fuchs' Endothelial Dystrophy\n",
      "4768 articles for Otosclerosis\n",
      "2277 articles for Conduct Disorder\n",
      "4440 articles for Glomerulosclerosis, Focal Segmental\n",
      "37451 articles for Dental Caries\n",
      "138233 articles for Anemia\n",
      "6096 articles for Panic Disorder\n",
      "72916 articles for Acquired Immunodeficiency Syndrome\n"
     ]
    }
   ],
   "source": [
    "rows_out = list()\n",
    "\n",
    "for i, row in disease_df.iterrows():\n",
    "    term_query = '{disease}[MeSH Terms]'.format(disease = row.mesh_name.lower())\n",
    "    payload = {'db': 'pubmed', 'term': term_query}\n",
    "    pmids = eutility.esearch_query(payload, retmax = 10000)\n",
    "    row['term_query'] = term_query\n",
    "    row['n_articles'] = len(pmids)\n",
    "    row['pubmed_ids'] = '|'.join(pmids)\n",
    "    rows_out.append(row)\n",
    "    print('{} articles for {}'.format(len(pmids), row.mesh_name))\n",
    "\n",
    "disease_pmids_df = pandas.DataFrame(rows_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "with gzip.open('data/disease-pmids-topic.tsv.gz', 'wt') as write_file:\n",
    "    disease_pmids_df.to_csv(write_file, sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "disease_df, disease_to_pmids = cooccurrence.read_pmids_tsv('data/disease-pmids-topic.tsv.gz', key='doid_code')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total articles containing a doid_code_0: 4161769\n",
      "Total articles containing a doid_code_1: 4161769\n",
      "Total articles containing both a doid_code_0 and doid_code_1: 4161769\n",
      "\n",
      "After removing terms without any cooccurences:\n",
      "+ 133 doid_code_0s remain\n",
      "+ 133 doid_code_1s remain\n",
      "\n",
      "Cooccurrence scores calculated for 17689 doid_code_0 -- doid_code_1 pairs\n"
     ]
    }
   ],
   "source": [
    "cooc_df = cooccurrence.score_pmid_cooccurrence(disease_to_pmids, disease_to_pmids, 'doid_code_0', 'doid_code_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_code</th>\n",
       "      <th>doid_name</th>\n",
       "      <th>mesh_id</th>\n",
       "      <th>mesh_name</th>\n",
       "      <th>term_query</th>\n",
       "      <th>n_articles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:2531</td>\n",
       "      <td>hematologic cancer</td>\n",
       "      <td>D019337</td>\n",
       "      <td>Hematologic Neoplasms</td>\n",
       "      <td>hematologic neoplasms[MeSH Terms]</td>\n",
       "      <td>10320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:1319</td>\n",
       "      <td>brain cancer</td>\n",
       "      <td>D001932</td>\n",
       "      <td>Brain Neoplasms</td>\n",
       "      <td>brain neoplasms[MeSH Terms]</td>\n",
       "      <td>122727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DOID:1324</td>\n",
       "      <td>lung cancer</td>\n",
       "      <td>D008175</td>\n",
       "      <td>Lung Neoplasms</td>\n",
       "      <td>lung neoplasms[MeSH Terms]</td>\n",
       "      <td>180844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DOID:263</td>\n",
       "      <td>kidney cancer</td>\n",
       "      <td>D007680</td>\n",
       "      <td>Kidney Neoplasms</td>\n",
       "      <td>kidney neoplasms[MeSH Terms]</td>\n",
       "      <td>60494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DOID:1793</td>\n",
       "      <td>pancreatic cancer</td>\n",
       "      <td>D010190</td>\n",
       "      <td>Pancreatic Neoplasms</td>\n",
       "      <td>pancreatic neoplasms[MeSH Terms]</td>\n",
       "      <td>57863</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   doid_code           doid_name  mesh_id              mesh_name  \\\n",
       "0  DOID:2531  hematologic cancer  D019337  Hematologic Neoplasms   \n",
       "1  DOID:1319        brain cancer  D001932        Brain Neoplasms   \n",
       "2  DOID:1324         lung cancer  D008175         Lung Neoplasms   \n",
       "3   DOID:263       kidney cancer  D007680       Kidney Neoplasms   \n",
       "4  DOID:1793   pancreatic cancer  D010190   Pancreatic Neoplasms   \n",
       "\n",
       "                          term_query  n_articles  \n",
       "0  hematologic neoplasms[MeSH Terms]       10320  \n",
       "1        brain neoplasms[MeSH Terms]      122727  \n",
       "2         lung neoplasms[MeSH Terms]      180844  \n",
       "3       kidney neoplasms[MeSH Terms]       60494  \n",
       "4   pancreatic neoplasms[MeSH Terms]       57863  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disease_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_code_0</th>\n",
       "      <th>doid_code_1</th>\n",
       "      <th>cooccurrence</th>\n",
       "      <th>expected</th>\n",
       "      <th>enrichment</th>\n",
       "      <th>odds_ratio</th>\n",
       "      <th>p_fisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:11615</td>\n",
       "      <td>DOID:11615</td>\n",
       "      <td>4612</td>\n",
       "      <td>5.110938</td>\n",
       "      <td>902.378361</td>\n",
       "      <td>inf</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:11615</td>\n",
       "      <td>DOID:8577</td>\n",
       "      <td>1</td>\n",
       "      <td>31.349378</td>\n",
       "      <td>0.031899</td>\n",
       "      <td>0.031654</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DOID:11615</td>\n",
       "      <td>DOID:5612</td>\n",
       "      <td>2</td>\n",
       "      <td>10.436864</td>\n",
       "      <td>0.191628</td>\n",
       "      <td>0.191106</td>\n",
       "      <td>0.999669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DOID:11615</td>\n",
       "      <td>DOID:14330</td>\n",
       "      <td>0</td>\n",
       "      <td>54.687703</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DOID:11615</td>\n",
       "      <td>DOID:0050425</td>\n",
       "      <td>0</td>\n",
       "      <td>3.040853</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  doid_code_0   doid_code_1  cooccurrence   expected  enrichment  odds_ratio  \\\n",
       "0  DOID:11615    DOID:11615          4612   5.110938  902.378361         inf   \n",
       "1  DOID:11615     DOID:8577             1  31.349378    0.031899    0.031654   \n",
       "2  DOID:11615     DOID:5612             2  10.436864    0.191628    0.191106   \n",
       "3  DOID:11615    DOID:14330             0  54.687703    0.000000    0.000000   \n",
       "4  DOID:11615  DOID:0050425             0   3.040853    0.000000    0.000000   \n",
       "\n",
       "   p_fisher  \n",
       "0  0.000000  \n",
       "1  1.000000  \n",
       "2  0.999669  \n",
       "3  1.000000  \n",
       "4  1.000000  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cooc_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "cooc_df = cooc_df[cooc_df['doid_code_0'] != cooc_df['doid_code_1']]\n",
    "doid_name_df = disease_df[['doid_code', 'doid_name']].drop_duplicates()\n",
    "cooc_df = doid_name_df.rename(columns={'doid_code': 'doid_code_1', 'doid_name': 'doid_name_1'}).merge(cooc_df)\n",
    "cooc_df = doid_name_df.rename(columns={'doid_code': 'doid_code_0', 'doid_name': 'doid_name_0'}).merge(cooc_df)\n",
    "cooc_df = cooc_df.sort_values(by=['doid_name_0', 'p_fisher'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doid_code_0</th>\n",
       "      <th>doid_name_0</th>\n",
       "      <th>doid_code_1</th>\n",
       "      <th>doid_name_1</th>\n",
       "      <th>cooccurrence</th>\n",
       "      <th>expected</th>\n",
       "      <th>enrichment</th>\n",
       "      <th>odds_ratio</th>\n",
       "      <th>p_fisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9444</th>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>DOID:14330</td>\n",
       "      <td>Parkinson's disease</td>\n",
       "      <td>2760</td>\n",
       "      <td>827.098152</td>\n",
       "      <td>3.336968</td>\n",
       "      <td>3.577398</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9465</th>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>DOID:11949</td>\n",
       "      <td>Creutzfeldt-Jakob disease</td>\n",
       "      <td>332</td>\n",
       "      <td>96.723002</td>\n",
       "      <td>3.432482</td>\n",
       "      <td>3.593306</td>\n",
       "      <td>3.377672e-80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9456</th>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>DOID:332</td>\n",
       "      <td>amyotrophic lateral sclerosis</td>\n",
       "      <td>451</td>\n",
       "      <td>227.754094</td>\n",
       "      <td>1.980206</td>\n",
       "      <td>2.020452</td>\n",
       "      <td>5.524978e-40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9496</th>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>DOID:11555</td>\n",
       "      <td>Fuchs' endothelial dystrophy</td>\n",
       "      <td>1</td>\n",
       "      <td>13.106461</td>\n",
       "      <td>0.076298</td>\n",
       "      <td>0.075102</td>\n",
       "      <td>9.999982e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9490</th>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>Alzheimer's disease</td>\n",
       "      <td>DOID:1595</td>\n",
       "      <td>endogenous depression</td>\n",
       "      <td>1221</td>\n",
       "      <td>1399.827043</td>\n",
       "      <td>0.872251</td>\n",
       "      <td>0.868045</td>\n",
       "      <td>9.999997e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     doid_code_0          doid_name_0 doid_code_1  \\\n",
       "9444  DOID:10652  Alzheimer's disease  DOID:14330   \n",
       "9465  DOID:10652  Alzheimer's disease  DOID:11949   \n",
       "9456  DOID:10652  Alzheimer's disease    DOID:332   \n",
       "9496  DOID:10652  Alzheimer's disease  DOID:11555   \n",
       "9490  DOID:10652  Alzheimer's disease   DOID:1595   \n",
       "\n",
       "                        doid_name_1  cooccurrence     expected  enrichment  \\\n",
       "9444            Parkinson's disease          2760   827.098152    3.336968   \n",
       "9465      Creutzfeldt-Jakob disease           332    96.723002    3.432482   \n",
       "9456  amyotrophic lateral sclerosis           451   227.754094    1.980206   \n",
       "9496   Fuchs' endothelial dystrophy             1    13.106461    0.076298   \n",
       "9490          endogenous depression          1221  1399.827043    0.872251   \n",
       "\n",
       "      odds_ratio      p_fisher  \n",
       "9444    3.577398  0.000000e+00  \n",
       "9465    3.593306  3.377672e-80  \n",
       "9456    2.020452  5.524978e-40  \n",
       "9496    0.075102  9.999982e-01  \n",
       "9490    0.868045  9.999997e-01  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cooc_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17556"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cooc_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1086"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cooc_df[cooc_df.p_fisher <= 0.005])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "cooc_df.to_csv('data/disease-disease-cooccurrence.tsv', index=False, sep='\\t')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}