TCIA_Client_Starter_Local.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <script>\n",
       "            function code_toggle_10314748398578157728() {\n",
       "                $('div.cell.code_cell.rendered.selected').find('div.input').toggle();\n",
       "            }\n",
       "\n",
       "            \n",
       "        </script>\n",
       "\n",
       "        <a href=\"javascript:code_toggle_10314748398578157728()\">Toggle show/hide</a>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# from pyspark.sql.types import BooleanType\n",
    "# from pyspark.sql import functions as F\n",
    "\n",
    "import os\n",
    "import urllib\n",
    "import requests\n",
    "import sys\n",
    "import math\n",
    "import json\n",
    "from zipfile import ZipFile\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import ipywidgets as widgets\n",
    "import random\n",
    "from IPython.display import HTML\n",
    "from ipywidgets import Box, Layout\n",
    "from IPython.display import display\n",
    "\n",
    "def hide_toggle(for_next=False):\n",
    "    this_cell = \"\"\"$('div.cell.code_cell.rendered.selected')\"\"\"\n",
    "    next_cell = this_cell + '.next()'\n",
    "\n",
    "    toggle_text = 'Toggle show/hide'  # text shown on toggle link\n",
    "    target_cell = this_cell  # target cell to control with toggle\n",
    "    js_hide_current = ''  # bit of JS to permanently hide code in current cell (only when toggling next cell)\n",
    "\n",
    "    if for_next:\n",
    "        target_cell = next_cell\n",
    "        toggle_text += ' next cell'\n",
    "        js_hide_current = this_cell + '.find(\"div.input\").hide();'\n",
    "\n",
    "    js_f_name = 'code_toggle_{}'.format(str(random.randint(1,2**64)))\n",
    "\n",
    "    html = \"\"\"\n",
    "        <script>\n",
    "            function {f_name}() {{\n",
    "                {cell_selector}.find('div.input').toggle();\n",
    "            }}\n",
    "\n",
    "            {js_hide_current}\n",
    "        </script>\n",
    "\n",
    "        <a href=\"javascript:{f_name}()\">{toggle_text}</a>\n",
    "    \"\"\".format(\n",
    "        f_name=js_f_name,\n",
    "        cell_selector=target_cell,\n",
    "        js_hide_current=js_hide_current, \n",
    "        toggle_text=toggle_text\n",
    "    )\n",
    "\n",
    "    return HTML(html)\n",
    "# ------------------------------------------------------------------------------------------------------------------\n",
    "#\n",
    "# Refer https://wiki.cancerimagingarchive.net/display/Public/TCIA+Programmatic+Interface+%28REST+API%29+Usage+Guide for complete list of API Resources\n",
    "#\n",
    "class TCIAClient:\n",
    "    GET_IMAGE = \"getImage\"\n",
    "    GET_MANUFACTURER_VALUES = \"getManufacturerValues\"\n",
    "    GET_MODALITY_VALUES = \"getModalityValues\"\n",
    "    GET_COLLECTION_VALUES = \"getCollectionValues\"\n",
    "    GET_BODY_PART_VALUES = \"getBodyPartValues\"\n",
    "    GET_PATIENT_STUDY = \"getPatientStudy\"\n",
    "    GET_SERIES = \"getSeries\"\n",
    "    GET_PATIENT = \"getPatient\"\n",
    "    GET_SERIES_SIZE = \"getSeriesSize\"\n",
    "    CONTENTS_BY_NAME = \"ContentsByName\"\n",
    "    GET_SOP_INSTANCE_UID = \"getSOPInstanceUIDs\"\n",
    "\n",
    "    def __init__(self, baseUrl, resource):\n",
    "        self.baseUrl = baseUrl + \"/\" + resource\n",
    "\n",
    "    def execute(self, url, queryParameters={}):\n",
    "        queryParameters = dict((k, v) for k, v in queryParameters.items() if v)\n",
    "        queryString = \"?%s\" % urllib.parse.urlencode(queryParameters)\n",
    "        requestUrl = url + queryString\n",
    "        resp = requests.get(url=requestUrl)\n",
    "        return resp\n",
    "      \n",
    "    def get_collection_values(self, outputFormat = \"json\" ):\n",
    "        # set of all collection names\n",
    "        serviceUrl = self.baseUrl + \"/query/\" + self.GET_COLLECTION_VALUES\n",
    "        queryParameters = { \"format\" : outputFormat }\n",
    "        resp = self.execute(serviceUrl , queryParameters)\n",
    "        return resp\n",
    "      \n",
    "    def get_series(self, collection = None , modality = None , studyInstanceUid = None , outputFormat = \"json\" ):\n",
    "        serviceUrl = self.baseUrl + \"/query/\" + self.GET_SERIES\n",
    "        queryParameters = {\"Collection\" : collection , \"StudyInstanceUID\" : studyInstanceUid , \"Modality\" : modality , \"format\" : outputFormat }\n",
    "        resp = self.execute(serviceUrl , queryParameters)\n",
    "        return resp\n",
    "      \n",
    "    def get_sop_instance_uids(self, seriesInstanceUid, outputFormat = \"json\" ):\n",
    "        serviceUrl = self.baseUrl + \"/query/\" + self.GET_SOP_INSTANCE_UID\n",
    "        queryParameters = {\"SeriesInstanceUID\" : seriesInstanceUid }\n",
    "        resp = self.execute(serviceUrl , queryParameters)\n",
    "        return resp\n",
    "      \n",
    "    \n",
    "    def get_image(self, seriesInstanceUid, local_path):\n",
    "        '''\n",
    "        seriesInstanceUid: the series of images to load\n",
    "        dbfsPath: the path to the mounted folder in dbfs to place the images (should follow convention collection/patient/study/series/image.dcm)\n",
    "        '''\n",
    "        serviceUrl = self.baseUrl + \"/query/\" + self.GET_IMAGE\n",
    "        queryParameters = { \"SeriesInstanceUID\" : seriesInstanceUid }\n",
    "        os.umask(0o022)\n",
    "        try:\n",
    "            queryParameters = dict((k, v) for k, v in queryParameters.items() if v)\n",
    "            queryString = \"?%s\" % urllib.parse.urlencode(queryParameters)\n",
    "            requestUrl = serviceUrl + queryString\n",
    "            zipresp = urllib.request.urlopen(requestUrl)\n",
    "            \n",
    "            # Specify path on driver for zip file and extraction file\n",
    "            tempZipPath = local_path +'/'+ seriesInstanceUid + '.zip'\n",
    "            os.makedirs(local_path +'/'+ seriesInstanceUid)\n",
    "    \n",
    "            bytesRead = zipresp.read()\n",
    "            fout = open(tempZipPath, \"wb\")\n",
    "            fout.write(bytesRead)\n",
    "            fout.close()\n",
    "    \n",
    "            old_dir = os.getcwd()\n",
    "            os.chdir(local_path)\n",
    "            zf = ZipFile(tempZipPath)\n",
    "            zf.extractall(path = local_path+'/'+seriesInstanceUid)\n",
    "#             print('Writing slice to: ', local_path+'/'+seriesInstanceUid)\n",
    "#             print()\n",
    "            zf.close()\n",
    "            os.remove(tempZipPath)\n",
    "            os.chdir(old_dir)\n",
    "            return True\n",
    "        except requests.HTTPError as e:\n",
    "            print(\"HTTP Error:\", e.code, serviceUrl)\n",
    "            return False\n",
    "\n",
    "\n",
    "## --------------------------------------- DEFINING FUNCTIONS ---------------------------------------------------- ##\n",
    "def get_collection_data(collection_name):\n",
    "    # Get a dataframe of patients based on collection name from widget\n",
    "    collection_df = pd.DataFrame(tcia_client.get_series(collection = collection_name, outputFormat = \"json\").json())\n",
    "    collection_df['Collection'] = np.repeat(collection_name, collection_df.shape[0])\n",
    "    print('Meta data for patients in collection ', collection_name, ' loaded. \\n')\n",
    "    return collection_df\n",
    "\n",
    "\n",
    "def load_image(collection, patient_id, study_instance_uid, series_instance_uid, dest_path):\n",
    "    if dest_path:\n",
    "        path = dest_path +'/'+ collection +'/'+ patient_id +'/'+ study_instance_uid\n",
    "    else:\n",
    "        path = os.getcwd() +'/'+ collection +'/'+ patient_id +'/'+ study_instance_uid\n",
    "        \n",
    "    return tcia_client.get_image(seriesInstanceUid=series_instance_uid, local_path=path)\n",
    "    \n",
    "def load_images(df, dest_path):\n",
    "    print('Loading images...')\n",
    "    ## Loops through all of the images in the collection_df, calls a function to store them at the specified location\n",
    "    ## is_img_loaded is a list of Boolean values (True means image was loaded - False means image failed to load)\n",
    "    is_img_loaded = list(map(load_image, df['Collection'], df['PatientID'], df['StudyInstanceUID'],\n",
    "                 df['SeriesInstanceUID'], np.repeat(dest_path, df.shape[0])))\n",
    "    \n",
    "    if all(flag == True for flag in is_img_loaded):\n",
    "        return \"Collection loaded successfully!\"\n",
    "\n",
    "    \n",
    "# ---------------------------------------- DEFINE WIDGETS --------------------------------------------------------- #       \n",
    "#Instantiate TCIA Client\n",
    "tcia_client = TCIAClient(baseUrl='https://services.cancerimagingarchive.net/services/v4', resource='TCIA')\n",
    "\n",
    "# Gather list of all collections to determine which one to load and create widget\n",
    "collections = sorted([obj['Collection'] for obj in tcia_client.get_collection_values().json()])\n",
    "\n",
    "collection_dropdown = widgets.Dropdown(\n",
    "    options=collections,\n",
    "    value=collections[0],\n",
    "    description='Collection:',\n",
    "    disabled=False,\n",
    ")\n",
    "\n",
    "collection_dropdown = widgets.Dropdown(\n",
    "    options=collections,\n",
    "    value=collections[0],\n",
    "    description='Collection:',\n",
    "    disabled=False,\n",
    "    layout = Layout(width='200px')\n",
    ")\n",
    "\n",
    "destination_path = widgets.Text(\n",
    "    value=os.getcwd(),\n",
    "    placeholder=os.getcwd(),\n",
    "    description='Destination Path:',\n",
    "    disabled=False,\n",
    "    layout = Layout(width='500px')\n",
    ")\n",
    "\n",
    "items = [collection_dropdown, destination_path]\n",
    "box = Box(children=items)   \n",
    "\n",
    "# output = widgets.Output()\n",
    "\n",
    "hide_toggle()     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Starter Notebook on Pulling Images From TCIA API Directly Into DataLake\n",
    "\n",
    "**The Cancer Imaging Archive (TCIA)**\n",
    "\n",
    "> The image data in The Cancer Imaging Archive (TCIA) is organized into purpose-built collections. \n",
    "> A collection typically includes studies from several subjects (patients) - typically patients’ imaging related by a common disease (e.g. lung cancer), \n",
    "> image modality or type (MRI, CT, digital histopathology, etc) or research focus.  In some collections, there may be only one study per subject. \n",
    "> In other collections, subjects may have been followed over time, in which case there will be multiple studies per subject. \n",
    "> The subjects typically have a disease and/or particular anatomical site (lung, brain, etc.) in common.  \n",
    "\n",
    "> TCIA submissions are organized in the following hierarchy, which is important to remember in creating your search query and reviewing search results:\n",
    "\n",
    "> Collection > Patient (Subject) > Study > Series > Images\n",
    "\n",
    "> In other words, a Collection is the largest organizing concept within TCIA and it includes data about Patients (also called Subjects). \n",
    "> As you continue to drill down to more granular concepts, Patients contain Studies, Studies contain (Image) Series, and Series contain individual Images.\n",
    "\n",
    "\n",
    "**Directions**\n",
    "To use this notebook, run the first cell and enter desired inputs.\n",
    "+ *Collection*: the collection that you want to load\n",
    "+ *Destination Path*: the folder in which you want to load your images. The images will load in the following file structure <br>\n",
    "    > Collection > Patient (Subject) > Study > Series > Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36f38f5b511f465e959b01eb44f6f4c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Box(children=(Dropdown(description='Collection:', layout=Layout(width='200px'), options=('4D-Lung', 'AAPM-RT-M…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8635fa521d284afca5f0b18c2dd0d6c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Button(description='Click Me!', style=ButtonStyle())"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a37b9f0e253345c39029d75c20a192fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "button = widgets.Button(description=\"Download Images\")\n",
    "output = widgets.Output()\n",
    "display(button, output)\n",
    "\n",
    "\n",
    "def on_button_clicked(b):\n",
    "    with output:\n",
    "        collection_df = get_collection_data(collection_dropdown.value).head(2)\n",
    "        dest_path = None if destination_path.value == '' else destination_path.value\n",
    "        load_images(collection_df, dest_path)\n",
    "\n",
    "button.on_click(on_button_clicked)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "name": "TCIA_Client_Starter",
  "notebookId": 3445613378330858
 },
 "nbformat": 4,
 "nbformat_minor": 1
}