STEP3a_Evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, '/home/paul/.conda/envs/tensorflow/lib/python3.6/site-packages')\n",
    "#sys.path.insert(0, '/usr/local/lib/python3.5/dist-packages')\n",
    "\n",
    "import glob\n",
    "import os\n",
    "import itertools\n",
    "import re\n",
    "from collections import Counter\n",
    "import csv\n",
    "\n",
    "import numpy\n",
    "import tensorflow\n",
    "import keras\n",
    "import sklearn.metrics\n",
    "\n",
    "import matplotlib.pyplot\n",
    "import pandas\n",
    "import seaborn\n",
    "\n",
    "import deepometry.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _shape(pathname):\n",
    "    return numpy.load(pathname).shape\n",
    "\n",
    "\n",
    "def load(pathnames, labels, patient_to_exclude):\n",
    "\n",
    "    #print('Before exclusion: ',len(pathnames))\n",
    "    #pathnames = [x for x in pathnames if patient_to_exclude not in x]\n",
    "    #print('After exclusion: ',len(pathnames))\n",
    "\n",
    "    x = numpy.empty((len(pathnames),) + _shape(pathnames[0]), dtype=numpy.uint8)\n",
    "\n",
    "    y = numpy.empty((len(pathnames),), dtype=numpy.uint8)\n",
    "\n",
    "    label_to_index = {label: index for index, label in enumerate(sorted(labels))}\n",
    "\n",
    "    for index, pathname in enumerate(pathnames):\n",
    "        if (os.path.isfile(pathname) == True):\n",
    "\n",
    "            label = os.path.split(os.path.dirname(pathname))[-1]\n",
    "\n",
    "            x[index] = numpy.load(pathname)\n",
    "\n",
    "            y[index] = label_to_index[label]\n",
    "\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample(directories):\n",
    "\n",
    "    pathnames = []\n",
    "\n",
    "    for directory in directories:\n",
    "        subdirectories = sorted(glob.glob(os.path.join(directory, \"*\")))\n",
    "\n",
    "        subdirectory_pathnames = [glob.glob(os.path.join(subdirectory, \"*\")) for subdirectory in subdirectories]\n",
    "\n",
    "        nsamples = max([len(pathnames) for pathnames in subdirectory_pathnames])\n",
    "        #nsamples = 200000\n",
    "\n",
    "        pathnames += [list(numpy.random.permutation(pathnames)[:nsamples]) for pathnames in subdirectory_pathnames]\n",
    "\n",
    "    pathnames = sum(pathnames, [])\n",
    "\n",
    "    return pathnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_class_weights(y):\n",
    "    counter = Counter(y)\n",
    "    majority = max(counter.values())\n",
    "    return  {cls: float(majority/count) for cls, count in counter.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_pathnames(directories, labels):\n",
    "\n",
    "    pathnames = []\n",
    "\n",
    "    for directory in directories:\n",
    "        subdirectories = sorted(glob.glob(os.path.join(directory, \"*\")))\n",
    "        \n",
    "        # transform the files of the same label into directory\n",
    "        subdirectory_pathnames = [glob.glob(\"{}/*.npy\".format(subdirectory)) for subdirectory in subdirectories ]      \n",
    "\n",
    "        nsamples = max([len(pathnames) for pathnames in subdirectory_pathnames])\n",
    "\n",
    "        pathnames += [list(numpy.random.permutation(pathnames)[:nsamples]) for pathnames in subdirectory_pathnames]\n",
    "\n",
    "    pathnames = sum(pathnames, [])\n",
    "\n",
    "    return pathnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_include(pathnames, labels, patient_to_include):\n",
    "\n",
    "    print('All cells in treated patients: ',len(pathnames))\n",
    "    pathnames = [x for x in pathnames if patient_to_include in x]\n",
    "    print('Cells in this patient: ',len(pathnames))\n",
    "\n",
    "    x = numpy.empty((len(pathnames),) + _shape(pathnames[0]), dtype=numpy.uint8)\n",
    "\n",
    "    y = numpy.empty((len(pathnames),), dtype=numpy.uint8)\n",
    "\n",
    "    label_to_index = {label: index for index, label in enumerate(sorted(labels))}\n",
    "\n",
    "    for index, pathname in enumerate(pathnames):\n",
    "        if (os.path.isfile(pathname) == True):\n",
    "\n",
    "            label = os.path.split(os.path.dirname(pathname))[-1]\n",
    "\n",
    "            x[index] = numpy.load(pathname)\n",
    "\n",
    "            y[index] = label_to_index[label]\n",
    "\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [\"Leukemic\", \"Normal\", \"Others\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "directories = [\"/parsed_data/\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples = sample(directories)\n",
    "len(samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "patients_to_test = ['157pres','157day8','157day15','171pres','171day11','172pres','172day29','175pres','175day8','177pres', '177day8']\n",
    "#patients_to_test = ['177day8']\n",
    "#selected_to_train = [x for x in samples if numpy.all([not z in x for z in patients_to_test])]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build session running on GPU 1\n",
    "configuration = tensorflow.ConfigProto()\n",
    "configuration.gpu_options.allow_growth = True\n",
    "configuration.gpu_options.visible_device_list = \"0\"\n",
    "session = tensorflow.Session(config = configuration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_include_all(pathnames, labels, patients_to_include):\n",
    "\n",
    "    print('All cells in treated patients: ',len(pathnames))\n",
    "    pathnames = [x for x in pathnames for patient_to_include in patients_to_include if patient_to_include in x]\n",
    "    print('Cells in all selected patient: ',len(pathnames))\n",
    "\n",
    "    x = numpy.empty((len(pathnames),) + _shape(pathnames[0]), dtype=numpy.uint8)\n",
    "\n",
    "    y = numpy.empty((len(pathnames),), dtype=numpy.uint8)\n",
    "\n",
    "    label_to_index = {label: index for index, label in enumerate(sorted(labels))}\n",
    "\n",
    "    for index, pathname in enumerate(pathnames):\n",
    "        if (os.path.isfile(pathname) == True):\n",
    "\n",
    "            label = os.path.split(os.path.dirname(pathname))[-1]\n",
    "\n",
    "            x[index] = numpy.load(pathname)\n",
    "\n",
    "            y[index] = label_to_index[label]\n",
    "\n",
    "    return x, y\n",
    "\n",
    "x_test, y_test = load_include_all(samples, labels, patients_to_test)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xx = numpy.delete(x_test[:,:,:,6:],-2,-1)\n",
    "\n",
    "del(x_test)\n",
    "x_test = xx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Testing set: \", x_test.shape)\n",
    "del(xx)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = deepometry.model.Model(shape=x_test.shape[1:], units=len(labels))\n",
    "\n",
    "model.compile()\n",
    "\n",
    "model_directory = str('/models/resnet_drop_' + str(6) + '_channels')\n",
    "\n",
    "model.model.load_weights(os.path.join(model_directory,'model.h5'))        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate_metrics = model.evaluate(x_test, y_test, batch_size=256, verbose=1)\n",
    "\n",
    "\n",
    "predicted = model.predict(\n",
    "    batch_size=50,\n",
    "    x=x_test\n",
    ")\n",
    "\n",
    "predicted = numpy.argmax(predicted, axis=1)\n",
    "# predicted = numpy.argmax(predicted, -1)\n",
    "# expected = numpy.argmax(y[:, :], -1)\n",
    "expected = y_test  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "confusion = sklearn.metrics.confusion_matrix(expected, predicted)\n",
    "\n",
    "norm_confusion = confusion.astype('float') / confusion.sum(axis=1)[:, numpy.newaxis]\n",
    "\n",
    "norm_confusion = pandas.DataFrame(norm_confusion)\n",
    "\n",
    "matplotlib.pyplot.figure(figsize=(12, 8))\n",
    "\n",
    "seaborn.heatmap(norm_confusion, annot=True)\n",
    "seaborn.set(font_scale = 1.5)\n",
    "\n",
    "matplotlib.pyplot.savefig( os.path.join(model_directory, str( 'confusion_matrix_all_tested_patients.eps')) , format='eps', dpi=300)\n",
    "\n",
    "\n",
    "with open(os.path.join(model_directory, str( 'metrics_all_tested_patients.csv')), \"w\") as metrics_csv:\n",
    "    metrics_writer = csv.writer(metrics_csv)\n",
    "    metrics_writer.writerow(model.model.metrics_names)\n",
    "    metrics_writer.writerow(evaluate_metrics)\n",
    "\n",
    "    numpy.save(os.path.join(model_directory, str( 'confusion_matrix_all_tested_patients.npy')), confusion)             \n",
    "\n",
    "del(x_test)\n",
    "#keras.backend.clear_session()    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for i in [6]: \n",
    "    \n",
    "    for patient_to_test in patients_to_test:\n",
    "\n",
    "        x_test, y_test = load_include(samples, labels, patient_to_test)        \n",
    "\n",
    "#         xx = x_test[:,:,:,i:]\n",
    "        xx = numpy.delete(x_test[:,:,:,6:],-2,-1)\n",
    "\n",
    "        del(x_test)\n",
    "        x_test = xx\n",
    "        print('Testing : ',patient_to_test)\n",
    "        print(\"Testing set: \", x_test.shape)\n",
    "        del(xx)\n",
    "        \n",
    "\n",
    "        model = deepometry.model.Model(shape=x_test.shape[1:], units=len(labels))\n",
    "\n",
    "        model.compile()\n",
    "\n",
    "        model_directory = str('/models/resnet_drop_' + str(i) + '_channels')\n",
    "\n",
    "        model.model.load_weights(os.path.join(model_directory,'model.h5'))        \n",
    "        \n",
    "        \n",
    "        evaluate_metrics = model.evaluate(x_test, y_test, batch_size=256, verbose=1)\n",
    "\n",
    "        \n",
    "        predicted = model.predict(\n",
    "            batch_size=50,\n",
    "            x=x_test\n",
    "        )\n",
    "\n",
    "        predicted = numpy.argmax(predicted, axis=1)\n",
    "        # predicted = numpy.argmax(predicted, -1)\n",
    "        # expected = numpy.argmax(y[:, :], -1)\n",
    "        expected = y_test        \n",
    "               \n",
    "        confusion = sklearn.metrics.confusion_matrix(expected, predicted)\n",
    "\n",
    "        norm_confusion = confusion.astype('float') / confusion.sum(axis=1)[:, numpy.newaxis]\n",
    "\n",
    "        norm_confusion = pandas.DataFrame(norm_confusion)\n",
    "\n",
    "        matplotlib.pyplot.figure(figsize=(12, 8))\n",
    "\n",
    "        seaborn.heatmap(norm_confusion, annot=True)\n",
    "        \n",
    "        matplotlib.pyplot.savefig( os.path.join(model_directory, str( 'confusion_matrix_'+ patient_to_test +'.eps')) , format='eps', dpi=300)\n",
    "              \n",
    "\n",
    "        with open(os.path.join(model_directory, str( 'metrics_'+ patient_to_test +'.csv')), \"w\") as metrics_csv:\n",
    "            metrics_writer = csv.writer(metrics_csv)\n",
    "            metrics_writer.writerow(model.model.metrics_names)\n",
    "            metrics_writer.writerow(evaluate_metrics)\n",
    "\n",
    "            numpy.save(os.path.join(model_directory, str( 'confusion_matrix_'+ patient_to_test +'.npy')), confusion)             \n",
    "\n",
    "        del(x_test)\n",
    "        keras.backend.clear_session()    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}