blood_gases_1_1100.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![alt text](./Cerny_logo_1.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Analysis of Cerny ventilation recordings\n",
    "\n",
    "## Processing blood gases\n",
    "\n",
    "This notebook imports and processes blood gas data and exports it into a pickle archive."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing the necessary libraries and setting options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import sklearn as sk\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import re\n",
    "import pickle\n",
    "\n",
    "from scipy import stats\n",
    "from pandas import Series, DataFrame\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "%matplotlib inline\n",
    "matplotlib.style.use('classic')\n",
    "matplotlib.rcParams['figure.facecolor'] = 'w'\n",
    "\n",
    "pd.set_option('display.max_rows', 100)\n",
    "pd.set_option('display.max_columns', 100)\n",
    "# pd.set_option('mode.chained_assignment', None) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version: 3.7.9 (default, Aug 31 2020, 07:22:35) \n",
      "[Clang 10.0.0 ]\n",
      "pandas version: 1.1.3\n",
      "matplotlib version: 3.3.2\n",
      "NumPy version: 1.19.2\n",
      "SciPy version: 1.5.2\n",
      "IPython version: 7.19.0\n",
      "scikit-learn version: 0.23.2\n"
     ]
    }
   ],
   "source": [
    "print(\"Python version: {}\".format(sys.version))\n",
    "print(\"pandas version: {}\".format(pd.__version__))\n",
    "print(\"matplotlib version: {}\".format(matplotlib.__version__))\n",
    "print(\"NumPy version: {}\".format(np.__version__))\n",
    "print(\"SciPy version: {}\".format(sp.__version__))\n",
    "print(\"IPython version: {}\".format(IPython.__version__))\n",
    "print(\"scikit-learn version: {}\".format(sk.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### List and set the working directory and the directory to write out data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Topic of the Notebook which will also be the name of the subfolder containing results\n",
    "TOPIC = 'fabian'\n",
    "\n",
    "# Name of the external hard drive\n",
    "DRIVE = 'GUSZTI'\n",
    "\n",
    "# Directory containing clinical and blood gas data\n",
    "CWD = '/Users/guszti/ventilation_fabian'\n",
    "\n",
    "# Directory on external drive to read the ventilation data from\n",
    "DIR_READ = '/Volumes/%s/Fabian/fabian_patient_data_all' % DRIVE\n",
    "\n",
    "DIR_WRITE = '%s/%s' % (CWD, 'Analyses')\n",
    "\n",
    "# Images and raw data will be written on an external hard drive\n",
    "if not os.path.isdir('/Volumes/%s/data_dump/%s' % (DRIVE, TOPIC)):\n",
    "    os.makedirs('/Volumes/%s/data_dump/%s' % (DRIVE, TOPIC))\n",
    "DATA_DUMP = '/Volumes/%s/data_dump/%s' % (DRIVE, TOPIC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/guszti/ventilation_fabian'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.chdir(CWD)\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('/Volumes/GUSZTI/Fabian/fabian_patient_data_all',\n",
       " '/Users/guszti/ventilation_fabian/Analyses',\n",
       " '/Volumes/GUSZTI/data_dump/fabian')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DIR_READ, DIR_WRITE, DATA_DUMP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import clinical DataFrame from pickle archive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('%s/%s.pickle' % (DATA_DUMP, 'clin_df_1_1100'), 'rb') as handle:\n",
    "    clin_df = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cases = sorted(clin_df.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "792"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import all clinical data containing blood gases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import text files in a dictionary\n",
    "clin_dict = {}\n",
    "for fname in os.listdir(DIR_READ):\n",
    "    if not fname.startswith('.'): # disregard hidden files\n",
    "        fhandle = open(os.path.join('%s' % DIR_READ, fname), 'r', encoding = 'cp1252')\n",
    "        clin_dict[fname[:-4]] = fhandle.read() # use the filenames without the .txt extension as keys\n",
    "        fhandle.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "925"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(clin_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "clin_dict = {key: value for key, value in clin_dict.items() if key in cases }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "792"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(clin_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_dict = {}\n",
    "# Remove clinical details preceding the blood gases\n",
    "for key, value in clin_dict.items():\n",
    "    try:\n",
    "        gas_dict[key] = value[value.index('Astrup'):]\n",
    "    except ValueError:\n",
    "        print(key, 'has no blood gas')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "792"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gas_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_dict_2 = {}\n",
    "\n",
    "for key, value in gas_dict.items():\n",
    "    gas_dict_2[key] = {}\n",
    "    \n",
    "    for i, gas in enumerate(value.split('Astrup')[1:]):\n",
    "        gas_dict_2[key][i] = {}\n",
    "        items = gas.split('\\n')[1:-1]\n",
    "        for item in items:\n",
    "            name, value = item.split(':')\n",
    "            if value.strip() == '':\n",
    "                break\n",
    "            else:\n",
    "                gas_dict_2[key][i][name.strip()] = value.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for case in gas_dict_2:\n",
    "    for gas in sorted(gas_dict_2[case].keys()):\n",
    "        if gas_dict_2[case][gas] == {}:\n",
    "            del gas_dict_2[case][gas]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "gas_frames = {}\n",
    "\n",
    "for case in gas_dict_2.keys():\n",
    "    gas_frames[case] = DataFrame(gas_dict_2[case])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_changer(rec):\n",
    "    a = clin_df.loc[rec]['Recording start'].date()\n",
    "    for column in gas_frames[rec]:\n",
    "        b = gas_frames[rec][column]['Time']\n",
    "        c = datetime.strptime(str(b), '%H%M').time()\n",
    "        # This str() is needed here because AL000665 (and only that) is interpreted as Datetime\n",
    "        d = datetime.combine(a, c)\n",
    "        gas_frames[rec][column]['Time'] = d  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for case in cases: \n",
    "    # AL000310 has an abnormal time stamp for first gas (\"70:19\")\n",
    "    if case == 'AL000310':\n",
    "        continue\n",
    "    #print(case)\n",
    "    time_changer(case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No blood gas for AL000042\n",
      "No blood gas for AL000051\n",
      "No blood gas for AL000070\n",
      "No blood gas for AL000133\n",
      "No blood gas for AL000144\n",
      "No blood gas for AL000169\n",
      "No blood gas for AL000172\n",
      "No blood gas for AL000290\n",
      "No blood gas for AL000294\n",
      "No blood gas for AL000308\n",
      "No blood gas for AL000333\n",
      "No blood gas for AL000353\n",
      "No blood gas for AL000360\n",
      "No blood gas for AL000369\n",
      "No blood gas for AL000428\n",
      "No blood gas for AL000436\n",
      "No blood gas for AL000440\n",
      "No blood gas for AL000464\n",
      "No blood gas for AL000471\n",
      "No blood gas for AL000475\n",
      "No blood gas for AL000492\n",
      "No blood gas for AL000504\n",
      "No blood gas for AL000537\n",
      "No blood gas for AL000548\n",
      "No blood gas for AL000554\n",
      "No blood gas for AL000557\n",
      "No blood gas for AL000606\n",
      "No blood gas for AL000627\n",
      "No blood gas for AL000653\n",
      "No blood gas for AL000667\n",
      "No blood gas for AL000668\n",
      "No blood gas for AL000683\n",
      "No blood gas for AL000703\n",
      "No blood gas for AL000748\n",
      "No blood gas for AL000758\n",
      "No blood gas for AL000777\n",
      "No blood gas for AL000783\n",
      "No blood gas for AL000794\n",
      "No blood gas for AL000843\n",
      "No blood gas for AL000845\n",
      "No blood gas for AL000855\n",
      "No blood gas for AL000873\n",
      "No blood gas for AL000878\n",
      "No blood gas for AL000900\n",
      "No blood gas for AL000906\n",
      "No blood gas for AL000924\n",
      "No blood gas for AL000944\n",
      "No blood gas for AL000947\n",
      "No blood gas for AL000960\n",
      "No blood gas for AL000991\n",
      "No blood gas for AL000998\n",
      "No blood gas for AL001010\n",
      "No blood gas for AL001022\n",
      "No blood gas for AL001030\n",
      "No blood gas for AL001034\n"
     ]
    }
   ],
   "source": [
    "for case in cases:\n",
    "    try:\n",
    "        gas_frames[case] =  gas_frames[case].T.set_index('Time')\n",
    "    \n",
    "    except:\n",
    "        print('No blood gas for %s' % case)\n",
    "        del gas_frames[case]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "737"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gas_frames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export bood gases as Excel files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save blood gases into a multi-sheet Excel file\n",
    "\n",
    "writer = pd.ExcelWriter('%s/%s' % (DIR_WRITE, 'blood_gases_1_1100.xlsx'))\n",
    "for case in sorted(gas_frames.keys()):\n",
    "    gas_frames[case].to_excel(writer, case)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Export processed data as pickle files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('%s/%s.pickle' % (DATA_DUMP, 'blood_gases_1_1100'), 'wb') as handle:\n",
    "    pickle.dump(gas_frames, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}