blood_gases_new_1_1397.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![alt text](./Cerny_logo_1.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "# Analysis of Cerny ventilation recordings\n",
    "\n",
    "#### Processing blood gases\n",
    "\n",
    "This notebook imports and processes blood gas data and exports it into a pickle archive.\n",
    "\n",
    "The data processed and analysed in this Notebook were collected by the **Neonatal Emergency and Transport Service of the Peter Cerny Foundation**, Budapest, Hungary\n",
    "\n",
    "**Author: Dr Gusztav Belteki**\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Total number of recordings: **1251**\n",
    "- Ventilation recordings longer than 15 minutes: **1035 cases**\n",
    "- Clinical data available in **1053 cases**\n",
    "- Only keep clinical data for cases where ventilation recordings (>15 minutes) are also available: **987 cases**\n",
    "- Blood gases available in **927 cases**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the required libraries and set options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import re\n",
    "import pickle\n",
    "\n",
    "from scipy import stats\n",
    "from pandas import Series, DataFrame\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "%matplotlib inline\n",
    "matplotlib.style.use('classic')\n",
    "matplotlib.rcParams['figure.facecolor'] = 'w'\n",
    "\n",
    "pd.set_option('display.max_rows', 100)\n",
    "pd.set_option('display.max_columns', 100)\n",
    "pd.set_option('mode.chained_assignment', None) \n",
    "\n",
    "# This is to turn off a warning message which is given when read_Excel() imports '.xlsx' files\n",
    "import warnings\n",
    "warnings.simplefilter(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version: 3.11.7 (main, Dec 15 2023, 12:09:04) [Clang 14.0.6 ]\n",
      "pandas version: 2.1.4\n",
      "matplotlib version: 3.8.0\n",
      "NumPy version: 1.26.3\n",
      "SciPy version: 1.11.4\n",
      "IPython version: 8.20.0\n"
     ]
    }
   ],
   "source": [
    "print(\"Python version: {}\".format(sys.version))\n",
    "print(\"pandas version: {}\".format(pd.__version__))\n",
    "print(\"matplotlib version: {}\".format(matplotlib.__version__))\n",
    "print(\"NumPy version: {}\".format(np.__version__))\n",
    "print(\"SciPy version: {}\".format(sp.__version__))\n",
    "print(\"IPython version: {}\".format(IPython.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. List and set the working directory and the directory to write out data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Name of the external hard drive\n",
    "DRIVE = 'GUSZTI'\n",
    "\n",
    "# Directory on external drive to read the clinical from\n",
    "DIR_READ = os.path.join(os.sep, 'Volumes', DRIVE, 'Fabian_new', 'fabian_patient_data_all_new')\n",
    "\n",
    "# Path to project folder containing ventilation research results\n",
    "PATH = os.path.join(os.sep, 'Users', 'guszti', 'Library', 'Mobile Documents', 'com~apple~CloudDocs', \n",
    "                            'Documents', 'Research', 'Ventilation')\n",
    "\n",
    "# Folder to export the result of analysis\n",
    "DIR_WRITE = os.path.join(PATH, 'ventilation_fabian_new', 'Analyses')\n",
    "os.makedirs(DIR_WRITE, exist_ok = True)\n",
    "\n",
    "# Folder on a USB stick to export data to and to import processed data exported by other Notebooks\n",
    "DATA_DUMP = os.path.join(os.sep, '/Volumes', DRIVE, 'data_dump', 'fabian_new',)\n",
    "os.makedirs(DATA_DUMP, exist_ok = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('/Volumes/GUSZTI/Fabian_new/fabian_patient_data_all_new',\n",
       " '/Users/guszti/Library/Mobile Documents/com~apple~CloudDocs/Documents/Research/Ventilation/ventilation_fabian_new/Analyses',\n",
       " '/Volumes/GUSZTI/data_dump/fabian_new')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DIR_READ, DIR_WRITE, DATA_DUMP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Import clinical DataFrame from pickle archive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_DUMP, 'clin_df_new.pickle'), 'rb') as handle:\n",
    "    clin_df = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cases = sorted(clin_df.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1079"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Import all clinical data containing blood gases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import text files in a dictionary\n",
    "clin_dict = {}\n",
    "for fname in os.listdir(DIR_READ):\n",
    "    if not fname.startswith('.'): # disregard hidden files\n",
    "        #print(fname)\n",
    "        fhandle = open(os.path.join(DIR_READ, fname), 'r', encoding = 'cp1252', errors='ignore')\n",
    "        clin_dict[fname[:-4]] = fhandle.read() # use the filenames without the .txt extension as keys\n",
    "        fhandle.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1199"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(clin_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "clin_dict = {key: value for key, value in clin_dict.items() if key in cases }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1079"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(clin_dict)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "gas_dict = {}\n",
    "# Remove clinical details preceding the blood gases\n",
    "for key, value in clin_dict.items():\n",
    "    try:\n",
    "        gas_dict[key] = value[value.index('Astrup'):]\n",
    "    except ValueError:\n",
    "        print(key, 'has no blood gas')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AT001299 has no blood gas\n",
      "AT001281 has no blood gas\n",
      "AT001372 has no blood gas\n"
     ]
    }
   ],
   "source": [
    "gas_dict = {}\n",
    "# Remove clinical details preceding the blood gases\n",
    "for key, value in clin_dict.items():\n",
    "    # For recordings starting with AT001263, 'Astrup' in the text file was changed to 'Labor\n",
    "    if 'Astrup' in value:\n",
    "        gas_dict[key] = value[value.index('Astrup'):]\n",
    "    elif 'Labor' in value:\n",
    "        gas_dict[key] = value[value.index('Labor'):]\n",
    "    else:\n",
    "        print(key, 'has no blood gas')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1076"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gas_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AT000005\n",
      "AT000006\n",
      "AT000007\n",
      "AT000008\n",
      "AT000009\n",
      "AT000011\n",
      "AT000012\n",
      "AT000013\n",
      "AT000014\n",
      "AT000015\n",
      "AT000016\n",
      "AT000017\n",
      "AT000018\n",
      "AT000019\n",
      "AT000020\n",
      "AT000022\n",
      "AT000023\n",
      "AT000025\n",
      "AT000027\n",
      "AT000029\n",
      "AT000030\n",
      "AT000031\n",
      "AT000032\n",
      "AT000033\n",
      "AT000034\n",
      "AT000035\n",
      "AT000036\n",
      "AT000037\n",
      "AT000038\n",
      "AT000039\n",
      "AT000040\n",
      "AT000042\n",
      "AT000043\n",
      "AT000048\n",
      "AT000049\n",
      "AT000050\n",
      "AT000051\n",
      "AT000052\n",
      "AT000053\n",
      "AT000054\n",
      "AT000055\n",
      "AT000056\n",
      "AT000057\n",
      "AT000058\n",
      "AT000059\n",
      "AT000060\n",
      "AT000061\n",
      "AT000062\n",
      "AT000063\n",
      "AT000065\n",
      "AT000066\n",
      "AT000068\n",
      "AT000069\n",
      "AT000070\n",
      "AT000072\n",
      "AT000073\n",
      "AT000074\n",
      "AT000075\n",
      "AT000076\n",
      "AT000077\n",
      "AT000078\n",
      "AT000079\n",
      "AT000080\n",
      "AT000081\n",
      "AT000082\n",
      "AT000083\n",
      "AT000084\n",
      "AT000085\n",
      "AT000086\n",
      "AT000087\n",
      "AT000088\n",
      "AT000089\n",
      "AT000090\n",
      "AT000091\n",
      "AT000094\n",
      "AT000095\n",
      "AT000096\n",
      "AT000098\n",
      "AT000099\n",
      "AT000100\n",
      "AT000101\n",
      "AT000103\n",
      "AT000104\n",
      "AT000105\n",
      "AT000106\n",
      "AT000107\n",
      "AT000108\n",
      "AT000110\n",
      "AT000111\n",
      "AT000112\n",
      "AT000113\n",
      "AT000115\n",
      "AT000116\n",
      "AT000117\n",
      "AT000118\n",
      "AT000120\n",
      "AT000121\n",
      "AT000122\n",
      "AT000123\n",
      "AT000124\n",
      "AT000125\n",
      "AT000126\n",
      "AT000127\n",
      "AT000129\n",
      "AT000130\n",
      "AT000131\n",
      "AT000132\n",
      "AT000133\n",
      "AT000134\n",
      "AT000135\n",
      "AT000137\n",
      "AT000138\n",
      "AT000139\n",
      "AT000140\n",
      "AT000141\n",
      "AT000142\n",
      "AT000143\n",
      "AT000144\n",
      "AT000146\n",
      "AT000147\n",
      "AT000148\n",
      "AT000149\n",
      "AT000151\n",
      "AT000152\n",
      "AT000153\n",
      "AT000154\n",
      "AT000155\n",
      "AT000156\n",
      "AT000157\n",
      "AT000158\n",
      "AT000159\n",
      "AT000160\n",
      "AT000161\n",
      "AT000162\n",
      "AT000163\n",
      "AT000165\n",
      "AT000168\n",
      "AT000169\n",
      "AT000170\n",
      "AT000171\n",
      "AT000172\n",
      "AT000173\n",
      "AT000174\n",
      "AT000175\n",
      "AT000176\n",
      "AT000177\n",
      "AT000178\n",
      "AT000179\n",
      "AT000180\n",
      "AT000182\n",
      "AT000184\n",
      "AT000186\n",
      "AT000187\n",
      "AT000189\n",
      "AT000191\n",
      "AT000192\n",
      "AT000195\n",
      "AT000196\n",
      "AT000198\n",
      "AT000199\n",
      "AT000200\n",
      "AT000201\n",
      "AT000203\n",
      "AT000204\n",
      "AT000205\n",
      "AT000206\n",
      "AT000207\n",
      "AT000208\n",
      "AT000210\n",
      "AT000212\n",
      "AT000213\n",
      "AT000214\n",
      "AT000215\n",
      "AT000216\n",
      "AT000217\n",
      "AT000219\n",
      "AT000220\n",
      "AT000222\n",
      "AT000223\n",
      "AT000224\n",
      "AT000225\n",
      "AT000226\n",
      "AT000227\n",
      "AT000228\n",
      "AT000230\n",
      "AT000232\n",
      "AT000233\n",
      "AT000234\n",
      "AT000235\n",
      "AT000236\n",
      "AT000238\n",
      "AT000239\n",
      "AT000240\n",
      "AT000241\n",
      "AT000242\n",
      "AT000244\n",
      "AT000245\n",
      "AT000246\n",
      "AT000247\n",
      "AT000248\n",
      "AT000249\n",
      "AT000250\n",
      "AT000251\n",
      "AT000252\n",
      "AT000253\n",
      "AT000254\n",
      "AT000258\n",
      "AT000259\n",
      "AT000260\n",
      "AT000263\n",
      "AT000264\n",
      "AT000265\n",
      "AT000266\n",
      "AT000267\n",
      "AT000269\n",
      "AT000270\n",
      "AT000271\n",
      "AT000272\n",
      "AT000273\n",
      "AT000275\n",
      "AT000276\n",
      "AT000277\n",
      "AT000279\n",
      "AT000280\n",
      "AT000281\n",
      "AT000282\n",
      "AT000283\n",
      "AT000284\n",
      "AT000285\n",
      "AT000286\n",
      "AT000287\n",
      "AT000288\n",
      "AT000291\n",
      "AT000292\n",
      "AT000294\n",
      "AT000295\n",
      "AT000297\n",
      "AT000299\n",
      "AT000300\n",
      "AT000303\n",
      "AT000304\n",
      "AT000307\n",
      "AT000308\n",
      "AT000309\n",
      "AT000311\n",
      "AT000313\n",
      "AT000314\n",
      "AT000315\n",
      "AT000316\n",
      "AT000317\n",
      "AT000318\n",
      "AT000319\n",
      "AT000320\n",
      "AT000321\n",
      "AT000322\n",
      "AT000323\n",
      "AT000324\n",
      "AT000325\n",
      "AT000327\n",
      "AT000328\n",
      "AT000330\n",
      "AT000331\n",
      "AT000332\n",
      "AT000333\n",
      "AT000334\n",
      "AT000337\n",
      "AT000338\n",
      "AT000339\n",
      "AT000340\n",
      "AT000341\n",
      "AT000342\n",
      "AT000345\n",
      "AT000347\n",
      "AT000349\n",
      "AT000350\n",
      "AT000352\n",
      "AT000354\n",
      "AT000355\n",
      "AT000357\n",
      "AT000358\n",
      "AT000359\n",
      "AT000360\n",
      "AT000361\n",
      "AT000362\n",
      "AT000363\n",
      "AT000364\n",
      "AT000365\n",
      "AT000367\n",
      "AT000368\n",
      "AT000369\n",
      "AT000371\n",
      "AT000372\n",
      "AT000373\n",
      "AT000375\n",
      "AT000377\n",
      "AT000378\n",
      "AT000379\n",
      "AT000380\n",
      "AT000381\n",
      "AT000382\n",
      "AT000383\n",
      "AT000384\n",
      "AT000385\n",
      "AT000386\n",
      "AT000387\n",
      "AT000388\n",
      "AT000389\n",
      "AT000390\n",
      "AT000391\n",
      "AT000392\n",
      "AT000393\n",
      "AT000394\n",
      "AT000396\n",
      "AT000397\n",
      "AT000399\n",
      "AT000402\n",
      "AT000404\n",
      "AT000405\n",
      "AT000406\n",
      "AT000408\n",
      "AT000409\n",
      "AT000410\n",
      "AT000411\n",
      "AT000412\n",
      "AT000413\n",
      "AT000414\n",
      "AT000415\n",
      "AT000416\n",
      "AT000417\n",
      "AT000418\n",
      "AT000420\n",
      "AT000423\n",
      "AT000424\n",
      "AT000425\n",
      "AT000426\n",
      "AT000428\n",
      "AT000429\n",
      "AT000430\n",
      "AT000431\n",
      "AT000432\n",
      "AT000433\n",
      "AT000435\n",
      "AT000436\n",
      "AT000437\n",
      "AT000438\n",
      "AT000439\n",
      "AT000440\n",
      "AT000441\n",
      "AT000442\n",
      "AT000444\n",
      "AT000445\n",
      "AT000446\n",
      "AT000448\n",
      "AT000449\n",
      "AT000450\n",
      "AT000451\n",
      "AT000452\n",
      "AT000453\n",
      "AT000454\n",
      "AT000455\n",
      "AT000456\n",
      "AT000458\n",
      "AT000459\n",
      "AT000460\n",
      "AT000461\n",
      "AT000462\n",
      "AT000463\n",
      "AT000465\n",
      "AT000466\n",
      "AT000467\n",
      "AT000468\n",
      "AT000469\n",
      "AT000470\n",
      "AT000471\n",
      "AT000472\n",
      "AT000473\n",
      "AT000474\n",
      "AT000475\n",
      "AT000476\n",
      "AT000477\n",
      "AT000478\n",
      "AT000480\n",
      "AT000481\n",
      "AT000482\n",
      "AT000483\n",
      "AT000484\n",
      "AT000485\n",
      "AT000486\n",
      "AT000487\n",
      "AT000488\n",
      "AT000489\n",
      "AT000490\n",
      "AT000491\n",
      "AT000493\n",
      "AT000494\n",
      "AT000495\n",
      "AT000496\n",
      "AT000497\n",
      "AT000498\n",
      "AT000499\n",
      "AT000500\n",
      "AT000501\n",
      "AT000502\n",
      "AT000503\n",
      "AT000504\n",
      "AT000507\n",
      "AT000508\n",
      "AT000509\n",
      "AT000510\n",
      "AT000511\n",
      "AT000512\n",
      "AT000513\n",
      "AT000516\n",
      "AT000517\n",
      "AT000518\n",
      "AT000519\n",
      "AT000522\n",
      "AT000523\n",
      "AT000524\n",
      "AT000525\n",
      "AT000526\n",
      "AT000527\n",
      "AT000528\n",
      "AT000529\n",
      "AT000530\n",
      "AT000531\n",
      "AT000532\n",
      "AT000533\n",
      "AT000534\n",
      "AT000535\n",
      "AT000536\n",
      "AT000537\n",
      "AT000538\n",
      "AT000539\n",
      "AT000540\n",
      "AT000541\n",
      "AT000542\n",
      "AT000543\n",
      "AT000545\n",
      "AT000546\n",
      "AT000547\n",
      "AT000548\n",
      "AT000549\n",
      "AT000550\n",
      "AT000551\n",
      "AT000552\n",
      "AT000553\n",
      "AT000554\n",
      "AT000555\n",
      "AT000556\n",
      "AT000557\n",
      "AT000560\n",
      "AT000562\n",
      "AT000564\n",
      "AT000565\n",
      "AT000567\n",
      "AT000568\n",
      "AT000569\n",
      "AT000570\n",
      "AT000571\n",
      "AT000573\n",
      "AT000574\n",
      "AT000575\n",
      "AT000576\n",
      "AT000577\n",
      "AT000578\n",
      "AT000579\n",
      "AT000580\n",
      "AT000581\n",
      "AT000582\n",
      "AT000583\n",
      "AT000584\n",
      "AT000585\n",
      "AT000586\n",
      "AT000587\n",
      "AT000588\n",
      "AT000589\n",
      "AT000590\n",
      "AT000591\n",
      "AT000594\n",
      "AT000595\n",
      "AT000597\n",
      "AT000598\n",
      "AT000599\n",
      "AT000600\n",
      "AT000601\n",
      "AT000602\n",
      "AT000603\n",
      "AT000604\n",
      "AT000605\n",
      "AT000606\n",
      "AT000607\n",
      "AT000609\n",
      "AT000611\n",
      "AT000612\n",
      "AT000613\n",
      "AT000614\n",
      "AT000615\n",
      "AT000616\n",
      "AT000617\n",
      "AT000618\n",
      "AT000619\n",
      "AT000620\n",
      "AT000621\n",
      "AT000623\n",
      "AT000624\n",
      "AT000626\n",
      "AT000627\n",
      "AT000628\n",
      "AT000629\n",
      "AT000631\n",
      "AT000632\n",
      "AT000633\n",
      "AT000634\n",
      "AT000635\n",
      "AT000636\n",
      "AT000637\n",
      "AT000638\n",
      "AT000639\n",
      "AT000641\n",
      "AT000644\n",
      "AT000645\n",
      "AT000648\n",
      "AT000649\n",
      "AT000650\n",
      "AT000651\n",
      "AT000652\n",
      "AT000653\n",
      "AT000654\n",
      "AT000655\n",
      "AT000657\n",
      "AT000658\n",
      "AT000659\n",
      "AT000660\n",
      "AT000661\n",
      "AT000662\n",
      "AT000666\n",
      "AT000668\n",
      "AT000669\n",
      "AT000670\n",
      "AT000672\n",
      "AT000673\n",
      "AT000674\n",
      "AT000675\n",
      "AT000676\n",
      "AT000678\n",
      "AT000679\n",
      "AT000680\n",
      "AT000681\n",
      "AT000682\n",
      "AT000683\n",
      "AT000684\n",
      "AT000685\n",
      "AT000686\n",
      "AT000687\n",
      "AT000688\n",
      "AT000689\n",
      "AT000690\n",
      "AT000691\n",
      "AT000693\n",
      "AT000694\n",
      "AT000695\n",
      "AT000696\n",
      "AT000697\n",
      "AT000699\n",
      "AT000700\n",
      "AT000701\n",
      "AT000703\n",
      "AT000707\n",
      "AT000708\n",
      "AT000709\n",
      "AT000710\n",
      "AT000711\n",
      "AT000712\n",
      "AT000713\n",
      "AT000714\n",
      "AT000715\n",
      "AT000717\n",
      "AT000718\n",
      "AT000719\n",
      "AT000720\n",
      "AT000721\n",
      "AT000722\n",
      "AT000723\n",
      "AT000724\n",
      "AT000729\n",
      "AT000730\n",
      "AT000732\n",
      "AT000733\n",
      "AT000735\n",
      "AT000737\n",
      "AT000738\n",
      "AT000741\n",
      "AT000742\n",
      "AT000743\n",
      "AT000744\n",
      "AT000745\n",
      "AT000746\n",
      "AT000747\n",
      "AT000749\n",
      "AT000750\n",
      "AT000751\n",
      "AT000752\n",
      "AT000753\n",
      "AT000754\n",
      "AT000756\n",
      "AT000757\n",
      "AT000758\n",
      "AT000759\n",
      "AT000760\n",
      "AT000761\n",
      "AT000762\n",
      "AT000763\n",
      "AT000765\n",
      "AT000766\n",
      "AT000767\n",
      "AT000768\n",
      "AT000769\n",
      "AT000770\n",
      "AT000771\n",
      "AT000772\n",
      "AT000773\n",
      "AT000775\n",
      "AT000776\n",
      "AT000778\n",
      "AT000779\n",
      "AT000781\n",
      "AT000782\n",
      "AT000783\n",
      "AT000784\n",
      "AT000785\n",
      "AT000786\n",
      "AT000787\n",
      "AT000788\n",
      "AT000789\n",
      "AT000790\n",
      "AT000791\n",
      "AT000792\n",
      "AT000793\n",
      "AT000794\n",
      "AT000795\n",
      "AT000796\n",
      "AT000797\n",
      "AT000798\n",
      "AT000799\n",
      "AT000800\n",
      "AT000801\n",
      "AT000802\n",
      "AT000803\n",
      "AT000804\n",
      "AT000806\n",
      "AT000807\n",
      "AT000808\n",
      "AT000809\n",
      "AT000811\n",
      "AT000812\n",
      "AT000813\n",
      "AT000815\n",
      "AT000819\n",
      "AT000820\n",
      "AT000821\n",
      "AT000823\n",
      "AT000825\n",
      "AT000826\n",
      "AT000829\n",
      "AT000830\n",
      "AT000831\n",
      "AT000832\n",
      "AT000833\n",
      "AT000834\n",
      "AT000837\n",
      "AT000838\n",
      "AT000839\n",
      "AT000840\n",
      "AT000841\n",
      "AT000842\n",
      "AT000845\n",
      "AT000847\n",
      "AT000851\n",
      "AT000853\n",
      "AT000854\n",
      "AT000855\n",
      "AT000856\n",
      "AT000858\n",
      "AT000859\n",
      "AT000860\n",
      "AT000861\n",
      "AT000862\n",
      "AT000863\n",
      "AT000865\n",
      "AT000866\n",
      "AT000867\n",
      "AT000868\n",
      "AT000869\n",
      "AT000870\n",
      "AT000871\n",
      "AT000873\n",
      "AT000874\n",
      "AT000875\n",
      "AT000876\n",
      "AT000877\n",
      "AT000878\n",
      "AT000880\n",
      "AT000881\n",
      "AT000882\n",
      "AT000883\n",
      "AT000884\n",
      "AT000885\n",
      "AT000886\n",
      "AT000887\n",
      "AT000888\n",
      "AT000889\n",
      "AT000890\n",
      "AT000891\n",
      "AT000892\n",
      "AT000894\n",
      "AT000895\n",
      "AT000896\n",
      "AT000897\n",
      "AT000898\n",
      "AT000899\n",
      "AT000902\n",
      "AT000903\n",
      "AT000904\n",
      "AT000905\n",
      "AT000906\n",
      "AT000907\n",
      "AT000908\n",
      "AT000910\n",
      "AT000912\n",
      "AT000913\n",
      "AT000914\n",
      "AT000915\n",
      "AT000916\n",
      "AT000918\n",
      "AT000919\n",
      "AT000920\n",
      "AT000921\n",
      "AT000922\n",
      "AT000923\n",
      "AT000924\n",
      "AT000925\n",
      "AT000926\n",
      "AT000927\n",
      "AT000928\n",
      "AT000929\n",
      "AT000932\n",
      "AT000933\n",
      "AT000934\n",
      "AT000935\n",
      "AT000936\n",
      "AT000938\n",
      "AT000940\n",
      "AT000941\n",
      "AT000942\n",
      "AT000943\n",
      "AT000944\n",
      "AT000945\n",
      "AT000946\n",
      "AT000948\n",
      "AT000949\n",
      "AT000950\n",
      "AT000952\n",
      "AT000953\n",
      "AT000954\n",
      "AT000956\n",
      "AT000957\n",
      "AT000959\n",
      "AT000960\n",
      "AT000961\n",
      "AT000962\n",
      "AT000963\n",
      "AT000965\n",
      "AT000966\n",
      "AT000967\n",
      "AT000969\n",
      "AT000971\n",
      "AT000973\n",
      "AT000974\n",
      "AT000976\n",
      "AT000977\n",
      "AT000978\n",
      "AT000980\n",
      "AT000982\n",
      "AT000983\n",
      "AT000984\n",
      "AT000985\n",
      "AT000986\n",
      "AT000987\n",
      "AT000988\n",
      "AT000990\n",
      "AT000991\n",
      "AT000992\n",
      "AT000995\n",
      "AT000996\n",
      "AT000997\n",
      "AT000998\n",
      "AT001000\n",
      "AT001002\n",
      "AT001004\n",
      "AT001005\n",
      "AT001006\n",
      "AT001007\n",
      "AT001008\n",
      "AT001009\n",
      "AT001010\n",
      "AT001012\n",
      "AT001014\n",
      "AT001015\n",
      "AT001016\n",
      "AT001017\n",
      "AT001018\n",
      "AT001020\n",
      "AT001021\n",
      "AT001022\n",
      "AT001023\n",
      "AT001024\n",
      "AT001025\n",
      "AT001027\n",
      "AT001028\n",
      "AT001029\n",
      "AT001031\n",
      "AT001032\n",
      "AT001033\n",
      "AT001034\n",
      "AT001035\n",
      "AT001036\n",
      "AT001037\n",
      "AT001038\n",
      "AT001039\n",
      "AT001040\n",
      "AT001043\n",
      "AT001044\n",
      "AT001045\n",
      "AT001046\n",
      "AT001048\n",
      "AT001049\n",
      "AT001050\n",
      "AT001051\n",
      "AT001052\n",
      "AT001054\n",
      "AT001055\n",
      "AT001056\n",
      "AT001057\n",
      "AT001058\n",
      "AT001059\n",
      "AT001060\n",
      "AT001061\n",
      "AT001062\n",
      "AT001063\n",
      "AT001064\n",
      "AT001066\n",
      "AT001067\n",
      "AT001068\n",
      "AT001070\n",
      "AT001071\n",
      "AT001074\n",
      "AT001075\n",
      "AT001076\n",
      "AT001077\n",
      "AT001078\n",
      "AT001080\n",
      "AT001082\n",
      "AT001086\n",
      "AT001087\n",
      "AT001088\n",
      "AT001091\n",
      "AT001093\n",
      "AT001094\n",
      "AT001095\n",
      "AT001096\n",
      "AT001097\n",
      "AT001098\n",
      "AT001100\n",
      "AT001102\n",
      "AT001103\n",
      "AT001104\n",
      "AT001105\n",
      "AT001106\n",
      "AT001107\n",
      "AT001109\n",
      "AT001110\n",
      "AT001112\n",
      "AT001113\n",
      "AT001115\n",
      "AT001116\n",
      "AT001117\n",
      "AT001118\n",
      "AT001119\n",
      "AT001120\n",
      "AT001121\n",
      "AT001122\n",
      "AT001123\n",
      "AT001124\n",
      "AT001125\n",
      "AT001126\n",
      "AT001127\n",
      "AT001128\n",
      "AT001129\n",
      "AT001130\n",
      "AT001131\n",
      "AT001132\n",
      "AT001134\n",
      "AT001135\n",
      "AT001136\n",
      "AT001137\n",
      "AT001138\n",
      "AT001139\n",
      "AT001140\n",
      "AT001141\n",
      "AT001143\n",
      "AT001146\n",
      "AT001147\n",
      "AT001148\n",
      "AT001150\n",
      "AT001152\n",
      "AT001153\n",
      "AT001154\n",
      "AT001155\n",
      "AT001156\n",
      "AT001157\n",
      "AT001159\n",
      "AT001161\n",
      "AT001162\n",
      "AT001163\n",
      "AT001166\n",
      "AT001167\n",
      "AT001168\n",
      "AT001169\n",
      "AT001171\n",
      "AT001172\n",
      "AT001173\n",
      "AT001175\n",
      "AT001177\n",
      "AT001178\n",
      "AT001179\n",
      "AT001251\n",
      "AT001180\n",
      "AT001181\n",
      "AT001184\n",
      "AT001185\n",
      "AT001186\n",
      "AT001188\n",
      "AT001189\n",
      "AT001191\n",
      "AT001193\n",
      "AT001194\n",
      "AT001195\n",
      "AT001196\n",
      "AT001197\n",
      "AT001198\n",
      "AT001200\n",
      "AT001201\n",
      "AT001202\n",
      "AT001204\n",
      "AT001206\n",
      "AT001210\n",
      "AT001211\n",
      "AT001212\n",
      "AT001214\n",
      "AT001215\n",
      "AT001216\n",
      "AT001217\n",
      "AT001218\n",
      "AT001221\n",
      "AT001222\n",
      "AT001223\n",
      "AT001228\n",
      "AT001229\n",
      "AT001230\n",
      "AT001231\n",
      "AT001232\n",
      "AT001234\n",
      "AT001235\n",
      "AT001236\n",
      "AT001237\n",
      "AT001238\n",
      "AT001239\n",
      "AT001240\n",
      "AT001241\n",
      "AT001242\n",
      "AT001243\n",
      "AT001244\n",
      "AT001245\n",
      "AT001246\n",
      "AT001248\n",
      "AT001249\n",
      "AT001263\n",
      "AT001266\n",
      "AT001270\n",
      "AT001271\n",
      "AT001272\n",
      "AT001274\n",
      "AT001275\n",
      "AT001276\n",
      "AT001291\n",
      "AT001292\n",
      "AT001285\n",
      "AT001305\n",
      "AT001307\n",
      "AT001295\n",
      "AT001296\n",
      "AT001308\n",
      "AT001298\n",
      "AT001288\n",
      "AT001277\n",
      "AT001278\n",
      "AT001279\n",
      "AT001300\n",
      "AT001282\n",
      "AT001289\n",
      "AT001284\n",
      "AT001309\n",
      "AT001302\n",
      "AT001303\n",
      "AT001304\n",
      "AT001310\n",
      "AT001311\n",
      "AT001312\n",
      "AT001313\n",
      "AT001314\n",
      "AT001316\n",
      "AT001317\n",
      "AT001318\n",
      "AT001320\n",
      "AT001321\n",
      "AT001322\n",
      "AT001323\n",
      "AT001324\n",
      "AT001325\n",
      "AT001326\n",
      "AT001327\n",
      "AT001329\n",
      "AT001330\n",
      "AT001331\n",
      "AT001341\n",
      "AT001342\n",
      "AT001334\n",
      "AT001335\n",
      "AT001336\n",
      "AT001337\n",
      "AT001338\n",
      "AT001343\n",
      "AT001344\n",
      "AT001345\n",
      "AT001346\n",
      "AT001347\n",
      "AT001348\n",
      "AT001349\n",
      "AT001360\n",
      "AT001361\n",
      "AT001362\n",
      "AT001363\n",
      "AT001364\n",
      "AT001365\n",
      "AT001374\n",
      "AT001367\n",
      "AT001368\n",
      "AT001369\n",
      "AT001370\n",
      "AT001371\n",
      "AT001373\n",
      "AT001375\n",
      "AT001376\n",
      "AT001377\n",
      "AT001378\n",
      "AT001379\n",
      "AT001380\n",
      "AT001383\n",
      "AT001385\n",
      "AT001388\n",
      "AT001389\n",
      "AT001391\n",
      "AT001395\n",
      "AT001396\n",
      "AT001397\n"
     ]
    }
   ],
   "source": [
    "gas_dict_2 = {}\n",
    "\n",
    "for key, value in gas_dict.items():\n",
    "    gas_dict_2[key] = {}\n",
    "    # Recordings before and after AT001263 have different formats and they need to be processed differently \n",
    "    if int(key[2:].lstrip('0')) < 1263:\n",
    "        print(key)\n",
    "        for i, gas in enumerate(value.split('Astrup')[1:]):\n",
    "            gas_dict_2[key][i] = {}\n",
    "            items = gas.split('\\n')[1:-1]\n",
    "            for item in items:\n",
    "                name, value = item.split(':')\n",
    "                if value.strip() == '':\n",
    "                    break\n",
    "                else:\n",
    "                    gas_dict_2[key][i][name.strip()] = value.strip()\n",
    "    else:\n",
    "        print(key)\n",
    "        for i, gas in enumerate(value.split('Labor')[1:]):\n",
    "            gas_dict_2[key][i] = {}\n",
    "            items = gas.split('\\n')[1:-1]\n",
    "            for item in items:\n",
    "                if item.startwith('Time'):\n",
    "                    name = item\n",
    "                name, value = item.split(':')[:2]\n",
    "                if value.strip() == '':\n",
    "                    break\n",
    "                else:\n",
    "                    gas_dict_2[key][i][name.strip()] = value.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_dict_2 = {}\n",
    "\n",
    "for key, value in gas_dict.items():\n",
    "    gas_dict_2[key] = {}\n",
    "    # Recordings before and after AT001263 have different formats and they need to be processed differently \n",
    "    if int(key[2:].lstrip('0')) < 1263:\n",
    "        #print(key)\n",
    "        for i, gas in enumerate(value.split('Astrup')[1:]):\n",
    "            gas_dict_2[key][i] = {}\n",
    "            items = gas.split('\\n')[1:-1]\n",
    "            for item in items:\n",
    "                name, value = item.split(':')\n",
    "                if value.strip() == '':\n",
    "                    break\n",
    "                else:\n",
    "                    gas_dict_2[key][i][name.strip()] = value.strip()\n",
    "    else:\n",
    "        #print(key)\n",
    "        for i, gas in enumerate(value.split('Labor')[1:]):\n",
    "            gas_dict_2[key][i] = {}\n",
    "            items = gas.split('\\n')[1:-1]\n",
    "            for item in items:\n",
    "                name, value = item.split(':', maxsplit=1)\n",
    "                if value.strip() == '':\n",
    "                    break\n",
    "                else:\n",
    "                    gas_dict_2[key][i][name.strip()] = value.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {'Time': '0702',\n",
       "  'pH': '7.331',\n",
       "  'pCO2': '46.4',\n",
       "  'pO2': '64.0',\n",
       "  'HCO3': '24.0',\n",
       "  'ABE': '-2.3',\n",
       "  'Saturatio': '90.5',\n",
       "  'FiO2': '0.30',\n",
       "  'Type': 'Capillaris'},\n",
       " 1: {},\n",
       " 2: {}}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gas_dict_2['AT001251']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {'Time': '2024-02-17 14:32',\n",
       "  'Sample site': 'KapillÃ¡ris',\n",
       "  'Hypothermia': 'nem',\n",
       "  'VÃ©rcukor': '7.8 mmol/l',\n",
       "  'pH': '7.037',\n",
       "  'pCO2': '85.3 Hgmm',\n",
       "  'pO2': '47.9 Hgmm',\n",
       "  'HCO3': '22.4 mmol/l',\n",
       "  'BE(ecf)': '-10.24 mmol/l',\n",
       "  'Lactat': '1.4 mmol/l'},\n",
       " 1: {'Time': '2024-02-17 15:13',\n",
       "  'Sample site': 'KapillÃ¡ris',\n",
       "  'Hypothermia': 'nem',\n",
       "  'VÃ©rcukor': '8.2 mmol/l',\n",
       "  'pH': '7.117',\n",
       "  'pCO2': '77.2 Hgmm',\n",
       "  'pO2': '33.2 Hgmm',\n",
       "  'HCO3': '24.9 mmol/l',\n",
       "  'BE(ecf)': '-4.5 mmol/l',\n",
       "  'cSO2': '43.5 %',\n",
       "  'Lactat': '3.8 mmol/l'},\n",
       " 2: {'Time': '2024-02-17 16:12',\n",
       "  'Sample site': 'KapillÃ¡ris',\n",
       "  'Hypothermia': 'nem',\n",
       "  'VÃ©rcukor': '8.9 mmol/l',\n",
       "  'pH': '7.065',\n",
       "  'pCO2': '74.3 Hgmm',\n",
       "  'pO2': '41.5 Hgmm',\n",
       "  'HCO3': '16 mmol/l',\n",
       "  'BE(ecf)': '-9 mmol/l',\n",
       "  'cSO2': '84.4 %',\n",
       "  'Lactat': '3.9 mmol/l'}}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gas_dict_2['AT001263']"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "gas_dict_2 = {}\n",
    "\n",
    "for key, value in gas_dict.items():\n",
    "    gas_dict_2[key] = {}\n",
    "    \n",
    "    for i, gas in enumerate(value.split('Astrup')[1:]):\n",
    "        gas_dict_2[key][i] = {}\n",
    "        items = gas.split('\\n')[1:-1]\n",
    "        for item in items:\n",
    "            name, value = item.split(':')\n",
    "            if value.strip() == '':\n",
    "                break\n",
    "            else:\n",
    "                gas_dict_2[key][i][name.strip()] = value.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "for case in gas_dict_2:\n",
    "    for gas in sorted(gas_dict_2[case].keys()):\n",
    "        if gas_dict_2[case][gas] == {}:\n",
    "            del gas_dict_2[case][gas]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames = {}\n",
    "for case in gas_dict_2.keys():\n",
    "    gas_frames[case] = DataFrame(gas_dict_2[case])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_changer(rec):\n",
    "    a = clin_df.loc[rec]['Recording start'].date()\n",
    "    for column in gas_frames[rec]:\n",
    "        b = gas_frames[rec][column]['Time']\n",
    "        c = datetime.strptime(str(b), '%H%M').time()\n",
    "        # This str() is needed here because AL000665 (and only that) is interpreted as Datetime\n",
    "        d = datetime.combine(a, c)\n",
    "        gas_frames[rec][column]['Time'] = d  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "unconverted data remains: -02-17 14:32",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[45], line 4\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m case \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28msorted\u001b[39m(gas_frames\u001b[38;5;241m.\u001b[39mkeys()):\n\u001b[1;32m      2\u001b[0m     \u001b[38;5;66;03m# Recordings after AT001263 do not have blood gas in the text file\u001b[39;00m\n\u001b[1;32m      3\u001b[0m     \u001b[38;5;66;03m#print(case)\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m     time_changer(case)\n",
      "Cell \u001b[0;32mIn[44], line 5\u001b[0m, in \u001b[0;36mtime_changer\u001b[0;34m(rec)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m column \u001b[38;5;129;01min\u001b[39;00m gas_frames[rec]:\n\u001b[1;32m      4\u001b[0m     b \u001b[38;5;241m=\u001b[39m gas_frames[rec][column][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTime\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[0;32m----> 5\u001b[0m     c \u001b[38;5;241m=\u001b[39m datetime\u001b[38;5;241m.\u001b[39mstrptime(\u001b[38;5;28mstr\u001b[39m(b), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mH\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mM\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mtime()\n\u001b[1;32m      6\u001b[0m     \u001b[38;5;66;03m# This str() is needed here because AL000665 (and only that) is interpreted as Datetime\u001b[39;00m\n\u001b[1;32m      7\u001b[0m     d \u001b[38;5;241m=\u001b[39m datetime\u001b[38;5;241m.\u001b[39mcombine(a, c)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.11/_strptime.py:568\u001b[0m, in \u001b[0;36m_strptime_datetime\u001b[0;34m(cls, data_string, format)\u001b[0m\n\u001b[1;32m    565\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_strptime_datetime\u001b[39m(\u001b[38;5;28mcls\u001b[39m, data_string, \u001b[38;5;28mformat\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%a\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mb \u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mH:\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mM:\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mS \u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mY\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m    566\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Return a class cls instance based on the input string and the\u001b[39;00m\n\u001b[1;32m    567\u001b[0m \u001b[38;5;124;03m    format string.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 568\u001b[0m     tt, fraction, gmtoff_fraction \u001b[38;5;241m=\u001b[39m _strptime(data_string, \u001b[38;5;28mformat\u001b[39m)\n\u001b[1;32m    569\u001b[0m     tzname, gmtoff \u001b[38;5;241m=\u001b[39m tt[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m:]\n\u001b[1;32m    570\u001b[0m     args \u001b[38;5;241m=\u001b[39m tt[:\u001b[38;5;241m6\u001b[39m] \u001b[38;5;241m+\u001b[39m (fraction,)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.11/_strptime.py:352\u001b[0m, in \u001b[0;36m_strptime\u001b[0;34m(data_string, format)\u001b[0m\n\u001b[1;32m    349\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtime data \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m does not match format \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m    350\u001b[0m                      (data_string, \u001b[38;5;28mformat\u001b[39m))\n\u001b[1;32m    351\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(data_string) \u001b[38;5;241m!=\u001b[39m found\u001b[38;5;241m.\u001b[39mend():\n\u001b[0;32m--> 352\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124munconverted data remains: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m    353\u001b[0m                       data_string[found\u001b[38;5;241m.\u001b[39mend():])\n\u001b[1;32m    355\u001b[0m iso_year \u001b[38;5;241m=\u001b[39m year \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    356\u001b[0m month \u001b[38;5;241m=\u001b[39m day \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n",
      "\u001b[0;31mValueError\u001b[0m: unconverted data remains: -02-17 14:32"
     ]
    }
   ],
   "source": [
    "for case in sorted(gas_frames.keys()):\n",
    "    # Recordings after AT001263 do not have blood gas in the text file\n",
    "    #print(case)\n",
    "    time_changer(case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Time</th>\n",
       "      <td>2023-01-09 02:35:00</td>\n",
       "      <td>2023-01-09 03:14:00</td>\n",
       "      <td>2023-01-09 05:44:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pH</th>\n",
       "      <td>6.867</td>\n",
       "      <td>7.023</td>\n",
       "      <td>7.287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pCO2</th>\n",
       "      <td>98.2</td>\n",
       "      <td>82.8</td>\n",
       "      <td>60.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pO2</th>\n",
       "      <td>37.6</td>\n",
       "      <td>45.1</td>\n",
       "      <td>36.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HCO3</th>\n",
       "      <td>17.8</td>\n",
       "      <td>21.5</td>\n",
       "      <td>28.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABE</th>\n",
       "      <td>-15.6</td>\n",
       "      <td>-9.4</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Saturatio</th>\n",
       "      <td>35.5</td>\n",
       "      <td>57.5</td>\n",
       "      <td>83.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FiO2</th>\n",
       "      <td>0.60</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Type</th>\n",
       "      <td>Capillaris</td>\n",
       "      <td>Capillaris</td>\n",
       "      <td>Capillaris</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             0                    1                    2\n",
       "Time       2023-01-09 02:35:00  2023-01-09 03:14:00  2023-01-09 05:44:00\n",
       "pH                       6.867                7.023                7.287\n",
       "pCO2                      98.2                 82.8                 60.8\n",
       "pO2                       37.6                 45.1                 36.3\n",
       "HCO3                      17.8                 21.5                 28.4\n",
       "ABE                      -15.6                 -9.4                  0.4\n",
       "Saturatio                 35.5                 57.5                 83.1\n",
       "FiO2                      0.60                 0.90                 0.38\n",
       "Type                Capillaris           Capillaris           Capillaris"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gas_frames['AT000935']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gas_frames['AT001285']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for case in sorted(gas_frames.keys()):\n",
    "    try:\n",
    "        gas_frames[case] =  gas_frames[case].T.set_index('Time')\n",
    "    \n",
    "    except:\n",
    "        print('No blood gas for %s' % case)\n",
    "        del gas_frames[case]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(gas_frames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Quality control of blood gases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Combine all gases to a single DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all = pd.concat(gas_frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in ['pH', 'pCO2', 'pO2', 'HCO3', 'ABE', 'Saturatio', 'FiO2']:\n",
    "    gas_frames_all[column] = gas_frames_all[column].astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all[['pH', 'pCO2', 'pO2', 'HCO3', 'ABE']].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Manually review outlier with impossible values in gases and correct as appropriate\n",
    "\n",
    "Only correct trivial ones. \n",
    "\n",
    "- For example, \"7\" is sometimes mis-recognised as \"2\" by OCR. \n",
    "- Other times, the decimal point is clearly in the wrong place.\n",
    "\n",
    "Only correct those ones where the other values in the blood gas are consistent with the change you are making. Otherwise, remove the clearly impossible values:\n",
    "\n",
    "- pH < 6 or pH > 7.7\n",
    "- pCO2 < 10 mmHg or > 200 mmHg\n",
    "- ABE < - 40 or > 50 \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all[gas_frames_all['pH'] < 6].sort_values('pH', ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all[gas_frames_all['pH'] > 7.7].sort_values('pH', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all[gas_frames_all['pCO2'] > 200].sort_values('pCO2', ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all[gas_frames_all['pCO2'] < 10].sort_values('pCO2', ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all[gas_frames_all['ABE'] < - 40].sort_values('ABE', ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_frames_all[gas_frames_all['ABE'] > 50].sort_values('ABE', ascending=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Export bood gases as Excel files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save blood gases into a multi-sheet Excel file\n",
    "\n",
    "writer = pd.ExcelWriter(os.path.join(DIR_WRITE, 'blood_gases_new.xlsx'))\n",
    "for case in sorted(gas_frames.keys()):\n",
    "    gas_frames[case].to_excel(writer, case)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "### 7. Export processed data as pickle files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_DUMP, 'blood_gases_new.pickle'), 'wb') as handle:\n",
    "    pickle.dump(gas_frames, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}