ventilator_data_new_451_600.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![alt text](./Cerny_logo_1.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis of Cerny ventilation recordings: `AT000451 - AT000600`\n",
    "\n",
    "The data processed and analysed in this Notebook were collected by the **Neonatal Emergency and Transport Service of the Peter Cerny Foundation**, Budapest, Hungary\n",
    "\n",
    "**Author: Dr Gusztav Belteki**\n",
    "\n",
    "____\n",
    "\n",
    "This notebook imports all ventilator data of these recordings (including ventilator parameters, settings, alarms (0.5Hz sampling rate) and waveform data (150Hz sampling rate).\n",
    "\n",
    "- Total: **150 cases**\n",
    "-  14 cases were removed as they were less than 15 minutes long (empty or partial recordings) \n",
    "- **136 cases** remaining\n",
    "\n",
    "A dictionary containing the processed ventilation data exported as pickle archive: **data_pars_451_600.pickle** "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the required libraries and set options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import pickle\n",
    "import datetime\n",
    "\n",
    "from pandas import Series, DataFrame\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "matplotlib.style.use('classic')\n",
    "matplotlib.rcParams['figure.facecolor'] = 'w'\n",
    "\n",
    "pd.set_option('display.max_rows', 300)\n",
    "pd.set_option('display.max_columns', 300)\n",
    "\n",
    "# This is to turn off a warning message which is given when read_Excel() imports '.xlsx' files\n",
    "import warnings\n",
    "warnings.simplefilter(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Python version: {}\".format(sys.version))\n",
    "print(\"pandas version: {}\".format(pd.__version__))\n",
    "print(\"matplotlib version: {}\".format(matplotlib.__version__))\n",
    "print(\"NumPy version: {}\".format(np.__version__))\n",
    "print(\"IPython version: {}\".format(IPython.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. List and set the working directory and the directory to write out data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Name of the external hard drive\n",
    "DRIVE = 'GUSZTI'\n",
    "\n",
    "# Directory on external drive to read the ventilation data from\n",
    "DIR_READ =  os.path.join(os.sep, 'Volumes', DRIVE, 'Fabian_new', 'fabian_ventilator_data_new_451_600')\n",
    "\n",
    "# Path to project folder containing ventilation research results\n",
    "PATH = os.path.join(os.sep, 'Users', 'guszti', 'Library', 'Mobile Documents', 'com~apple~CloudDocs', \n",
    "                            'Documents', 'Research', 'Ventilation')\n",
    "\n",
    "# Folder to export the result of analysis\n",
    "DIR_WRITE = os.path.join(PATH, 'ventilation_fabian_new', 'Analyses')\n",
    "os.makedirs(DIR_WRITE, exist_ok = True)\n",
    "\n",
    "# Folder on a USB stick to export data to\n",
    "DATA_DUMP = os.path.join(os.sep, '/Volumes', DRIVE, 'data_dump', 'fabian_new',)\n",
    "os.makedirs(DATA_DUMP, exist_ok = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR_READ, DIR_WRITE, DATA_DUMP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Import ventilation data as text and create a dictionary of the different recordings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cases = os.listdir(DIR_READ)\n",
    "cases = sorted(case for case in cases if case.startswith('AT')) # remove hidden other files\n",
    "# print(cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# import all data file in the vent_dict dictionary\n",
    "\n",
    "vent_dict = {}\n",
    "for case in cases:\n",
    "    flist = os.listdir(os.path.join(DIR_READ, case))\n",
    "    # print(flist)\n",
    "    for fle in flist:\n",
    "        if not fle.startswith('.') and fle.endswith('txt'):\n",
    "            fle_handle = open(os.path.join(DIR_READ, case, fle), 'r', encoding = 'latin1')\n",
    "            vent_dict['%s_%s' % (case, fle[-5])] = fle_handle.read()\n",
    "            fle_handle.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(vent_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Split recordings into records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "with open(os.path.join(DIR_WRITE, 'error_log_1.txt'), 'a') as fileout:\n",
    "\n",
    "    data_dict = {} # In this dict the keys are time points and the values are vent data\n",
    "    for key, value in sorted(vent_dict.items()):\n",
    "        # print('Working on %s' % key)\n",
    "        data_list = value.split('\\n') # Individual records are separated by a 'newline' character\n",
    "        data_dict[key] = {} # an inner dictionary for the given recording\n",
    "        for number, record in enumerate(data_list[:-1]):\n",
    "            try:\n",
    "                time_stamp, data_str = record.split(';') # splitting record to time stamp and ventilation data\n",
    "                data_dict[key][time_stamp] = data_str       \n",
    "            except:\n",
    "                #print('In %s, record #%d cannot be parsed: \\n %s' % (key, number, record[:75]), '\\n')\n",
    "                print('In %s, record #%d cannot be parsed: \\n %s' % (key, number, record[:75]), '\\n', file=fileout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Combine data dictionaries from the same cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# data_dict_2 contains the combined ventilation data for each case\n",
    "\n",
    "data_dict_2 = {}\n",
    "for case in cases:\n",
    "    dicts_to_combine = []\n",
    "    for recording in vent_dict.keys():\n",
    "        if recording.startswith(case):\n",
    "            dicts_to_combine.append(data_dict[recording])\n",
    "    data_dict_2[case] = {k: v for dct in dicts_to_combine for k, v in dct.items()} "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Separate parameter data and waves data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Records containing parameter data start with '<', records containing waves data always have a space (' ')\n",
    "# after the first byte (two characters in hexadecimal notation)\n",
    "\n",
    "with open(os.path.join(DIR_WRITE, 'error_log_2.txt'), 'a') as fileout:\n",
    "\n",
    "    data_dict_waves = {}\n",
    "    data_dict_pars = {}\n",
    "    for case in cases:\n",
    "        data_dict_waves[case] = {}\n",
    "        data_dict_pars[case] = {}\n",
    "        for key, value in sorted(data_dict_2[case].items()):\n",
    "            if value.startswith('<'): # These are ventilator parameter slow data  \n",
    "                data_dict_pars[case][key] = value[13:-12] # removing 13 characters from the \n",
    "                # beginnings and 12 characters from the end (they are not parameters)\n",
    "            elif value[2] == ' ': # Waves data have a space character at the third position\n",
    "                data_dict_waves[case][key] = value\n",
    "            else:\n",
    "                #print('In %s, record at %s cannot be parsed as waves or parameters:' % (case, key ))\n",
    "                #print(value, '\\n')\n",
    "                \n",
    "                print('In %s, record at %s cannot be parsed as waves or parameters:' % (case, key ), \n",
    "                     file = fileout)\n",
    "                print(value, '\\n', file = fileout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Create nested dictionary for the various parameters and their values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "with open(os.path.join(DIR_WRITE, 'error_log_3.txt'), 'a') as fileout:\n",
    "\n",
    "    data_dict_pars_2 = {}\n",
    "    for case in cases:\n",
    "        #print('Working on %s' % case)\n",
    "        data_dict_pars_2[case] = {}\n",
    "        for time, values in data_dict_pars[case].items():\n",
    "            time = datetime.strptime(time[:-4], '%m/%d/%Y %I:%M:%S %p')\n",
    "            data_dict_pars_2[case][time] = {} # inner dictionary with the time stamps used as keys\n",
    "            for pair in values.split('|'):\n",
    "                if ',' in pair: # ventilator software version field contains a comma\n",
    "                    continue\n",
    "                \n",
    "                try:\n",
    "                    code, value = pair.split('=') # split records into parameter keys and values\n",
    "                except:\n",
    "                    print('In case %s this record at cannot be unpacked:\\n %s, %s\\n' \n",
    "                                  % (case, time, pair), file = fileout)\n",
    "                    continue\n",
    "                \n",
    "                if code.startswith('0'): # The codes <10 start with zeros (e.g. 00, 01, 02,...)\n",
    "                                     # and the leading zeros need to be removed\n",
    "                    code = code[1:]  \n",
    "                \n",
    "                try:\n",
    "                    parameter = int(code)\n",
    "                except ValueError:\n",
    "                    print('In case %s at %s error during coverting value to int:\\n%r\\n' \n",
    "                          % (case, time,code), file = fileout)\n",
    "                    continue\n",
    "            \n",
    "                if code == '145': # Device ID variant, hexadecimal number\n",
    "                    data_dict_pars_2[case][time][parameter] = value\n",
    "                    \n",
    "                elif code == '125': \n",
    "                    # convert Mode options 1 to binary number to retrieve bits\n",
    "                    data_dict_pars_2[case][time][parameter] = bin(int(value))[2:].zfill(16)\n",
    "                    \n",
    "                elif code == '126': \n",
    "                    # convert Mode options 2 to binary number to retrieve bits\n",
    "                    data_dict_pars_2[case][time][parameter] = bin(int(value))[2:].zfill(16)\n",
    "            \n",
    "                elif '.' in value or value == '0':\n",
    "                \n",
    "                    try:\n",
    "                        data_dict_pars_2[case][time][parameter] = float(value)\n",
    "                    except ValueError:\n",
    "                        #print('Value cannot be converted to float\\n:%r\\n' % value)\n",
    "                        print('In case %s at %s, value cannot be converted to float\\n:%r\\n' \n",
    "                              % (case, time, value), file = fileout)\n",
    "                        continue\n",
    "                else: \n",
    "                    try:\n",
    "                        data_dict_pars_2[case][time][parameter] = int(value)\n",
    "                    except:\n",
    "                        #print('Value cannot be converted to int\\n:%r\\n' % value)\n",
    "                        print('In case %s at %s, value cannot be converted to int\\n:%r\\n' \n",
    "                              % (case, time, value), file = fileout)\n",
    "                        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Parameter #125 is 'Mode option 1': its different bits are meaning different parameters\n",
    "# Parameter #126 is 'Mode option 2': its different bits are meaning different parameters\n",
    "# See Aculink protocol for more details\n",
    "\n",
    "for case in cases:\n",
    "    #print('Working on %s' % case)\n",
    "    for time in sorted(data_dict_pars_2[case].keys()):\n",
    "        try:\n",
    "            # Ventilation_stopped; 0 = no, 1 = yes\n",
    "            data_dict_pars_2[case][time][276] = int(data_dict_pars_2[case][time][125][-1])\n",
    "            \n",
    "            # VG_state: 0 = off, 1 = on\n",
    "            data_dict_pars_2[case][time][277] = int(data_dict_pars_2[case][time][125][-2])\n",
    "            \n",
    "            # Volume limit state: 0 = off, 1 = on\n",
    "            data_dict_pars_2[case][time][278] = int(data_dict_pars_2[case][time][125][-3])\n",
    "            \n",
    "            # Ventilator_range: 0  = neonatal, 1 = paediatric\n",
    "            data_dict_pars_2[case][time][279] = int(data_dict_pars_2[case][time][125][-4])\n",
    "            \n",
    "            # trigger_mode: 0 = volumetrigger, 1 = flowtrigger\n",
    "            data_dict_pars_2[case][time][280] = int(data_dict_pars_2[case][time][125][-8])\n",
    "    \n",
    "            # I_E_HFOV (HFOV I:E ratio): 0=1:3, 1=1:2, 2=1:1\n",
    "            if data_dict_pars_2[case][time][125][-14:-12] == '00':\n",
    "                data_dict_pars_2[case][time][281] = 0\n",
    "            elif data_dict_pars_2[case][time][125][-14:-12] == '01':\n",
    "                data_dict_pars_2[case][time][281] = 1\n",
    "            elif data_dict_pars_2[case][time][125][-14:-12] == '10':\n",
    "                data_dict_pars_2[case][time][281] = 2\n",
    "    \n",
    "            # pressure_rise_control: 0=I-flow, 1=Ramp, 2=AutoIFlow\n",
    "            if data_dict_pars_2[case][time][126][-2:]   == '00':\n",
    "                data_dict_pars_2[case][time][282] = 0\n",
    "            elif data_dict_pars_2[case][time][126][-2:] == '01':\n",
    "                data_dict_pars_2[case][time][282] = 1\n",
    "            elif data_dict_pars_2[case][time][126][-2:] == '10':\n",
    "                data_dict_pars_2[case][time][282] = 2\n",
    "    \n",
    "            # HFOV recruitment: 0 = off, 1 = on\n",
    "            data_dict_pars_2[case][time][283] = int(data_dict_pars_2[case][time][126][-3])\n",
    "            \n",
    "            # HFOV flow: 0 = off, 1 = on\n",
    "            data_dict_pars_2[case][time][284] = int(data_dict_pars_2[case][time][126][-4])\n",
    "        \n",
    "            # Bias flow: 0 = off, 1 = on\n",
    "            data_dict_pars_2[case][time][285] = int(data_dict_pars_2[case][time][126][-6])\n",
    "            \n",
    "            # Trigger mode 2: 0=Volumetrigger, 1=Flowtrigger, 2=Pressuretrigger\n",
    "            if data_dict_pars_2[case][time][126][-8:-6]   == '00':\n",
    "                data_dict_pars_2[case][time][286] = 0\n",
    "            elif data_dict_pars_2[case][time][126][-8:-6] == '01':\n",
    "                data_dict_pars_2[case][time][286] = 1\n",
    "            elif data_dict_pars_2[case][time][126][-8:-6] == '10':\n",
    "                data_dict_pars_2[case][time][286] = 2\n",
    "                \n",
    "            # FOT oscillation running: 0 = no, 1 = yes\n",
    "            data_dict_pars_2[case][time][287] = int(data_dict_pars_2[case][time][126][-10])\n",
    "            \n",
    "            # Leak compensation: 0=off, 1=Low, 2=Middle, 3=High\n",
    "            if data_dict_pars_2[case][time][126][-12:-10]   == '00':\n",
    "                data_dict_pars_2[case][time][288] = 0\n",
    "            elif data_dict_pars_2[case][time][126][-12:-10] == '01':\n",
    "                data_dict_pars_2[case][time][288] = 1\n",
    "            elif data_dict_pars_2[case][time][126][-12:-10] == '10':\n",
    "                data_dict_pars_2[case][time][288] = 2\n",
    "            elif data_dict_pars_2[case][time][126][-12:-10] == '11':\n",
    "                data_dict_pars_2[case][time][288] = 3\n",
    "        \n",
    "        except:\n",
    "            print('Error in %s, %s' % (case, time))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Create DataFrame from Parameters Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "data_pars = {}\n",
    "for case in cases:\n",
    "    data_pars[case] = DataFrame(data_dict_pars_2[case]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Replace codes for text (see Aculink protocol)\n",
    "\n",
    "for case in cases:\n",
    "    a = data_pars[case].copy()\n",
    "    a= a.replace(-32764, 'off')\n",
    "    a= a.replace(-32765, 'not valid')\n",
    "    a= a.replace(-32766, 'out of range')\n",
    "    a= a.replace(-32767, 'unused')\n",
    "    data_pars[case] = a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recording_duration = []\n",
    "\n",
    "for case in cases:\n",
    "    # The sampling rate is 0.5 Hz (1 in 2 seconds), e.g. if the lentgh of the DataFrame is 450, its duration\n",
    "    # is 15 minutes (900 seconds)\n",
    "    recording_duration.append((case, 2 * len(data_pars[case])))  \n",
    "\n",
    "recording_duration = DataFrame(recording_duration)\n",
    "recording_duration.columns = ['case', 'seconds']\n",
    "recording_duration.index = recording_duration['case']\n",
    "recording_duration.drop('case', axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recording_duration['seconds'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recording_duration.sort_values(by = 'seconds', ascending = True)[:25]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. Remove those recordings which are less than 15 minutes long\n",
    "\n",
    "Recordings less than 15 minutes (900 seconds) long are very likely incomplete and sometimes completely empty."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_pars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The sampling rate is 0.5 Hz (1 in 2 seocnds), if the lentgh of the DataFrame is 450, its duration\n",
    "# is 15 minutes (900 seconds)\n",
    "\n",
    "for case in cases:\n",
    "    if len(data_pars[case]) < 450:\n",
    "        print('Removing %s' % case)\n",
    "        del data_pars[case]\n",
    "\n",
    "cases = sorted(data_pars.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_pars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10. Replace codes for categorical variables with informative category names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_patient_range = {1: 'Neonatal', 2: 'Pediatric'}\n",
    "for case in cases:\n",
    "    data_pars[case][100].replace(mapping_patient_range, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_vent_mode = {0: None, 1: 'IPPV', 2: 'SIPPV', 3: 'SIMV', 4: 'SIMVPSV', 5: 'PSV', \n",
    "                     6: 'CPAP', 7: 'NCPAP', 8: 'DUOPAP', 9: 'HFO', 10: 'O2therapy', 15: 'service'}\n",
    "for case in cases:\n",
    "    data_pars[case][101].replace(mapping_vent_mode, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_power = {0: 'Network', 1: 'Battery'}\n",
    "for case in cases:\n",
    "    data_pars[case][127].replace(mapping_power, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_off_on = {0: 'off', 1: 'on'}\n",
    "for case in cases:\n",
    "    data_pars[case][157].replace(mapping_off_on, inplace = True)\n",
    "    data_pars[case][158].replace(mapping_off_on, inplace = True)\n",
    "    data_pars[case][277].replace(mapping_off_on, inplace = True)\n",
    "    data_pars[case][278].replace(mapping_off_on, inplace = True)\n",
    "    data_pars[case][283].replace(mapping_off_on, inplace = True)\n",
    "    data_pars[case][284].replace(mapping_off_on, inplace = True)\n",
    "    data_pars[case][285].replace(mapping_off_on, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_no_yes = {0: 'no', 1: 'yes'}\n",
    "for case in cases:\n",
    "    data_pars[case][276].replace(mapping_no_yes, inplace = True)\n",
    "    data_pars[case][287].replace(mapping_no_yes, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_trigger = {0: 'Volumetrigger', 1: 'Flowtrigger'}\n",
    "for case in cases:\n",
    "    data_pars[case][280].replace(mapping_trigger, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_IE_HFOV = {0: '1:3', 1: '1:2', 2: '1:1'}\n",
    "for case in cases:\n",
    "    data_pars[case][281].replace(mapping_IE_HFOV, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_pressure_rise_ctrl = {0: 'I-flow', 1: 'Ramp', 2: 'AutoIFlow'}\n",
    "for case in cases:\n",
    "    data_pars[case][282].replace(mapping_pressure_rise_ctrl, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_pressure_unit = {0: 'mbar', 1: 'cmH2O',}\n",
    "for case in cases:\n",
    "    data_pars[case][140].replace(mapping_pressure_unit, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_CO2 = {0: 'mmHg', 1: 'kPa', 2: 'Vol%'}\n",
    "for case in cases:\n",
    "    data_pars[case][141].replace(mapping_CO2, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_trigger_2 = {0: 'Volumetrigger', 1: 'Flowtrigger', 2: 'Pressuretrigger'}\n",
    "for case in cases:\n",
    "    data_pars[case][286].replace(mapping_trigger_2, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_leak_comp = {0: 'off', 1: 'low', 2: 'middle', 3: 'high'}\n",
    "for case in cases:\n",
    "    data_pars[case][288].replace(mapping_leak_comp, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 11. Parse the parameter values using Fabian parameter library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "par_key_table = pd.read_excel(os.path.join(PATH, 'ventilation_fabian_new','Fabian_parameters_new.xlsx'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "par_key_table;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "par_key_dict = {}\n",
    "for row in par_key_table.index:\n",
    "    par_key_dict[par_key_table.code[row]] = par_key_table.name[row]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for case in cases:\n",
    "    data_pars[case].rename(columns = par_key_dict, inplace = True )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 12. Sort DataFrames according to time stamp index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for case in cases:\n",
    "    data_pars[case].sort_index(inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 13. Write individual text files with the ventilator modes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create sub-directories for each case if it does not yet exist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Images and raw data will be written on an external hard drive\n",
    "os.makedirs(os.path.join(DATA_DUMP, 'fabian_cases_new'), exist_ok = True)\n",
    "\n",
    "for case in cases: \n",
    "    os.makedirs(os.path.join(DATA_DUMP, 'fabian_cases_new', case), exist_ok = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for case in cases:\n",
    "    a = data_pars[case]\n",
    "    \n",
    "    o2therapy = len(a[a['Ventilator_mode'] == 'O2therapy'])\n",
    "    ncpap = len(a[a['Ventilator_mode'] == 'NCPAP'])\n",
    "    duopap = len(a[a['Ventilator_mode'] == 'DUOPAP'])\n",
    "    simv = len(a[a['Ventilator_mode'] == 'SIMV'])\n",
    "    ippv = len(a[a['Ventilator_mode'] == 'IPPV'])\n",
    "    sippv = len(a[a['Ventilator_mode'] == 'SIPPV'])\n",
    "    simvpsv = len(a[a['Ventilator_mode'] == 'SIMVPSV'])\n",
    "    vg_on = len(data_pars[case][data_pars[case]['VG_state'] == 'on'])\n",
    "  \n",
    "    \n",
    "    fileout = open(os.path.join(DATA_DUMP, 'fabian_cases_new', case, \n",
    "                                f'{case}_vent_info.txt'), 'w')\n",
    "    \n",
    "    fileout.write('O2 therapy: %d sec \\n' % (o2therapy * 2))\n",
    "    fileout.write('NCPAP:      %d sec \\n' % (ncpap * 2))\n",
    "    fileout.write('DUOPAP:     %d sec \\n' % (duopap * 2))\n",
    "    fileout.write('IPPV:       %d sec \\n' % (ippv * 2))\n",
    "    fileout.write('SIPPV:      %d sec \\n' % (sippv * 2))\n",
    "    fileout.write('SIMV:       %d sec \\n' % (simv * 2))\n",
    "    fileout.write('SIMVPSV:    %d sec \\n\\n' % (simvpsv * 2))\n",
    "    fileout.write('VG on:      %d sec \\n' % (vg_on * 2))\n",
    "    \n",
    "    fileout.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 14. Export processed data as pickle files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_DUMP, 'data_pars_451_600.pickle'), 'wb') as handle:\n",
    "    pickle.dump(data_pars, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_DUMP, 'cases_451_600.pickle'), 'wb') as handle:\n",
    "    pickle.dump(cases, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}