ventilator_data_301_450.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![alt text](./Cerny_logo_1.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis of Cerny ventilation recordings: `AL000301 - AL000450`\n",
    "\n",
    "## Importing and parsing raw ventilator data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Author: Dr Gusztav Belteki**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing the necessary libraries and setting options"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook preprocessed and analyses the ventilator parameter data obtained with **0.5Hz sampling rate** from the Fabian ventilators at the Cerny neonatal transport service. It exports desrciptive statistics into Excel files and the preprocessed data as pickle archive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import sklearn as sk\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import re\n",
    "import pickle\n",
    "import datetime\n",
    "\n",
    "from scipy import stats\n",
    "from pandas import Series, DataFrame\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "matplotlib.style.use('classic')\n",
    "matplotlib.rcParams['figure.facecolor'] = 'w'\n",
    "\n",
    "pd.set_option('display.max_rows', 300)\n",
    "pd.set_option('display.max_columns', 300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Python version: {}\".format(sys.version))\n",
    "print(\"pandas version: {}\".format(pd.__version__))\n",
    "print(\"matplotlib version: {}\".format(matplotlib.__version__))\n",
    "print(\"NumPy version: {}\".format(np.__version__))\n",
    "print(\"SciPy version: {}\".format(sp.__version__))\n",
    "print(\"IPython version: {}\".format(IPython.__version__))\n",
    "print(\"scikit-learn version: {}\".format(sk.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### List and set the working directory and the directory to write out data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Topic of the Notebook which will also be the name of the subfolder containing results\n",
    "TOPIC = 'fabian'\n",
    "\n",
    "# Name of the external hard drive\n",
    "DRIVE = 'GUSZTI'\n",
    "\n",
    "# Directory containing clinical and blood gas data\n",
    "CWD = '/Users/guszti/ventilation_fabian'\n",
    "\n",
    "# Directory on external drive to read the ventilation data from\n",
    "DIR_READ = '/Volumes/%s/Fabian/fabian_ventilator_data_301_450' % DRIVE\n",
    "\n",
    "DIR_WRITE = '%s/%s' % (CWD, 'Analyses')\n",
    "\n",
    "# Images and raw data will be written on an external hard drive\n",
    "if not os.path.isdir('/Volumes/%s/data_dump/%s' % (DRIVE, TOPIC)):\n",
    "    os.makedirs('/Volumes/%s/data_dump/%s' % (DRIVE, TOPIC))\n",
    "DATA_DUMP = '/Volumes/%s/data_dump/%s' % (DRIVE, TOPIC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(CWD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR_READ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR_WRITE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DUMP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import ventilation data as text and create a dictionary of the different recordings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cases = os.listdir(DIR_READ)\n",
    "cases = sorted(case for case in cases if case.startswith('AL')) # remove hidden other files\n",
    "# print(cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# import all data file in the vent_dict dictionary\n",
    "\n",
    "vent_dict = {}\n",
    "for case in cases:\n",
    "    flist = os.listdir(os.path.join(DIR_READ, case))\n",
    "    # print(flist)\n",
    "    for fle in flist:\n",
    "        if not fle.startswith('.'):\n",
    "            fle_handle = open(os.path.join(DIR_READ, case, fle), 'r', encoding = 'latin1')\n",
    "            vent_dict['%s_%s' % (case, fle[-5])] = fle_handle.read()\n",
    "            fle_handle.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(vent_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split recordings into records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "data_dict = {} # In this dict the keys are time points and the values are vent data\n",
    "for key, value in sorted(vent_dict.items()):\n",
    "    # print('Working on %s' % key)\n",
    "    data_list = value.split('\\n') # Individual records are separated by a 'newline' character\n",
    "    data_dict[key] = {} # an inner dictionary for the given recording\n",
    "    for number, record in enumerate(data_list[:-1]):\n",
    "        try:\n",
    "            time_stamp, data_str = record.split(';') # splitting record to time stamp and ventilation data\n",
    "            data_dict[key][time_stamp] = data_str       \n",
    "        except:\n",
    "            print('In %s, record #%d cannot be parsed: \\n %s' % (key, number, record[:75]), '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combine data dictionaries from the same cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# data_dict_2 contains the combined ventilation data for each case\n",
    "\n",
    "data_dict_2 = {}\n",
    "for case in cases:\n",
    "    dicts_to_combine = []\n",
    "    for recording in vent_dict.keys():\n",
    "        if recording.startswith(case):\n",
    "            dicts_to_combine.append(data_dict[recording])\n",
    "    data_dict_2[case] = {k: v for dct in dicts_to_combine for k, v in dct.items()} "
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "scrolled": false
   },
   "source": [
    "# show the cases with start and end time\n",
    "for case in cases:\n",
    "    print('%-20s%-35s%s' % (case, sorted(data_dict_2[case].keys())[0], \n",
    "                              sorted(data_dict_2[case].keys())[-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Separate parameter data and waves data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Records containing parameter data start with '<', records containing waves data always have a space (' ')\n",
    "# after the first byte (two characters in hexadecimal notation)\n",
    "\n",
    "with open('%s/%s' % (DIR_WRITE, 'abnormal_records_301_450_120519.txt'), 'a') as fileout:\n",
    "\n",
    "    data_dict_waves = {}\n",
    "    data_dict_pars = {}\n",
    "    for case in cases:\n",
    "        data_dict_waves[case] = {}\n",
    "        data_dict_pars[case] = {}\n",
    "        for key, value in sorted(data_dict_2[case].items()):\n",
    "            if value.startswith('<'): # These are ventilator parameter slow data  \n",
    "                data_dict_pars[case][key] = value[13:-12] # removing 13 characters from the \n",
    "                # beginnings and 12 characters from the end (they are not parameters)\n",
    "            elif value[2] == ' ': # Waves data have a space character at the third position\n",
    "                data_dict_waves[case][key] = value\n",
    "            else:\n",
    "                print('In case %s, record at %s cannot be parsed as waves or parameters:' % (case, key ), \n",
    "                     file = fileout)\n",
    "                print(value, '\\n', file = fileout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters data (sampling rate: 0.5Hz)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create embedded dictionary for the various parameters and their values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "with open('%s/%s' % (DIR_WRITE, 'abnormal_values_301_450_120519.txt'), 'a') as fileout:\n",
    "\n",
    "    data_dict_pars_2 = {}\n",
    "    for case in cases:\n",
    "        print('Working on %s' % case)\n",
    "        data_dict_pars_2[case] = {}\n",
    "        for time, values in data_dict_pars[case].items():\n",
    "            time = datetime.strptime(time[:-4], '%Y. %m. %d. %H:%M:%S')\n",
    "            data_dict_pars_2[case][time] = {} # inner dictionary with the time stamps used as keys\n",
    "            for pair in values.split('|'):\n",
    "                if ',' in pair: # ventilator software version field contains a comma\n",
    "                    continue\n",
    "                \n",
    "                try:\n",
    "                    code, value = pair.split('=') # split records into parameter keys and values\n",
    "                except:\n",
    "                    print('Record cannot be unpacked:\\n %s, %s\\n' % (time, pair), file = fileout)\n",
    "                    continue\n",
    "                \n",
    "                if code.startswith('0'): # The codes <10 start with zeros (e.g. 00, 01, 02,...)\n",
    "                                     # and the leading zeros need to be removed\n",
    "                    code = code[1:]  \n",
    "                \n",
    "                try:\n",
    "                    parameter = int(code)\n",
    "                except ValueError:\n",
    "                    print('Error during coverting value to int:\\n%r\\n' % code, file = fileout)\n",
    "                    continue\n",
    "            \n",
    "                if code == '145': # Device ID variant, hexadecimal number\n",
    "                    data_dict_pars_2[case][time][parameter] = value\n",
    "                    \n",
    "                elif code in ['125', '126']: \n",
    "                    # convert Mode options 1 & 2 to binary number to retrieve bits\n",
    "                    data_dict_pars_2[case][time][parameter] = bin(int(value))[2:].zfill(14)\n",
    "            \n",
    "                elif '.' in value or value == '0':\n",
    "                \n",
    "                    try:\n",
    "                        data_dict_pars_2[case][time][parameter] = float(value)\n",
    "                    except ValueError:\n",
    "                        print('Value cannot be converted to float\\n:%r\\n' % value, file = fileout)\n",
    "                        continue\n",
    "                else: \n",
    "                    try:\n",
    "                        data_dict_pars_2[case][time][parameter] = int(value)\n",
    "                    except:\n",
    "                        print('Value cannot be converted to int\\n:%r\\n' % value, file = fileout)\n",
    "                        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Parameter #125 is 'Mode option 1': its different bits are meaning different parameters\n",
    "# Parameter #126 is 'Mode option 2': its different bits are meaning different parameters\n",
    "# See Aculink protocol for more details\n",
    "\n",
    "for case in cases:\n",
    "    print('Working on %s' % case)\n",
    "    for time in sorted(data_dict_pars_2[case].keys()):\n",
    "        try:\n",
    "            # Ventilation_stopped; 0 = no, 1 = yes\n",
    "            data_dict_pars_2[case][time][270] = int(data_dict_pars_2[case][time][125][-1])\n",
    "            # VG_state: 0 = off, 1 = on\n",
    "            data_dict_pars_2[case][time][271] = int(data_dict_pars_2[case][time][125][-2])\n",
    "            # Volume limit state: 0 = off, 1 = on\n",
    "            data_dict_pars_2[case][time][272] = int(data_dict_pars_2[case][time][125][-3])\n",
    "            # Ventilator_range: 0  = neonatal, 1 = paediatric\n",
    "            data_dict_pars_2[case][time][273] = int(data_dict_pars_2[case][time][125][-4])\n",
    "            # trigger_mode: 0 = volumetrigger, 1 = flowtrigger\n",
    "            data_dict_pars_2[case][time][274] = int(data_dict_pars_2[case][time][125][-8])\n",
    "    \n",
    "            # I_E_HFOV (HFOV I:E ratio): 0=1:3, 1=1:2, 2=1:1\n",
    "            if data_dict_pars_2[case][time][125][-14:-12] == '00':\n",
    "                data_dict_pars_2[case][time][275] = 0\n",
    "            elif data_dict_pars_2[case][time][125][-14:-12] == '01':\n",
    "                data_dict_pars_2[case][time][275] = 1\n",
    "            elif data_dict_pars_2[case][time][125][-14:-12] == '10':\n",
    "                data_dict_pars_2[case][time][275] = 2\n",
    "    \n",
    "            # pressure_rise_control: 0=I-flow, 1=Ramp, 2=AutoIFlow\n",
    "            if data_dict_pars_2[case][time][126][-2:]   == '00':\n",
    "                data_dict_pars_2[case][time][276] = 0\n",
    "            elif data_dict_pars_2[case][time][126][-2:] == '01':\n",
    "                data_dict_pars_2[case][time][276] = 1\n",
    "            elif data_dict_pars_2[case][time][126][-2:] == '10':\n",
    "                data_dict_pars_2[case][time][276] = 2\n",
    "    \n",
    "            # HFOV recruitment: 0 = off, 1 = on\n",
    "            data_dict_pars_2[case][time][277] = int(data_dict_pars_2[case][time][126][-3])\n",
    "        \n",
    "        except:\n",
    "            print('Error in %s, %s' % (case, time))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create DataFrame from Parameters Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "data_pars = {}\n",
    "for case in cases:\n",
    "    data_pars[case] = DataFrame(data_dict_pars_2[case]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Replace codes for text (see Aculink protocol)\n",
    "\n",
    "for case in cases:\n",
    "    a = data_pars[case].copy()\n",
    "    a = a.replace(-32764, 'off')\n",
    "    a = a.replace(-32765, 'not valid')\n",
    "    a = a.replace(-32766, 'out of range')\n",
    "    a = a.replace(-32767, 'unused')\n",
    "    data_pars[case] = a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recording_duration = []\n",
    "\n",
    "for case in cases:\n",
    "    # The sampling rate is 0.5 Hz (1 in 2 seocnds), e.g. if the lentgh of the DataFrame is 450, its duration\n",
    "    # is 15 minutes (900 seconds)\n",
    "    recording_duration.append((case, 2 * len(data_pars[case])))  \n",
    "\n",
    "recording_duration = DataFrame(recording_duration)\n",
    "recording_duration.columns = ['case', 'seconds']\n",
    "recording_duration.index = recording_duration['case']\n",
    "recording_duration.drop('case', axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recording_duration['seconds'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "recording_duration.sort_values(by = 'seconds', ascending = True)[:25]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove those recordings which are less than 15 minutes long\n",
    "\n",
    "Recordings less than 15 minutes (900 seconds) long are very likely incomplete and sometimes completely empty."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_pars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# The sampling rate is 0.5 Hz (1 in 2 seocnds), if the lentgh of the DataFrame is 450, its duration\n",
    "# is 15 minutes (900 seconds)\n",
    "\n",
    "for case in cases:\n",
    "    if len(data_pars[case]) < 450:\n",
    "        print('Removing %s' % case)\n",
    "        del data_pars[case]\n",
    "\n",
    "cases = sorted(data_pars.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_pars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Replace codes for categorical variables with informative category names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mapping_vent_mode = {0: None, 1: 'IPPV', 2: 'SIPPV', 3: 'SIMV', 4: 'SIMVPSV', 5: 'PSV', \n",
    "                     6: 'CPAP', 7: 'NCPAP', 8: 'DUOPAP', 9: 'HFO', 10: 'O2therapy', 15: 'service'}\n",
    "for case in cases:\n",
    "    data_pars[case][101].replace(mapping_vent_mode, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_patient_range = {1: 'Neonatal', 2: 'Pediatric'}\n",
    "for case in cases:\n",
    "    data_pars[case][100].replace(mapping_patient_range, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_patient_range_2 = {0: 'Neonatal', 1: 'Pediatric'}\n",
    "for case in cases:\n",
    "    data_pars[case][273].replace(mapping_patient_range_2, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_power = {0: 'Network', 1: 'Battery'}\n",
    "for case in cases:\n",
    "    data_pars[case][127].replace(mapping_power, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_off_on = {0: 'off', 1: 'on'}\n",
    "for case in cases:\n",
    "    data_pars[case][157].replace(mapping_off_on, inplace = True)\n",
    "    data_pars[case][158].replace(mapping_off_on, inplace = True)\n",
    "    data_pars[case][271].replace(mapping_off_on, inplace = True)\n",
    "    data_pars[case][272].replace(mapping_off_on, inplace = True)\n",
    "    data_pars[case][277].replace(mapping_off_on, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_no_yes = {0: 'no', 1: 'yes'}\n",
    "for case in cases:\n",
    "    data_pars[case][270].replace(mapping_no_yes, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_trigger = {0: 'Volumetrigger', 1: 'Flowtrigger'}\n",
    "for case in cases:\n",
    "    data_pars[case][274].replace(mapping_trigger, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_IE_HFOV = {0: '1:3', 1: '1:2', 2: '1:1'}\n",
    "for case in cases:\n",
    "    data_pars[case][275].replace(mapping_IE_HFOV, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_pressure_rise_ctrl = {0: 'I-flow', 1: 'Ramp', 2: 'AutoIFlow'}\n",
    "for case in cases:\n",
    "    data_pars[case][276].replace(mapping_pressure_rise_ctrl, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_pressure_unit = {0: 'mbar', 1: 'cmH2O',}\n",
    "for case in cases:\n",
    "    data_pars[case][140].replace(mapping_pressure_unit, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_CO2 = {0: 'mmHg', 1: 'kPa', 2: 'Vol%'}\n",
    "for case in cases:\n",
    "    data_pars[case][141].replace(mapping_CO2, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parse the parameter values using Fabian parameter library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "par_key_table = pd.read_excel('Fabian_parameters.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "par_key_table;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "par_key_dict = {}\n",
    "for row in par_key_table.index:\n",
    "    par_key_dict[par_key_table.code[row]] = par_key_table.name[row]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for case in cases:\n",
    "    data_pars[case].rename(columns = par_key_dict, inplace = True )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sort DataFrames according to time stamp index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for case in cases:\n",
    "    data_pars[case].sort_index(inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write individual text files with the ventilator modes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create sub-directories for each case if it does not yet exist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Images and raw data will be written on an external hard drive\n",
    "if not os.path.isdir('%s/%s' % (DATA_DUMP, 'fabian_cases')):\n",
    "    os.makedirs('%s/%s' % (DATA_DUMP, 'fabian_cases'))\n",
    "\n",
    "for case in cases: \n",
    "    if not os.path.isdir('%s/%s/%s' % (DATA_DUMP, 'fabian_cases', case)):\n",
    "        os.makedirs('%s/%s/%s' % (DATA_DUMP, 'fabian_cases', case))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for case in cases:\n",
    "    \n",
    "    a = data_pars[case]\n",
    "    \n",
    "    o2therapy = len(a[a['Ventilator_mode'] == 'O2therapy'])\n",
    "    ncpap = len(a[a['Ventilator_mode'] == 'NCPAP'])\n",
    "    duopap = len(a[a['Ventilator_mode'] == 'DUOPAP'])\n",
    "    simv = len(a[a['Ventilator_mode'] == 'SIMV'])\n",
    "    ippv = len(a[a['Ventilator_mode'] == 'IPPV'])\n",
    "    sippv = len(a[a['Ventilator_mode'] == 'SIPPV'])\n",
    "    simvpsv = len(a[a['Ventilator_mode'] == 'SIMVPSV'])\n",
    "    vg_on = len(data_pars[case][data_pars[case]['VG_state'] == 'on'])\n",
    "  \n",
    "    \n",
    "    fileout = open('%s/%s/%s/%s_%s.%s' % (DATA_DUMP, 'fabian_cases', case, case, 'vent_info', 'txt'), 'w')\n",
    "    \n",
    "    fileout.write('O2 therapy: %d sec \\n' % (o2therapy * 2))\n",
    "    fileout.write('NCPAP:      %d sec \\n' % (ncpap * 2))\n",
    "    fileout.write('DUOPAP:     %d sec \\n' % (duopap * 2))\n",
    "    fileout.write('IPPV:       %d sec \\n' % (ippv * 2))\n",
    "    fileout.write('SIPPV:      %d sec \\n' % (sippv * 2))\n",
    "    fileout.write('SIMV:       %d sec \\n' % (simv * 2))\n",
    "    fileout.write('SIMVPSV:    %d sec \\n\\n' % (simvpsv * 2))\n",
    "    fileout.write('VG on:      %d sec \\n' % (vg_on * 2))\n",
    "    \n",
    "    fileout.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export processed data as pickle files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_301_450'), 'wb') as handle:\n",
    "    pickle.dump(data_pars, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('%s/%s.pickle' % (DATA_DUMP, 'cases_301_450'), 'wb') as handle:\n",
    "    pickle.dump(cases, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Waves data (sampling rate = 150Hz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_dict_waves.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_dict_waves.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove those cases which are < 15 minute long\n",
    "data_dict_waves_selected = {key : value for key, value in data_dict_waves.items() if key in cases}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_dict_waves_selected.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Export sample waves data to pickle files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rec1 = cases[10:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dict_waves_selected_ex = { key: value for key, value in data_dict_waves_selected.items() if key in rec1}\n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_dict_waves_selected_ex_4'), 'wb') as handle:\n",
    "    pickle.dump(data_dict_waves_selected_ex, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}