ventilator_data_processing_new_1_1305.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![alt text](./Cerny_logo_1.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis of Cerny ventilation recordings\n",
    "\n",
    "#### Further processing and analysis of ventilator parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "This notebook import the preprocessed ventilator data from piclkle archive and analyses all the ventilator parameter data and alarms data obtained with **0.5Hz sampling rate** from the Fabian ventilators at the Cerny neonatal transport service. It exports desrciptive statistics into Excel files and the further processed data as pickle archive.\n",
    "\n",
    "The data processed and analysed in this Notebook were collected by the **Neonatal Emergency and Transport Service of the Peter Cerny Foundation**, Budapest, Hungary\n",
    "\n",
    "**Author: Dr Gusztav Belteki**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the required libraries and set options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "import IPython\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import pickle\n",
    "\n",
    "from pandas import Series, DataFrame\n",
    "from datetime import datetime, timedelta\n",
    "from matplotlib import dates\n",
    "\n",
    "%matplotlib inline\n",
    "matplotlib.style.use('classic')\n",
    "matplotlib.rcParams['figure.facecolor'] = 'w'\n",
    "\n",
    "pd.set_option('display.max_rows', 100)\n",
    "pd.set_option('display.max_columns', 300)\n",
    "pd.set_option('mode.chained_assignment', None) \n",
    "\n",
    "# This is to turn off a warning message which is given when read_Excel() imports '.xlsx' files\n",
    "import warnings\n",
    "warnings.simplefilter(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Python version: {}\".format(sys.version))\n",
    "print(\"pandas version: {}\".format(pd.__version__))\n",
    "print(\"matplotlib version: {}\".format(matplotlib.__version__))\n",
    "print(\"NumPy version: {}\".format(np.__version__))\n",
    "print(\"IPython version: {}\".format(IPython.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. List and set the working directory and the directory to write out data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Name of the external hard drive\n",
    "DRIVE = 'GUSZTI'\n",
    "\n",
    "# Directory on external drive to read the clinical from\n",
    "DIR_READ = os.path.join(os.sep, 'Volumes', DRIVE, 'Fabian_new', 'fabian_patient_data_all_new')\n",
    "\n",
    "# Path to project folder containing ventilation research results\n",
    "PATH = os.path.join(os.sep, 'Users', 'guszti', 'Library', 'Mobile Documents', 'com~apple~CloudDocs', \n",
    "                            'Documents', 'Research', 'Ventilation')\n",
    "\n",
    "# Folder to export the result of analysis\n",
    "DIR_WRITE = os.path.join(PATH, 'ventilation_fabian_new', 'Analyses')\n",
    "os.makedirs(DIR_WRITE, exist_ok = True)\n",
    "\n",
    "# Folder on a USB stick to export data to and to import processed data exported by other Notebooks\n",
    "DATA_DUMP = os.path.join(os.sep, '/Volumes', DRIVE, 'data_dump', 'fabian_new',)\n",
    "os.makedirs(DATA_DUMP, exist_ok = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR_READ, DIR_WRITE, DATA_DUMP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Import pickle archives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "with open(os.path.join(DATA_DUMP, 'data_pars_1_150.pickle'), 'rb') as handle:\n",
    "    data_pars_1_150 = pickle.load(handle)\n",
    "\n",
    "with open(os.path.join(DATA_DUMP, 'data_pars_151_300.pickle'), 'rb') as handle:\n",
    "    data_pars_151_300 = pickle.load(handle)\n",
    "\n",
    "with open(os.path.join(DATA_DUMP, 'data_pars_301_450.pickle'), 'rb') as handle:\n",
    "    data_pars_301_450 = pickle.load(handle)\n",
    "    \n",
    "with open(os.path.join(DATA_DUMP, 'data_pars_451_600.pickle'), 'rb') as handle:\n",
    "    data_pars_451_600 = pickle.load(handle)\n",
    "    \n",
    "with open(os.path.join(DATA_DUMP, 'data_pars_601_750.pickle'), 'rb') as handle:\n",
    "    data_pars_601_750 = pickle.load(handle)\n",
    "    \n",
    "with open(os.path.join(DATA_DUMP, 'data_pars_751_900.pickle'), 'rb') as handle:\n",
    "    data_pars_751_900 = pickle.load(handle)\n",
    "    \n",
    "with open(os.path.join(DATA_DUMP, 'data_pars_901_1050.pickle'), 'rb') as handle:\n",
    "    data_pars_901_1050 = pickle.load(handle)\n",
    "    \n",
    "with open(os.path.join(DATA_DUMP, 'data_pars_1051_1200.pickle'), 'rb') as handle:\n",
    "    data_pars_1051_1200 = pickle.load(handle)\n",
    "\n",
    "with open(os.path.join(DATA_DUMP, 'data_pars_1201_1350.pickle'), 'rb') as handle:\n",
    "    data_pars_1201_1350 = pickle.load(handle)\n",
    "    \n",
    "data_pars = {**data_pars_1_150, **data_pars_151_300, **data_pars_301_450, **data_pars_451_600,\n",
    "             **data_pars_601_750, **data_pars_751_900, **data_pars_901_1050, **data_pars_1051_1200,\n",
    "            **data_pars_1201_1350,}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Total number of ventilator recordings longer than 15 minutes\n",
    "len(data_pars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clinical data\n",
    "with open(os.path.join(DATA_DUMP, 'clin_df_new.pickle'), 'rb') as handle:\n",
    "    clin_df = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of cases where clinical data are available\n",
    "len(clin_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Limit ventilator data to cases for which clinical data and appropriate (>15 minutes) ventilator data are available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = sorted(set(clin_df.index) & set(data_pars.keys()))\n",
    "data_pars = {key : value for key, value in data_pars.items() if key in combined}\n",
    "len(data_pars)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Create a dictionary of Dataframes with measured ventilator parameters only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ventilator_measurements = ['PIP', 'MAP', 'PEEP', 'Ti_PSV', 'Cdyn', 'C20_C', 'R', \n",
    "                           'MV', 'MVresp', 'VTemand', 'VTemand_resp', 'VTespon_pat', \n",
    "                           'Leak', 'RR', 'Trigger', 'VTimand', 'deltaP', 'VThf_emand', \n",
    "                           'DCO2', 'HFOV_freq', 'FiO2', 'etCO2', 'Flow_demand', \n",
    "                           'Flow_insp', 'Flow_exp', 'TC', 'RR_CO2', \n",
    "                           'SpO2', 'Perfusion_index', 'Pulse_rate', 'PVI', 'RSBI', 'FiCO2', 'VTe_kg',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ventilator_measurements = ['PIP', 'MAP', 'PEEP', 'Ti_PSV', 'Cdyn', 'C20_C', 'R', \n",
    "                           'MV', 'MVresp', 'VTemand', 'VTemand_resp', 'VTespon_pat', \n",
    "                           'Leak', 'RR', 'Trigger', 'VTimand', 'deltaP', 'VThf_emand', \n",
    "                           'DCO2', 'HFOV_freq', 'FiO2', 'etCO2', 'Flow_demand', \n",
    "                           'Flow_insp', 'Flow_exp', 'TC', 'RR_CO2', \n",
    "                           'SpO2', 'Perfusion_index', 'Pulse_rate',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pars_measurements = {}\n",
    "for case in data_pars:\n",
    "    data_pars_measurements[case] = data_pars[case][ventilator_measurements].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Replace some uninformative fields with np.nan\n",
    "repl_dct = {'not valid': np.nan, 'out of range': np.nan, 'unused': np.nan}\n",
    "for case in data_pars_measurements:\n",
    "    data_pars_measurements[case].replace(repl_dct, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize relevant parameters to body weight\n",
    "pars_per_kg = ['MV', 'VTimand', 'VTemand', 'VTespon_pat', 'VTemand_resp', 'VThf_emand', 'Cdyn']\n",
    "\n",
    "for case in data_pars_measurements:\n",
    "    for par in pars_per_kg:\n",
    "        data_pars_measurements[case]['%s_kg' % par] = \\\n",
    "        data_pars_measurements[case][par] / (clin_df.loc[case]['Weight'] / 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize DCO2 to body weight\n",
    "pars_per_kg2 = ['DCO2']\n",
    "\n",
    "for case in data_pars_measurements:\n",
    "    for par in pars_per_kg2:\n",
    "        data_pars_measurements[case]['%s_kg2' % par] = \\\n",
    "        data_pars_measurements[case][par] / (clin_df.loc[case]['Weight'] / 1000) ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop columns which only have NaN values\n",
    "for case in data_pars_measurements:\n",
    "    data_pars_measurements[case].dropna(axis = 1, how = 'all', inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Export dictionary containing measured ventilator parameters to a pickle archive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_DUMP, 'data_pars_measurements_new_1_1305.pickle'), 'wb') as handle:\n",
    "    pickle.dump(data_pars_measurements, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Create a dictionary of Dataframes with ventilator settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are two `Trigger_mode` parameters. After manual observation of the data and discussion with the clinical team, `Trigger_mode_2` contains the actual ventilator triggering mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ventilator_settings = ['Patient_range', 'Ventilator_mode', 'PIP_set', 'PIP_set_PSV', \n",
    "    'PEEP_set', 'FiO2_set', 'Flow_insp_set', 'Slope_set', 'Flow_exp_set', 'Ti_set', \n",
    "    'Te_set', 'RR_set', 'IE_I_set', 'IE_E_set', 'deltaP_set', 'Freq_set_HFOV', \n",
    "    'MAP_set_HFOV', 'MAP_set_HFOV_recr', 'Volume_limit_set', 'VG_set',\n",
    "    'Term_criteria_PSV_set', 'Apnea_time_set', 'RR_backup_set', 'Trigger_sens_set', \n",
    "    'Powerstate', 'Battery_rem_time', 'Battery_rem_pc', 'MV_lim_high_set', \n",
    "    'MV_lim_low_set', 'PIP_lim_high_set', 'PEEP_lim_low_set', 'RR_lim_set', \n",
    "    'Leakage_lim_set', 'DCO2_lim_high_set', 'DCO2_lim_low_set', 'etCO2_lim_high_set', \n",
    "    'etCO2_lim_low_set', 'Measuring_unit_pressure_set', 'Measuring_unit_CO2_set', \n",
    "    'O2_compensation_set', 'CO2_baropressure_set',\n",
    "    'Flow_sensor_state', 'Oxy_sensor_state', 'P_man_breath_CPAP_HFO_set', \n",
    "    'P_man_breath_duoPAP_NCPAP_set', 'FiO2_flush_time_set', 'FiO2_flush_set',\n",
    "    'Ventilation_stopped', 'VG_state', 'Volume_limit_state', 'Ventilator_range',\n",
    "    'Trigger_mode', 'I_E_HFOV', 'Pressure_rise_control',\n",
    "    'HFO_recruitment', 'HFO_flow', 'Bias_flow', 'Trigger_mode_2', 'FOT_running', 'Leak_comp']    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pars_settings = {}\n",
    "\n",
    "for case in data_pars:\n",
    "    data_pars_settings[case] = data_pars[case][ventilator_settings].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace textual data with np.nan\n",
    "repl_dct = {'not valid': np.nan, 'out of range': np.nan, 'unused': np.nan}\n",
    "for case in data_pars_settings:\n",
    "    data_pars_settings[case].replace(repl_dct, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize relevant parameters to body weight\n",
    "pars_per_kg = ['Volume_limit_set', 'VG_set', 'MV_lim_high_set', 'MV_lim_low_set',]\n",
    "\n",
    "for case in data_pars_settings:\n",
    "    #print(case)\n",
    "    for par in pars_per_kg:\n",
    "        dta = data_pars_settings[case][par]\n",
    "        # Exclude data points when the parameter was 'off'\n",
    "        data_pars_settings[case]['%s_kg' % par] = \\\n",
    "        dta[dta != 'off'] / (clin_df.loc[case]['Weight'] / 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize DCO2 limits parameters to body weight square\n",
    "pars_per_kg = ['DCO2_lim_high_set', 'DCO2_lim_low_set']\n",
    "\n",
    "for case in data_pars_settings:\n",
    "    for par in pars_per_kg:\n",
    "        dta = data_pars_settings[case][par]\n",
    "        # Exclude data points when the parameter was 'off'\n",
    "        data_pars_settings[case]['%s_kg2' % par] = \\\n",
    "        dta[dta != 'off'] / (clin_df.loc[case]['Weight'] / 1000) ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop columns which only have NaN values\n",
    "for case in data_pars_settings:\n",
    "    data_pars_settings[case].dropna(axis=1, how='all', inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Export dictionary containing ventilator settings to a pickle archive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_DUMP, 'data_pars_settings_new_1_1305.pickle'), 'wb') as handle:\n",
    "    pickle.dump(data_pars_settings, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Create a dictionary of Dataframes with ventilator alarms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ventilator_alarms = ['Alarm_susp', 'Alarm_Flat_battery', 'Alarm_Checksum_ctrl_PIC',\n",
    "                     'Alarm_Checksum_monitor_PIC', 'Alarm_Safety_relay_defect',\n",
    "                     'Alarm_Sens_dev_prox_pressure', 'Alarm_input_pressure_blender',\n",
    "                     'Alarm_excess_pressure', 'Alarm_voltage_monit',\n",
    "                     'Alarm_SPI_interface', 'Alarm_DIO2_interface',\n",
    "                     'Alarm_COM_interface', 'Alarm_I2C_interface',\n",
    "                     'Alarm_parallel_interface', 'Alarm_serial_tem_interface',\n",
    "                     'Alarm_low_physical_memory', 'Alarm_Fan_defect',\n",
    "                     'Alarm_CO2_interface', 'Alarm_blender_defect',\n",
    "                     'Alarm_battery_defect', 'Alarm_input_pressure_O2_supply',\n",
    "                     'Alarm_input_pressure_air_supply', 'Alarm_tube_occlusion',\n",
    "                     'Alarm_patient_disconnected', 'Alarm_ETT_blocked',\n",
    "                     'Alarm_flow_sensor_defect', 'Alarm_flow_sensor_clean',\n",
    "                     'Alarm_flow_sensor_disconnected', 'Oxygen_sensor_defect',\n",
    "                     'Oxygen_sensor_used_up', 'Oxyen_value_divergence',\n",
    "                     'Alarm_O2_sensor_cal_error', 'Alarm_MV_high', 'Alarm_MV_low',\n",
    "                     'Alarm_pressure_high', 'Alarm_pressure_low', 'Alarm_PEEP_high',\n",
    "                     'Alarm_RR_high', 'Alarm_ETT_leak_high', 'Alarm_apnea',\n",
    "                     'Alarm_DCO2_high', 'Alarm_DCO2_low', 'Alarm_etCO2_high',\n",
    "                     'Alarm_etCO2_low', 'Alarm_PIP_not_reached', 'Alarm_limited_volume',\n",
    "                     'Alarm_volume_not_reached', 'Alarm_power_failure',\n",
    "                     'Alarm_charge_battery_60min', 'Alarm_charge_battery_30min',\n",
    "                     'Alarm_charge_battery_15min', 'Alarm_nebulizer_disconnection',\n",
    "                     'Alarm_nebulizer_system_error', 'Alarm_CO2_module_not_connected',\n",
    "                     'Alarm_CO2_filterline_not_connected', 'Alarm_CO2_check_sampleline',\n",
    "                     'Alarm_CO2_check_airway_adapter', 'Alarm_CO2_sensor_faulty',\n",
    "                     'Alarm_CO2_high', 'Alarm_CO2_low', 'Alarm_PRICO_FiO2_our_of_range',\n",
    "                     'Alarm_SPO2_module_not_connected', 'Alarm_SPO2_sensor_faulty',\n",
    "                     'Alarm_check_SPO2_sensor', 'Alarm_SPO2_low_SIQ', 'Alarm_high_SPO2',\n",
    "                     'Alarm_low_SPO2', 'Alarm_SPO2_low_PI', 'Alarm_low_PIP',\n",
    "                     'Alarm_high_FICO2', 'Alarm_low_FICO2', 'Alarm_high_pulserate',\n",
    "                     'Alarm_low_pulserate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ventilator_alarms = ['Alarm_susp', 'Alarm_Flat_battery', 'Alarm_Checksum_ctrl_PIC',\n",
    "                     'Alarm_Checksum_monitor_PIC', 'Alarm_Safety_relay_defect',\n",
    "                     'Alarm_Sens_dev_prox_pressure', 'Alarm_input_pressure_blender',\n",
    "                     'Alarm_excess_pressure', 'Alarm_voltage_monit',\n",
    "                     'Alarm_SPI_interface', 'Alarm_DIO2_interface',\n",
    "                     'Alarm_COM_interface', 'Alarm_I2C_interface',\n",
    "                     'Alarm_parallel_interface', 'Alarm_serial_tem_interface',\n",
    "                     'Alarm_low_physical_memory', 'Alarm_Fan_defect',\n",
    "                     'Alarm_CO2_interface', 'Alarm_blender_defect',\n",
    "                     'Alarm_battery_defect', 'Alarm_input_pressure_O2_supply',\n",
    "                     'Alarm_input_pressure_air_supply', 'Alarm_tube_occlusion',\n",
    "                     'Alarm_patient_disconnected', 'Alarm_ETT_blocked',\n",
    "                     'Alarm_flow_sensor_defect', 'Alarm_flow_sensor_clean',\n",
    "                     'Alarm_flow_sensor_disconnected', 'Oxygen_sensor_defect',\n",
    "                     'Oxygen_sensor_used_up', 'Oxyen_value_divergence',\n",
    "                     'Alarm_O2_sensor_cal_error', 'Alarm_MV_high', 'Alarm_MV_low',\n",
    "                     'Alarm_pressure_high', 'Alarm_pressure_low', 'Alarm_PEEP_high',\n",
    "                     'Alarm_RR_high', 'Alarm_ETT_leak_high', 'Alarm_apnea',\n",
    "                     'Alarm_DCO2_high', 'Alarm_DCO2_low', 'Alarm_etCO2_high',\n",
    "                     'Alarm_etCO2_low', 'Alarm_PIP_not_reached', 'Alarm_limited_volume',\n",
    "                     'Alarm_volume_not_reached', 'Alarm_power_failure',\n",
    "                     'Alarm_charge_battery_60min', 'Alarm_charge_battery_30min',\n",
    "                     'Alarm_charge_battery_15min', 'Alarm_nebulizer_disconnection',\n",
    "                     'Alarm_nebulizer_system_error', 'Alarm_CO2_module_not_connected',\n",
    "                     'Alarm_CO2_filterline_not_connected', 'Alarm_CO2_check_sampleline',\n",
    "                     'Alarm_CO2_check_airway_adapter', 'Alarm_CO2_sensor_faulty',\n",
    "                     'Alarm_CO2_high', 'Alarm_CO2_low', 'Alarm_PRICO_FiO2_our_of_range',\n",
    "                     'Alarm_SPO2_module_not_connected', 'Alarm_SPO2_sensor_faulty',\n",
    "                     'Alarm_check_SPO2_sensor', 'Alarm_SPO2_low_SIQ', 'Alarm_high_SPO2',\n",
    "                     'Alarm_low_SPO2', 'Alarm_SPO2_low_PI', 'Alarm_low_PIP',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pars_alarms = {}\n",
    "\n",
    "for case in data_pars:\n",
    "    data_pars_alarms[case] = data_pars[case][ventilator_alarms].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace textual data with np.nan\n",
    "repl_dct = {'off': np.nan, 'not valid': np.nan, 'out of range': np.nan, 'unused': np.nan}\n",
    "for case in data_pars_alarms:\n",
    "    data_pars_alarms[case].replace(repl_dct, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Drop columns which only have NaN values (these alarms never went off)\n",
    "for case in data_pars_alarms:\n",
    "    for column in data_pars_alarms[case].columns:\n",
    "        if data_pars_alarms[case][column].sum() == 0:\n",
    "            del data_pars_alarms[case][column]        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Export dictionary containing ventilator alarms to a pickle archive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_DUMP, 'data_pars_alarms_new_1_1305.pickle'), 'wb') as handle:\n",
    "    pickle.dump(data_pars_alarms, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Exploratory analysis on ventilation modes in the unedited recordings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### How many cases of the different ventilation modes occur"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vent_modes = {}\n",
    "for case in data_pars_settings:\n",
    "    # Multiply by two to get the number of seconds\n",
    "    vent_modes[case] = data_pars_settings[case]['Ventilator_mode'].value_counts() * 2\n",
    "    \n",
    "vent_modes = DataFrame(vent_modes).T\n",
    "vent_modes.replace(np.nan, 0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add the duration of the recordings\n",
    "\n",
    "recording_duration = {}\n",
    "\n",
    "for case in data_pars_settings:\n",
    "    recording_duration[case] = 2 * len(data_pars_settings[case])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vent_modes['ventilation'] = vent_modes['IPPV']  + vent_modes['SIMV'] + \\\n",
    "                            vent_modes['PSV']   + vent_modes['HFO'] + \\\n",
    "                            vent_modes['SIPPV'] + vent_modes['SIMVPSV']\n",
    "\n",
    "vent_modes['noninvasive'] = vent_modes['CPAP']  + vent_modes['DUOPAP'] + \\\n",
    "                            vent_modes['NCPAP'] + vent_modes['O2therapy']  \n",
    "\n",
    "vent_modes['total'] = Series(recording_duration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How long was VG turned on\n",
    "VG = {}\n",
    "for case in data_pars_settings:\n",
    "    VG[case] = sum(data_pars_settings[case]['VG_state'] == 'on') * 2\n",
    "        \n",
    "VG = Series(VG)\n",
    "vent_modes['VG'] = VG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vent_modes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('SIMV:', sum(vent_modes['SIMV'] > 0))\n",
    "print('SIPPV:', sum(vent_modes['SIPPV'] > 0))\n",
    "print('SIMVPSV:', sum(vent_modes['SIMVPSV'] > 0))\n",
    "print('PSV:', sum(vent_modes['PSV'] > 0))\n",
    "print('HFO:', sum(vent_modes['HFO'] > 0))\n",
    "print('IPPV:', sum(vent_modes['IPPV'] > 0))\n",
    "print('NCPAP:', sum(vent_modes['NCPAP'] > 0))\n",
    "print('CPAP:', sum(vent_modes['CPAP'] > 0))\n",
    "print('DUOPAP:', sum(vent_modes['DUOPAP'] > 0))\n",
    "print('O2therapy:', sum(vent_modes['O2therapy'] > 0))\n",
    "print('ventilation:', sum(vent_modes['ventilation'] > 0))\n",
    "print('VG on:', sum(vent_modes['VG'] > 0))\n",
    "print('noninvasive:', sum(vent_modes['noninvasive'] > 0))\n",
    "print('total:', len(vent_modes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How many seconds of each ventilation mode in total ?\n",
    "total_duration = DataFrame(vent_modes.sum(axis = 0), columns = ['duration (seconds)'])\n",
    "total_duration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Export Dataframes containing ventilator modes to Excel files and pickle archives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter(os.path.join(DIR_WRITE, 'ventilation_modes_unedited_new_1_1305.xlsx'))\n",
    "vent_modes.to_excel(writer, 'vent_modes')\n",
    "total_duration.to_excel(writer, 'total_duration')\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_DUMP, 'vent_modes_unedited_new_1_1305.pickle'), 'wb') as handle:\n",
    "    pickle.dump(vent_modes, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Export data about ventilation modes in the unedited recordings as text files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for case in data_pars_settings:\n",
    "    a = data_pars_settings[case]\n",
    "    \n",
    "    o2therapy = len(a[a['Ventilator_mode'] == 'O2therapy'])\n",
    "    ncpap = len(a[a['Ventilator_mode'] == 'NCPAP'])\n",
    "    duopap = len(a[a['Ventilator_mode'] == 'DUOPAP'])\n",
    "    simv = len(a[a['Ventilator_mode'] == 'SIMV'])\n",
    "    ippv = len(a[a['Ventilator_mode'] == 'IPPV'])\n",
    "    sippv = len(a[a['Ventilator_mode'] == 'SIPPV'])\n",
    "    simvpsv = len(a[a['Ventilator_mode'] == 'SIMVPSV'])\n",
    "    hfo = len(a[a['Ventilator_mode'] == 'HFO'])\n",
    "    vg_on = len(data_pars[case][data_pars[case]['VG_state'] == 'on'])\n",
    "  \n",
    "    \n",
    "    fileout = open(os.path.join(DATA_DUMP, 'fabian_cases_new', case, f'{case}_vent_info_unedited.txt'), 'w')\n",
    "    \n",
    "    # Sampling is 0.5 Hz, hence multiply it by 2 to get the number of seconds\n",
    "    fileout.write('O2 therapy: %d sec \\n' % (o2therapy * 2))\n",
    "    fileout.write('NCPAP:      %d sec \\n' % (ncpap * 2))\n",
    "    fileout.write('DUOPAP:     %d sec \\n' % (duopap * 2))\n",
    "    fileout.write('IPPV:       %d sec \\n' % (ippv * 2))\n",
    "    fileout.write('SIPPV:      %d sec \\n' % (sippv * 2))\n",
    "    fileout.write('SIMV:       %d sec \\n' % (simv * 2))\n",
    "    fileout.write('SIMVPSV:    %d sec \\n' % (simvpsv * 2))\n",
    "    fileout.write('HFO:        %d sec \\n\\n' % (hfo * 2))\n",
    "    fileout.write('VG on:      %d sec \\n' % (vg_on * 2))\n",
    "    \n",
    "    fileout.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Make some graphs about all recordings in an unedited way"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "majorFmt = dates.DateFormatter('%H:%M')\n",
    "filetype = 'jpg'\n",
    "dpi = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "par = 'MV_kg'\n",
    "dim = 'L/kg/min'; \n",
    "for case in data_pars_measurements:\n",
    "    \n",
    "    #if int(case[2:].lstrip('0')) <= 1179:\n",
    "    #    continue\n",
    "\n",
    "    try:\n",
    "        print('Saving %s' % case)\n",
    "        fig = plt.figure()\n",
    "        fig.set_size_inches(8, 4)\n",
    "        fig.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=None, hspace=0.1)\n",
    "        ax = fig.add_subplot(1, 1, 1)\n",
    "        data_pars_measurements[case][par].plot(ax = ax, label = par, x_compat = True)\n",
    "        ax.set_xlabel('Time', size = 14, color = 'black')\n",
    "        ax.set_ylabel(dim, size = 14, color = 'black')\n",
    "        ax.set_title(case,  size = 14, color = 'black')\n",
    "        ax.legend()\n",
    "        ax.grid('on', linestyle='-', linewidth=0.5, color = 'gray')\n",
    "        ax.xaxis.set_major_formatter(majorFmt)\n",
    "        plt.setp(ax.xaxis.get_majorticklabels(), rotation=0, fontsize = 12, horizontalalignment = 'center')\n",
    "        ax.tick_params(which = 'both', labelsize=12)\n",
    "            \n",
    "        fig.savefig(os.path.join(DATA_DUMP, 'fabian_cases_new', case, f'{case}_unedited_{par}.{filetype}'), \n",
    "            dpi = dpi, format = filetype, bbox_inches='tight', pad_inches=0.1,);\n",
    "\n",
    "        if case != 'AT000980':\n",
    "            plt.close()\n",
    "        \n",
    "    except Exception as e:\n",
    "        print('%s does not have ventilation' % case)\n",
    "        print(e)\n",
    "        plt.close()\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "par = 'MAP'\n",
    "dim = 'cmH$_2$O'; filetype = 'jpg'; dpi = 200\n",
    "\n",
    "for case in data_pars_measurements:\n",
    "    \n",
    "    #if int(case[2:].lstrip('0')) <= 1180:\n",
    "    #    continue\n",
    "    \n",
    "    try:\n",
    "        print('Saving %s' % case)\n",
    "        fig = plt.figure()\n",
    "        fig.set_size_inches(8, 4)\n",
    "        fig.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=None, hspace=0.1)\n",
    "        ax = fig.add_subplot(1, 1, 1)\n",
    "        data_pars_measurements[case][par].plot(ax = ax, label = par, x_compat = True)\n",
    "        ax.set_xlabel('Time', size = 14, color = 'black')\n",
    "        ax.set_ylabel(dim, size = 14, color = 'black')\n",
    "        ax.set_title(case,  size = 14, color = 'black')\n",
    "        ax.legend()\n",
    "        ax.grid('on', linestyle='-', linewidth=0.5, color = 'gray')\n",
    "    \n",
    "        majorFmt = dates.DateFormatter('%H:%M')  \n",
    "        ax.xaxis.set_major_formatter(majorFmt)\n",
    "        plt.setp(ax.xaxis.get_majorticklabels(), rotation=0, fontsize = 12, horizontalalignment = 'center')\n",
    "        ax.tick_params(which = 'both', labelsize=12)\n",
    "            \n",
    "        fig.savefig(os.path.join(DATA_DUMP, 'fabian_cases_new', case, f'{case}_unedited_{par}.{filetype}'), \n",
    "            dpi = dpi, format = filetype, bbox_inches='tight', pad_inches=0.1,);\n",
    "\n",
    "        if case != 'AT000980':\n",
    "            plt.close()\n",
    "            \n",
    "    except Exception as e:\n",
    "        print('%s does not have MAP' % case)\n",
    "        print(e)\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_vent_mode = {1: 'IPPV', 2: 'SIPPV', 3: 'SIMV', 4: 'SIMVPSV', 5: 'PSV', \n",
    "                     6: 'CPAP', 7: 'NCPAP', 8: 'DUOPAP', 9: 'HFO', 10: 'O2therapy'}\n",
    "mapping_vent_mode_rev = {value:key for key, value in mapping_vent_mode.items()}\n",
    "mapping_vent_mode_rev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping_VG = {11: 'off', 12: 'on'}\n",
    "mapping_VG_rev = {value:key for key, value in mapping_VG.items()}\n",
    "mapping_VG_rev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "par1 = 'Ventilator_mode'\n",
    "par2 = 'VG_state'\n",
    "dim = 'cmH$_2$O'\n",
    "\n",
    "yticks = np.arange(1, 13)\n",
    "yticklabels = ['IPPV', 'SIPPV','SIMV','SIMVPSV', 'PSV', 'CPAP', 'NCPAP', 'DUOPAP', 'HFO', 'O2therapy',\n",
    "              'VG OFF', 'VG ON']\n",
    "\n",
    "for case in data_pars_settings:\n",
    "    \n",
    "    #if int(case[2:].lstrip('0')) <= 1180:\n",
    "    #    continue\n",
    "\n",
    "    try:\n",
    "        \n",
    "        print('Saving %s' % case)\n",
    "        fig = plt.figure()\n",
    "        fig.set_size_inches(8, 4)\n",
    "        fig.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=None, hspace=0.1)\n",
    "        ax = fig.add_subplot(1, 1, 1)\n",
    "        \n",
    "        data = data_pars_settings[case][par1].replace(mapping_vent_mode_rev)\n",
    "        data.plot(ax = ax, label = par1, color='blue', x_compat = True)\n",
    "        \n",
    "        data_2 = data_pars_settings[case][par2].replace(mapping_VG_rev)\n",
    "        data_2.plot(ax = ax, label = par2, color='blue', linestyle='dashed', x_compat = True)\n",
    "        \n",
    "        ax.set_xlabel('Time', size = 14, color = 'black')\n",
    "        ax.set_ylabel('', size = 14, color = 'black')\n",
    "        ax.set_ylim(0,17)\n",
    "        ax.set_yticks(yticks)\n",
    "        ax.set_yticklabels(yticklabels)\n",
    "        ax.set_title(case,  size = 14, color = 'black')\n",
    "        ax.legend(ncols=2)\n",
    "        ax.grid('on', linestyle='-', linewidth=0.5, color = 'gray')\n",
    "        ax.xaxis.set_major_formatter(majorFmt)\n",
    "        plt.setp(ax.xaxis.get_majorticklabels(), rotation=0, fontsize = 12, horizontalalignment = 'center')\n",
    "        ax.tick_params(which = 'both', labelsize=12)\n",
    "            \n",
    "        fig.savefig(os.path.join(DATA_DUMP, 'fabian_cases_new', case, f'{case}_unedited_ventilation_mode.{filetype}'), \n",
    "            dpi = dpi, format = filetype, bbox_inches='tight', pad_inches=0.1,);\n",
    "\n",
    "        if case != 'AT000980':\n",
    "            plt.close()\n",
    "            \n",
    "    except Exception as e:\n",
    "        print('error with %s' % case)\n",
    "        print(e)\n",
    "        plt.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}