accelerometer_ventilated_processing_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "![alt text](./Cerny_logo_1.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The effect of ambulance acceleration on mechanical ventilation during neonatal transport\n",
    "\n",
    "#### Author: Dr Gusztav Belteki"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the required libraries and set options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import pickle\n",
    "\n",
    "from pandas import Series, DataFrame\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "%matplotlib inline\n",
    "matplotlib.style.use('classic')\n",
    "matplotlib.rcParams['figure.facecolor'] = 'w'\n",
    "\n",
    "pd.set_option('display.max_rows', 250)\n",
    "pd.set_option('display.max_columns', 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Python version: {}\".format(sys.version))\n",
    "print(\"pandas version: {}\".format(pd.__version__))\n",
    "print(\"matplotlib version: {}\".format(matplotlib.__version__))\n",
    "print(\"NumPy version: {}\".format(np.__version__))\n",
    "print(\"SciPy version: {}\".format(sp.__version__))\n",
    "print(\"IPython version: {}\".format(IPython.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. List and set the working directory and the directory to write out data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Topic of the Notebook which will also be the name of the subfolder containing results\n",
    "TOPIC = 'accelerometer_ventilated'\n",
    "\n",
    "# Name of the external hard drives\n",
    "DRIVE = 'GUSZTI'\n",
    "\n",
    "# Directory containing clinical and blood gas data\n",
    "CWD = os.path.join('/Users','guszti', 'ventilation_fabian')\n",
    "\n",
    "# Directory on external drive to read the ventilation data from\n",
    "#DIR_READ = '/Volumes/%s/Raw_data/Fabian/accelerometer_data' % DRIVE1\n",
    "DIR_READ = os.path.join('/Volumes', DRIVE, 'Fabian', 'accelerometer_data')\n",
    "\n",
    "DIR_WRITE = os.path.join(CWD, 'Analyses', TOPIC)\n",
    "if not os.path.exists(DIR_WRITE):\n",
    "    os.mkdir(DIR_WRITE)\n",
    "\n",
    "DATA_DUMP = os.path.join('/Volumes', DRIVE, 'data_dump', 'fabian')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR_READ, DIR_WRITE, DATA_DUMP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Import the start time of the acceleration recordings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time_accel = {}\n",
    "\n",
    "flist = [file for file in os.listdir(DIR_READ) if not file.startswith('.')]\n",
    "flist  = sorted(flist, key = lambda x: int(x.split('.')[0].split('__')[1]))\n",
    "len(flist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# Get the start time of the acceleration recordings as Timestamp\n",
    "for fle in flist:\n",
    "    with open(os.path.join(DIR_READ, fle), 'r') as infile:\n",
    "        infile.readline()\n",
    "        line = infile.readline().split('@')[1].split()\n",
    "        line = ' '.join(line[:4] + line[5:])\n",
    "        start_time_accel[fle] = pd.to_datetime(line)     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Calculate stop time of accelerometer recordings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "stop_time_accel = {}\n",
    "for fle in flist:\n",
    "    if int(fle[9:-4]) % 10 == 0:\n",
    "        print('Working on %s' % fle)\n",
    "    temp_frame = pd.read_csv('%s/%s' % (DIR_READ, fle), header = None, \n",
    "        names = ['X', 'Y', 'Z', 'timedelta'], delim_whitespace = True, comment = '#', low_memory = False)\n",
    "    cum_time = temp_frame['timedelta'].sum()\n",
    "    stop_time_accel[fle] = start_time_accel[fle] + pd.to_timedelta(cum_time, unit='ms')\n",
    "    temp_frame = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accel_time_frame = DataFrame([start_time_accel, stop_time_accel]).T\n",
    "accel_time_frame.columns = ['accelerometer_start_time', 'accelerometer_stop_time']\n",
    "accel_time_frame.sort_values('accelerometer_start_time', inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Remove or correct wrong timestamps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`default__100.txt` and `default__101.txt` have got completely wrong time stamps that cannot be tracked down. Remove them. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del start_time_accel['default__100.txt'], start_time_accel['default__101.txt']\n",
    "del stop_time_accel['default__100.txt'], stop_time_accel['default__101.txt']\n",
    "\n",
    "accel_time_frame.drop(index = ['default__100.txt', 'default__101.txt'], inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Recordings `default__102.txt` - `default__122.txt` have been wrongly recorded with time stamps one year ahead actual time. Shift the time back by one year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "accel_time_frame.sort_values('accelerometer_start_time', inplace = True )\n",
    "accel_time_frame[375:430]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rec_list = list(accel_time_frame.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On inspection of the start and stop times it is clear that for recordings `default__102.txt` to `default__122.txt` the year is one year ahead of the actual time of the recording (2019 instead or 2018). Therefore, the timestamp needs to be shifted back by one year (365 days)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_shift = ['default__102.txt', 'default__103.txt', 'default__104.txt', 'default__105.txt', 'default__106.txt',\n",
    "            'default__107.txt', 'default__108.txt', 'default__109.txt', 'default__110.txt', 'default__111.txt',\n",
    "            'default__112.txt', 'default__113.txt', 'default__114.txt', 'default__115.txt', 'default__116.txt',\n",
    "            'default__117.txt', 'default__118.txt', 'default__119.txt', 'default__120.txt', 'default__121.txt',\n",
    "            'default__122.txt',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for rec in to_shift:\n",
    "    start_time_accel[rec] = start_time_accel[rec] - pd.Timedelta(value = 365, unit = 'D')\n",
    "    stop_time_accel[rec] = stop_time_accel[rec] - pd.Timedelta(value = 365, unit = 'D')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recreate DataFrame now with the correct start and stop times\n",
    "\n",
    "accel_time_frame = DataFrame([start_time_accel, stop_time_accel]).T\n",
    "accel_time_frame.columns = ['accelerometer_start_time', 'accelerometer_stop_time']\n",
    "accel_time_frame.sort_values('accelerometer_start_time', inplace = True)\n",
    "\n",
    "accel_time_frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Import ventilator recordings with mechanical ventilation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Import ventilator parameters, settings and alarms\n",
    "\n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_measurements_ventilated_1_300'), 'rb') as handle:\n",
    "    data_pars_measurements_ventilated_1_300 = pickle.load(handle)    \n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_settings_ventilated_1_300'), 'rb') as handle:\n",
    "    data_pars_settings_ventilated_1_300 = pickle.load(handle)   \n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_alarms_ventilated_1_300'), 'rb') as handle:\n",
    "    data_pars_alarms_ventilated_1_300 = pickle.load(handle)\n",
    "    \n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_measurements_ventilated_301_600'), 'rb') as handle:\n",
    "    data_pars_measurements_ventilated_301_600 = pickle.load(handle)    \n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_settings_ventilated_301_600'), 'rb') as handle:\n",
    "    data_pars_settings_ventilated_301_600 = pickle.load(handle)    \n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_alarms_ventilated_301_600'), 'rb') as handle:\n",
    "    data_pars_alarms_ventilated_301_600 = pickle.load(handle)\n",
    "    \n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_measurements_ventilated_601_900'), 'rb') as handle:\n",
    "    data_pars_measurements_ventilated_601_900 = pickle.load(handle)   \n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_settings_ventilated_601_900'), 'rb') as handle:\n",
    "    data_pars_settings_ventilated_601_900 = pickle.load(handle)    \n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_alarms_ventilated_601_900'), 'rb') as handle:\n",
    "    data_pars_alarms_ventilated_601_900 = pickle.load(handle)\n",
    "        \n",
    "data_pars_measurements_ventilated = {**data_pars_measurements_ventilated_1_300, \n",
    "                                     **data_pars_measurements_ventilated_301_600,\n",
    "                                     **data_pars_measurements_ventilated_601_900}\n",
    "\n",
    "data_pars_settings_ventilated = {**data_pars_settings_ventilated_1_300, \n",
    "                                 **data_pars_settings_ventilated_301_600,\n",
    "                                 **data_pars_settings_ventilated_601_900}\n",
    "\n",
    "data_pars_alarms_ventilated = {**data_pars_alarms_ventilated_1_300, \n",
    "                               **data_pars_alarms_ventilated_301_600,\n",
    "                               **data_pars_alarms_ventilated_601_900}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_pars_measurements_ventilated), len(data_pars_settings_ventilated), len(data_pars_alarms_ventilated),"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Generate dictionaries with the start and stop time of ventilator recordings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time_vent = {}\n",
    "stop_time_vent = {}\n",
    "\n",
    "for recording in data_pars_measurements_ventilated:\n",
    "    start_time_vent[recording] = data_pars_measurements_ventilated[recording].index[0]\n",
    "    stop_time_vent[recording] = data_pars_measurements_ventilated[recording].index[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create DataFrame now with the start and stop times of ventilator data for ventilated recordings\n",
    "\n",
    "vent_time_frame = DataFrame([start_time_vent, stop_time_vent]).T\n",
    "vent_time_frame.columns = ['ventilator_start_time', 'ventilator_stop_time']\n",
    "vent_time_frame.sort_values('ventilator_start_time', inplace = True)\n",
    "\n",
    "#vent_time_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the acceleremoter and ventilator recording times in an excel file\n",
    "\n",
    "writer = pd.ExcelWriter('%s/%s' % (DIR_WRITE, 'accelerometer_ventilator_times.xlsx'))\n",
    "accel_time_frame.to_excel(writer, 'accelerometer')\n",
    "vent_time_frame.to_excel(writer, 'ventilator')\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Identify matches between accelerometer and ventilator recordings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A ventilator and accelerometer recording can only overlap if one does not start after the other ended or does not end before the other started"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matches_ventilated = []\n",
    "\n",
    "for accel_key in start_time_accel:\n",
    "    for vent_key in start_time_vent:\n",
    "        # Do not include if it starts later or ends earlier\n",
    "        if start_time_accel[accel_key] > stop_time_vent[vent_key] or \\\n",
    "           stop_time_accel[accel_key] < start_time_vent[vent_key]:\n",
    "            continue\n",
    "        \n",
    "        else:\n",
    "            matches_ventilated.append((accel_key, vent_key))\n",
    "\n",
    "len(matches_ventilated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "matches_ventilated_frame = DataFrame(matches_ventilated, columns = ['accel_rec', 'vent_rec'])\n",
    "matches_ventilated_frame = matches_ventilated_frame.merge(right = accel_time_frame, \n",
    "                                    how = 'left', left_on = 'accel_rec', right_index = True)\n",
    "matches_ventilated_frame = matches_ventilated_frame.merge(right = vent_time_frame, \n",
    "                                               how = 'left', left_on = 'vent_rec', right_index = True)\n",
    "\n",
    "#matches_ventilated_frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the acceleromoter and ventilator recording times in an excel file\n",
    "writer = pd.ExcelWriter('%s/%s' % (DIR_WRITE, 'accel_vent_matching_times.xlsx'))\n",
    "matches_ventilated_frame.to_excel(writer, 'matches')\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. Import relevant accelerometer data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How many accelerometer recordings have a matching ventilator recording\n",
    "len(sorted(set(matches_ventilated_frame['accel_rec'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Recordings with mechanical ventilation\n",
    "accelero_ventilated = {}\n",
    "\n",
    "for i, fle in enumerate(sorted(set(matches_ventilated_frame['accel_rec']), key = lambda x : int(x[9:-4]))):\n",
    "    if i % 10 == 0:\n",
    "        print('Imported %d recordings' % (i+1))\n",
    "    accelero_ventilated[fle] = pd.read_csv('%s/%s' % (DIR_READ, fle), header = None, \n",
    "        names = ['X', 'Y', 'Z', 'timedelta'], delim_whitespace = True, comment = '#', low_memory = False)\n",
    "\n",
    "len(accelero_ventilated)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10. Generate acceleration DataFrames with timestamps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Recordings with mechanical ventilation\n",
    "for i, rec in enumerate(accelero_ventilated):\n",
    "    if i % 10 == 0:\n",
    "        print('Processed %d recordings' % i)\n",
    "        \n",
    "    # Calculate cumulative times from timedelta [ms]\n",
    "    accelero_ventilated[rec]['ms'] = accelero_ventilated[rec]['timedelta'].cumsum()\n",
    "    \n",
    "    # Create timestamps from milisec data, this starts at '1970:00:00:00 00:00:00'\n",
    "    accelero_ventilated[rec]['timestamp'] = pd.to_datetime(accelero_ventilated[rec]['ms'], unit='ms')\n",
    "    \n",
    "    # Correct timestamp to actual one\n",
    "    init_time = pd.Timestamp('1970-01-01 00:00:00.000')\n",
    "    delta = start_time_accel[rec] - init_time\n",
    "    accelero_ventilated[rec]['timestamp_2'] = accelero_ventilated[rec]['timestamp'] + delta\n",
    "    \n",
    "    # use times as index\n",
    "    accelero_ventilated[rec].index = accelero_ventilated[rec]['timestamp_2']\n",
    "    accelero_ventilated[rec].index.name = 'time'\n",
    "    \n",
    "    # Remove unnecessary columns\n",
    "    accelero_ventilated[rec] = accelero_ventilated[rec][['X', 'Y', 'Z', 'timedelta', 'ms']]  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Drop rows with na values\n",
    "for accel_rec in accelero_ventilated:\n",
    "    accelero_ventilated[accel_rec].dropna(how = 'any', inplace = True)\n",
    "\n",
    "len(accelero_ventilated)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 11. Limit the accelerometer and ventilator recordings to the parts when data are avaliable for both"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time_comb_vent = {}\n",
    "stop_time_comb_vent = {}\n",
    "\n",
    "for accel_rec, vent_rec in matches_ventilated:\n",
    "    if vent_rec in data_pars_measurements_ventilated.keys():\n",
    "        start_time_comb_vent[(accel_rec, vent_rec)] = max(accelero_ventilated[accel_rec].index[0], \n",
    "                                                 data_pars_measurements_ventilated[vent_rec].index[0])\n",
    "        stop_time_comb_vent[(accel_rec, vent_rec)]  = min(accelero_ventilated[accel_rec].index[-1], \n",
    "                                                 data_pars_measurements_ventilated[vent_rec].index[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 12. Trim ventilator and accelerometer data to contain only the overlapping regions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some accelerometer recordings are long and pair up with several consecutive ventilator recordings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "accelero_ventilated_trimmed = {}\n",
    "data_pars_measurements_ventilated_accelero = {}\n",
    "data_pars_settings_ventilated_accelero = {}\n",
    "data_pars_alarms_ventilated_accelero = {}\n",
    "\n",
    "for accel_rec, vent_rec in matches_ventilated:\n",
    "    \n",
    "    if vent_rec in data_pars_measurements_ventilated:\n",
    "        \n",
    "        start_time = start_time_comb_vent[accel_rec, vent_rec]\n",
    "        stop_time = stop_time_comb_vent[accel_rec, vent_rec]\n",
    "    \n",
    "        accelero_ventilated_trimmed[accel_rec, vent_rec] = \\\n",
    "            accelero_ventilated[accel_rec][start_time : stop_time].copy()\n",
    "        \n",
    "        data_pars_measurements_ventilated_accelero[accel_rec, vent_rec] =  \\\n",
    "            data_pars_measurements_ventilated[vent_rec][start_time : stop_time].copy()\n",
    "        \n",
    "        data_pars_settings_ventilated_accelero[accel_rec, vent_rec] =  \\\n",
    "            data_pars_settings_ventilated[vent_rec][start_time : stop_time].copy()\n",
    "        \n",
    "        data_pars_alarms_ventilated_accelero[accel_rec, vent_rec] =  \\\n",
    "            data_pars_alarms_ventilated[vent_rec][start_time : stop_time].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove those recordings where there is too much (>10%) missing data\n",
    "\n",
    "As the sampling rate of ventilator data is 0.5 Hz and the accelerometer data is 100 Hz, the proportion of the number of data points should be ideally ~200. Because of missing data this does not hold exactly. Allow for ~10% mismatch, that is a proportion between 180 - 220. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for rec, value in data_pars_measurements_ventilated_accelero.items():\n",
    "    print(rec, len(value), len(accelero_ventilated_trimmed[rec]),\n",
    "              len(accelero_ventilated_trimmed[rec]) / len(value), sep =  ' ' * 8 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_pars_measurements_ventilated_accelero)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pars_measurements_ventilated_accelero_sel = { key : value for key, value \n",
    "            in data_pars_measurements_ventilated_accelero.items()\n",
    "            if 180 < len(accelero_ventilated_trimmed[key]) / len(value) < 220 }\n",
    "\n",
    "data_pars_settings_ventilated_accelero_sel = { key : value for key, value \n",
    "            in data_pars_settings_ventilated_accelero.items()\n",
    "            if 180 < len(accelero_ventilated_trimmed[key]) / \n",
    "                     len(data_pars_measurements_ventilated_accelero[key]) < 220 }\n",
    "\n",
    "data_pars_alarms_ventilated_accelero_sel = { key : value for key, value \n",
    "            in data_pars_alarms_ventilated_accelero.items()\n",
    "            if 180 < len(accelero_ventilated_trimmed[key]) / \n",
    "                     len(data_pars_measurements_ventilated_accelero[key]) < 220 }\n",
    "\n",
    "accelero_ventilated_trimmed_sel = { key : value for key, value \n",
    "            in accelero_ventilated_trimmed.items()\n",
    "            if 180 < len(accelero_ventilated_trimmed[key]) / \n",
    "                     len(data_pars_measurements_ventilated_accelero[key]) < 220 }\n",
    "\n",
    "(len(data_pars_measurements_ventilated_accelero_sel), len(data_pars_settings_ventilated_accelero_sel),\n",
    "len(data_pars_alarms_ventilated_accelero_sel), len(accelero_ventilated_trimmed_sel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for rec, value in data_pars_measurements_ventilated_accelero_sel.items():\n",
    "    print(rec, len(value), len(accelero_ventilated_trimmed_sel[rec]),\n",
    "              len(accelero_ventilated_trimmed_sel[rec]) / len(value), sep =  ' ' * 8 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 13. Correct data types as appropriate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accelero_ventilated_trimmed_sel[('default__393.txt', 'AL000628')].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pars_measurements_ventilated_accelero_sel[('default__393.txt', 'AL000628')].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for rec, item in data_pars_measurements_ventilated_accelero_sel.items():\n",
    "    for col in item.columns:\n",
    "        item[col] = item[col].astype('float')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pars_measurements_ventilated_accelero_sel[('default__393.txt', 'AL000628')].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pars_settings_ventilated_accelero_sel['default__393.txt', 'AL000628'].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numeric = ['PIP_set', 'PEEP_set', 'FiO2_set',\n",
    "           'Flow_insp_set', 'Flow_exp_set', 'Ti_set', 'Te_set', 'RR_set',\n",
    "           'IE_I_set', 'IE_E_set', 'VG_set', 'Trigger_sens_set',\n",
    "           'PIP_lim_high_set', 'PIP_lim_low_set', \n",
    "           'P_man_breath_duoPAP_NCPAP_set', 'FiO2_flush_time_set',\n",
    "           'FiO2_flush_set', 'VG_set_kg']\n",
    "\n",
    "categorical = ['Patient_range', 'Ventilator_mode', 'Powerstate', 'Measuring_unit_pressure_set',\n",
    "               'Flow_sensor_state', 'Oxy_sensor_state', 'Ventilation_stopped',  'VG_state', \n",
    "               'Ventilator_range', 'Trigger_mode', 'Pressure_rise_control',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for rec, item in data_pars_settings_ventilated_accelero_sel.items():\n",
    "    for col in item.columns:\n",
    "        if col in numeric:\n",
    "            item[col] = item[col].astype('float')\n",
    "        elif col in categorical:\n",
    "            item[col] = item[col].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pars_settings_ventilated_accelero_sel['default__393.txt', 'AL000628'].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pars_alarms_ventilated_accelero_sel['default__393.txt', 'AL000628'].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 14. Export trimmed ventilator and accelerometer data to pickle archives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_measurements_ventilated_accelero'), 'wb') as handle:\n",
    "    pickle.dump(data_pars_measurements_ventilated_accelero_sel, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_settings_ventilated_accelero'), 'wb') as handle:\n",
    "    pickle.dump(data_pars_settings_ventilated_accelero_sel, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'data_pars_alarms_ventilated_accelero'), 'wb') as handle:\n",
    "    pickle.dump(data_pars_alarms_ventilated_accelero_sel, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "with open('%s/%s.pickle' % (DATA_DUMP, 'accelero_ventilated_1'), 'wb') as handle:\n",
    "    pickle.dump(accelero_ventilated_trimmed_sel, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}