Supervised Learning OOP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis of Ladder Walk Videos\n",
    "\n",
    "BIOF 509 Spring 2021\n",
    "\n",
    "This notebook contains all the code needed to train and run machine learning models to predict the different scores of interest on the horizontal ladder walk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load packages\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "import datetime as dt\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.svm import SVR\n",
    "from dateutil import parser\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import joblib, os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing\n",
    "\n",
    "The function separates the scores for the different limbs and processes the raw output from DeepLabCut. For each video, all features are normalized, frames are made the same length, and flattend."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Processing:\n",
    "    def __init__(self):\n",
    "        #list with a string order of the output of processing\n",
    "        self.keys = [\"dominant_front_hit\", \"dominant_front_miss\", \"dominant_front_step\", \n",
    " \"nondominant_front_hit\", \"nondominant_front_miss\", \"nondominant_front_step\",\n",
    "    \"dominant_back_hit\", \"dominant_back_miss\", \"dominant_back_step\", \n",
    " \"nondominant_back_hit\", \"nondominant_back_miss\", \"nondominant_back_step\"]\n",
    "    def process_with_labels(data_path,label_path,project_name,rung_network,rung_it,max_len,show_skipped):\n",
    "        '''\n",
    "        Processes input data that has associated labels for training\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        data_path: string\n",
    "            String containing path to the DLC output data (.h5 files)\n",
    "        label_path: string\n",
    "            String containing path to the manual score csv\n",
    "        project_name: string\n",
    "            String containing the name of the DLC project for rat tracking\n",
    "        rung_network: string\n",
    "            String containing the project and date info of the DLC project for rung tracking\n",
    "        rung_it: int\n",
    "            Integer that shows the number of iterations of the snapshot that was used to analyze the videos for rungs\n",
    "        max_len: int\n",
    "            Integer that defines the maximum number of frames of all videos in the dataset\n",
    "        show_skipped: bool\n",
    "            If this is set to True, then the script will print out the names of the videos that were skipped\n",
    "        '''\n",
    "        \n",
    "        scores_df = pd.read_csv(label_path)\n",
    "\n",
    "        #hit scores\n",
    "        hit_df = scores_df[[\"subject\",\"date\",\"crossing number\",\"run\",\"limb\",\"avg_human_hit\"]]\n",
    "\n",
    "        #miss\n",
    "        miss_df = scores_df[[\"subject\",\"date\",\"crossing number\",\"run\",\"limb\",\"avg_human_miss\"]]\n",
    "\n",
    "        #steps\n",
    "        step_df = scores_df[[\"subject\",\"date\",\"crossing number\",\"run\",\"limb\",\"avg_human_steps\"]]\n",
    "\n",
    "        #score extraction\n",
    "        dom_f_hit = hit_df.loc[hit_df[\"limb\"]==\"Dominant Front\"]\n",
    "        dom_f_miss = miss_df.loc[miss_df[\"limb\"]==\"Dominant Front\"]\n",
    "        dom_f_step = step_df.loc[step_df[\"limb\"]==\"Dominant Front\"]\n",
    "\n",
    "        ndom_f_hit = hit_df.loc[hit_df[\"limb\"]==\"Nondominant Front\"]\n",
    "        ndom_f_miss = miss_df.loc[miss_df[\"limb\"]==\"Nondominant Front\"]\n",
    "        ndom_f_step = step_df.loc[step_df[\"limb\"]==\"Nondominant Front\"]\n",
    "\n",
    "        dom_b_hit = hit_df.loc[hit_df[\"limb\"]==\"Dominant Back\"]\n",
    "        dom_b_miss = miss_df.loc[miss_df[\"limb\"]==\"Dominant Back\"]\n",
    "        dom_b_step = step_df.loc[step_df[\"limb\"]==\"Dominant Back\"]\n",
    "\n",
    "        ndom_b_hit = hit_df.loc[hit_df[\"limb\"]==\"Nondominant Back\"]\n",
    "        ndom_b_miss = miss_df.loc[miss_df[\"limb\"]==\"Nondominant Back\"]\n",
    "        ndom_b_step = step_df.loc[step_df[\"limb\"]==\"Nondominant Back\"]\n",
    "        dom_f_hit_x = []\n",
    "        dom_f_miss_x = []\n",
    "        dom_f_step_x =[]\n",
    "\n",
    "        ndom_f_hit_x = []\n",
    "        ndom_f_miss_x = []\n",
    "        ndom_f_step_x = []\n",
    "\n",
    "        dom_b_hit_x = []\n",
    "        dom_b_miss_x = []\n",
    "        dom_b_step_x = []\n",
    "\n",
    "        ndom_b_hit_x = []\n",
    "        ndom_b_miss_x = []\n",
    "        ndom_b_step_x = []\n",
    "\n",
    "        #sets of labels\n",
    "        dom_f_hit_y = []\n",
    "        dom_f_miss_y = []\n",
    "        dom_f_step_y =[]\n",
    "\n",
    "        ndom_f_hit_y = []\n",
    "        ndom_f_miss_y = []\n",
    "        ndom_f_step_y = []\n",
    "\n",
    "        dom_b_hit_y = []\n",
    "        dom_b_miss_y = []\n",
    "        dom_b_step_y = []\n",
    "\n",
    "        ndom_b_hit_y = []\n",
    "        ndom_b_miss_y = []\n",
    "        ndom_b_step_y = []\n",
    "\n",
    "        lengths = []\n",
    "        \n",
    "        #video_list\n",
    "        dom_f_hit_videos = []\n",
    "        dom_f_miss_videos = []\n",
    "        dom_f_step_videos =[]\n",
    "\n",
    "        ndom_f_hit_videos = []\n",
    "        ndom_f_miss_videos = []\n",
    "        ndom_f_step_videos = []\n",
    "\n",
    "        dom_b_hit_videos = []\n",
    "        dom_b_miss_videos = []\n",
    "        dom_b_step_videos = []\n",
    "\n",
    "        ndom_b_hit_videos = []\n",
    "        ndom_b_miss_videos = []\n",
    "        ndom_b_step_videos = []\n",
    "        \n",
    "        rat_folder = glob.glob(data_path)\n",
    "        \n",
    "        for file in rat_folder:\n",
    "            #rung file info\n",
    "            rung_name_list = file.split(\"/\")[-1].split(\"_\")[0:8]+[rung_network,str(rung_it)+\".h5\"]\n",
    "            rung_file = '_'.join(rung_name_list)\n",
    "            #rat tracking file\n",
    "            #open the file\n",
    "            rat_df = pd.read_hdf(file)[project_name]\n",
    "            #properties of the file\n",
    "            subject = file.split(\"/\")[0]\n",
    "            date_raw = rung_name_list[1]\n",
    "            date = parser.parse(date_raw).date().strftime(\"%Y-%m-%d\")\n",
    "            run = rung_name_list[2]\n",
    "            crossing = [int(s) for s in rung_name_list[3] if s.isdigit()][0]\n",
    "                \n",
    "            \n",
    "            #\n",
    "            dom_f_hit_score = dom_f_hit[(dom_f_hit[\"subject\"]==subject) & (dom_f_hit[\"date\"]==date) & (dom_f_hit[\"run\"]==run)].reset_index()\n",
    "            dom_f_miss_score = dom_f_miss[(dom_f_miss[\"subject\"]==subject) & (dom_f_miss[\"date\"]==date) & (dom_f_miss[\"run\"]==run)].reset_index()\n",
    "            dom_f_step_score = dom_f_step[(dom_f_step[\"subject\"]==subject) & (dom_f_step[\"date\"]==date) & (dom_f_step[\"run\"]==run)].reset_index()\n",
    "\n",
    "            ndom_f_hit_score = ndom_f_hit[(ndom_f_hit[\"subject\"]==subject) & (ndom_f_hit[\"date\"]==date) & (ndom_f_hit[\"run\"]==run)].reset_index()\n",
    "            ndom_f_miss_score = ndom_f_miss[(ndom_f_miss[\"subject\"]==subject) & (ndom_f_miss[\"date\"]==date) & (ndom_f_miss[\"run\"]==run)].reset_index()\n",
    "            ndom_f_step_score = ndom_f_step[(ndom_f_step[\"subject\"]==subject) & (ndom_f_step[\"date\"]==date) & (ndom_f_step[\"run\"]==run)].reset_index()\n",
    "\n",
    "            dom_b_hit_score = dom_b_hit[(dom_b_hit[\"subject\"]==subject) & (dom_b_hit[\"date\"]==date) & (dom_b_hit[\"run\"]==run)].reset_index()\n",
    "            dom_b_miss_score = dom_b_miss[(dom_b_miss[\"subject\"]==subject) & (dom_b_miss[\"date\"]==date) & (dom_b_miss[\"run\"]==run)].reset_index()\n",
    "            dom_b_step_score = dom_b_step[(dom_b_step[\"subject\"]==subject) & (dom_b_step[\"date\"]==date) & (dom_b_step[\"run\"]==run)].reset_index()\n",
    "\n",
    "            ndom_b_hit_score = ndom_b_hit[(ndom_b_hit[\"subject\"]==subject) & (ndom_b_hit[\"date\"]==date) & (ndom_b_hit[\"run\"]==run)].reset_index()\n",
    "            ndom_b_miss_score = ndom_b_miss[(ndom_b_miss[\"subject\"]==subject) & (ndom_b_miss[\"date\"]==date) & (ndom_b_miss[\"run\"]==run)].reset_index()\n",
    "            ndom_b_step_score = ndom_b_step[(ndom_b_step[\"subject\"]==subject) & (ndom_b_step[\"date\"]==date) & (ndom_b_step[\"run\"]==run)].reset_index()\n",
    "\n",
    "            \n",
    "            video_df = scores_df[[\"subject\",\"date\",\"crossing number\",\"run\"]]\n",
    "            video_info = video_df[(video_df[\"subject\"]==subject) & (video_df[\"date\"]==date) & (video_df[\"run\"]==run)].reset_index(drop=True)\n",
    "            \n",
    "            \n",
    "            #join the rat and rung dataframes\n",
    "            df = rat_df\n",
    "            df_cols = df.columns.tolist()\n",
    "            df_temp = df\n",
    "\n",
    "            if df.shape[1] == 0:\n",
    "                continue\n",
    "            #if len(dom_f_low_like) >0:\n",
    "                #print(dom_f_low_like)\n",
    "            df = df.drop('likelihood', axis=1, level=1)\n",
    "\n",
    "            #scale data\n",
    "            #Uses the values function of pandas - converts any dataframe to an array\n",
    "            data_for_scaling = df.values\n",
    "            scaler = MinMaxScaler()\n",
    "            scaled_data = scaler.fit_transform(data_for_scaling)\n",
    "\n",
    "            if len(scaled_data) < max_len:\n",
    "                newlength = (max_len-len(scaled_data))\n",
    "                zero = np.zeros((newlength,6))\n",
    "                scaled_temp = pd.DataFrame(scaled_data).append(pd.DataFrame(zero),ignore_index=True)\n",
    "                scaled_temp = scaled_temp.fillna(0)\n",
    "                scaled_data2 = scaled_temp.values\n",
    "            else:\n",
    "                scaled_data2 = scaled_data\n",
    "\n",
    "            video_data = scaled_data2.flatten()\n",
    "            if len(dom_f_hit_score) !=0:\n",
    "                dom_f_hit_y.append(dom_f_hit_score[\"avg_human_hit\"][0])\n",
    "                dom_f_hit_x.append(video_data)\n",
    "                dom_f_hit_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Dom Front Hit\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(dom_f_miss_score) != 0:\n",
    "                dom_f_miss_y.append(dom_f_miss_score[\"avg_human_miss\"][0])\n",
    "                dom_f_miss_x.append(video_data)\n",
    "                dom_f_miss_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Dom Front Miss\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(dom_f_step_score) != 0:\n",
    "                dom_f_step_y.append(dom_f_step_score[\"avg_human_steps\"][0])\n",
    "                dom_f_step_x.append(video_data)\n",
    "                dom_f_step_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Dom Front Step\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(ndom_f_hit_score) != 0:\n",
    "                ndom_f_hit_y.append(ndom_f_hit_score[\"avg_human_hit\"][0])\n",
    "                ndom_f_hit_x.append(video_data)\n",
    "                ndom_f_hit_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Nondom Front Hit\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(ndom_f_miss_score) != 0:\n",
    "                ndom_f_miss_y.append(ndom_f_miss_score[\"avg_human_miss\"][0])\n",
    "                ndom_f_miss_x.append(video_data)\n",
    "                ndom_f_miss_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Nondom Front Miss\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(ndom_f_step_score) != 0:\n",
    "                ndom_f_step_y.append(ndom_f_step_score[\"avg_human_steps\"][0])\n",
    "                ndom_f_step_x.append(video_data)\n",
    "                ndom_f_step_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Nondom Front Step\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(dom_b_hit_score) != 0:\n",
    "                dom_b_hit_y.append(dom_b_hit_score[\"avg_human_hit\"][0])\n",
    "                dom_b_hit_x.append(video_data)\n",
    "                dom_b_hit_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Dom Back Hit\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(dom_b_miss_score) != 0:\n",
    "                dom_b_miss_y.append(dom_b_miss_score[\"avg_human_miss\"][0])\n",
    "                dom_b_miss_x.append(video_data)\n",
    "                dom_b_miss_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Dom Back Miss\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(dom_b_step_score) != 0:\n",
    "                dom_b_step_y.append(dom_b_step_score[\"avg_human_steps\"][0])\n",
    "                dom_b_step_x.append(video_data)\n",
    "                dom_b_step_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Dom Back Step\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(ndom_b_hit_score) != 0:\n",
    "                ndom_b_hit_y.append(ndom_b_hit_score[\"avg_human_hit\"][0])\n",
    "                ndom_b_hit_x.append(video_data)\n",
    "                ndom_b_hit_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Nondom Back Hit\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(ndom_b_miss_score) != 0:\n",
    "                ndom_b_miss_y.append(ndom_b_miss_score[\"avg_human_miss\"][0])\n",
    "                ndom_b_miss_x.append(video_data)\n",
    "                ndom_b_miss_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Nondom Back Miss\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "            if len(ndom_b_step_score) != 0:\n",
    "                ndom_b_step_y.append(ndom_b_step_score[\"avg_human_steps\"][0])\n",
    "                ndom_b_step_x.append(video_data)\n",
    "                ndom_b_step_videos.append(video_info.iloc[0])\n",
    "            else:\n",
    "                if show_skipped:\n",
    "                    print(\"Nondom Back Step\"+\"Missing scores: \"+subject + \" \" + date + \" \" + run)\n",
    "                continue\n",
    "        return [dom_f_hit_x, dom_f_miss_x, dom_f_step_x, ndom_f_hit_x, ndom_f_miss_x, ndom_f_step_x,\n",
    "    dom_b_hit_x, dom_b_miss_x, dom_b_step_x, ndom_b_hit_x, ndom_b_miss_x, ndom_b_step_x],[dom_f_hit_y, dom_f_miss_y, dom_f_step_y, ndom_f_hit_y, ndom_f_miss_y, ndom_f_step_y, dom_b_hit_y,\n",
    "    dom_b_miss_y, dom_b_step_y, ndom_b_hit_y, ndom_b_miss_y, ndom_b_step_y],[dom_f_hit_videos, dom_f_miss_videos, dom_f_step_videos, ndom_f_hit_videos, ndom_f_miss_videos, ndom_f_step_videos, dom_b_hit_videos,\n",
    "    dom_b_miss_videos, dom_b_step_videos, ndom_b_hit_videos, ndom_b_miss_videos, ndom_b_step_videos]\n",
    "\n",
    "    def process_data(data_path,project_name,rung_network,rung_it,max_len,show_skipped):\n",
    "        '''\n",
    "        Processes input data that does not have labels. Only for prediction.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        data_path: string\n",
    "            String containing path to the DLC output data (.h5 files)\n",
    "        label_path: string\n",
    "            String containing path to the manual score csv\n",
    "        project_name: string\n",
    "            String containing the name of the DLC project for rat tracking\n",
    "        rung_network: string\n",
    "            String containing the project and date info of the DLC project for rung tracking\n",
    "        rung_it: int\n",
    "            Integer that shows the number of iterations of the snapshot that was used to analyze the videos for rungs\n",
    "        max_len: int\n",
    "            Integer that defines the maximum number of frames of all videos in the dataset\n",
    "        show_skipped: bool\n",
    "            If this is set to True, then the script will print out the names of the videos that were skipped\n",
    "        '''\n",
    "            \n",
    "        X = []\n",
    "        video_list = []\n",
    "\n",
    "        rat_folder = glob.glob(data_path)\n",
    "        \n",
    "        for file in rat_folder:\n",
    "            #rung file info\n",
    "            rung_name_list = file.split(\"/\")[-1].split(\"_\")[0:8]+[rung_network,str(rung_it)+\".h5\"]\n",
    "            rung_file = '_'.join(rung_name_list)\n",
    "            #rat tracking file\n",
    "            #open the file\n",
    "            rat_df = pd.read_hdf(file)[project_name]\n",
    "            #properties of the file\n",
    "            subject = file.split(\"/\")[0]\n",
    "            date_raw = rung_name_list[1]\n",
    "            date = parser.parse(date_raw).date().strftime(\"%Y-%m-%d\")\n",
    "            run = rung_name_list[2]\n",
    "            crossing = [int(s) for s in rung_name_list[3] if s.isdigit()][0]\n",
    "                \n",
    "            video_info = pd.DataFrame({\"subject\":[subject],\"date\":[date],\"crossing\":[crossing],\"run\":[run]})\n",
    "\n",
    "            df = rat_df\n",
    "            df_cols = df.columns.tolist()\n",
    "            df_temp = df\n",
    "\n",
    "            if df.shape[1] == 0:\n",
    "                continue\n",
    "            #if len(dom_f_low_like) >0:\n",
    "                #print(dom_f_low_like)\n",
    "            df = df.drop('likelihood', axis=1, level=1)\n",
    "\n",
    "            #scale data\n",
    "            #Uses the values function of pandas - converts any dataframe to an array\n",
    "            data_for_scaling = df.values\n",
    "            scaler = MinMaxScaler()\n",
    "            scaled_data = scaler.fit_transform(data_for_scaling)\n",
    "\n",
    "            if len(scaled_data) < max_len:\n",
    "                newlength = (max_len-len(scaled_data))\n",
    "                zero = np.zeros((newlength,6))\n",
    "                scaled_temp = pd.DataFrame(scaled_data).append(pd.DataFrame(zero),ignore_index=True)\n",
    "                scaled_temp = scaled_temp.fillna(0)\n",
    "                scaled_data2 = scaled_temp.values\n",
    "            else:\n",
    "                scaled_data2 = scaled_data\n",
    "\n",
    "            video_data = scaled_data2.flatten(order='C')\n",
    "            X.append(video_data)\n",
    "            video_list.append(video_info)\n",
    "            \n",
    "        return X,video_list\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scoring\n",
    "\n",
    "This class makes scoring possible with a support vector regressor and random forest regreesor. The result is some metrics of interest and a saved model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Scoring:\n",
    "    def support_vector(X,y,name,data_path,project_name,kern,save=True):\n",
    "        '''\n",
    "        Trains a support vector machine on regression. Returns accuracy, RMSE, and feature importances\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        X : list\n",
    "            List containing data of one limb\n",
    "        y: list\n",
    "            List containing the labels of the corresponding limb\n",
    "        name: string\n",
    "            String containing the name of what the model will be saved as\n",
    "        data_path: string\n",
    "            String containing the path to the original .h5 files\n",
    "        project_name: string\n",
    "            String containing the name of the DLC project for rat tracking\n",
    "        kern: string\n",
    "            String containing the name of the desired kernel\n",
    "        save: bool, default: True\n",
    "            If true, then a copy of the trained model will be saved. \n",
    "        '''\n",
    "        data = np.array(X)\n",
    "        labels = np.array(y)\n",
    "        \n",
    "        train_features, test_features, train_labels, test_labels = train_test_split(data, labels, test_size = 0.2, random_state = 31)\n",
    "        regressor = SVR(kernel=kern)\n",
    "        regressor.fit(train_features, train_labels)\n",
    "        predictions = regressor.predict(test_features)\n",
    "        \n",
    "        rat_folder = glob.glob(data_path)\n",
    "        df = pd.read_hdf(rat_folder[0])[project_name]\n",
    "        df = df.drop('likelihood', axis=1, level=1)\n",
    "        \n",
    "        all_feature_importances = np.reshape(regressor.coef_,(356,32))\n",
    "        \n",
    "        average_feature_importances = []\n",
    "        for i in range(all_feature_importances.shape[1]):\n",
    "            average_value = 0\n",
    "            for array in all_feature_importances:\n",
    "                average_value += abs(array[i])\n",
    "            average_feature_importances.append(average_value)\n",
    "        \n",
    "        # dictionary to hold {feature_name: feature_importance}\n",
    "        feats = {} \n",
    "        for feature, importance in zip(df.columns, average_feature_importances):\n",
    "            #add the name/value pair\n",
    "            feats[feature] = importance \n",
    "        #make it into a dataframe\n",
    "        importances = pd.DataFrame.from_dict(feats, orient='index')\n",
    "\n",
    "        # Calculate the absolute errors\n",
    "        errors = abs(predictions - test_labels)\n",
    "        # Calculate mean absolute percentage error (MAPE)\n",
    "        mape = 100 * (errors / test_labels)# Calculate and display accuracy\n",
    "        rmse = mean_squared_error(test_labels,predictions,squared=True)\n",
    "        accuracy = 100 - np.mean(mape)\n",
    "        if save == True:\n",
    "            joblib.dump(regressor, name+\".joblib\", compress=0)\n",
    "        return accuracy,rmse,importances\n",
    "\n",
    "    def forest(X,y,name,data_path,project_name, njobs=None,save=True):\n",
    "        '''\n",
    "        Trains a random forest on regression. Returns accuracy, RMSE, and feature importances\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        X : list\n",
    "            List containing data of one limb\n",
    "        y: list\n",
    "            List containing the labels of the corresponding limb\n",
    "        name: string\n",
    "            String containing the name of what the model will be saved as\n",
    "        data_path: string\n",
    "            String containing the path to the original .h5 files\n",
    "        project_name: string\n",
    "            String containing the name of the DLC project for rat tracking\n",
    "        njobs: int, default = None\n",
    "            The number of jobs that will be run in parallel. \n",
    "        save: bool, default= True\n",
    "            If true, then a copy of the trained model will be saved. \n",
    "        '''\n",
    "        data = np.array(X)\n",
    "        labels = np.array(y)\n",
    "        train_features, test_features, train_labels, test_labels = train_test_split(data, labels, test_size = 0.2, random_state = 31)\n",
    "        # Instantiate model with n decision trees\n",
    "        rf = RandomForestRegressor(n_estimators = 20, random_state = 42,n_jobs=njobs)# Train the model on training data\n",
    "        rf.fit(train_features, train_labels)\n",
    "        # Use the forest's predict method on the test data\n",
    "        # Calculate the absolute errors\n",
    "        predictions = rf.predict(test_features)\n",
    "        errors = abs(predictions - test_labels)\n",
    "        # Calculate mean absolute percentage error (MAPE)\n",
    "        mape = 100 * (errors / test_labels)\n",
    "        # Calculate and display accuracy\n",
    "        accuracy = 100 - np.mean(mape)\n",
    "        rmse = mean_squared_error(test_labels,predictions,squared=True)\n",
    "        \n",
    "        if save == True:\n",
    "            joblib.dump(rf, name+\".joblib\", compress=0) \n",
    "        \n",
    "        rat_folder = glob.glob(data_path)\n",
    "        df = pd.read_hdf(rat_folder[0])[project_name]\n",
    "        df = df.drop('likelihood', axis=1, level=1)\n",
    "        # dictionary to hold feature_name: feature_importance\n",
    "        feats = {} \n",
    "        for feature, importance in zip(df.columns, rf.feature_importances_):\n",
    "            #add the name/value pair \n",
    "            feats[feature] = importance \n",
    "        importances = pd.DataFrame.from_dict(feats, orient='index').rename(columns={0: 'Gini-importance'})\n",
    "        return predictions,errors,mape,accuracy,rmse,importances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "datas = Processing.process_with_labels(\"*/dlc_output_resnet50/*.h5\",\"LW_Manual_scores_for_ICC_2020-05-20.csv\",\"DLC_resnet50_LadderWalkFeb13shuffle1_450000\",\"LadderWalkMar12shuffle1\",rung_it=450000,max_len=356,show_skipped = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split the processing data into data, labels, and video information\n",
    "data_x = datas[0]\n",
    "labels_y = datas[1]\n",
    "videos = datas[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dominant_front_hit</td>\n",
       "      <td>dominant_front_miss</td>\n",
       "      <td>dominant_front_step</td>\n",
       "      <td>nondominant_front_hit</td>\n",
       "      <td>nondominant_front_miss</td>\n",
       "      <td>nondominant_front_step</td>\n",
       "      <td>dominant_back_hit</td>\n",
       "      <td>dominant_back_miss</td>\n",
       "      <td>dominant_back_step</td>\n",
       "      <td>nondominant_back_hit</td>\n",
       "      <td>nondominant_back_miss</td>\n",
       "      <td>nondominant_back_step</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2.33333</td>\n",
       "      <td>1.66667</td>\n",
       "      <td>0</td>\n",
       "      <td>2.33333</td>\n",
       "      <td>1.66667</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.33333</td>\n",
       "      <td>0</td>\n",
       "      <td>2.33333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>7.33333</td>\n",
       "      <td>10.6667</td>\n",
       "      <td>8.33333</td>\n",
       "      <td>4.66667</td>\n",
       "      <td>9.66667</td>\n",
       "      <td>7</td>\n",
       "      <td>4.66667</td>\n",
       "      <td>7.66667</td>\n",
       "      <td>6.33333</td>\n",
       "      <td>2</td>\n",
       "      <td>6.33333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   0                    1                    2   \\\n",
       "0  dominant_front_hit  dominant_front_miss  dominant_front_step   \n",
       "1                   1                    0              2.33333   \n",
       "2                   6              7.33333              10.6667   \n",
       "\n",
       "                      3                       4                       5   \\\n",
       "0  nondominant_front_hit  nondominant_front_miss  nondominant_front_step   \n",
       "1                1.66667                       0                 2.33333   \n",
       "2                8.33333                 4.66667                 9.66667   \n",
       "\n",
       "                  6                   7                   8   \\\n",
       "0  dominant_back_hit  dominant_back_miss  dominant_back_step   \n",
       "1            1.66667                   0                   2   \n",
       "2                  7             4.66667             7.66667   \n",
       "\n",
       "                     9                      10                     11  \n",
       "0  nondominant_back_hit  nondominant_back_miss  nondominant_back_step  \n",
       "1               1.33333                      0                2.33333  \n",
       "2               6.33333                      2                6.33333  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get the range of the labels for each measurement\n",
    "'''limbs = []\n",
    "mins = []\n",
    "maxs = []\n",
    "for i,x in enumerate(labels_y):\n",
    "    ind = i\n",
    "    limb = Processing().keys[i]\n",
    "    minimum = min(x)\n",
    "    maximum = max(x)\n",
    "    limbs.append(limb)\n",
    "    mins.append(minimum)\n",
    "    maxs.append(maximum)\n",
    "pd.DataFrame([limbs,mins,maxs])'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide warnings when desired\n",
    "import warnings\n",
    "\n",
    "#hide warnings function\n",
    "def action_with_warnings():\n",
    "    warnings.warn(\"should not appear\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and save the support vector machines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 dominant_front_hit\n",
      "1 dominant_front_miss\n",
      "2 dominant_front_step\n",
      "3 nondominant_front_hit\n",
      "4 nondominant_front_miss\n",
      "5 nondominant_front_step\n",
      "6 dominant_back_hit\n",
      "7 dominant_back_miss\n",
      "8 dominant_back_step\n",
      "9 nondominant_back_hit\n",
      "10 nondominant_back_miss\n",
      "11 nondominant_back_step\n"
     ]
    }
   ],
   "source": [
    "with warnings.catch_warnings(record=True):\n",
    "    #hide warnings for dividing by 0\n",
    "    action_with_warnings()\n",
    "    for i,k in enumerate(Processing().keys):\n",
    "        print(i,k)\n",
    "        svr_results = Scoring.support_vector(data_x[i],labels_y[i],k+\"_svr\",\"*/dlc_output_resnet50/*.h5\",\"DLC_resnet50_LadderWalkFeb13shuffle1_450000\",kern=\"linear\",save=True)\n",
    "        #svr_results[2].to_csv(k+\"_svr_importances.csv\")\n",
    "        #print(\"accuracy: \", svr_results[0],\" RMSE: \", svr_results[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and save random forests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 dominant_front_hit\n",
      "1 dominant_front_miss\n",
      "2 dominant_front_step\n",
      "3 nondominant_front_hit\n",
      "4 nondominant_front_miss\n",
      "5 nondominant_front_step\n",
      "6 dominant_back_hit\n",
      "7 dominant_back_miss\n",
      "8 dominant_back_step\n",
      "9 nondominant_back_hit\n",
      "10 nondominant_back_miss\n",
      "11 nondominant_back_step\n"
     ]
    }
   ],
   "source": [
    "with warnings.catch_warnings(record=True):\n",
    "    for i,k in enumerate(Processing().keys):\n",
    "        print(i,k)\n",
    "        forest_results = Scoring.forest(data_x[i],labels_y[i],k+\"_random_forest\",\"*/dlc_output_resnet50/*.h5\",\"DLC_resnet50_LadderWalkFeb13shuffle1_450000\",njobs=6,save=True)\n",
    "        #forest_results[5].to_csv(k+\"_rf_importances.csv\")\n",
    "        #print(\"accuracy: \", forest_results[3],\" RMSE: \", forest_results[4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation with KFold Cross-Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.914527913893575, 0.6990113721337498, 0.731303677711002, 0.8060484538644213, 0.9054053665674096] 0.8112593568340316\n"
     ]
    }
   ],
   "source": [
    "'''Only 1 model'''\n",
    "\n",
    "loaded_model = joblib.load(\"dominant_front_hit_svr.joblib\")\n",
    "\n",
    "temp_data = np.array(data_x[0])\n",
    "temp_labels = np.array(labels_y[0])\n",
    "temp_vids = np.array(videos[0])\n",
    "\n",
    "scores = []\n",
    "\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "\n",
    "for train_index,test_index in kf.split(temp_data):\n",
    "    X_train, X_test, y_train, y_test = temp_data[train_index], temp_data[test_index], temp_labels[train_index], temp_labels[test_index]\n",
    "    scores.append(loaded_model.score(X_test, y_test))\n",
    "print(scores,np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 dominant_front_hit\n",
      "Scores:  [0.914527913893575, 0.6990113721337498, 0.731303677711002, 0.8060484538644213, 0.9054053665674096]\n",
      "1 dominant_front_miss\n",
      "Scores:  [0.7558111737967048, 0.9683383506506983, 0.9014736823243106, 0.9311350673382512, 0.8983084858355225]\n",
      "2 dominant_front_step\n",
      "Scores:  [0.9206718646080475, 0.8850080171775608, 0.6790149000770475, 0.8858613885218157, 0.9359420826192857]\n",
      "3 nondominant_front_hit\n",
      "Scores:  [0.8854762792123323, 0.9259004796107195, 0.7447476283760996, 0.9091808409116596, 0.879025728939417]\n",
      "4 nondominant_front_miss\n",
      "Scores:  [0.5487701762259453, 0.6130921226216595, 0.49597976157000134, 0.9448666221450881, 0.6592927566119544]\n",
      "5 nondominant_front_step\n",
      "Scores:  [0.9206815829191874, 0.9305994005089346, 0.8031316270436214, 0.9180540709643148, 0.9298643492871516]\n",
      "6 dominant_back_hit\n",
      "Scores:  [0.8944416801176944, 0.7991984612849072, 0.6699123167949894, 0.9110586040174083, 0.7921049579862843]\n",
      "7 dominant_back_miss\n",
      "Scores:  [0.9571108592570481, 0.9621600559858559, 0.7704759899503036, 0.8608978013903155, 0.764268961776324]\n",
      "8 dominant_back_step\n",
      "Scores:  [0.9008221740779071, 0.9097874190837891, 0.44603333004619794, 0.9259928342899707, 0.8808346171169289]\n",
      "9 nondominant_back_hit\n",
      "Scores:  [0.6655780333593865, 0.8623426357452202, 0.574964325598286, 0.9386639491926043, 0.8163590756015612]\n",
      "10 nondominant_back_miss\n",
      "Scores:  [0.8652402760687015, 0.7513651616391127, 0.48987293327759884, 0.8563320490712435, 0.9203067571130672]\n",
      "11 nondominant_back_step\n",
      "Scores:  [0.7998497727811621, 0.8502579013676477, 0.5674221845296397, 0.9332396618946386, 0.8799625286083892]\n"
     ]
    }
   ],
   "source": [
    "'''KFold validation for all models; SVR'''\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "\n",
    "for i,k in enumerate(Processing().keys):\n",
    "    scores = []\n",
    "    print(i,k)\n",
    "    data = np.array(data_x[i])\n",
    "    labels = np.array(labels_y[i])\n",
    "    vids = np.array(videos[i])\n",
    "    loaded_model = joblib.load(k+\"_svr.joblib\")\n",
    "    for train_index,test_index in kf.split(data):\n",
    "        X_train, X_test, y_train, y_test = data[train_index], data[test_index], labels[train_index], labels[test_index]\n",
    "        scores.append(abs(loaded_model.score(X_test, y_test)))\n",
    "    print(\"Scores: \",scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 dominant_front_hit\n",
      "Scores:  [0.7397889331038934, 0.786734802912775, 0.7848894367629117, 0.7973208656644035, 0.7731755856027422]\n",
      "1 dominant_front_miss\n",
      "Scores:  [0.6881090651558075, 0.9081484354585837, 0.7135435452573908, 0.8197753324163228, 0.8216593059936907]\n",
      "2 dominant_front_step\n",
      "Scores:  [0.8776161127471603, 0.9043544773678702, 0.6894219994102035, 0.8368514784946238, 0.8386187845303867]\n",
      "3 nondominant_front_hit\n",
      "Scores:  [0.8561082632903261, 0.9061222274657857, 0.7285815205509403, 0.9185861742660059, 0.8765194828841953]\n",
      "4 nondominant_front_miss\n",
      "Scores:  [0.46930858054825386, 0.6225891877058178, 0.6908428620928622, 0.8343836552478132, 0.6406083815028901]\n",
      "5 nondominant_front_step\n",
      "Scores:  [0.8837752619760478, 0.9017599885946466, 0.8266992905937993, 0.915281762295082, 0.9235320382868434]\n",
      "6 dominant_back_hit\n",
      "Scores:  [0.7380478046778827, 0.7523338552540014, 0.5846096018249689, 0.8078865944251348, 0.8314729598220494]\n",
      "7 dominant_back_miss\n",
      "Scores:  [0.785, 0.8138977918568899, 0.5822681101309811, 0.7138325022000587, 0.6870762331838565]\n",
      "8 dominant_back_step\n",
      "Scores:  [0.7853630465694986, 0.8711378718088845, 0.5042098241358399, 0.8440369608399546, 0.8565954788816181]\n",
      "9 nondominant_back_hit\n",
      "Scores:  [0.5586425603081289, 0.7919300210859251, 0.5724375347029429, 0.8401868327402133, 0.7917553893532774]\n",
      "10 nondominant_back_miss\n",
      "Scores:  [0.7192414004914005, 0.7453156565656566, 0.6283328844600053, 0.7422734627831715, 0.8705853370122192]\n",
      "11 nondominant_back_step\n",
      "Scores:  [0.8021371156547525, 0.8121731986368063, 0.7208009321722064, 0.8496311371152379, 0.7834596948150516]\n"
     ]
    }
   ],
   "source": [
    "'''KFold validation for all models; Random forest'''\n",
    "\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "\n",
    "for i,k in enumerate(Processing().keys):\n",
    "    scores = []\n",
    "    print(i,k)\n",
    "    data = np.array(data_x[i])\n",
    "    labels = np.array(labels_y[i])\n",
    "    vids = np.array(videos[i])\n",
    "    loaded_model = joblib.load(k+\"_random_forest.joblib\")\n",
    "    for train_index,test_index in kf.split(data):\n",
    "        X_train, X_test, y_train, y_test = data[train_index], data[test_index], labels[train_index], labels[test_index]\n",
    "        scores.append(abs(loaded_model.score(X_test, y_test)))\n",
    "    print(\"Scores: \",scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example of using loaded model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7524629831750013\n"
     ]
    }
   ],
   "source": [
    "loaded_model = joblib.load(\"dominant_front_hit_svr.joblib\")\n",
    "temp_data = np.array(data_x[0])\n",
    "temp_labels = np.array(labels_y[0])\n",
    "temp_vids = np.array(videos[0])\n",
    "\n",
    "\n",
    "train_features, test_features, train_labels, test_labels = train_test_split(temp_data, temp_labels, test_size = 0.5, random_state = 1)\n",
    "\n",
    "result = loaded_model.score(test_features, test_labels)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example of use\n",
    "\n",
    "I imagine that this tool can be used to score videos and quickly output a dataframe with the video information and scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Subject        Date Crossing Run  predictions\n",
      "0     MC70  2019-04-09        7  L4     4.899937\n",
      "1     MC30  2019-12-04        3  R2     4.353295\n",
      "2     MC87  2018-12-10        3  R2     3.433569\n",
      "3     MC78  2019-03-28        7  R4     5.233646\n",
      "4     MC61  2019-05-22        7  R4     4.100091\n",
      "5     MC78  2019-04-23        2  L1     3.433569\n",
      "6     MC61  2019-05-21        5  R3     3.182038\n",
      "7     MC30  2019-12-04        7  R4     3.433464\n",
      "8     MC87  2018-12-12        1  R1     4.568946\n",
      "9     MC78  2019-04-22        6  L3     3.899816\n",
      "10    MC30  2019-11-06        6  L3     3.099924\n",
      "11    MC70  2019-03-14        1  R1     3.726849\n",
      "12    MC87  2018-12-12        7  R4     3.791385\n",
      "13    MC30  2019-11-04        7  R4     4.899755\n",
      "14    MC78  2019-03-21        6  L3     3.371577\n",
      "15    MC70  2019-03-14        2  L1     3.566815\n",
      "16    MC61  2019-05-28        1  R1     3.099924\n",
      "17    MC87  2018-12-14        4  L2     2.433230\n",
      "18    MC30  2019-12-03        4  L2     4.766538\n",
      "19    MC30  2019-11-06        8  L4     3.566627\n",
      "20    MC78  2019-04-25        7  R4     3.233246\n",
      "21    MC61  2019-07-02        8  L4     2.433477\n",
      "22    MC61  2019-07-01        3  R2     3.168476\n"
     ]
    }
   ],
   "source": [
    "'''Example of using the code to predict the results of a random sample of the dataset without the corresponding labels'''\n",
    "import random\n",
    "\n",
    "#process data without labels\n",
    "datas = Processing.process_data(\"*/dlc_output_resnet50/*.h5\",\"DLC_resnet50_LadderWalkFeb13shuffle1_450000\",\"LadderWalkMar12shuffle1\",rung_it=450000,max_len=356,show_skipped = False)\n",
    "\n",
    "data_x = datas[0]\n",
    "videos = datas[1]\n",
    "\n",
    "temp_data = np.array(data_x)\n",
    "temp_vids = np.array(videos)\n",
    "\n",
    "\n",
    "#get a random sample of the dataset\n",
    "random.seed(23)\n",
    "random.shuffle(temp_data)\n",
    "indexes = []\n",
    "values = []\n",
    "#just take 1/10th of the dataset\n",
    "for idx, val in random.sample(list(enumerate(temp_data)), (len(temp_data)//10)):\n",
    "    indexes.append(idx)\n",
    "    values.append(val)\n",
    "\n",
    "#get video info\n",
    "vids = temp_vids[indexes]\n",
    "\n",
    "#load model\n",
    "loaded_model = joblib.load(\"dominant_front_hit_svr.joblib\")\n",
    "pred_labels = loaded_model.predict(values)\n",
    "\n",
    "ex_df = pd.DataFrame(np.concatenate(vids),columns=[\"Subject\",\"Date\",\"Crossing\",\"Run\"])\n",
    "ex_df[\"predictions\"] = pred_labels\n",
    "print(ex_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}