random_forest_classification/koos_classification_by_random_forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "00ed976a-9a99-4ea0-a371-b3dc654c9ae6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "from natsort import natsorted\n",
    "import os\n",
    "import pandas as pd\n",
    "import re\n",
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix, f1_score, classification_report\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "random_state=42\n",
    "\n",
    "pd.set_option('display.max_rows', 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1f77b696-87ac-402a-82cc-4b643a85cfe2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set path to training set features\n",
    "train_set_path = os.path.join(\"features\", \"extracted_from_GIF_output\")\n",
    "\n",
    "# set paths to test sets \n",
    "# for each of the test sets, training and inference will be performed\n",
    "# if there is an overlap between a test set and the training set, \n",
    "# the corresponding cases will be removed from the training set\n",
    "test_set_paths = [os.path.join(\"features\", \"extracted_from_nnUNet_prediction\", \"T1\"),\n",
    "                 os.path.join(\"features\", \"extracted_from_nnUNet_prediction\", \"T2\"),\n",
    "                 os.path.join(\"features\", \"extracted_from_nnUNet_prediction\", \"T1T2\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "09e594cb-4728-4282-9289-993ffed6a5cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to process the training and test sets\n",
    "def process_set(csv_dfs, train_feature_df_mean=None):\n",
    "    # get pairs of right/left labels (check if \"Right\" or \"Left\" in label name, and find the corresponding label)\n",
    "    LRpairs = []\n",
    "    for caseID in list(csv_dfs.keys())[0:]:\n",
    "        curr_csv_df = csv_dfs[caseID]\n",
    "        label_names = curr_csv_df[\"name\"].values\n",
    "        label_numbers = curr_csv_df[\"labelnumber\"].values\n",
    "\n",
    "        for name, n in zip(label_names, label_numbers):\n",
    "            if \"Right\" in name:\n",
    "                corresponding_left_name = name.replace(\"Right\", \"Left\")\n",
    "                try:\n",
    "                    corresponding_left_labelnumber = curr_csv_df[curr_csv_df[\"name\"]==corresponding_left_name][\"labelnumber\"].values[0]\n",
    "                    LRpairs.append([n, corresponding_left_labelnumber])\n",
    "                except:\n",
    "                    print(f\"Error: No corresponding left parcel found for caseID {caseID}, labelnumber {n}, name {name}\")\n",
    "\n",
    "            elif \"Left\" in name:\n",
    "                corresponding_right_name = name.replace(\"Left\", \"Right\")\n",
    "                try:\n",
    "                    corresponding_right_labelnumber = curr_csv_df[curr_csv_df[\"name\"]==corresponding_right_name][\"labelnumber\"].values[0]\n",
    "                    LRpairs.append([corresponding_right_labelnumber, n])\n",
    "                except:\n",
    "                    print(f\"Error: No corresponding right parcel found for caseID {caseID}, labelnumber {n}, name {name}\")\n",
    "\n",
    "    LRpairs = sorted([y for y in set([tuple(x) for x in LRpairs])])\n",
    "    RtoLdict = {r:l for (r,l) in LRpairs}\n",
    "    LtoRdict = {l:r for (r,l) in LRpairs}\n",
    "\n",
    "    # relabel left/right structures with ipsi/contra lateral labels\n",
    "    # ipsilateral labels will be labelled with the up-to-here \"right\" label, contralateral with the up-to-here \"left\" label\n",
    "    \n",
    "    # load left-right list (required to convert right/left to ipsi/contra-lateral labels)\n",
    "    LR_df = pd.read_csv(os.path.join(\"..\", \"TumorLR.csv\"))\n",
    "    csv_dfs_ipscon = {}\n",
    "\n",
    "    for caseID in tqdm(list(csv_dfs.keys())[0:]):\n",
    "        curr_csv_df = csv_dfs[caseID]\n",
    "        curr_csv_df = curr_csv_df.set_index(\"labelnumber\")\n",
    "        label_names = curr_csv_df[\"name\"].values\n",
    "        label_numbers = curr_csv_df.index.values\n",
    "\n",
    "        tumorside = LR_df[LR_df[\"patient_ID\"] == caseID][\"TumorLR\"].values[0]\n",
    "        ipscon_dict = {}\n",
    "        for i, [ln, n] in enumerate(zip(label_names, label_numbers)):\n",
    "            if \"Left\" in ln and tumorside==\"r\":\n",
    "                new_name = ln.replace(\"Left\", \"Contralateral\")\n",
    "                new_labelnumber = n\n",
    "                pass # don't do anything, since left labels are contralateral already\n",
    "            elif \"Right\" in ln and tumorside==\"r\":\n",
    "                new_name = ln.replace(\"Right\", \"Ipsilateral\")\n",
    "                new_labelnumber = n\n",
    "                pass # don't do anything, since right labels are ipsilateral already\n",
    "            elif \"Left\" in ln and tumorside==\"l\":\n",
    "                # label should switch to the old right label, since it is ipsilateral\n",
    "                new_name = ln.replace(\"Left\", \"Ipsilateral\")\n",
    "                new_labelnumber = LtoRdict[n]\n",
    "            elif \"Right\" in ln and tumorside==\"l\":\n",
    "                # label should switch to the old left label since it is contralateral\n",
    "                new_name = ln.replace(\"Right\", \"Contralateral\")\n",
    "                new_labelnumber = RtoLdict[n]\n",
    "            else:\n",
    "                new_name = ln\n",
    "                new_labelnumber = n\n",
    "\n",
    "            # add new row to ipscon_dict\n",
    "            ipscon_dict_row = {}\n",
    "\n",
    "            ipscon_dict_row[\"labelnumber\"] = new_labelnumber\n",
    "            ipscon_dict_row[\"name\"] = new_name\n",
    "            ipscon_dict_row[\"vol\"] = curr_csv_df.loc[n][\"vol\"]\n",
    "            ipscon_dict_row[\"surfVS\"] = curr_csv_df.loc[n][\"surfVS\"]\n",
    "            ipscon_dict_row[\"distVS\"] = curr_csv_df.loc[n][\"distVS\"]\n",
    "            ipscon_dict_row[\"set\"] = curr_csv_df.loc[n][\"set\"]\n",
    "\n",
    "            ipscon_dict[i] = ipscon_dict_row\n",
    "\n",
    "        # this dataframe is the same as csv_dfs, but with right/left labels replaced by ipsi/contra-lateral labels\n",
    "        csv_dfs_ipscon[caseID] = pd.DataFrame.from_dict(ipscon_dict, orient='index')\n",
    "        \n",
    "        \n",
    "    # read the ground truth Koos grades into a new dataframe\n",
    "    df = pd.read_csv(os.path.join(\"..\", \"Koos_grades_groundtruth.csv\"))\n",
    "    df = df.rename(columns = {'final groundtruth':'Koos'})\n",
    "    df_with_koos = df[df[\"Koos\"].isin([1, 2, 3, 4, \"1\", \"2\", \"3\", \"4\"])].copy()\n",
    "\n",
    "    df_with_koos['patient_ID'] = df_with_koos['patient_ID'].apply(lambda x: int(re.findall(r\"vs_gk_([\\d]+)\", x)[0]))\n",
    "    df_with_koos[\"Koos\"] = df_with_koos[\"Koos\"].astype(int)\n",
    "    df_with_koos[\"patient_ID\"] = df_with_koos[\"patient_ID\"].astype(int)\n",
    "\n",
    "    caseIDs = df_with_koos[\"patient_ID\"].values\n",
    "\n",
    "    # make sure no post-operative cases and no repeated cases (ID>=400) are included\n",
    "    post_op_caseIDs = [11, 17, 26, 27, 41, 45, 49, 74, 75, 92, 96, 98, 100, 108, 114, 122, 128, 129, 135, 147, 150, 164, 171, 193, 204, 214, 218, 223, 225, 238, 246, 248, 249, 250, 259, 269, 275, 280, 283, 290, 297, 300, 340, 353, 354, 356, 375, 377, 380, 391, 2, 7, 47, 62, 80, 85, 89, 109, 127, 131, 134, 152, 161, 178, 196, 203, 209, 213, 231, 247, 253, 359, 360, 379]\n",
    "    caseIDs = [c for c in caseIDs if c not in post_op_caseIDs and c < 400 and c in csv_dfs]\n",
    "\n",
    "    koos = {caseID: df_with_koos[df_with_koos[\"patient_ID\"] == caseID][\"Koos\"].values[0] for caseID in caseIDs}\n",
    "    print(f\"Found {len(koos)} cases with Koos grades.\")\n",
    "\n",
    "    # select features for random forest training\n",
    "    featureIDs = \\\n",
    "    [(4, 'distVS'), # cerebellar vermal lobules \n",
    "     (7, 'surfVS'), # ipsilateral cerebellum\n",
    "     (1, 'vol'), # tumour\n",
    "     (6, 'distVS'), # cerebellar vermal lobules \n",
    "     (5, 'distVS'), # cerebellar vermal lobules \n",
    "     (2, 'distVS'), # Pons\n",
    "     (8, 'distVS'), # contralateral cerebellum\n",
    "     (0, 'surfVS'), # background\n",
    "     (3, 'distVS'), # Brainstem\n",
    "     (7, 'distVS'), # ipsilateral cerebellum\n",
    "    ]\n",
    "    featureIDs\n",
    "\n",
    "    # create feature dataframe of selected features only\n",
    "    feature_df = pd.DataFrame()\n",
    "    \n",
    "    for (n,t) in featureIDs:\n",
    "        feature_vals = []\n",
    "        for caseID in caseIDs:\n",
    "            try:\n",
    "                feature_vals.append(csv_dfs_ipscon[caseID][csv_dfs_ipscon[caseID][\"labelnumber\"] == n][t].values[0])\n",
    "            except:\n",
    "                # decide how to deal with missing values (can happen when a structure was not in the FOV)\n",
    "                print(\"no value available:\", f\"{caseID=}\", f\"{n=}\", f\"{t=}\")\n",
    "                if t == \"distVS\":\n",
    "                    feature_vals.append(\"fill mean\")  # if distance has to be guessed, assume mean of all other corresponding values of other patients in training set\n",
    "                elif t == \"vol\":\n",
    "                    feature_vals.append(\"fill mean\")  # if volume has to be guessed, assume mean of all other corresponding values of other patients in training set\n",
    "                elif t == \"surfVS\":\n",
    "                    feature_vals.append(0)  # contact surface is most likely 0 if the structure is not it the FOV\n",
    "                else:\n",
    "                    raise Exception(\"No rule defined for this missing feature.\")\n",
    "        \n",
    "        if any(feature_vals):  # makes sure this feature is ignored if it's zero for all cases\n",
    "            col_name = str(n)+\",\"+t\n",
    "            feature_df[col_name] = feature_vals\n",
    "\n",
    "    # replace \"fill mean\" either with mean of current set (training) or passed mean (test set)\n",
    "    assert(not feature_df.isnull().values.any()), \"dataframe has nans\"\n",
    "    feature_df = feature_df.replace('fill mean',np.NaN)  # replace \"fill mean\" with nan\n",
    "    if train_feature_df_mean is None:  # if no means have been passed (training set) \n",
    "        feature_df_mean = feature_df.mean()\n",
    "        feature_df=feature_df.fillna(feature_df_mean)  # replace nan with mean of column\n",
    "    else: # if means have been passed \n",
    "        feature_df_mean = feature_df.mean()\n",
    "        feature_df=feature_df.fillna(train_feature_df_mean)  # replace nan with passed means\n",
    "    \n",
    "    # replace linear dataframe index with caseID\n",
    "    feature_df.set_index(np.array(caseIDs), inplace=True)\n",
    "    feature_df.index.name = \"caseID\"\n",
    "    \n",
    "    data = feature_df.to_numpy()\n",
    "    target = np.array([koos[caseID] for caseID in koos])\n",
    "    \n",
    "    return data, target, feature_df_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "91791928-5f83-4513-b6ca-d9a74c6cb828",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "features/extracted_from_nnUNet_prediction/T1\n",
      "load training set...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8081e84291ee49248e4e8cf1fba53d5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2 cases with Koos grades.\n",
      "load test set...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3fabd65aef045f19155ff3802246b14",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1 cases with Koos grades.\n",
      "calculate metrics...\n",
      "confusion matrix:\n",
      "[[0 1]\n",
      " [0 0]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           3       0.00      0.00      0.00       1.0\n",
      "           4       0.00      0.00      0.00       0.0\n",
      "\n",
      "    accuracy                           0.00       1.0\n",
      "   macro avg       0.00      0.00      0.00       1.0\n",
      "weighted avg       0.00      0.00      0.00       1.0\n",
      "\n",
      "----------------------------------------------------------------\n",
      "features/extracted_from_nnUNet_prediction/T2\n",
      "load training set...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1394e067e5a94b2cb134dc923a930006",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2 cases with Koos grades.\n",
      "load test set...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed2dde37e8034898b4f8819593486eef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1 cases with Koos grades.\n",
      "calculate metrics...\n",
      "confusion matrix:\n",
      "[[0 1]\n",
      " [0 0]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           3       0.00      0.00      0.00       1.0\n",
      "           4       0.00      0.00      0.00       0.0\n",
      "\n",
      "    accuracy                           0.00       1.0\n",
      "   macro avg       0.00      0.00      0.00       1.0\n",
      "weighted avg       0.00      0.00      0.00       1.0\n",
      "\n",
      "----------------------------------------------------------------\n",
      "features/extracted_from_nnUNet_prediction/T1T2\n",
      "load training set...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40c8b6f305d74d36a85b027e6c82d06b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2 cases with Koos grades.\n",
      "load test set...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "334cac639c2f4f6eb2338cfe10efa8c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1 cases with Koos grades.\n",
      "calculate metrics...\n",
      "confusion matrix:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1]\n",
      " [0 0]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           3       0.00      0.00      0.00       1.0\n",
      "           4       0.00      0.00      0.00       0.0\n",
      "\n",
      "    accuracy                           0.00       1.0\n",
      "   macro avg       0.00      0.00      0.00       1.0\n",
      "weighted avg       0.00      0.00      0.00       1.0\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/aaron/.local/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "all_results = {}\n",
    "for idx, test_set_path in enumerate(test_set_paths):\n",
    "    print(\"----------------------------------------------------------------\")\n",
    "    print(test_set_path)\n",
    "    \n",
    "    print(\"load training set...\")\n",
    "    # load extracted GIF features from origial GIF output (with manual VS segmentation)\n",
    "    csv_files = natsorted(glob(os.path.join(train_set_path, \"*.csv\")))\n",
    "\n",
    "    csv_dfs = {}  # save loaded dataframes in dictionary\n",
    "    for f in csv_files:\n",
    "        caseID = int(re.findall(r\"vs_gk_(\\d+)\", f)[0])\n",
    "        csv_df = pd.read_csv(f)\n",
    "        csv_df['set'] = 'train'\n",
    "        csv_dfs[caseID] = csv_df\n",
    "\n",
    "    # remove csv entries of the current test set from the training set\n",
    "    csv_files_test = natsorted(glob(os.path.join(test_set_path, \"*.csv\")))\n",
    "\n",
    "    for f in csv_files_test:\n",
    "        caseID = int(re.findall(r\"vs_gk_(\\d+)\", f)[0])\n",
    "        del csv_dfs[caseID]\n",
    "    \n",
    "    # process the training set\n",
    "    X_train, y_train_groundtruth, train_feature_df_mean = process_set(csv_dfs)\n",
    "\n",
    "    # train random forest\n",
    "    clf = RandomForestClassifier(n_estimators=100000, max_depth=5, bootstrap=True, min_samples_leaf=2)\n",
    "    clf.fit(X_train,y_train_groundtruth)\n",
    "    \n",
    "    print(\"load test set...\")\n",
    "    # load test set\n",
    "    csv_test_dfs = {}\n",
    "    for f in csv_files_test:\n",
    "        caseID = int(re.findall(r\"vs_gk_(\\d+)\", f)[0])\n",
    "        csv_df = pd.read_csv(f)\n",
    "        csv_df['set'] = 'test'\n",
    "        csv_test_dfs[caseID] = csv_df\n",
    "    \n",
    "    # make sure no overlap between training and test set\n",
    "    assert([c for c in csv_test_dfs if c in csv_dfs] == []), \"Error: found training cases in test set\"\n",
    "    \n",
    "    X_test, y_test_groundtruth, test_feature_df_mean = process_set(csv_test_dfs, train_feature_df_mean=train_feature_df_mean)\n",
    "\n",
    "    # predict with trained random forest\n",
    "    y_pred=clf.predict(X_test)\n",
    "\n",
    "    print(\"calculate metrics...\")\n",
    "    # print confusion matrices and classification report\n",
    "    all_results[idx] = (y_test_groundtruth, y_pred)\n",
    "    print(\"confusion matrix:\")\n",
    "    confusion_mat = confusion_matrix(y_test_groundtruth, y_pred)\n",
    "    class_report = classification_report(y_test_groundtruth, y_pred, )\n",
    "    print(confusion_mat)\n",
    "    print(class_report)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}