notebooks/6_ChemStructure_MLP_XGB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0a58cf53",
   "metadata": {},
   "outputs": [],
   "source": [
    "import rdkit\n",
    "import os\n",
    "import copy\n",
    "import numpy as np\n",
    "from rdkit import Chem, DataStructs\n",
    "from rdkit.Chem import AllChem\n",
    "import pandas as pd\n",
    "from sklearn.utils import class_weight\n",
    "from modules.utils import prepare_data\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "from sklearn.metrics import classification_report\n",
    "from modules.viz import conf_mat\n",
    "from modules.utils import load_cv\n",
    "from modules.model_utils import EarlyStopper\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, precision_score, recall_score, average_precision_score\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "import itertools\n",
    "\n",
    "rand_seed = 42\n",
    "moa_dict = {'PI3K' : 0, 'p38 MAPK': 1, 'RAF': 2, 'AURK': 3, 'CDK': 4, 'EGFR': 5, 'ROCK': 6,\n",
    "             'MEK': 7, 'GSK': 8, 'mTOR': 9}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "443de664",
   "metadata": {},
   "source": [
    "Following source code contained here:\n",
    "\n",
    "    https://github.com/pharmbio/CP-Chem-MoA/blob/main/Compound_structure_based_models/MLP.ipynb\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5d87378",
   "metadata": {},
   "source": [
    "# Static Variables:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4bc9c6f6",
   "metadata": {},
   "source": [
    "# Load Data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b5018dad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(635, 4778)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Metadata_Source</th>\n",
       "      <th>Metadata_Plate</th>\n",
       "      <th>Metadata_Well</th>\n",
       "      <th>Cells_AreaShape_Area</th>\n",
       "      <th>Cells_AreaShape_BoundingBoxArea</th>\n",
       "      <th>Cells_AreaShape_BoundingBoxMaximum_X</th>\n",
       "      <th>Cells_AreaShape_BoundingBoxMaximum_Y</th>\n",
       "      <th>Cells_AreaShape_BoundingBoxMinimum_X</th>\n",
       "      <th>Cells_AreaShape_BoundingBoxMinimum_Y</th>\n",
       "      <th>Cells_AreaShape_Center_X</th>\n",
       "      <th>...</th>\n",
       "      <th>smiles</th>\n",
       "      <th>clinical_phase</th>\n",
       "      <th>moa_src</th>\n",
       "      <th>Metadata_JCP2022</th>\n",
       "      <th>Metadata_InChIKey</th>\n",
       "      <th>Metadata_PlateType</th>\n",
       "      <th>blur_score</th>\n",
       "      <th>sat_score</th>\n",
       "      <th>focus_score</th>\n",
       "      <th>comp_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>source_3</td>\n",
       "      <td>JCPQC023</td>\n",
       "      <td>G14</td>\n",
       "      <td>3227.817708</td>\n",
       "      <td>5310.328125</td>\n",
       "      <td>589.21875</td>\n",
       "      <td>541.552083</td>\n",
       "      <td>519.484375</td>\n",
       "      <td>471.942708</td>\n",
       "      <td>553.446757</td>\n",
       "      <td>...</td>\n",
       "      <td>Nc1cc(c(cn1)-c1cc(nc(n1)N1CCOCC1)N1CCOCC1)C(F)...</td>\n",
       "      <td>Phase 3</td>\n",
       "      <td>dr_hub</td>\n",
       "      <td>JCP2022_013856</td>\n",
       "      <td>CWHUFRVAEUJCEF-UHFFFAOYSA-N</td>\n",
       "      <td>TARGET2</td>\n",
       "      <td>0.430742</td>\n",
       "      <td>0.453621</td>\n",
       "      <td>0.517562</td>\n",
       "      <td>1.401925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>source_4</td>\n",
       "      <td>BR00121424</td>\n",
       "      <td>G14</td>\n",
       "      <td>4255.300000</td>\n",
       "      <td>7338.300000</td>\n",
       "      <td>572.70000</td>\n",
       "      <td>554.500000</td>\n",
       "      <td>488.420000</td>\n",
       "      <td>470.490000</td>\n",
       "      <td>530.260000</td>\n",
       "      <td>...</td>\n",
       "      <td>Nc1cc(c(cn1)-c1cc(nc(n1)N1CCOCC1)N1CCOCC1)C(F)...</td>\n",
       "      <td>Phase 3</td>\n",
       "      <td>dr_hub</td>\n",
       "      <td>JCP2022_013856</td>\n",
       "      <td>CWHUFRVAEUJCEF-UHFFFAOYSA-N</td>\n",
       "      <td>TARGET2</td>\n",
       "      <td>0.436727</td>\n",
       "      <td>0.144924</td>\n",
       "      <td>0.386009</td>\n",
       "      <td>0.967661</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 4778 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Metadata_Source Metadata_Plate Metadata_Well  Cells_AreaShape_Area  \\\n",
       "0        source_3       JCPQC023           G14           3227.817708   \n",
       "1        source_4     BR00121424           G14           4255.300000   \n",
       "\n",
       "   Cells_AreaShape_BoundingBoxArea  Cells_AreaShape_BoundingBoxMaximum_X  \\\n",
       "0                      5310.328125                             589.21875   \n",
       "1                      7338.300000                             572.70000   \n",
       "\n",
       "   Cells_AreaShape_BoundingBoxMaximum_Y  Cells_AreaShape_BoundingBoxMinimum_X  \\\n",
       "0                            541.552083                            519.484375   \n",
       "1                            554.500000                            488.420000   \n",
       "\n",
       "   Cells_AreaShape_BoundingBoxMinimum_Y  Cells_AreaShape_Center_X  ...  \\\n",
       "0                            471.942708                553.446757  ...   \n",
       "1                            470.490000                530.260000  ...   \n",
       "\n",
       "                                              smiles  clinical_phase  moa_src  \\\n",
       "0  Nc1cc(c(cn1)-c1cc(nc(n1)N1CCOCC1)N1CCOCC1)C(F)...         Phase 3   dr_hub   \n",
       "1  Nc1cc(c(cn1)-c1cc(nc(n1)N1CCOCC1)N1CCOCC1)C(F)...         Phase 3   dr_hub   \n",
       "\n",
       "   Metadata_JCP2022            Metadata_InChIKey  Metadata_PlateType  \\\n",
       "0    JCP2022_013856  CWHUFRVAEUJCEF-UHFFFAOYSA-N             TARGET2   \n",
       "1    JCP2022_013856  CWHUFRVAEUJCEF-UHFFFAOYSA-N             TARGET2   \n",
       "\n",
       "   blur_score  sat_score  focus_score  comp_score  \n",
       "0    0.430742   0.453621     0.517562    1.401925  \n",
       "1    0.436727   0.144924     0.386009    0.967661  \n",
       "\n",
       "[2 rows x 4778 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ki_ibp = pd.read_csv('data/ibp/ki_ibp.csv')\n",
    "print(ki_ibp.shape)\n",
    "ki_ibp.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a25b3a6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load CV split data:\n",
    "folds = 5\n",
    "cv_path = 'data/cv_val_split/'\n",
    "cv_data = load_cv(cv_path, folds, ki_ibp, moa_dict, norm=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5cb1d09",
   "metadata": {},
   "source": [
    "# Convert to Fingerprints:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bf33ffa",
   "metadata": {},
   "source": [
    "## Morgan Fingerprints:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "52772a7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# change SMILES to Morgan Fingerprints \n",
    "def morgan_fprints(smiles):\n",
    "    molecules = Chem.MolFromSmiles(smiles) \n",
    "    fingerprints = AllChem.GetMorganFingerprintAsBitVect(molecules, 2)\n",
    "    x_array = []\n",
    "    arrays = np.zeros(0,)\n",
    "    DataStructs.ConvertToNumpyArray(fingerprints, arrays)\n",
    "    x_array.append(arrays)\n",
    "    x_array = np.asarray(x_array)\n",
    "    x_array = ((np.squeeze(x_array)).astype(int))\n",
    "    \n",
    "    return x_array"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da34e6bd",
   "metadata": {},
   "source": [
    "## Convert Fingerprints:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "66abfc6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_fprints(smiles, fprint_type):\n",
    "    # Convert to cannonical smiles\n",
    "    can_smiles = [Chem.MolToSmiles(Chem.MolFromSmiles(smi), True) for smi in smiles]\n",
    "    \n",
    "    # Convert canonical smiles to fingerprints:\n",
    "    fprints = np.zeros((len(can_smiles), 2048), dtype = np.float32)\n",
    "    \n",
    "    for f in range(fprints.shape[0]):\n",
    "        fprints[f] = morgan_fprints(can_smiles[f])\n",
    "\n",
    "    return torch.tensor(fprints)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99a40698",
   "metadata": {},
   "source": [
    "# Create Class Weights:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fec878a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_class_weights(y_train):\n",
    "    # List of unique y_values:\n",
    "    y_unique = np.unique(np.array(y_train))\n",
    "\n",
    "    # Computing class weights based on data:\n",
    "    class_weights = class_weight.compute_class_weight(class_weight = 'balanced', classes = y_unique,\n",
    "                    y = np.array(y_train)) \n",
    "\n",
    "    # Create a dictionary of weights:\n",
    "    class_weights_dict = dict(enumerate(class_weights))\n",
    "    \n",
    "    return class_weights_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9ec0bbe",
   "metadata": {},
   "source": [
    "# Create Model:\n",
    "Adapted the code to pytorch from its original Tensorflow format.\n",
    "\n",
    "Source: https://github.com/pharmbio/CP-Chem-MoA/blob/main/Compound_structure_based_models/MLP.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1ce785d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MlpModel(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size, dropout):\n",
    "        super(MlpModel, self).__init__()\n",
    "        self.layer1 = nn.Linear(input_size, hidden_size)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.layer2 = nn.Linear(hidden_size, output_size)\n",
    "        self.softmax = nn.Softmax(dim=1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = self.layer1(x)\n",
    "        out = self.relu(out)\n",
    "        out = self.dropout(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.softmax(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "021579d6",
   "metadata": {},
   "source": [
    "# Model Training:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5b8c2d0",
   "metadata": {},
   "source": [
    "### Metric Function(s):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3e1d1558",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the metrics (accuracy)\n",
    "def calc_acc(output, target):\n",
    "    _, predicted = torch.max(output, 1)\n",
    "    correct = (predicted == target).sum().item()\n",
    "    total = target.size(0)\n",
    "    return (correct / total)*100"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5eb69c79",
   "metadata": {},
   "source": [
    "### Training Loop:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e345fd0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_mlp(model, train_loader, valid_loader, criterion, optimizer, num_epochs, \n",
    "          checkpoint_mlp, reduce_lr_loss, early_stopper, verbose, criteria):\n",
    "    # Setting initial minimum val. loss to infinity:\n",
    "    min_val_loss = np.Inf\n",
    "    max_val_acc = 0\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        train_loss_epoch = 0\n",
    "        train_acc_epoch = 0\n",
    "        val_loss_epoch = 0\n",
    "        val_acc_epoch = 0\n",
    "        \n",
    "        # ---- Set model to TRAINING mode ---- #\n",
    "        model.train()\n",
    "        \n",
    "        for inputs, targets in train_loader:\n",
    "            optimizer.zero_grad()\n",
    "            y_pred_batch = model(inputs)\n",
    "            train_batch_loss = criterion(y_pred_batch, targets.to(torch.int64))\n",
    "            train_batch_acc = calc_acc(y_pred_batch, targets.to(torch.int64))\n",
    "            train_batch_loss.backward()\n",
    "            optimizer.step()\n",
    "            train_loss_epoch += train_batch_loss.item()\n",
    "            train_acc_epoch += train_batch_acc\n",
    "        \n",
    "        model.eval()\n",
    "        valid_loss = 0.0\n",
    "        valid_acc = 0.0\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            for inputs, targets in valid_loader:\n",
    "                val_pred_batch = model(inputs)\n",
    "                val_batch_loss = criterion(val_pred_batch, targets.to(torch.int64))\n",
    "                val_batch_acc = calc_acc(val_pred_batch, targets.to(torch.int64))\n",
    "                \n",
    "                val_loss_epoch += val_batch_loss.item()\n",
    "                val_acc_epoch += val_batch_acc\n",
    "        \n",
    "        # ---------------------\n",
    "        #  Log Results\n",
    "        # ---------------------\n",
    "        avg_train_loss = train_loss_epoch / len(train_loader)\n",
    "        avg_train_acc = train_acc_epoch / len(train_loader)\n",
    "        avg_val_loss = val_loss_epoch / len(valid_loader)\n",
    "        avg_val_acc = val_acc_epoch / len(valid_loader)\n",
    "        \n",
    "        if verbose == 1:\n",
    "            print(f'Epoch {epoch+1}/{num_epochs} | Train Loss: {avg_train_loss:.4f} |'\n",
    "                  f'Train Acc: {avg_train_acc:.2f} | Valid Loss: {avg_val_loss:.4f} | '\n",
    "                  f'Valid Acc: {avg_val_acc:.2f}')\n",
    "\n",
    "        # Check if the current validation loss/accuracy is the best so far\n",
    "        if criteria == 'loss':\n",
    "            if avg_val_loss < min_val_loss:\n",
    "                min_val_loss = avg_val_loss\n",
    "                torch.save(model.state_dict(), checkpoint_mlp['file_path'])\n",
    "        elif criteria == 'acc':\n",
    "            if avg_val_acc > max_val_acc:\n",
    "                max_val_acc = avg_val_acc\n",
    "                torch.save(model.state_dict(), checkpoint_mlp['file_path'])\n",
    "\n",
    "        # Adjust learning rate using the learning rate scheduler\n",
    "        reduce_lr_loss.step(avg_val_loss)\n",
    "        \n",
    "        # Break loop if \n",
    "        if early_stopper.early_stop(avg_val_loss):\n",
    "            print(\"Training stopped early due to early stopper.\")\n",
    "            break\n",
    "    \n",
    "    if criteria == 'loss':\n",
    "        print('Training completed...\\nLowest Val Loss: %s.' % min_val_loss)\n",
    "        return min_val_loss\n",
    "    elif criteria == 'acc':\n",
    "        print('Training completed...\\nHighest Val Acc.: %s.' % max_val_acc)\n",
    "        return max_val_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffd1ee0c",
   "metadata": {},
   "source": [
    "# Create Dataset and Dataloaders:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b63f402c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDataset(Dataset):\n",
    "    def __init__(self, data, labels):\n",
    "        self.data = data\n",
    "        self.labels = labels\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        x = self.data[index]\n",
    "        y = self.labels[index]\n",
    "        return x, y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71f91ffd",
   "metadata": {},
   "source": [
    "# Train Model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "493d68ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "def summary_stats(df):\n",
    "    \"\"\"\n",
    "    Return performance stats for the model\n",
    "    \"\"\"\n",
    "    if 'actual' in df.columns:\n",
    "        df.rename(columns={'actual': 'actuals'}, inplace=True)\n",
    "        \n",
    "    acc = accuracy_score(df.actuals, df.preds)\n",
    "    f1 = f1_score(df.actuals, df.preds, average='macro')\n",
    "    precision = precision_score(df.actuals, df.preds, average='macro')\n",
    "    recall = recall_score(df.actuals, df.preds, average='macro')\n",
    "    \n",
    "    return {'acc': acc, 'f1': f1, 'precision': precision, 'recall': recall}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d57b5e8",
   "metadata": {},
   "source": [
    "## Static Variables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "d2777635",
   "metadata": {},
   "outputs": [],
   "source": [
    "run_dir = 'output/Struct_MLP/'\n",
    "if not os.path.exists(run_dir):\n",
    "        os.makedirs(run_dir)\n",
    "\n",
    "class Args:\n",
    "    def __init__(self):\n",
    "        self.num_classes = 10\n",
    "        self.num_epochs = 1000\n",
    "        self.batch_size = 8\n",
    "        self.input_size = 2048   # length of morgan fingerprints, don't change\n",
    "        self.hidden_size = 64\n",
    "        self.lr = 1e-3\n",
    "        self.lrd_fac = 0.2\n",
    "        self.lr_pat = 15\n",
    "        self.lr_min = 1e-12\n",
    "        self.eq_weights = False\n",
    "        self.opt_wd = 1e-1\n",
    "        self.dropout = 0.6\n",
    "        self.early_stop = 50\n",
    "        self.criteria = 'loss' # choices ['loss', 'acc']\n",
    "        \n",
    "args = Args()\n",
    "\n",
    "with open('{}/args_vars.txt'.format(run_dir), 'w') as file:\n",
    "    for key, value in vars(args).items():\n",
    "        file.write(f\"{key}: {value}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec36774a",
   "metadata": {},
   "source": [
    "## Training Loop:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ca7e4e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_res = []\n",
    "res_dicts = []\n",
    "\n",
    "for f in range(folds):\n",
    "   \n",
    "    # ---------------------\n",
    "    #  LOAD DATA\n",
    "    # ---------------------\n",
    "    # Training and validation data:\n",
    "    train_meta, val_meta = cv_data[f]['train_meta'], cv_data[f]['val_meta']\n",
    "    \n",
    "    # Remove duplicate smiles caused by replicates:\n",
    "    tm_no_dupes = train_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    vm_no_dupes = val_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    \n",
    "    # Convert smiles in meta data to morgan fingerprints:\n",
    "    train_x = convert_fprints(tm_no_dupes.smiles.to_list(), 'morgan')\n",
    "    val_x = convert_fprints(vm_no_dupes.smiles.to_list(), 'morgan')\n",
    "    \n",
    "    # Return moa labels:\n",
    "    y_train = np.array([moa_dict[element] for element in tm_no_dupes.moa.tolist()])\n",
    "    y_val = np.array([moa_dict[element] for element in vm_no_dupes.moa.tolist()])\n",
    "    \n",
    "    # Return dictionary of relative class weights:\n",
    "    class_weights_dict = get_class_weights(y_train)\n",
    "    \n",
    "    # Convert weights dict to tensor:\n",
    "    class_weights = torch.tensor(list(class_weights_dict.values()), dtype=torch.float32)\n",
    "    \n",
    "    # ---------------------\n",
    "    \n",
    "    #  CREATE DATA OBJECTS\n",
    "    # ---------------------\n",
    "    # Create Pytorch dataset:\n",
    "    train_dataset = MyDataset(train_x, y_train)\n",
    "    val_dataset = MyDataset(val_x, y_val)\n",
    "    \n",
    "    # Create data loaders for training and validation datasets\n",
    "    train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)\n",
    "    valid_loader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False)\n",
    "    \n",
    "    # ---------------------\n",
    "    #  INIT. MODEL\n",
    "    # ---------------------\n",
    "    model = MlpModel(args.input_size, args.hidden_size, args.num_classes, args.dropout)\n",
    "#     print(model)\n",
    "\n",
    "    # Define the optimizer and loss function\n",
    "    optimizer = optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.opt_wd)\n",
    "    if args.eq_weights:\n",
    "        criterion = nn.CrossEntropyLoss(weight=class_weights)\n",
    "    else:\n",
    "        criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    # Define the checkpoint path\n",
    "    filepath_mlp = os.path.join(run_dir, f'CV_Fold{f}.pth')\n",
    "\n",
    "    # Define the model checkpoint\n",
    "    if args.criteria == 'loss':\n",
    "        checkpoint_mlp = {\n",
    "            'file_path': filepath_mlp, 'monitor': 'val_loss',\n",
    "            'verbose': 0,'save_best_only': True,'mode': 'min'}\n",
    "    elif args.criteria == 'acc':\n",
    "        checkpoint_mlp = {\n",
    "            'file_path': filepath_mlp, 'monitor': 'val_acc',\n",
    "            'verbose': 0,'save_best_only': True,'mode': 'max'}\n",
    "\n",
    "    # Define the learning rate scheduler\n",
    "    reduce_lr_loss = ReduceLROnPlateau(optimizer, factor=args.lrd_fac, patience=args.lr_pat, \n",
    "                                   verbose=1, mode='min', min_lr=args.lr_min)\n",
    "    \n",
    "    # Define early stopping:\n",
    "    early_stopper = EarlyStopper(patience=args.early_stop, min_delta=1e-5)\n",
    "    \n",
    "    # ---------------------\n",
    "    #  TRAIN MODEL\n",
    "    # ---------------------\n",
    "    train_mlp(model, train_loader, valid_loader, criterion, optimizer, args.num_epochs, \n",
    "          checkpoint_mlp, reduce_lr_loss, early_stopper, verbose=1, criteria=args.criteria)\n",
    "    \n",
    "    # ---------------------\n",
    "    #  EVALUATE MODEL\n",
    "    # ---------------------\n",
    "    # Load test data:\n",
    "    test_meta = cv_data[f]['test_meta']\n",
    "    test_no_dupes = test_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    test_x = convert_fprints(test_no_dupes.smiles.to_list(), 'morgan')\n",
    "    y_test = np.array([moa_dict[element] for element in test_no_dupes.moa.tolist()])\n",
    "    \n",
    "    # Load Trained Model:\n",
    "    eval_model = MlpModel(args.input_size, args.hidden_size, args.num_classes, args.dropout)\n",
    "    eval_model.load_state_dict(torch.load(checkpoint_mlp['file_path']))\n",
    "\n",
    "    # Evaluate Model on test set:\n",
    "    eval_model.eval()\n",
    "            \n",
    "    with torch.no_grad():\n",
    "        outputs = eval_model(test_x)\n",
    "        test_acc = calc_acc(outputs, torch.tensor(y_test))\n",
    "        print(f\"Cpnd-level Test Accuracy = {test_acc:.2f}%\")\n",
    "        preds = torch.argmax(outputs, dim=1).numpy()\n",
    "        \n",
    "    # Saving results:\n",
    "    test_proba = []\n",
    "    [test_proba.append(p.detach().cpu().numpy()) for p in outputs]\n",
    "    res_df = pd.DataFrame({'preds': preds, 'proba': test_proba, 'actual': y_test, \n",
    "                      'cpnd_ids': test_no_dupes.Metadata_JCP2022})\n",
    "    res_df['proba'] = res_df['proba'].apply(lambda x: ', '.join(map(str, x)))\n",
    "    ss_dict = summary_stats(res_df)\n",
    "    \n",
    "    # Log cv results:\n",
    "    res_dicts.append(ss_dict)\n",
    "    cv_res.append(res_df)\n",
    "    cm_save_loc = os.path.join(run_dir, f\"CV{f}_CM_{test_acc:.2f}.png\")\n",
    "    conf_mat(y_test, preds, f'Structural MLP - Fold {f}\\n', cm_save_loc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b5f2030",
   "metadata": {},
   "source": [
    "## Amalgamate Model Results (across folds):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "851837cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>preds</th>\n",
       "      <th>proba</th>\n",
       "      <th>actuals</th>\n",
       "      <th>cpnd_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.7228253, 0.11624495, 0.06420749, 0.024424778...</td>\n",
       "      <td>2</td>\n",
       "      <td>JCP2022_006029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.2522545, 0.3364648, 0.022225024, 0.08587418,...</td>\n",
       "      <td>1</td>\n",
       "      <td>JCP2022_009919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0035663005, 0.0069937008, 0.009755337, 0.335...</td>\n",
       "      <td>5</td>\n",
       "      <td>JCP2022_002910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.039721202, 0.61459315, 0.02256958, 0.0512067...</td>\n",
       "      <td>1</td>\n",
       "      <td>JCP2022_023860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.03488773, 0.08748526, 0.037528146, 0.1476160...</td>\n",
       "      <td>4</td>\n",
       "      <td>JCP2022_032771</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   preds                                              proba  actuals  \\\n",
       "0      0  0.7228253, 0.11624495, 0.06420749, 0.024424778...        2   \n",
       "1      1  0.2522545, 0.3364648, 0.022225024, 0.08587418,...        1   \n",
       "2      5  0.0035663005, 0.0069937008, 0.009755337, 0.335...        5   \n",
       "3      1  0.039721202, 0.61459315, 0.02256958, 0.0512067...        1   \n",
       "4      4  0.03488773, 0.08748526, 0.037528146, 0.1476160...        4   \n",
       "\n",
       "         cpnd_ids  \n",
       "0  JCP2022_006029  \n",
       "1  JCP2022_009919  \n",
       "2  JCP2022_002910  \n",
       "3  JCP2022_023860  \n",
       "4  JCP2022_032771  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Concatenate the cross-validated results across folds:\n",
    "cpnd_comb = pd.concat(cv_res, axis=0).reset_index(drop=True)\n",
    "cpnd_comb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f0850295",
   "metadata": {},
   "outputs": [],
   "source": [
    "def string_to_float_list(s):\n",
    "    return [float(val) for val in s.split(',')]\n",
    "\n",
    "# Convert probability column back into floats:\n",
    "cpnd_comb['proba'] = cpnd_comb['proba'].apply(string_to_float_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2f0844d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 58.33%\n",
      "F1 Score: 56.08%\n",
      "Precision: 58.19%\n",
      "Recall: 55.94%\n",
      "ROC AUC: 84.34%\n",
      "AUPR: 60.31%\n"
     ]
    }
   ],
   "source": [
    "# Calculate performance metrics at a compound-level:\n",
    "acc = accuracy_score(cpnd_comb.actuals, cpnd_comb.preds)\n",
    "f1 = f1_score(cpnd_comb.actuals, cpnd_comb.preds, average='macro')\n",
    "precision = precision_score(cpnd_comb.actuals, cpnd_comb.preds, average='macro', zero_division=0)\n",
    "recall = recall_score(cpnd_comb.actuals, cpnd_comb.preds, average='macro')\n",
    "\n",
    "# Create a prediction array from probs:\n",
    "proba_arr = np.array(cpnd_comb['proba'].tolist())\n",
    "roc_auc = roc_auc_score(cpnd_comb.actuals, proba_arr, average='macro', multi_class='ovr')\n",
    "\n",
    "# Calculate AUPR for each class\n",
    "aupr_scores = [average_precision_score(cpnd_comb.actuals == class_index, proba_arr[:, class_index]\n",
    "                                       ) for class_index in range(proba_arr.shape[1])]\n",
    "mean_aupr = np.mean(aupr_scores)\n",
    "\n",
    "# Print metrics:\n",
    "print('Accuracy: {:.2f}%'.format(acc*100))\n",
    "print('F1 Score: {:.2f}%'.format(f1*100))\n",
    "print('Precision: {:.2f}%'.format(precision * 100))\n",
    "print('Recall: {:.2f}%'.format(recall * 100))\n",
    "print('ROC AUC: {:.2f}%'.format(roc_auc * 100))\n",
    "print('AUPR: {:.2f}%'.format(mean_aupr * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "261953c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "conf_mat(cpnd_comb.actuals, cpnd_comb.preds, 'Struct MLP - Compounds', \n",
    "         'output/Struct_MLP/cpnd_cm.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "9cb28df1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save results:\n",
    "cpnd_comb.to_csv('output/Struct_MLP/results_df.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6db3d01e",
   "metadata": {},
   "source": [
    "# XGBoost:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "4fe90ebf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define model:\n",
    "xgb_model = XGBClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "45617f87",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- CV 0: Accuracy 60.00%\n",
      "--- CV 0: F1 Score 54.38%\n",
      "\n",
      "--- CV 1: Accuracy 57.89%\n",
      "--- CV 1: F1 Score 40.22%\n",
      "\n",
      "--- CV 2: Accuracy 47.37%\n",
      "--- CV 2: F1 Score 32.38%\n",
      "\n",
      "--- CV 3: Accuracy 36.84%\n",
      "--- CV 3: F1 Score 40.36%\n",
      "\n",
      "--- CV 4: Accuracy 47.37%\n",
      "--- CV 4: F1 Score 33.71%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for f in range(folds):\n",
    "    # Training and validation data:\n",
    "    train_meta, val_meta = cv_data[f]['train_meta'], cv_data[f]['val_meta']\n",
    "    test_meta = cv_data[f]['test_meta']\n",
    "    \n",
    "    # Remove duplicate smiles caused by replicates:\n",
    "    tm_no_dupes = train_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    vm_no_dupes = val_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    test_no_dupes = test_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    \n",
    "    # Convert smiles in meta data to morgan fingerprints:\n",
    "    train_x = convert_fprints(tm_no_dupes.smiles.to_list(), 'morgan')\n",
    "    val_x = convert_fprints(vm_no_dupes.smiles.to_list(), 'morgan')\n",
    "    test_x = convert_fprints(test_no_dupes.smiles.to_list(), 'morgan').numpy()\n",
    "\n",
    "    # Combine training and val. set for XGBoost:\n",
    "    training_set = torch.cat((train_x, val_x), dim=0).numpy()\n",
    "    \n",
    "    # Return moa labels:\n",
    "    y_train = np.array([moa_dict[element] for element in tm_no_dupes.moa.tolist()])\n",
    "    y_val = np.array([moa_dict[element] for element in vm_no_dupes.moa.tolist()])\n",
    "    training_labels = np.hstack((y_train, y_val))\n",
    "    y_test = np.array([moa_dict[element] for element in test_no_dupes.moa.tolist()])\n",
    "    \n",
    "    # Make a deep copy of the model and fit it to the training data:\n",
    "    cv_mod = copy.deepcopy(xgb_model)\n",
    "    cv_mod.fit(training_set, training_labels)\n",
    "\n",
    "    # Make predictions on the test/validation set:\n",
    "    y_pred = cv_mod.predict(test_x)\n",
    "    preds = [round(value) for value in y_pred]\n",
    "#         y_pred_proba = cv_mod.predict_proba(X_test) # for roc_auc\n",
    "\n",
    "    # Calculate well-level accuracy of predictions:\n",
    "    acc = accuracy_score(y_test, preds)*100\n",
    "    f1 = f1_score(y_test, preds, average='macro')*100\n",
    "    print(f\"--- CV {f}: Accuracy {acc:.2f}%\")\n",
    "    print(f\"--- CV {f}: F1 Score {f1:.2f}%\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3737bc9b",
   "metadata": {},
   "source": [
    "## Random Search for Parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "fdf9ff33",
   "metadata": {},
   "outputs": [],
   "source": [
    "rs_params = {'max_depth': [3, 5, 6, 10, 15, 20],\n",
    "           'learning_rate': [0.01, 0.1, 0.2, 0.3],\n",
    "           'subsample': np.arange(0.5, 1.0, 0.1),\n",
    "           'colsample_bytree': np.arange(0.4, 1.0, 0.1),\n",
    "           'colsample_bylevel': np.arange(0.4, 1.0, 0.1),\n",
    "           'n_estimators': np.arange(50, 1000, 200)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "a412ac97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n",
      "Best CV Accuracy:  0.5117948717948718\n",
      "Best Parameters:  {'subsample': 0.8999999999999999, 'n_estimators': 450, 'max_depth': 5, 'learning_rate': 0.1, 'colsample_bytree': 0.4, 'colsample_bylevel': 0.4}\n",
      "--------------------------------------------\n",
      "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n",
      "Best CV Accuracy:  0.4682051282051282\n",
      "Best Parameters:  {'subsample': 0.7999999999999999, 'n_estimators': 250, 'max_depth': 5, 'learning_rate': 0.01, 'colsample_bytree': 0.6, 'colsample_bylevel': 0.6}\n",
      "--------------------------------------------\n",
      "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n",
      "Best CV Accuracy:  0.4794871794871794\n",
      "Best Parameters:  {'subsample': 0.8999999999999999, 'n_estimators': 850, 'max_depth': 10, 'learning_rate': 0.01, 'colsample_bytree': 0.7, 'colsample_bylevel': 0.4}\n",
      "--------------------------------------------\n",
      "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n",
      "Best CV Accuracy:  0.4676923076923077\n",
      "Best Parameters:  {'subsample': 0.7, 'n_estimators': 650, 'max_depth': 5, 'learning_rate': 0.01, 'colsample_bytree': 0.8999999999999999, 'colsample_bylevel': 0.8999999999999999}\n",
      "--------------------------------------------\n",
      "Fitting 3 folds for each of 50 candidates, totalling 150 fits\n",
      "Best CV Accuracy:  0.52\n",
      "Best Parameters:  {'subsample': 0.5, 'n_estimators': 450, 'max_depth': 15, 'learning_rate': 0.1, 'colsample_bytree': 0.5, 'colsample_bylevel': 0.4}\n",
      "--------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for f in range(folds):\n",
    "    # Define model and search:\n",
    "    rand_search_XGB = XGBClassifier(random_state=rand_seed) \n",
    "    \n",
    "    # Search across 100 different combinations, and use all available cores\n",
    "    xgb_random = RandomizedSearchCV(estimator=rand_search_XGB, param_distributions=rs_params, \n",
    "                               n_iter=50, verbose=1, random_state=rand_seed, cv=3)\n",
    "    \n",
    "    # Training and validation data:\n",
    "    train_meta, val_meta = cv_data[f]['train_meta'], cv_data[f]['val_meta']\n",
    "    \n",
    "    # Remove duplicate smiles caused by replicates:\n",
    "    tm_no_dupes = train_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    vm_no_dupes = val_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    # Convert smiles in meta data to morgan fingerprints:\n",
    "    train_x = convert_fprints(tm_no_dupes.smiles.to_list(), 'morgan')\n",
    "    val_x = convert_fprints(vm_no_dupes.smiles.to_list(), 'morgan')\n",
    "    # Combine training and val. set for XGBoost:\n",
    "    training_set = torch.cat((train_x, val_x), dim=0).numpy()\n",
    "    \n",
    "    # Return moa labels:\n",
    "    y_train = np.array([moa_dict[element] for element in tm_no_dupes.moa.tolist()])\n",
    "    y_val = np.array([moa_dict[element] for element in vm_no_dupes.moa.tolist()])\n",
    "    training_labels = np.hstack((y_train, y_val))\n",
    "    \n",
    "    # Fit Random Search Model:\n",
    "    xgb_random.fit(training_set, training_labels)\n",
    "    \n",
    "    print(\"Best CV Accuracy: \", xgb_random.best_score_)\n",
    "    print(\"Best Parameters: \", xgb_random.best_params_)\n",
    "    print(\"--------------------------------------------\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a174e4e",
   "metadata": {},
   "source": [
    "## Fit Model with Opt. Hyperparameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1f53cf3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define hyp. searched model:\n",
    "xgb_model = XGBClassifier(subsample=0.9, n_estimators=450, max_depth=5, \n",
    "                          learning_rate=0.01, colsample_bytree=0.5, colsample_bylevel=0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cdf3a3f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 55.21%\n",
      "F1 Score: 48.45%\n",
      "Precision: 49.58%\n",
      "Recall: 49.68%\n",
      "ROC AUC: 83.25%\n",
      "AUPR: 55.90%\n"
     ]
    }
   ],
   "source": [
    "fold_preds = []\n",
    "fold_actuals = []\n",
    "fold_probas = []\n",
    "\n",
    "for f in range(folds):\n",
    "    # Training and validation data:\n",
    "    train_meta, val_meta = cv_data[f]['train_meta'], cv_data[f]['val_meta']\n",
    "    test_meta = cv_data[f]['test_meta']\n",
    "    \n",
    "    # Remove duplicate smiles caused by replicates:\n",
    "    tm_no_dupes = train_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    vm_no_dupes = val_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    test_no_dupes = test_meta.drop_duplicates(['smiles']).reset_index(drop=True)\n",
    "    \n",
    "    # Convert smiles in meta data to morgan fingerprints:\n",
    "    train_x = convert_fprints(tm_no_dupes.smiles.to_list(), 'morgan')\n",
    "    val_x = convert_fprints(vm_no_dupes.smiles.to_list(), 'morgan')\n",
    "    test_x = convert_fprints(test_no_dupes.smiles.to_list(), 'morgan').numpy()\n",
    "\n",
    "    # Combine training and val. set for XGBoost:\n",
    "    training_set = torch.cat((train_x, val_x), dim=0).numpy()\n",
    "    \n",
    "    # Return moa labels:\n",
    "    y_train = np.array([moa_dict[element] for element in tm_no_dupes.moa.tolist()])\n",
    "    y_val = np.array([moa_dict[element] for element in vm_no_dupes.moa.tolist()])\n",
    "    training_labels = np.hstack((y_train, y_val))\n",
    "    y_test = np.array([moa_dict[element] for element in test_no_dupes.moa.tolist()])\n",
    "    \n",
    "    # Make a deep copy of the model and fit it to the training data:\n",
    "    cv_mod = copy.deepcopy(xgb_model)\n",
    "    cv_mod.fit(training_set, training_labels)\n",
    "\n",
    "    # Make predictions on the test/validation set:\n",
    "    y_pred = cv_mod.predict(test_x)\n",
    "    preds = [round(value) for value in y_pred]\n",
    "    y_pred_proba = cv_mod.predict_proba(test_x) # for roc_auc\n",
    "    \n",
    "    # Update lists outside loop:\n",
    "    fold_preds.append(preds)\n",
    "    fold_actuals.append(y_test)\n",
    "    fold_probas.append(y_pred_proba)\n",
    "    \n",
    "# Concatenate the arrays within the results lists:\n",
    "pred_arr = np.concatenate(fold_preds)\n",
    "act_arr = np.concatenate(fold_actuals)\n",
    "proba_arr = np.concatenate(fold_probas)\n",
    "\n",
    "# Calculate performance metrics at a compound-level:\n",
    "acc = accuracy_score(act_arr, pred_arr)\n",
    "f1 = f1_score(act_arr, pred_arr, average='macro')\n",
    "precision = precision_score(act_arr, pred_arr, average='macro', zero_division=0)\n",
    "recall = recall_score(act_arr, pred_arr, average='macro')\n",
    "roc_auc = roc_auc_score(act_arr, proba_arr, average='macro', multi_class='ovr')\n",
    "\n",
    "# Calculate AUPR for each class\n",
    "aupr_scores = [average_precision_score(act_arr == class_index, proba_arr[:, class_index]\n",
    "                                       ) for class_index in range(proba_arr.shape[1])]\n",
    "mean_aupr = np.mean(aupr_scores)\n",
    "\n",
    "# Print metrics:\n",
    "print('Accuracy: {:.2f}%'.format(acc*100))\n",
    "print('F1 Score: {:.2f}%'.format(f1*100))\n",
    "print('Precision: {:.2f}%'.format(precision * 100))\n",
    "print('Recall: {:.2f}%'.format(recall * 100))\n",
    "print('ROC AUC: {:.2f}%'.format(roc_auc * 100))\n",
    "print('AUPR: {:.2f}%'.format(mean_aupr * 100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}