Traditional Classifiers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "34f7fabd",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>V5</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "      <th>V9</th>\n",
       "      <th>...</th>\n",
       "      <th>V21</th>\n",
       "      <th>V22</th>\n",
       "      <th>V23</th>\n",
       "      <th>V24</th>\n",
       "      <th>V25</th>\n",
       "      <th>V26</th>\n",
       "      <th>V27</th>\n",
       "      <th>V28</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.359807</td>\n",
       "      <td>-0.072781</td>\n",
       "      <td>2.536347</td>\n",
       "      <td>1.378155</td>\n",
       "      <td>-0.338321</td>\n",
       "      <td>0.462388</td>\n",
       "      <td>0.239599</td>\n",
       "      <td>0.098698</td>\n",
       "      <td>0.363787</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.018307</td>\n",
       "      <td>0.277838</td>\n",
       "      <td>-0.110474</td>\n",
       "      <td>0.066928</td>\n",
       "      <td>0.128539</td>\n",
       "      <td>-0.189115</td>\n",
       "      <td>0.133558</td>\n",
       "      <td>-0.021053</td>\n",
       "      <td>149.62</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.191857</td>\n",
       "      <td>0.266151</td>\n",
       "      <td>0.166480</td>\n",
       "      <td>0.448154</td>\n",
       "      <td>0.060018</td>\n",
       "      <td>-0.082361</td>\n",
       "      <td>-0.078803</td>\n",
       "      <td>0.085102</td>\n",
       "      <td>-0.255425</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.225775</td>\n",
       "      <td>-0.638672</td>\n",
       "      <td>0.101288</td>\n",
       "      <td>-0.339846</td>\n",
       "      <td>0.167170</td>\n",
       "      <td>0.125895</td>\n",
       "      <td>-0.008983</td>\n",
       "      <td>0.014724</td>\n",
       "      <td>2.69</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.358354</td>\n",
       "      <td>-1.340163</td>\n",
       "      <td>1.773209</td>\n",
       "      <td>0.379780</td>\n",
       "      <td>-0.503198</td>\n",
       "      <td>1.800499</td>\n",
       "      <td>0.791461</td>\n",
       "      <td>0.247676</td>\n",
       "      <td>-1.514654</td>\n",
       "      <td>...</td>\n",
       "      <td>0.247998</td>\n",
       "      <td>0.771679</td>\n",
       "      <td>0.909412</td>\n",
       "      <td>-0.689281</td>\n",
       "      <td>-0.327642</td>\n",
       "      <td>-0.139097</td>\n",
       "      <td>-0.055353</td>\n",
       "      <td>-0.059752</td>\n",
       "      <td>378.66</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.966272</td>\n",
       "      <td>-0.185226</td>\n",
       "      <td>1.792993</td>\n",
       "      <td>-0.863291</td>\n",
       "      <td>-0.010309</td>\n",
       "      <td>1.247203</td>\n",
       "      <td>0.237609</td>\n",
       "      <td>0.377436</td>\n",
       "      <td>-1.387024</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.108300</td>\n",
       "      <td>0.005274</td>\n",
       "      <td>-0.190321</td>\n",
       "      <td>-1.175575</td>\n",
       "      <td>0.647376</td>\n",
       "      <td>-0.221929</td>\n",
       "      <td>0.062723</td>\n",
       "      <td>0.061458</td>\n",
       "      <td>123.50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>-1.158233</td>\n",
       "      <td>0.877737</td>\n",
       "      <td>1.548718</td>\n",
       "      <td>0.403034</td>\n",
       "      <td>-0.407193</td>\n",
       "      <td>0.095921</td>\n",
       "      <td>0.592941</td>\n",
       "      <td>-0.270533</td>\n",
       "      <td>0.817739</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.009431</td>\n",
       "      <td>0.798278</td>\n",
       "      <td>-0.137458</td>\n",
       "      <td>0.141267</td>\n",
       "      <td>-0.206010</td>\n",
       "      <td>0.502292</td>\n",
       "      <td>0.219422</td>\n",
       "      <td>0.215153</td>\n",
       "      <td>69.99</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284802</th>\n",
       "      <td>172786.0</td>\n",
       "      <td>-11.881118</td>\n",
       "      <td>10.071785</td>\n",
       "      <td>-9.834783</td>\n",
       "      <td>-2.066656</td>\n",
       "      <td>-5.364473</td>\n",
       "      <td>-2.606837</td>\n",
       "      <td>-4.918215</td>\n",
       "      <td>7.305334</td>\n",
       "      <td>1.914428</td>\n",
       "      <td>...</td>\n",
       "      <td>0.213454</td>\n",
       "      <td>0.111864</td>\n",
       "      <td>1.014480</td>\n",
       "      <td>-0.509348</td>\n",
       "      <td>1.436807</td>\n",
       "      <td>0.250034</td>\n",
       "      <td>0.943651</td>\n",
       "      <td>0.823731</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284803</th>\n",
       "      <td>172787.0</td>\n",
       "      <td>-0.732789</td>\n",
       "      <td>-0.055080</td>\n",
       "      <td>2.035030</td>\n",
       "      <td>-0.738589</td>\n",
       "      <td>0.868229</td>\n",
       "      <td>1.058415</td>\n",
       "      <td>0.024330</td>\n",
       "      <td>0.294869</td>\n",
       "      <td>0.584800</td>\n",
       "      <td>...</td>\n",
       "      <td>0.214205</td>\n",
       "      <td>0.924384</td>\n",
       "      <td>0.012463</td>\n",
       "      <td>-1.016226</td>\n",
       "      <td>-0.606624</td>\n",
       "      <td>-0.395255</td>\n",
       "      <td>0.068472</td>\n",
       "      <td>-0.053527</td>\n",
       "      <td>24.79</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284804</th>\n",
       "      <td>172788.0</td>\n",
       "      <td>1.919565</td>\n",
       "      <td>-0.301254</td>\n",
       "      <td>-3.249640</td>\n",
       "      <td>-0.557828</td>\n",
       "      <td>2.630515</td>\n",
       "      <td>3.031260</td>\n",
       "      <td>-0.296827</td>\n",
       "      <td>0.708417</td>\n",
       "      <td>0.432454</td>\n",
       "      <td>...</td>\n",
       "      <td>0.232045</td>\n",
       "      <td>0.578229</td>\n",
       "      <td>-0.037501</td>\n",
       "      <td>0.640134</td>\n",
       "      <td>0.265745</td>\n",
       "      <td>-0.087371</td>\n",
       "      <td>0.004455</td>\n",
       "      <td>-0.026561</td>\n",
       "      <td>67.88</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284805</th>\n",
       "      <td>172788.0</td>\n",
       "      <td>-0.240440</td>\n",
       "      <td>0.530483</td>\n",
       "      <td>0.702510</td>\n",
       "      <td>0.689799</td>\n",
       "      <td>-0.377961</td>\n",
       "      <td>0.623708</td>\n",
       "      <td>-0.686180</td>\n",
       "      <td>0.679145</td>\n",
       "      <td>0.392087</td>\n",
       "      <td>...</td>\n",
       "      <td>0.265245</td>\n",
       "      <td>0.800049</td>\n",
       "      <td>-0.163298</td>\n",
       "      <td>0.123205</td>\n",
       "      <td>-0.569159</td>\n",
       "      <td>0.546668</td>\n",
       "      <td>0.108821</td>\n",
       "      <td>0.104533</td>\n",
       "      <td>10.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284806</th>\n",
       "      <td>172792.0</td>\n",
       "      <td>-0.533413</td>\n",
       "      <td>-0.189733</td>\n",
       "      <td>0.703337</td>\n",
       "      <td>-0.506271</td>\n",
       "      <td>-0.012546</td>\n",
       "      <td>-0.649617</td>\n",
       "      <td>1.577006</td>\n",
       "      <td>-0.414650</td>\n",
       "      <td>0.486180</td>\n",
       "      <td>...</td>\n",
       "      <td>0.261057</td>\n",
       "      <td>0.643078</td>\n",
       "      <td>0.376777</td>\n",
       "      <td>0.008797</td>\n",
       "      <td>-0.473649</td>\n",
       "      <td>-0.818267</td>\n",
       "      <td>-0.002415</td>\n",
       "      <td>0.013649</td>\n",
       "      <td>217.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>284807 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Time         V1         V2        V3        V4        V5  \\\n",
       "0            0.0  -1.359807  -0.072781  2.536347  1.378155 -0.338321   \n",
       "1            0.0   1.191857   0.266151  0.166480  0.448154  0.060018   \n",
       "2            1.0  -1.358354  -1.340163  1.773209  0.379780 -0.503198   \n",
       "3            1.0  -0.966272  -0.185226  1.792993 -0.863291 -0.010309   \n",
       "4            2.0  -1.158233   0.877737  1.548718  0.403034 -0.407193   \n",
       "...          ...        ...        ...       ...       ...       ...   \n",
       "284802  172786.0 -11.881118  10.071785 -9.834783 -2.066656 -5.364473   \n",
       "284803  172787.0  -0.732789  -0.055080  2.035030 -0.738589  0.868229   \n",
       "284804  172788.0   1.919565  -0.301254 -3.249640 -0.557828  2.630515   \n",
       "284805  172788.0  -0.240440   0.530483  0.702510  0.689799 -0.377961   \n",
       "284806  172792.0  -0.533413  -0.189733  0.703337 -0.506271 -0.012546   \n",
       "\n",
       "              V6        V7        V8        V9  ...       V21       V22  \\\n",
       "0       0.462388  0.239599  0.098698  0.363787  ... -0.018307  0.277838   \n",
       "1      -0.082361 -0.078803  0.085102 -0.255425  ... -0.225775 -0.638672   \n",
       "2       1.800499  0.791461  0.247676 -1.514654  ...  0.247998  0.771679   \n",
       "3       1.247203  0.237609  0.377436 -1.387024  ... -0.108300  0.005274   \n",
       "4       0.095921  0.592941 -0.270533  0.817739  ... -0.009431  0.798278   \n",
       "...          ...       ...       ...       ...  ...       ...       ...   \n",
       "284802 -2.606837 -4.918215  7.305334  1.914428  ...  0.213454  0.111864   \n",
       "284803  1.058415  0.024330  0.294869  0.584800  ...  0.214205  0.924384   \n",
       "284804  3.031260 -0.296827  0.708417  0.432454  ...  0.232045  0.578229   \n",
       "284805  0.623708 -0.686180  0.679145  0.392087  ...  0.265245  0.800049   \n",
       "284806 -0.649617  1.577006 -0.414650  0.486180  ...  0.261057  0.643078   \n",
       "\n",
       "             V23       V24       V25       V26       V27       V28  Amount  \\\n",
       "0      -0.110474  0.066928  0.128539 -0.189115  0.133558 -0.021053  149.62   \n",
       "1       0.101288 -0.339846  0.167170  0.125895 -0.008983  0.014724    2.69   \n",
       "2       0.909412 -0.689281 -0.327642 -0.139097 -0.055353 -0.059752  378.66   \n",
       "3      -0.190321 -1.175575  0.647376 -0.221929  0.062723  0.061458  123.50   \n",
       "4      -0.137458  0.141267 -0.206010  0.502292  0.219422  0.215153   69.99   \n",
       "...          ...       ...       ...       ...       ...       ...     ...   \n",
       "284802  1.014480 -0.509348  1.436807  0.250034  0.943651  0.823731    0.77   \n",
       "284803  0.012463 -1.016226 -0.606624 -0.395255  0.068472 -0.053527   24.79   \n",
       "284804 -0.037501  0.640134  0.265745 -0.087371  0.004455 -0.026561   67.88   \n",
       "284805 -0.163298  0.123205 -0.569159  0.546668  0.108821  0.104533   10.00   \n",
       "284806  0.376777  0.008797 -0.473649 -0.818267 -0.002415  0.013649  217.00   \n",
       "\n",
       "        Class  \n",
       "0           0  \n",
       "1           0  \n",
       "2           0  \n",
       "3           0  \n",
       "4           0  \n",
       "...       ...  \n",
       "284802      0  \n",
       "284803      0  \n",
       "284804      0  \n",
       "284805      0  \n",
       "284806      0  \n",
       "\n",
       "[284807 rows x 31 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"creditcard.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "30040d10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting xgboost\n",
      "  Downloading https://files.pythonhosted.org/packages/97/ef/05245964011e4fc5aa0d86e2285a41de122ee1c30d69df05ecfd594bd608/xgboost-1.5.2-py3-none-win_amd64.whl (106.6MB)\n",
      "Requirement already satisfied: numpy in c:\\users\\wj\\appdata\\local\\continuum\\anaconda3\\lib\\site-packages (from xgboost) (1.21.5)\n",
      "Requirement already satisfied: scipy in c:\\users\\wj\\appdata\\local\\continuum\\anaconda3\\lib\\site-packages (from xgboost) (1.7.3)\n",
      "Installing collected packages: xgboost\n",
      "Successfully installed xgboost-1.5.2\n",
      "Collecting lightgbm\n",
      "  Using cached https://files.pythonhosted.org/packages/ba/24/2f83a1008c8add8cd9da03163f911be6b555eb2b9166b5ab74e1ad63ff40/lightgbm-3.3.2-py3-none-win_amd64.whl\n",
      "Requirement already satisfied: scikit-learn!=0.22.0 in c:\\users\\wj\\appdata\\local\\continuum\\anaconda3\\lib\\site-packages (from lightgbm) (0.24.1)\n",
      "Requirement already satisfied: scipy in c:\\users\\wj\\appdata\\local\\continuum\\anaconda3\\lib\\site-packages (from lightgbm) (1.7.3)\n",
      "Requirement already satisfied: wheel in c:\\users\\wj\\appdata\\local\\continuum\\anaconda3\\lib\\site-packages (from lightgbm) (0.33.4)\n",
      "Requirement already satisfied: numpy in c:\\users\\wj\\appdata\\local\\continuum\\anaconda3\\lib\\site-packages (from lightgbm) (1.21.5)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\users\\wj\\appdata\\local\\continuum\\anaconda3\\lib\\site-packages (from scikit-learn!=0.22.0->lightgbm) (0.13.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\wj\\appdata\\local\\continuum\\anaconda3\\lib\\site-packages (from scikit-learn!=0.22.0->lightgbm) (2.1.0)\n",
      "Installing collected packages: lightgbm\n",
      "Successfully installed lightgbm-3.3.2\n"
     ]
    }
   ],
   "source": [
    "!pip install xgboost\n",
    "!pip install lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "10bce3c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\dask\\dataframe\\utils.py:15: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
      "  import pandas.util.testing as tm\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, StratifiedKFold\n",
    "from imblearn.over_sampling import SMOTE, ADASYN \n",
    "from imblearn.under_sampling import NearMiss\n",
    "from imblearn.pipeline import Pipeline as imbpipeline\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "#from sklearn import cross_validation, linear_model\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "#For Decision Tree implementation\n",
    "from scipy.stats import entropy\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn import tree\n",
    "\n",
    "#For KNN implementation\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "#For NB implementation \n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.preprocessing import PowerTransformer\n",
    "\n",
    "#For Logistic Regression implementation \n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#For Random Forest implementation \n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "#For SVM implementation \n",
    "from sklearn.svm import SVC\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "#For MLP implementation \n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "#For XGBoost \n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "#For LightGBM \n",
    "import lightgbm as lgbm\n",
    "\n",
    "#For Stacking \n",
    "from sklearn.ensemble import StackingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fdea3787",
   "metadata": {},
   "outputs": [],
   "source": [
    "# stratified sampling, 10 from each class\n",
    "#df = df.groupby('Class', group_keys=False).apply(lambda x: x.sample(100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0f2e218e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop all time from feature. assume linear time for classification\n",
    "X = df.iloc[:, 1:30]\n",
    "y = df['Class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e7cb220b",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size=0.2,\n",
    "                                                    stratify=y,\n",
    "                                                    random_state=424)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b1e51597",
   "metadata": {},
   "outputs": [],
   "source": [
    "stratified_kfold = StratifiedKFold(n_splits=10,shuffle=True,random_state=424)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "03823a31",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Logistic Regression: 0.7240250178089848\n",
      "Test score of Logistic Regression: 0.7615665414675451\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.89      0.65      0.75        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.94      0.83      0.88     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    }
   ],
   "source": [
    "log_reg = LogisticRegression(random_state=424)\n",
    "\n",
    "c_values = [0.1, 10, 100]\n",
    "parameters = dict(C=c_values)\n",
    "grid_search = GridSearchCV(estimator=log_reg,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Logistic Regression: {cv_score}\\nTest score of Logistic Regression: {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a159ce6a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Logistic Regression (SMOTE): 0.2741703907402613\n",
      "Test score of Logistic Regression (SMOTE): 0.2766211253595709\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.98      0.99     56864\n",
      "           1       0.09      0.91      0.16        98\n",
      "\n",
      "    accuracy                           0.98     56962\n",
      "   macro avg       0.54      0.95      0.57     56962\n",
      "weighted avg       1.00      0.98      0.99     56962\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    }
   ],
   "source": [
    "log_reg_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', LogisticRegression(random_state=424)]])\n",
    "\n",
    "\n",
    "\n",
    "param_grid = {'classifier__C':[0.1, 10, 100]}\n",
    "grid_search = GridSearchCV(estimator=log_reg_smote_pipeline,\n",
    "                           param_grid=param_grid,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Logistic Regression (SMOTE): {cv_score}\\nTest score of Logistic Regression (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "13de7706",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Logistic Regression (ADASYN): 0.17142940484603372\n",
      "Test score of Logistic Regression (ADASYN): 0.1629787893679948\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.95      0.97     56864\n",
      "           1       0.03      0.91      0.06        98\n",
      "\n",
      "    accuracy                           0.95     56962\n",
      "   macro avg       0.52      0.93      0.52     56962\n",
      "weighted avg       1.00      0.95      0.97     56962\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    }
   ],
   "source": [
    "log_reg_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', LogisticRegression(random_state=424)]])\n",
    "\n",
    "\n",
    "param_grid = {'classifier__C':[0.1, 10, 100]}\n",
    "grid_search = GridSearchCV(estimator=log_reg_adasyn_pipeline,\n",
    "                           param_grid=param_grid,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Logistic Regression (ADASYN): {cv_score}\\nTest score of Logistic Regression (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6d722a38",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
      "/common/home/projectgrps/IS450/IS450G10/jupyterlab-venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Logistic Regression (NearMiss): 0.019958345251695354\n",
      "Test score of Logistic Regression (NearMiss): 0.02029561095524574\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.25      0.40     56864\n",
      "           1       0.00      0.96      0.00        98\n",
      "\n",
      "    accuracy                           0.26     56962\n",
      "   macro avg       0.50      0.61      0.20     56962\n",
      "weighted avg       1.00      0.26      0.40     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "log_reg_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['classifier', LogisticRegression(random_state=424)]])\n",
    "\n",
    "\n",
    "param_grid = {'classifier__C':[0.1, 10, 100]}\n",
    "grid_search = GridSearchCV(estimator=log_reg_nearmiss_pipeline,\n",
    "                           param_grid=param_grid,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Logistic Regression (NearMiss): {cv_score}\\nTest score of Logistic Regression (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7eaac6f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Naive Bayes: 0.3699223482503689\n",
      "Test score of Naive Bayes: 0.35433026128132045\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.43      0.30      0.35        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.71      0.65      0.67     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb = GaussianNB()\n",
    "parameters = {'var_smoothing': np.logspace(0,-9, num=100)}\n",
    "grid_search = GridSearchCV(estimator=nb,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Naive Bayes: {cv_score}\\nTest score of Naive Bayes: {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "75f0d85a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Naive Bayes (SMOTE): 0.03791814657150627\n",
      "Test score of Naive Bayes (SMOTE): 0.043751082859970876\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.87      0.93     56864\n",
      "           1       0.01      0.48      0.01        98\n",
      "\n",
      "    accuracy                           0.87     56962\n",
      "   macro avg       0.50      0.68      0.47     56962\n",
      "weighted avg       1.00      0.87      0.93     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                   ['classifier', GaussianNB()]])\n",
    "\n",
    "parameters = {'classifier__var_smoothing': [0.1, 10, 100]}\n",
    "grid_search = GridSearchCV(estimator=nb_smote_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Naive Bayes (SMOTE): {cv_score}\\nTest score of Naive Bayes (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b50ed135",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Naive Bayes (ADASYN): 0.01077461991874442\n",
      "Test score of Naive Bayes (ADASYN): 0.010057087134220028\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.94      0.97     56864\n",
      "           1       0.00      0.12      0.01        98\n",
      "\n",
      "    accuracy                           0.94     56962\n",
      "   macro avg       0.50      0.53      0.49     56962\n",
      "weighted avg       1.00      0.94      0.96     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                   ['classifier', GaussianNB()]])\n",
    "\n",
    "parameters = {'classifier__var_smoothing': [0.1, 10, 100]}\n",
    "grid_search = GridSearchCV(estimator=nb_adasyn_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Naive Bayes (ADASYN): {cv_score}\\nTest score of Naive Bayes (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7e9196ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Naive Bayes (NearMiss): 0.01043747635098244\n",
      "Test score of Naive Bayes (NearMiss): 0.009403688445245265\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.93      0.97     56864\n",
      "           1       0.00      0.12      0.01        98\n",
      "\n",
      "    accuracy                           0.93     56962\n",
      "   macro avg       0.50      0.53      0.49     56962\n",
      "weighted avg       1.00      0.93      0.96     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                   ['classifier', GaussianNB()]])\n",
    "\n",
    "parameters = {'classifier__var_smoothing': [0.1, 10, 100]}\n",
    "grid_search = GridSearchCV(estimator=nb_nearmiss_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Naive Bayes (NearMiss): {cv_score}\\nTest score of Naive Bayes (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2bd663a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Decision Tree: 0.8172302444385761\n",
      "Test score of Decision Tree: 0.8810846044516867\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.96      0.81      0.88        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.98      0.90      0.94     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dt= DecisionTreeClassifier(random_state=424)\n",
    "\n",
    "parameters = {\n",
    "    'max_depth': [5, 10, 20],\n",
    "    'min_samples_leaf': [5, 20, 50],\n",
    "    'criterion': [\"gini\", \"entropy\"]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=dt,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Decision Tree: {cv_score}\\nTest score of Decision Tree: {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a27c112a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Decision Tree (SMOTE): 0.5539521802341207\n",
      "Test score of Decision Tree (SMOTE): 0.5964596771481685\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.42      0.85      0.56        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.71      0.92      0.78     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dt_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', DecisionTreeClassifier(random_state=424)]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__max_depth': [5, 10, 20],\n",
    "    'classifier__min_samples_leaf': [5, 20, 50],\n",
    "    'classifier__criterion': [\"gini\", \"entropy\"]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=dt_smote_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Decision Tree (SMOTE): {cv_score}\\nTest score of Decision Tree (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1bf8dafb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Decision Tree (ADASYN): 0.5539521802341207\n",
      "Test score of Decision Tree (ADASYN): 0.5964596771481685\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.42      0.85      0.56        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.71      0.92      0.78     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dt_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', DecisionTreeClassifier(random_state=424)]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__max_depth': [5, 10, 20],\n",
    "    'classifier__min_samples_leaf': [5, 20, 50],\n",
    "    'classifier__criterion': [\"gini\", \"entropy\"]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=dt_smote_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Decision Tree (ADASYN): {cv_score}\\nTest score of Decision Tree (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "31cea3a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Decision Tree (NearMiss): 0.020840141804736093\n",
      "Test score of Decision Tree (NearMiss): 0.027174439784000193\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.33      0.49     56864\n",
      "           1       0.00      0.98      0.01        98\n",
      "\n",
      "    accuracy                           0.33     56962\n",
      "   macro avg       0.50      0.65      0.25     56962\n",
      "weighted avg       1.00      0.33      0.49     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dt_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['classifier', DecisionTreeClassifier(random_state=424)]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__max_depth': [5, 10, 20],\n",
    "    'classifier__min_samples_leaf': [5, 20, 50],\n",
    "    'classifier__criterion': [\"gini\", \"entropy\"]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=dt_nearmiss_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Decision Tree (NearMiss): {cv_score}\\nTest score of Decision Tree (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b291f828",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Random Forest: 0.8605893158971591\n",
      "Test score of Random Forest: 0.8873111065584978\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.95      0.83      0.89        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.98      0.91      0.94     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(random_state=424)\n",
    "\n",
    "parameters = {\n",
    "    'max_depth': [5, 10, 20],\n",
    "    'criterion': [\"gini\", \"entropy\"]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=rf,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Random Forest: {cv_score}\\nTest score of Random Forest: {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "10625af0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of Random Forest (SMOTE): 0.8457494639064507\n",
      "Test score of Random Forest (SMOTE): 0.8729230972010839\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.91      0.84      0.87        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.96      0.92      0.94     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rf_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', RandomForestClassifier(random_state=424)]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__max_depth': [5, 10, 20],\n",
    "    'classifier__criterion': [\"gini\", \"entropy\"]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=rf_smote_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Random Forest (SMOTE): {cv_score}\\nTest score of Random Forest (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e094784a",
   "metadata": {},
   "source": [
    "## here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5475bdfb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of Random Forest (ADASYN): {'fit_time': array([705.09207392, 718.27538729, 701.71338105, 701.27119374,\n",
      "       678.8841579 , 654.92452025, 663.51967359, 643.08598185,\n",
      "       647.11083651, 641.11685228]), 'score_time': array([0.8253684 , 0.59500456, 0.60478854, 0.63196301, 0.6232667 ,\n",
      "       0.57636237, 0.603122  , 0.59758186, 0.61224222, 0.61520791]), 'test_mcc': array([0.86591578, 0.83151699, 0.84364134, 0.83522446, 0.8461694 ,\n",
      "       0.7735814 , 0.81169452, 0.83598837, 0.88594936, 0.8805095 ])}\n",
      "Test score of Random Forest (ADASYN): 0.9995786664794073\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.92      0.83      0.87        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.96      0.91      0.94     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rf_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', RandomForestClassifier(random_state=424)]])\n",
    "rf_adasyn_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(rf_adasyn_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = rf_adasyn_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of Random Forest (ADASYN): {cv_score}\\nTest score of Random Forest (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, rf_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "259c391a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8720226670871056\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, rf_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12b338cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Code with Hyperparamter Tuning\n",
    "rf_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', RandomForestClassifier(random_state=424)]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__max_depth': [5, 10, 20],\n",
    "    'classifier__criterion': [\"gini\", \"entropy\"]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=rf_adasyn_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Random Forest (ADASYN): {cv_score}\\nTest score of Random Forest (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ec530b30",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of Random Forest (NearMiss): {'fit_time': array([4.20813107, 4.36897874, 4.20237899, 4.25022078, 4.18331313,\n",
      "       4.2347393 , 4.23171973, 4.37209153, 4.25327516, 4.20124292]), 'score_time': array([0.3498528 , 0.34589052, 0.33714771, 0.32536721, 0.32414913,\n",
      "       0.34073353, 0.32940102, 0.34424901, 0.33224916, 0.32857084]), 'test_mcc': array([ 0.00973315,  0.00971265,  0.00942698,  0.00870876,  0.00965074,\n",
      "        0.00937534, -0.00427925,  0.0040559 ,  0.00917398,  0.00937534])}\n",
      "Test score of Random Forest (NearMiss): 0.04562690916751518\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.92      0.83      0.87        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.96      0.91      0.94     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rf_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['classifier', RandomForestClassifier(random_state=424)]])\n",
    "rf_nearmiss_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(rf_nearmiss_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = rf_nearmiss_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of Random Forest (NearMiss): {cv_score}\\nTest score of Random Forest (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, rf_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "5a60b74a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.006833247034260497\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, rf_nearmiss_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f32f5096",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Code with Hyperparamter Tuning\n",
    "rf_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['classifier', RandomForestClassifier(random_state=424)]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__max_depth': [5, 10, 20],\n",
    "    'classifier__criterion': [\"gini\", \"entropy\"]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=rf_nearmiss_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of Random Forest (NearMiss): {cv_score}\\nTest score of Random Forest (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5273321a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation score of K Nearest Neighbours: 0.7525364583430515\n",
      "Test score of K Nearest Neighbours: 0.8092567547581148\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.94      0.69      0.80        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.97      0.85      0.90     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "\n",
    "parameters = {\n",
    "    'n_neighbors': [5, 10, 15]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=knn,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of K Nearest Neighbours: {cv_score}\\nTest score of K Nearest Neighbours: {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "723813da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of K Nearest Neighbours (SMOTE): {'fit_time': array([1.4799335 , 1.32516503, 1.2673471 , 1.19734049, 1.64265323,\n",
      "       0.76911998, 0.6079874 , 0.55543256, 0.55198359, 0.5540061 ]), 'score_time': array([401.35399699, 401.23115587, 409.96008682, 384.28239751,\n",
      "       403.09966993, 187.50196886, 176.74125981, 177.94586372,\n",
      "       180.68097234, 193.33489943]), 'test_mcc': array([0.46151392, 0.52889167, 0.48942622, 0.48330888, 0.47931898,\n",
      "       0.48556462, 0.50726368, 0.42305409, 0.55363835, 0.62632513])}\n",
      "Test score of K Nearest Neighbours (SMOTE): 0.9964537762016783\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.31      0.85      0.45        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.65      0.92      0.72     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "knn_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', KNeighborsClassifier()]])\n",
    "knn_smote_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(knn_smote_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = knn_smote_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of K Nearest Neighbours (SMOTE): {cv_score}\\nTest score of K Nearest Neighbours (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, knn_smote_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b1fc75fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5090393990416132\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, knn_smote_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e0d4ea1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Code with Hyperparamter Tuning\n",
    "knn_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', KNeighborsClassifier()]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__n_neighbors': [5, 10, 15]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=knn_smote_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of K Nearest Neighbours (SMOTE): {cv_score}\\nTest score of K Nearest Neighbours (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d6d1a884",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of K Nearest Neighbours (ADASYN): {'fit_time': array([1.84614635, 1.90696049, 1.82566571, 1.96509004, 2.20043349,\n",
      "       1.8588438 , 2.00579858, 1.92530107, 2.39564657, 1.98110533]), 'score_time': array([164.0919261 , 160.86106491, 162.36943269, 157.4395926 ,\n",
      "       167.65654707, 174.33239388, 173.93988919, 166.13350153,\n",
      "       171.03258801, 172.65061665]), 'test_mcc': array([0.41203113, 0.48589053, 0.44184339, 0.42448997, 0.43340145,\n",
      "       0.45844721, 0.45676481, 0.38283145, 0.51029775, 0.59743456])}\n",
      "Test score of K Nearest Neighbours (ADASYN): 0.9953828868368386\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.25      0.85      0.39        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.63      0.92      0.69     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "knn_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', KNeighborsClassifier()]])\n",
    "knn_adasyn_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(knn_adasyn_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = knn_adasyn_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of K Nearest Neighbours (ADASYN): {cv_score}\\nTest score of K Nearest Neighbours (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, knn_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a675dc54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4594100404414329\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, knn_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "053f075b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Code with Hyperparamter Tuning\n",
    "knn_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', KNeighborsClassifier()]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__n_neighbors': [5, 10, 15]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=knn_adasyn_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of K Nearest Neighbours (ADASYN): {cv_score}\\nTest score of K Nearest Neighbours (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "110cbd33",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of K Nearest Neighbours (NearMiss): {'fit_time': array([2.04062796, 2.0172286 , 2.38285303, 2.19066358, 2.02655435,\n",
      "       2.14873433, 1.97282743, 1.92156911, 1.95988822, 1.91719913]), 'score_time': array([0.97684717, 1.32722211, 1.28426456, 0.91762853, 0.83600068,\n",
      "       0.89630556, 0.91187716, 0.94810104, 0.95302272, 0.88800168]), 'test_mcc': array([0.02032591, 0.01809059, 0.01850103, 0.01782093, 0.01820448,\n",
      "       0.021464  , 0.0133316 , 0.01208812, 0.01848956, 0.02305349])}\n",
      "Test score of K Nearest Neighbours (NearMiss): 0.23359432604192268\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.23      0.38     56864\n",
      "           1       0.00      0.97      0.00        98\n",
      "\n",
      "    accuracy                           0.23     56962\n",
      "   macro avg       0.50      0.60      0.19     56962\n",
      "weighted avg       1.00      0.23      0.38     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "knn_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['classifier', KNeighborsClassifier()]])\n",
    "knn_nearmiss_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(knn_nearmiss_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = knn_nearmiss_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of K Nearest Neighbours (NearMiss): {cv_score}\\nTest score of K Nearest Neighbours (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, knn_nearmiss_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "015efecb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01980484943380588\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, knn_nearmiss_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fc2b1f75",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of SVM: {'fit_time': array([60.94254184, 61.2372992 , 60.98366189, 60.38236952, 60.4531765 ,\n",
      "       59.81750083, 60.94850779, 61.49723339, 85.76798606, 99.77861118]), 'score_time': array([0.02011633, 0.01604509, 0.01630139, 0.01699853, 0.02100229,\n",
      "       0.01696324, 0.01846433, 0.01732326, 0.04313087, 0.04175568]), 'test_mcc': array([0.80031344, 0.66897996, 0.63514774, 0.79022187, 0.64083953,\n",
      "       0.66552595, 0.75223182, 0.7174651 , 0.75336619, 0.784206  ])}\n",
      "Test score of SVM: 0.9993504441557529\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.89      0.71      0.79        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.94      0.86      0.90     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "svm = LinearSVC()\n",
    "svm.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(svm,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = svm.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of SVM: {cv_score}\\nTest score of SVM: {test_score}')\n",
    "print(classification_report(y_test, svm.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "10f3ad33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7952481562864933\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, svm.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1afd6b5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for Hyperparameter Tuning \n",
    "svm = SVC(kernel='linear')\n",
    "\n",
    "parameters = {'kernel': ['rbf', 'poly', 'sigmoid']}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=svm,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of SVM: {cv_score}\\nTest score of SVM: {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a2e0f81a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of SVM (SMOTE): {'fit_time': array([269.23809576, 277.81777048, 251.27482152, 259.95279002,\n",
      "       266.50369096, 271.70513344, 258.17182589, 265.03283453,\n",
      "       275.5042243 , 253.93009877]), 'score_time': array([0.04245162, 0.04146481, 0.04917765, 0.04114699, 0.03513336,\n",
      "       0.03735423, 0.02715826, 0.0621357 , 0.0380671 , 0.03941917]), 'test_mcc': array([0.29660925, 0.35573531, 0.29767763, 0.09464981, 0.10562814,\n",
      "       0.2822011 , 0.26756596, 0.26312513, 0.31425297, 0.28607334])}\n",
      "Test score of SVM (SMOTE): 0.9793019908008848\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.98      0.99     56864\n",
      "           1       0.07      0.90      0.13        98\n",
      "\n",
      "    accuracy                           0.98     56962\n",
      "   macro avg       0.53      0.94      0.56     56962\n",
      "weighted avg       1.00      0.98      0.99     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "svm_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', LinearSVC()]])\n",
    "svm_smote_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(svm_smote_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = svm_smote_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of SVM (SMOTE): {cv_score}\\nTest score of SVM (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, svm_smote_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "6200f23d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2475228613284395\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, svm_smote_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdd4549c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for Hyperparameter Tuning\n",
    "svm_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', SVC(kernel='linear')]])\n",
    "\n",
    "parameters = {'classifier__kernel': ['rbf', 'poly', 'sigmoid']}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=svm_smote_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of SVM (SMOTE): {cv_score}\\nTest score of SVM (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "65711714",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of SVM (ADASYN): {'fit_time': array([285.04710889, 288.8594954 , 269.61709738, 273.27327681,\n",
      "       273.13952947, 270.27068353, 263.79954267, 291.2842443 ,\n",
      "       290.35313797, 287.28987718]), 'score_time': array([0.05139041, 0.04732656, 0.04756665, 0.0402112 , 0.06564236,\n",
      "       0.04068184, 0.04020548, 0.04108572, 0.04824877, 0.04623342]), 'test_mcc': array([0.15807488, 0.16163885, 0.17102415, 0.10113882, 0.17338289,\n",
      "       0.16168586, 0.16969701, 0.05779078, 0.17835991, 0.03532212])}\n",
      "Test score of SVM (ADASYN): 0.9247919665742074\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.92      0.96     56864\n",
      "           1       0.02      0.91      0.04        98\n",
      "\n",
      "    accuracy                           0.92     56962\n",
      "   macro avg       0.51      0.92      0.50     56962\n",
      "weighted avg       1.00      0.92      0.96     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "svm_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', LinearSVC()]])\n",
    "svm_adasyn_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(svm_adasyn_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = svm_adasyn_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of SVM (ADASYN): {cv_score}\\nTest score of SVM (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, svm_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4f482f78",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.12979003714991516\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, svm_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51ece64e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code with Hyperparameter Tuning\n",
    "svm_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', SVC(kernel='linear')]])\n",
    "\n",
    "parameters = {'classifier__kernel': ['rbf', 'poly', 'sigmoid']}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=svm_adasyn_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of SVM (ADASYN): {cv_score}\\nTest score of SVM (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "435eee10",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of SVM (NearMiss): {'fit_time': array([3.87364531, 3.67390966, 3.63814187, 4.10603762, 3.71766186,\n",
      "       3.6091888 , 3.8073833 , 3.81608677, 3.8911469 , 3.75558853]), 'score_time': array([0.03354359, 0.04773426, 0.04149723, 0.04610729, 0.03840423,\n",
      "       0.04100633, 0.04777145, 0.02212548, 0.03587031, 0.05380368]), 'test_mcc': array([0.01721541, 0.01860612, 0.01656354, 0.01723672, 0.02128675,\n",
      "       0.0149713 , 0.01322443, 0.01582652, 0.0157264 , 0.01637146])}\n",
      "Test score of SVM (NearMiss): 0.14902917734630106\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.15      0.26     56864\n",
      "           1       0.00      0.97      0.00        98\n",
      "\n",
      "    accuracy                           0.15     56962\n",
      "   macro avg       0.50      0.56      0.13     56962\n",
      "weighted avg       1.00      0.15      0.26     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "svm_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['classifier', LinearSVC()]])\n",
    "svm_nearmiss_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(svm_nearmiss_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = svm_nearmiss_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of SVM (NearMiss): {cv_score}\\nTest score of SVM (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, svm_nearmiss_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6fcb3d4a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.013677458427148381\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, svm_nearmiss_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f433048",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for Hyperparameter Tuning\n",
    "svm_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['classifier', SVC(kernel='linear')]])\n",
    "\n",
    "parameters = {'classifier__kernel': ['rbf', 'poly', 'sigmoid']}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=svm_nearmiss_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of SVM (NearMiss): {cv_score}\\nTest score of SVM (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "76ecc711",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of MLP: {'fit_time': array([44.45431972, 47.71399808, 47.21620107, 48.87117219, 48.9606986 ,\n",
      "       47.95490694, 45.98178101, 49.75159431, 48.18730116, 46.64757872]), 'score_time': array([0.19566703, 0.10289884, 0.09761882, 0.12350726, 0.09004855,\n",
      "       0.09355402, 0.10584974, 0.09899044, 0.08851433, 0.09960961]), 'test_mcc': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])}\n",
      "Test score of MLP: 0.9982795547909132\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.00      0.00      0.00        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.50      0.50      0.50     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "mlp = MLPClassifier(random_state=424, hidden_layer_sizes=(8,4,2,1), max_iter=100,activation = 'relu',solver='adam') \n",
    "mlp.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(mlp,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = mlp.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of MLP: {cv_score}\\nTest score of MLP: {test_score}')\n",
    "print(classification_report(y_test, mlp.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "f76b900f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, mlp.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cfd3ae9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code with Hyperparameter Tuning \n",
    "mlp = MLPClassifier(random_state=424) \n",
    "\n",
    "parameters = {\n",
    "    'hidden_layer_sizes': [(50,50,50), (50,100,50)],\n",
    "    'activation': ['tanh', 'relu'],\n",
    "    'learning_rate': ['constant','adaptive']\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=mlp,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of MLP: {cv_score}\\nTest score of MLP: {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9f4ccf74",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of MLP (SMOTE): {'fit_time': array([69.99091554, 68.14710093, 66.83545971, 67.45177245, 66.17068982,\n",
      "       67.43436241, 68.02238131, 67.719666  , 72.95098996, 61.34789181]), 'score_time': array([0.19003892, 0.08316708, 0.09995079, 0.08235025, 0.08916044,\n",
      "       0.10185957, 0.10271716, 0.09143543, 0.08194733, 0.03996897]), 'test_mcc': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])}\n",
      "Test score of MLP (SMOTE): 0.0017204452090867595\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00     56864\n",
      "           1       0.00      1.00      0.00        98\n",
      "\n",
      "    accuracy                           0.00     56962\n",
      "   macro avg       0.00      0.50      0.00     56962\n",
      "weighted avg       0.00      0.00      0.00     56962\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "mlp_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', MLPClassifier(random_state=424, hidden_layer_sizes=(8,4,2,1), max_iter=100,activation = 'relu',solver='adam')]])\n",
    "\n",
    "\n",
    "mlp_smote_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(mlp_smote_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = mlp_smote_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of MLP (SMOTE): {cv_score}\\nTest score of MLP (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, mlp_smote_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "7dc091cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, mlp_smote_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba9131b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for Hyperparameter Tuning\n",
    "mlp_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', MLPClassifier(random_state=424)]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__hidden_layer_sizes': [(50,50,50), (50,100,50)],\n",
    "    'classifier__activation': ['tanh', 'relu'],\n",
    "    'classifier__learning_rate': ['constant','adaptive']\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=mlp_smote_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of MLP (SMOTE): {cv_score}\\nTest score of MLP (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5f96b52a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of MLP (ADASYN): {'fit_time': array([25.66511106, 25.44943857, 27.18744254, 25.89575529, 25.16446376,\n",
      "       26.13890982, 25.37693524, 25.98228407, 25.25449562, 25.1505909 ]), 'score_time': array([0.03945112, 0.04975295, 0.05805373, 0.03948331, 0.03921247,\n",
      "       0.04584289, 0.04150152, 0.04714513, 0.04462004, 0.04521608]), 'test_mcc': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])}\n",
      "Test score of MLP (ADASYN): 0.9982795547909132\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.00      0.00      0.00        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.50      0.50      0.50     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "mlp_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', MLPClassifier(random_state=424, hidden_layer_sizes=(8,4,2,1), max_iter=100,activation = 'relu',solver='adam')]])\n",
    "\n",
    "mlp_adasyn_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(mlp_adasyn_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = mlp_adasyn_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of MLP (ADASYN): {cv_score}\\nTest score of MLP (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, mlp_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "c3d4461b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, mlp_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7f24599",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for Hyperparameter Tuning\n",
    "mlp_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', MLPClassifier(random_state=424)]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__hidden_layer_sizes': [(50,50,50), (50,100,50)],\n",
    "    'classifier__activation': ['tanh', 'relu'],\n",
    "    'classifier__learning_rate': ['constant','adaptive']\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=mlp_adasyn_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of MLP (ADASYN): {cv_score}\\nTest score of MLP (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "90784688",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of MLP (NearMiss): {'fit_time': array([1.99292707, 1.9593544 , 1.96868181, 1.96983719, 1.97482944,\n",
      "       1.96838474, 1.96068048, 1.95623326, 1.98410463, 1.9878912 ]), 'score_time': array([0.01930356, 0.02034903, 0.01920962, 0.0206151 , 0.01944041,\n",
      "       0.02049708, 0.01936102, 0.02027845, 0.0201695 , 0.02626705]), 'test_mcc': array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])}\n",
      "Test score of MLP (NearMiss): 0.9982795547909132\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.00      0.00      0.00        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.50      0.50      0.50     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "mlp_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['classifier', MLPClassifier(random_state=424, hidden_layer_sizes=(8,4,2,1), max_iter=100,activation = 'relu',solver='adam')]])\n",
    "mlp_nearmiss_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(mlp_nearmiss_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = mlp_nearmiss_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of MLP (NearMiss): {cv_score}\\nTest score of MLP (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, mlp_nearmiss_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "cca6a510",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, mlp_nearmiss_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae6d5d68",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for Hyperparameter Tuning\n",
    "mlp_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['classifier', MLPClassifier(random_state=424)]])\n",
    "\n",
    "parameters = {\n",
    "    'classifier__hidden_layer_sizes': [(50,50,50), (50,100,50)],\n",
    "    'classifier__activation': ['tanh', 'relu'],\n",
    "    'classifier__learning_rate': ['constant','adaptive']\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=mlp_nearmiss_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of MLP (NearMiss): {cv_score}\\nTest score of MLP (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "70ee3df2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:01] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:03] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:04] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:08] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:12] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:14] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:15] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of XGBoost: {'fit_time': array([1.33016253, 1.24499178, 1.28268433, 1.31967974, 1.26141667,\n",
      "       1.30670285, 1.27490425, 1.34400296, 1.26324701, 1.26534438]), 'score_time': array([0.04841161, 0.04630542, 0.04742908, 0.04235888, 0.04533386,\n",
      "       0.04640126, 0.04636455, 0.04327464, 0.04644394, 0.04632831]), 'test_mcc': array([0.65902049, 0.70423582, 0.74200411, 0.68681255, 0.71673792,\n",
      "       0.66737412, 0.59612332, 0.75336619, 0.68601217, 0.71581596])}\n",
      "Test score of XGBoost: 0.9992099996488887\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.91      0.60      0.72        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.95      0.80      0.86     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "xg_clf = XGBClassifier(objective ='binary:logistic', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = 5, alpha = 10, n_estimators = 10)\n",
    "xg_clf.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(xg_clf,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = xg_clf.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of XGBoost: {cv_score}\\nTest score of XGBoost: {test_score}')\n",
    "print(classification_report(y_test, xg_clf.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "c5bf99c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7388906866269531\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, xg_clf.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7f3b8e15",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:17] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:22] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:26] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:30] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:35] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:39] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:44] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:53] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:33:57] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:01] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of XGBoost (SMOTE): {'fit_time': array([3.86386919, 4.43788052, 4.56420517, 4.38189197, 4.62011671,\n",
      "       4.20060205, 4.18002343, 4.49459743, 3.96187091, 3.97921753]), 'score_time': array([0.04332471, 0.04899979, 0.04699826, 0.04601479, 0.04299998,\n",
      "       0.04638624, 0.04300094, 0.04699969, 0.04732823, 0.0464561 ]), 'test_mcc': array([0.47511298, 0.52504203, 0.46643648, 0.46471125, 0.41713823,\n",
      "       0.48105866, 0.44999303, 0.43218036, 0.47449589, 0.49801473])}\n",
      "Test score of XGBoost (SMOTE): 0.996243109441382\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.30      0.86      0.44        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.65      0.93      0.72     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "xgb_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', XGBClassifier(objective ='binary:logistic', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = 5, alpha = 10, n_estimators = 10)]])\n",
    "xgb_smote_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(xgb_smote_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = xgb_smote_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of XGBoost (SMOTE): {cv_score}\\nTest score of XGBoost (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, xgb_smote_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "4d3292cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5022648557278483\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, xgb_smote_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cf2b9e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for Hyperparamater Tuning\n",
    "xgb_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['classifier', XGBClassifier(objective ='binary:logistic', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = 5, alpha = 10, n_estimators = 10)]])\n",
    "\n",
    "parameters = {\n",
    " 'classifier__max_depth':[10,20,30]\n",
    "}\n",
    "grid_search = GridSearchCV(estimator=xgb_smote_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of XGBoost (SMOTE): {cv_score}\\nTest score of XGBoost (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "74484597",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:12] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:28] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:34] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:39] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:44] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:34:56] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:35:03] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of XGBoost (ADASYN): {'fit_time': array([5.22585869, 5.23554349, 5.22897983, 5.30822206, 5.24326038,\n",
      "       5.251508  , 5.4286201 , 6.06076741, 5.94363904, 6.5335772 ]), 'score_time': array([0.04741669, 0.04842377, 0.04744673, 0.04543734, 0.04635739,\n",
      "       0.04636216, 0.04899955, 0.05863953, 0.04890752, 0.0505054 ]), 'test_mcc': array([0.24763773, 0.29705017, 0.26002278, 0.30568118, 0.27021533,\n",
      "       0.27037858, 0.27497474, 0.26501207, 0.28063125, 0.28809254])}\n",
      "Test score of XGBoost (ADASYN): 0.9832695481197992\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.98      0.99     56864\n",
      "           1       0.08      0.89      0.15        98\n",
      "\n",
      "    accuracy                           0.98     56962\n",
      "   macro avg       0.54      0.94      0.57     56962\n",
      "weighted avg       1.00      0.98      0.99     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "xgb_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', XGBClassifier(objective ='binary:logistic', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = 5, alpha = 10, n_estimators = 10)]])\n",
    "xgb_adasyn_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(xgb_adasyn_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = xgb_adasyn_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of XGBoost (ADASYN): {cv_score}\\nTest score of XGBoost (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, xgb_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "86d5f8e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.271083141500256\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, xgb_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cbf6c73",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for Hyperparameter Tuning\n",
    "xgb_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', XGBClassifier(objective ='binary:logistic', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = 5, alpha = 10, n_estimators = 10)]])\n",
    "\n",
    "parameters = {\n",
    " 'classifier__max_depth':[10,20,30]\n",
    "}\n",
    "grid_search = GridSearchCV(estimator=xgb_adasyn_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of XGBoost (ADASYN): {cv_score}\\nTest score of XGBoost (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b4cbc88e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:35:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:35:15] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:35:21] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:35:26] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:35:32] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:35:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:35:44] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:35:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:35:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:36:00] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\WJ\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:36:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of XGBoost (NearMiss): {'fit_time': array([5.72105384, 5.74238205, 5.47278762, 5.65628338, 5.87726951,\n",
      "       5.38309836, 5.37322903, 5.12586093, 5.70744705, 5.21901131]), 'score_time': array([0.0522213 , 0.04739213, 0.04841328, 0.04346967, 0.04699469,\n",
      "       0.04843473, 0.04431796, 0.04442072, 0.04736567, 0.04742861]), 'test_mcc': array([0.24763773, 0.29705017, 0.26002278, 0.30568118, 0.27021533,\n",
      "       0.27037858, 0.27497474, 0.26501207, 0.28063125, 0.28809254])}\n",
      "Test score of XGBoost (NearMiss): 0.9832695481197992\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.98      0.99     56864\n",
      "           1       0.08      0.89      0.15        98\n",
      "\n",
      "    accuracy                           0.98     56962\n",
      "   macro avg       0.54      0.94      0.57     56962\n",
      "weighted avg       1.00      0.98      0.99     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "xgb_nearmiss_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['classifier', XGBClassifier(objective ='binary:logistic', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = 5, alpha = 10, n_estimators = 10)]])\n",
    "xgb_nearmiss_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(xgb_nearmiss_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = xgb_adasyn_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of XGBoost (NearMiss): {cv_score}\\nTest score of XGBoost (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, xgb_nearmiss_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "f6890a52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.271083141500256\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, xgb_nearmiss_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "202d192c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for Hyperparameter Tuning\n",
    "xgb_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['classifier', XGBClassifier(objective ='binary:logistic', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = 5, alpha = 10, n_estimators = 10)]])\n",
    "\n",
    "parameters = {\n",
    " 'classifier__max_depth':[10,20,30]\n",
    "}\n",
    "grid_search = GridSearchCV(estimator=xgb_nearmiss_pipeline,\n",
    "                           param_grid=parameters,\n",
    "                           scoring=make_scorer(matthews_corrcoef),\n",
    "                           cv=stratified_kfold, error_score='raise')\n",
    "\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of XGBoost (NearMiss): {cv_score}\\nTest score of XGBoost (NearMiss): {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "dfe613e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of XGBoost: {'fit_time': array([1.31781197, 1.48348331, 1.52644157, 1.44033885, 1.3932178 ,\n",
      "       1.37438965, 1.48688364, 1.4465661 , 1.34292316, 1.49170661]), 'score_time': array([0.08565283, 0.08678865, 0.08285761, 0.08564925, 0.08843565,\n",
      "       0.07859015, 0.07865882, 0.08678269, 0.08068013, 0.09152818]), 'test_mcc': array([0.42056535, 0.24482031, 0.12164022, 0.49377701, 0.35199774,\n",
      "       0.38521405, 0.43612343, 0.27283489, 0.51688821, 0.18016427])}\n",
      "Test score of XGBoost: 0.9937853305712581\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.99      1.00     56864\n",
      "           1       0.15      0.57      0.24        98\n",
      "\n",
      "    accuracy                           0.99     56962\n",
      "   macro avg       0.58      0.78      0.62     56962\n",
      "weighted avg       1.00      0.99      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lgbm.Dataset(X, categorical_feature='Class')\n",
    "lgb_clf=lgbm.LGBMClassifier()\n",
    "lgb_clf.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(lgb_clf,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = lgb_clf.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of XGBoost: {cv_score}\\nTest score of XGBoost: {test_score}')\n",
    "print(classification_report(y_test, lgb_clf.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "d60e2a28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2927483156465747\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, lgb_clf.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8241385e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for Hyperparameter Tuning\n",
    "lgb.Dataset(X, categorical_feature='Class')\n",
    "lgb_clf=lgb.LGBMClassifier()\n",
    "#Define the parameters\n",
    "parameters = {'num_leaves':[20,40,60,80,100], 'min_child_samples':[5,10,15],'max_depth':[-1,5,10,20],\n",
    "             'learning_rate':[0.05,0.1,0.2],'reg_alpha':[0,0.01,0.03]}\n",
    "#Define the scoring\n",
    "clf=GridSearchCV(lgb_clf,parameters,scoring=make_scorer(matthews_corrcoef), cv=stratified_kfold)\n",
    "grid_search.fit(X_train, y_train)\n",
    "cv_score = grid_search.best_score_\n",
    "test_score = grid_search.score(X_test, y_test)\n",
    "print(f'Cross-validation score of XGBoost: {cv_score}\\nTest score of XGBoost: {test_score}')\n",
    "print(classification_report(y_test, grid_search.best_estimator_.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "708a5e5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of XGBoost (SMOTE): {'fit_time': array([3.52803612, 3.51320314, 5.66277599, 5.86883926, 5.9112668 ,\n",
      "       5.98108816, 5.85172391, 5.89522195, 5.92672777, 5.7836504 ]), 'score_time': array([0.08755016, 0.0888381 , 0.16954517, 0.18202424, 0.14732623,\n",
      "       0.13769841, 0.16556454, 0.15912223, 0.17041659, 0.13648701]), 'test_mcc': array([0.74682338, 0.74500178, 0.69110497, 0.70947069, 0.65711855,\n",
      "       0.68123407, 0.69670959, 0.71002638, 0.78405057, 0.80858258])}\n",
      "Test score of XGBoost (SMOTE): 0.9988237772550121\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.61      0.86      0.71        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.81      0.93      0.86     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lgbm.Dataset(X, categorical_feature='Class')\n",
    "lgb_smote_pipeline = imbpipeline(steps = [['smote', SMOTE(random_state=424)],\n",
    "                                        ['lgbm', lgbm.LGBMClassifier()]])\n",
    "\n",
    "\n",
    "lgb_smote_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(lgb_smote_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = lgb_smote_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of XGBoost (SMOTE): {cv_score}\\nTest score of XGBoost (SMOTE): {test_score}')\n",
    "print(classification_report(y_test, lgb_smote_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "e33878e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.724406671212077\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, lgb_smote_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4bbbbcf4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of XGBoost (ADASYN): {'fit_time': array([9.01361847, 8.7849443 , 8.65360737, 8.84106612, 8.75927687,\n",
      "       8.54151106, 8.92574549, 8.6901598 , 8.64828849, 9.46942878]), 'score_time': array([0.1586926 , 0.14383245, 0.14250541, 0.13853264, 0.16012001,\n",
      "       0.16879249, 0.13692307, 0.1350894 , 0.1628418 , 0.14078641]), 'test_mcc': array([0.67591887, 0.74500178, 0.64253257, 0.64708739, 0.66195205,\n",
      "       0.63555438, 0.72699738, 0.65982741, 0.76308001, 0.73944312])}\n",
      "Test score of XGBoost (ADASYN): 0.9984024437344194\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     56864\n",
      "           1       0.52      0.82      0.64        98\n",
      "\n",
      "    accuracy                           1.00     56962\n",
      "   macro avg       0.76      0.91      0.82     56962\n",
      "weighted avg       1.00      1.00      1.00     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lgbm.Dataset(X, categorical_feature='Class')\n",
    "lgb_adasyn_pipeline = imbpipeline(steps = [['adasyn', ADASYN(random_state=424)],\n",
    "                                        ['lgbm', lgbm.LGBMClassifier()]])\n",
    "\n",
    "\n",
    "lgb_adasyn_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(lgb_adasyn_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = lgb_adasyn_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of XGBoost (ADASYN): {cv_score}\\nTest score of XGBoost (ADASYN): {test_score}')\n",
    "print(classification_report(y_test, lgb_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "d2a59c0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.65261620022208\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, lgb_adasyn_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ab6e9473",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\n",
      "Cross-validation score of XGBoost (Nearmiss): {'fit_time': array([3.89243412, 3.87438059, 3.87367964, 3.88412237, 3.82736754,\n",
      "       3.97484064, 3.96139288, 3.97924924, 4.00492311, 4.00824022]), 'score_time': array([0.11433649, 0.11604905, 0.11508608, 0.1109302 , 0.10586357,\n",
      "       0.1106751 , 0.12296987, 0.12338138, 0.11589909, 0.11093187]), 'test_mcc': array([0.00920513, 0.00882934, 0.01048693, 0.00159952, 0.01232639,\n",
      "       0.01075156, 0.00395513, 0.00683136, 0.00929968, 0.009798  ])}\n",
      "Test score of XGBoost (Nearmiss): 0.04834802148800955\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.05      0.09     56864\n",
      "           1       0.00      0.99      0.00        98\n",
      "\n",
      "    accuracy                           0.05     56962\n",
      "   macro avg       0.50      0.52      0.05     56962\n",
      "weighted avg       1.00      0.05      0.09     56962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lgbm.Dataset(X, categorical_feature='Class')\n",
    "lgb_nearmiss_pipeline = imbpipeline(steps = [['nearmiss', NearMiss()],\n",
    "                                        ['lgbm', lgbm.LGBMClassifier()]])\n",
    "\n",
    "\n",
    "lgb_nearmiss_pipeline.fit(X_train, y_train)\n",
    "mcc = {'mcc': make_scorer(matthews_corrcoef)}\n",
    "cv_score = cross_validate(lgb_nearmiss_pipeline,  X_train, y_train, cv = stratified_kfold, scoring = mcc)\n",
    "# cv_score = cross_val_score(pipeline, X_train, y_train, cv = stratified_kfold)\n",
    "test_score = lgb_nearmiss_pipeline.score(X_test, y_test)\n",
    "\n",
    "print(\"Due to computational limitation, this model will not be tuned. The classification report shows the test set result.\")\n",
    "print(f'Cross-validation score of XGBoost (Nearmiss): {cv_score}\\nTest score of XGBoost (Nearmiss): {test_score}')\n",
    "print(classification_report(y_test, lgb_nearmiss_pipeline.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "1b80a4aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.007176062660310684\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(y_test, lgb_nearmiss_pipeline.predict(X_test)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}