Scikilt-Learn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to Scikit-learn\n",
    "This demonstrated some of the most useful functions of the Sklearn libaray\n",
    "\n",
    "What it covers:\n",
    "0. Sklearn Workflow\n",
    "1. Getting datat ready\n",
    "2. Choose the right estimator / algorithm / model for our problem\n",
    "3. Fit the model/algorithm and use it to make predictions on our data \n",
    "4. Evalutating a model\n",
    "5. Improve a model\n",
    "6. Save and load a trained model\n",
    "7. Putting it all together!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. An end-to-end Sk-Learn Workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>241</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>123</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>110</td>\n",
       "      <td>264</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>68</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>144</td>\n",
       "      <td>193</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>141</td>\n",
       "      <td>0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>130</td>\n",
       "      <td>131</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>115</td>\n",
       "      <td>1</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>174</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>303 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0     63    1   3       145   233    1        0      150      0      2.3   \n",
       "1     37    1   2       130   250    0        1      187      0      3.5   \n",
       "2     41    0   1       130   204    0        0      172      0      1.4   \n",
       "3     56    1   1       120   236    0        1      178      0      0.8   \n",
       "4     57    0   0       120   354    0        1      163      1      0.6   \n",
       "..   ...  ...  ..       ...   ...  ...      ...      ...    ...      ...   \n",
       "298   57    0   0       140   241    0        1      123      1      0.2   \n",
       "299   45    1   3       110   264    0        1      132      0      1.2   \n",
       "300   68    1   0       144   193    1        1      141      0      3.4   \n",
       "301   57    1   0       130   131    0        1      115      1      1.2   \n",
       "302   57    0   1       130   236    0        0      174      0      0.0   \n",
       "\n",
       "     slope  ca  thal  target  \n",
       "0        0   0     1       1  \n",
       "1        0   0     2       1  \n",
       "2        2   0     2       1  \n",
       "3        2   0     2       1  \n",
       "4        2   0     2       1  \n",
       "..     ...  ..   ...     ...  \n",
       "298      1   0     3       0  \n",
       "299      1   0     3       0  \n",
       "300      1   2     3       0  \n",
       "301      1   1     3       0  \n",
       "302      1   1     2       0  \n",
       "\n",
       "[303 rows x 14 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # 1. Get the data ready\n",
    "import pandas as pd\n",
    "heart_disease=pd.read_csv(\"data/heart-disease.csv\")\n",
    "heart_disease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create X (features matrix)\n",
    "X = heart_disease.drop(\"target\",axis=1)\n",
    "\n",
    "\n",
    "#create Y (labels)\n",
    "y=heart_disease[\"target\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>241</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>123</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>110</td>\n",
       "      <td>264</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>68</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>144</td>\n",
       "      <td>193</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>141</td>\n",
       "      <td>0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>130</td>\n",
       "      <td>131</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>115</td>\n",
       "      <td>1</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>174</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>303 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0     63    1   3       145   233    1        0      150      0      2.3   \n",
       "1     37    1   2       130   250    0        1      187      0      3.5   \n",
       "2     41    0   1       130   204    0        0      172      0      1.4   \n",
       "3     56    1   1       120   236    0        1      178      0      0.8   \n",
       "4     57    0   0       120   354    0        1      163      1      0.6   \n",
       "..   ...  ...  ..       ...   ...  ...      ...      ...    ...      ...   \n",
       "298   57    0   0       140   241    0        1      123      1      0.2   \n",
       "299   45    1   3       110   264    0        1      132      0      1.2   \n",
       "300   68    1   0       144   193    1        1      141      0      3.4   \n",
       "301   57    1   0       130   131    0        1      115      1      1.2   \n",
       "302   57    0   1       130   236    0        0      174      0      0.0   \n",
       "\n",
       "     slope  ca  thal  \n",
       "0        0   0     1  \n",
       "1        0   0     2  \n",
       "2        2   0     2  \n",
       "3        2   0     2  \n",
       "4        2   0     2  \n",
       "..     ...  ..   ...  \n",
       "298      1   0     3  \n",
       "299      1   0     3  \n",
       "300      1   2     3  \n",
       "301      1   1     3  \n",
       "302      1   1     2  \n",
       "\n",
       "[303 rows x 13 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      1\n",
       "1      1\n",
       "2      1\n",
       "3      1\n",
       "4      1\n",
       "      ..\n",
       "298    0\n",
       "299    0\n",
       "300    0\n",
       "301    0\n",
       "302    0\n",
       "Name: target, Length: 303, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'ccp_alpha': 0.0,\n",
       " 'class_weight': None,\n",
       " 'criterion': 'gini',\n",
       " 'max_depth': None,\n",
       " 'max_features': 'auto',\n",
       " 'max_leaf_nodes': None,\n",
       " 'max_samples': None,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 100,\n",
       " 'n_jobs': None,\n",
       " 'oob_score': False,\n",
       " 'random_state': None,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2. Choose the right model and hyperparameters\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf = RandomForestClassifier(n_estimators=100) #Initialize RandomForest # clf -> classifier\n",
    "\n",
    "# We 'll keep the default hyperparameters\n",
    "clf.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#3. Fit the model to the training data\n",
    "from sklearn.model_selection import train_test_split\n",
    "# This is gonna spilt our data X and y for training and testing # 0.2 = 80% of data\n",
    "# 1000 data -> 800 for training and 200 for testing\n",
    "X_train , X_test , y_train ,y_test = train_test_split(X,y,test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.fit(X_train , y_train); #Saying clf to find a pattern on training data\n",
    "# classification model randomforest, model is fit to training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a prediction\n",
    "#y_label =clf.predict(np.array([0,2,3,4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1,\n",
       "       0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1,\n",
       "       1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1], dtype=int64)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_preds=clf.predict(X_test)\n",
    "y_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "61"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>130</td>\n",
       "      <td>330</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>132</td>\n",
       "      <td>1</td>\n",
       "      <td>1.8</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>67</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>152</td>\n",
       "      <td>277</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>241</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>123</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>59</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>170</td>\n",
       "      <td>326</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>1</td>\n",
       "      <td>3.4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>58</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>170</td>\n",
       "      <td>225</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>146</td>\n",
       "      <td>1</td>\n",
       "      <td>2.8</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>150</td>\n",
       "      <td>126</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>173</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>199</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>179</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>160</td>\n",
       "      <td>302</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>162</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>64</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>145</td>\n",
       "      <td>212</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>132</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>120</td>\n",
       "      <td>240</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>194</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>61 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "217   63    1   0       130   330    1        0      132      1      1.8   \n",
       "127   67    0   2       152   277    0        1      172      0      0.0   \n",
       "298   57    0   0       140   241    0        1      123      1      0.2   \n",
       "195   59    1   0       170   326    0        0      140      1      3.4   \n",
       "292   58    0   0       170   225    1        0      146      1      2.8   \n",
       "..   ...  ...  ..       ...   ...  ...      ...      ...    ...      ...   \n",
       "111   57    1   2       150   126    1        1      173      0      0.2   \n",
       "124   39    0   2        94   199    0        1      179      0      0.0   \n",
       "25    71    0   1       160   302    0        1      162      0      0.4   \n",
       "242   64    1   0       145   212    0        0      132      0      2.0   \n",
       "103   42    1   2       120   240    1        1      194      0      0.8   \n",
       "\n",
       "     slope  ca  thal  \n",
       "217      2   3     3  \n",
       "127      2   1     2  \n",
       "298      1   0     3  \n",
       "195      0   0     3  \n",
       "292      1   2     1  \n",
       "..     ...  ..   ...  \n",
       "111      2   1     3  \n",
       "124      2   0     2  \n",
       "25       2   2     2  \n",
       "242      1   2     1  \n",
       "103      0   0     3  \n",
       "\n",
       "[61 rows x 13 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 4. Evaluate the model\n",
    "clf.score(X_train, y_train) #model has found patterns in the data so we it returns 100%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8688524590163934"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test,y_test) #77% accuracy with test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.88      0.85        26\n",
      "           1       0.91      0.86      0.88        35\n",
      "\n",
      "    accuracy                           0.87        61\n",
      "   macro avg       0.87      0.87      0.87        61\n",
      "weighted avg       0.87      0.87      0.87        61\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report , confusion_matrix ,accuracy_score\n",
    "\n",
    "print(classification_report(y_test,y_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[23,  3],\n",
       "       [ 5, 30]], dtype=int64)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8688524590163934"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test,y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trying with 10 estimators...\n",
      "Model accuary on test set : 83.606557%\n",
      "Trying with 20 estimators...\n",
      "Model accuary on test set : 81.967213%\n",
      "Trying with 30 estimators...\n",
      "Model accuary on test set : 86.885246%\n",
      "Trying with 40 estimators...\n",
      "Model accuary on test set : 85.245902%\n",
      "Trying with 50 estimators...\n",
      "Model accuary on test set : 85.245902%\n",
      "Trying with 60 estimators...\n",
      "Model accuary on test set : 81.967213%\n",
      "Trying with 70 estimators...\n",
      "Model accuary on test set : 86.885246%\n",
      "Trying with 80 estimators...\n",
      "Model accuary on test set : 86.885246%\n",
      "Trying with 90 estimators...\n",
      "Model accuary on test set : 90.163934%\n"
     ]
    }
   ],
   "source": [
    "# 5. Improve a model\n",
    "# Try different amount of n_estimators\n",
    "np.random.seed(42)\n",
    "for i in range(10,100,10):\n",
    "    print(f\"Trying with {i} estimators...\")\n",
    "    clf=RandomForestClassifier(n_estimators=i).fit(X_train ,y_train )\n",
    "    print(f\"Model accuary on test set : {clf.score(X_test, y_test) * 100 :2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6. Save a model and load it \n",
    "import pickle\n",
    "pickle.dump(clf,open(\"random_forestmodel_1.pkl\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9016393442622951"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded_model = pickle.load(open(\"random_forestmodel_1.pkl\",\"rb\"))\n",
    "loaded_model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Getting our data ready to be used with machine learning\n",
    "\n",
    "Three main things we have to do:\n",
    "  1. Spilt the data into feature and labels ( usually 'X'and 'y')\n",
    "  2. Filling ( also called imputing ) or disregarding missing values\n",
    "  3. Converting non-numerical values to numerical values (also called feature encoding)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  \n",
       "0   0     1  \n",
       "1   0     2  \n",
       "2   0     2  \n",
       "3   0     2  \n",
       "4   0     2  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = heart_disease.drop(\"target\",axis=1)\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      1\n",
       "1      1\n",
       "2      1\n",
       "3      1\n",
       "4      1\n",
       "      ..\n",
       "298    0\n",
       "299    0\n",
       "300    0\n",
       "301    0\n",
       "302    0\n",
       "Name: target, Length: 303, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y= heart_disease[\"target\"]\n",
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split the datat into training and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train , X_test , y_train , y_test = train_test_split(X,y,test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((242, 13), (61, 13), (242,), (61,))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape , X_test.shape , y_train.shape ,y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(303, 13)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "242.4"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape[0] * 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "303"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(heart_disease)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 Make sure it's all numerical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431</td>\n",
       "      <td>4</td>\n",
       "      <td>15323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714</td>\n",
       "      <td>5</td>\n",
       "      <td>19943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714</td>\n",
       "      <td>4</td>\n",
       "      <td>28343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365</td>\n",
       "      <td>4</td>\n",
       "      <td>13434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577</td>\n",
       "      <td>3</td>\n",
       "      <td>14043</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors  Price\n",
       "0   Honda  White          35431      4  15323\n",
       "1     BMW   Blue         192714      5  19943\n",
       "2   Honda  White          84714      4  28343\n",
       "3  Toyota  White         154365      4  13434\n",
       "4  Nissan   Blue         181577      3  14043"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales =pd.read_csv(\"data/car-sales-extended.csv\")\n",
    "car_sales.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(car_sales)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Spilt the  data into X and y\n",
    "X = car_sales.drop(\"Price\", axis= 1)\n",
    "y= car_sales[\"Price\"]\n",
    "\n",
    "# Slit into trainng and test\n",
    "X_train , X_test , y_train , y_test = train_test_split(X,y,test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3235867221569877"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# # Build machine learning model \n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "model = RandomForestRegressor()\n",
    "model.fit(X_train, y_train)\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors\n",
       "0   Honda  White          35431      4\n",
       "1     BMW   Blue         192714      5\n",
       "2   Honda  White          84714      4\n",
       "3  Toyota  White         154365      4\n",
       "4  Nissan   Blue         181577      3"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 3.54310e+04],\n",
       "       [1.00000e+00, 0.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        1.00000e+00, 1.92714e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 8.47140e+04],\n",
       "       ...,\n",
       "       [0.00000e+00, 0.00000e+00, 1.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 6.66040e+04],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 2.15883e+05],\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 2.48360e+05]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Turn the categories into numbers\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]\n",
    "one_hot = OneHotEncoder()\n",
    "transformer = ColumnTransformer([(\"one_hot\",\n",
    "                                   one_hot,\n",
    "                                   categorical_features)],\n",
    "                                   remainder=\"passthrough\")\n",
    "\n",
    "transformed_X = transformer.fit_transform(X)\n",
    "transformed_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35431.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>192714.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>84714.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>154365.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>181577.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35820.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>155144.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66604.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>215883.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>248360.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      0    1    2    3    4    5    6    7    8    9    10   11        12\n",
       "0    0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0   35431.0\n",
       "1    1.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0  192714.0\n",
       "2    0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0   84714.0\n",
       "3    0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0  154365.0\n",
       "4    0.0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  0.0  1.0  0.0  0.0  181577.0\n",
       "..   ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...       ...\n",
       "995  0.0  0.0  0.0  1.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0   35820.0\n",
       "996  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0  1.0  0.0  0.0  155144.0\n",
       "997  0.0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  1.0  0.0   66604.0\n",
       "998  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0  215883.0\n",
       "999  0.0  0.0  0.0  1.0  0.0  1.0  0.0  0.0  0.0  0.0  1.0  0.0  248360.0\n",
       "\n",
       "[1000 rows x 13 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(transformed_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Doors</th>\n",
       "      <th>Make_BMW</th>\n",
       "      <th>Make_Honda</th>\n",
       "      <th>Make_Nissan</th>\n",
       "      <th>Make_Toyota</th>\n",
       "      <th>Colour_Black</th>\n",
       "      <th>Colour_Blue</th>\n",
       "      <th>Colour_Green</th>\n",
       "      <th>Colour_Red</th>\n",
       "      <th>Colour_White</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Doors  Make_BMW  Make_Honda  Make_Nissan  Make_Toyota  Colour_Black  \\\n",
       "0        4         0           1            0            0             0   \n",
       "1        5         1           0            0            0             0   \n",
       "2        4         0           1            0            0             0   \n",
       "3        4         0           0            0            1             0   \n",
       "4        3         0           0            1            0             0   \n",
       "..     ...       ...         ...          ...          ...           ...   \n",
       "995      4         0           0            0            1             1   \n",
       "996      3         0           0            1            0             0   \n",
       "997      4         0           0            1            0             0   \n",
       "998      4         0           1            0            0             0   \n",
       "999      4         0           0            0            1             0   \n",
       "\n",
       "     Colour_Blue  Colour_Green  Colour_Red  Colour_White  \n",
       "0              0             0           0             1  \n",
       "1              1             0           0             0  \n",
       "2              0             0           0             1  \n",
       "3              0             0           0             1  \n",
       "4              1             0           0             0  \n",
       "..           ...           ...         ...           ...  \n",
       "995            0             0           0             0  \n",
       "996            0             0           0             1  \n",
       "997            1             0           0             0  \n",
       "998            0             0           0             1  \n",
       "999            1             0           0             0  \n",
       "\n",
       "[1000 rows x 10 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies = pd.get_dummies(car_sales[[\"Make\",\"Colour\",\"Doors\"]])\n",
    "dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor()"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's refit the model\n",
    "np.random.seed(42)\n",
    "X_train , X_test , y_train , y_test = train_test_split(transformed_X,y,test_size=0.2)\n",
    "model.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3235867221569877"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_test , y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### If there were mising values\n",
    "\n",
    "1. Fill them with some value (also known as imputing data)\n",
    "2. Remove the samples with missing data altogether\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors    Price\n",
       "0   Honda  White        35431.0    4.0  15323.0\n",
       "1     BMW   Blue       192714.0    5.0  19943.0\n",
       "2   Honda  White        84714.0    4.0  28343.0\n",
       "3  Toyota  White       154365.0    4.0  13434.0\n",
       "4  Nissan   Blue       181577.0    3.0  14043.0"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Imoport car sales missing data\n",
    "car_sales_misssing=pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "car_sales_misssing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_misssing.isna().sum()  #How many missing values are there"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create X and y\n",
    "X=car_sales_misssing.drop(\"Price\",axis=1)\n",
    "y=car_sales_misssing[\"Price\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 1: Fill missing datat with pandas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill the \"Make\" column \n",
    "car_sales_misssing[\"Make\"].fillna(\"missing\", inplace=True)\n",
    "\n",
    "# Fill the \"colour\" column \n",
    "car_sales_misssing[\"Colour\"].fillna(\"missing\", inplace=True)\n",
    "\n",
    "#Fill the \"Odometer(KM)\" column\n",
    "\n",
    "car_sales_misssing[\"Odometer (KM)\"].fillna(car_sales_misssing[\"Odometer (KM)\"].mean(), inplace=True)\n",
    "\n",
    "# Fill the \"Doors\" column\n",
    "car_sales_misssing[\"Doors\"].fillna(4,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make              0\n",
       "Colour            0\n",
       "Odometer (KM)     0\n",
       "Doors             0\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check our daatFrame again \n",
    "car_sales_misssing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove rows with missing Price Value\n",
    "car_sales_misssing.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Odometer (KM)    0\n",
       "Doors            0\n",
       "Price            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_misssing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "950"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(car_sales_misssing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=car_sales_misssing.drop(\"Price\",axis=1)\n",
    "y=car_sales_misssing[\"Price\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        3.54310e+04, 1.53230e+04],\n",
       "       [1.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        1.92714e+05, 1.99430e+04],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        8.47140e+04, 2.83430e+04],\n",
       "       ...,\n",
       "       [0.00000e+00, 0.00000e+00, 1.00000e+00, ..., 0.00000e+00,\n",
       "        6.66040e+04, 3.15700e+04],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        2.15883e+05, 4.00100e+03],\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        2.48360e+05, 1.27320e+04]])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let' s try and convert our data into numbers\n",
    "# Turn the categories into numbers\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]\n",
    "one_hot = OneHotEncoder()\n",
    "transformer = ColumnTransformer([(\"one_hot\",\n",
    "                                   one_hot,\n",
    "                                   categorical_features)],\n",
    "                                   remainder=\"passthrough\")\n",
    "\n",
    "transformed_X = transformer.fit_transform(car_sales_misssing)\n",
    "transformed_X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 2: Fill missing values with Sk-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors    Price\n",
       "0   Honda  White        35431.0    4.0  15323.0\n",
       "1     BMW   Blue       192714.0    5.0  19943.0\n",
       "2   Honda  White        84714.0    4.0  28343.0\n",
       "3  Toyota  White       154365.0    4.0  13434.0\n",
       "4  Nissan   Blue       181577.0    3.0  14043.0"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_missing=pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "car_sales_missing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             47\n",
       "Colour           46\n",
       "Odometer (KM)    48\n",
       "Doors            47\n",
       "Price             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Drop the rows with no labels\n",
    "car_sales_missing.dropna(subset=[\"Price\"],inplace=True)\n",
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into X & y\n",
    "X = car_sales_missing.drop(\"Price\",axis=1)\n",
    "y=car_sales_missing[\"Price\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Honda', 'White', 35431.0],\n",
       "       ['BMW', 'Blue', 192714.0],\n",
       "       ['Honda', 'White', 84714.0],\n",
       "       ...,\n",
       "       ['Nissan', 'Blue', 66604.0],\n",
       "       ['Honda', 'White', 215883.0],\n",
       "       ['Toyota', 'Blue', 248360.0]], dtype=object)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Fill missing values with Scikilt-Learn\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "#FIll categorical values with \"missing \" and numerical values with mean\n",
    "\n",
    "cat_imputer = SimpleImputer(strategy=\"constant\",fill_value='missing')\n",
    "door_imputer=SimpleImputer(strategy=\"constant\", fill_value=4)\n",
    "num_imputer=SimpleImputer(strategy=\"mean\")\n",
    "\n",
    "# Define Column \n",
    "\n",
    "cat_features=[\"Make\",\"Colour\"]\n",
    "door_feature=[\"Door\"]\n",
    "num_features =[\"Odometer (KM)\"]\n",
    "\n",
    "#Create an imputer (Something that fills missing data)\n",
    "\n",
    "imputer = ColumnTransformer([\n",
    "    (\"cat_imputer\", cat_imputer,cat_features),\n",
    "    (\"num_imputer\", num_imputer, num_features)\n",
    "])\n",
    "\n",
    "# Transform the data\n",
    "filled_X=imputer.fit_transform(X)\n",
    "filled_X\n",
    "                              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour Odometer (KM)\n",
       "0   Honda  White         35431\n",
       "1     BMW   Blue        192714\n",
       "2   Honda  White         84714\n",
       "3  Toyota  White        154365\n",
       "4  Nissan   Blue        181577"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_filled =pd.DataFrame(filled_X, columns=[\"Make\",\"Colour\",\"Odometer (KM)\"])\n",
    "car_sales_filled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Odometer (KM)    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_filled.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Choosing the right estimator/algorithm for our problem\n",
    "\n",
    "Scikit-Learn uses estimator as another term for machine learning model or algorithm\n",
    "\n",
    "* Classification- Predicting whether a sample is  one thing or another .\n",
    "* Regression - predicting a number\n",
    "\n",
    "Step 1 - Check the Scikit-Learn machine Learning map...https://scikit-learn.org/stable/tutorial/machine_learning_map/index.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Picking a machine Learning model for a regression probelem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Boston housing datatset\n",
    "from sklearn.datasets import load_boston\n",
    "boston = load_boston()\n",
    "boston; # boston is a dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  target  \n",
       "0     15.3  396.90   4.98    24.0  \n",
       "1     17.8  396.90   9.14    21.6  \n",
       "2     17.8  392.83   4.03    34.7  \n",
       "3     18.7  394.63   2.94    33.4  \n",
       "4     18.7  396.90   5.33    36.2  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston_df = pd.DataFrame(boston[\"data\"], columns=boston[\"feature_names\"])\n",
    "boston_df[\"target\"] = pd.Series(boston[\"target\"])\n",
    "boston_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "506"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How many samples?\n",
    "len(boston_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6662221670168519"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's try the Ridge Regression model\n",
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate Ridge model\n",
    "model = Ridge()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Check the score of the Ridge model on test data\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    " # How do we improve this score?\n",
    "# What if RIdge was not working"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8922527442109116"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's try the Random Forst Regressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split the data\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instatiate Random Forest Regressor\n",
    "rf = RandomForestRegressor(n_estimators=100)\n",
    "rf.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate the Random Forest Regressor\n",
    "rf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6662221670168519"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the Ridge model again\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Chosing and estimator for a classification problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease = pd.read_csv(\"data/heart-disease.csv\")\n",
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "303"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(heart_disease)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Consulting the map and it says to try `Linear SVC`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\manik\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8688524590163934"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the LinearSVC estimator class\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Make the data\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate LinearSVC\n",
    "clf = LinearSVC(max_iter=1000)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate the LinearSVC\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    165\n",
       "0    138\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease[\"target\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the RandomForestClassifier estimator class\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Make the data\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate Random Forest Classifier\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate the Random Forest Classifier\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tidbit:\n",
    "\n",
    "1. If you have structured data, used ensemble methods\n",
    "2. If you have unstructured data, use deep learning or transfer learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Fit the model/algorithm on our data and use it to make predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Different names for:\n",
    "\n",
    "* `X` = features, features variables, data\n",
    "* `y` = labels, targets, target variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the RandomForestClassifier estimator class\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Make the data\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate Random Forest Classifier\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "\n",
    "# Fit the model to the data (training the machine learning model)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate the Random Forest Classifier (use the patterns the model has learned)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  \n",
       "0   0     1  \n",
       "1   0     2  \n",
       "2   0     2  \n",
       "3   0     2  \n",
       "4   0     2  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Make predictions using a machine learning mode\n",
    "2 ways to make predictions:\n",
    "\n",
    "1. `predict()`\n",
    "2. `predict_proba`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use a trained model \n",
    "#clf.predict(np.array([1,2,3,4,5,5])) #Error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>276</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>112</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>59</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>170</td>\n",
       "      <td>288</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>159</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>150</td>\n",
       "      <td>126</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>173</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>134</td>\n",
       "      <td>409</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>1</td>\n",
       "      <td>1.9</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>110</td>\n",
       "      <td>265</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>130</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "179   57    1   0       150   276    0        0      112      1      0.6   \n",
       "228   59    1   3       170   288    0        0      159      0      0.2   \n",
       "111   57    1   2       150   126    1        1      173      0      0.2   \n",
       "246   56    0   0       134   409    0        0      150      1      1.9   \n",
       "60    71    0   2       110   265    1        0      130      0      0.0   \n",
       "\n",
       "     slope  ca  thal  \n",
       "179      1   1     1  \n",
       "228      1   0     3  \n",
       "111      2   1     3  \n",
       "246      1   2     3  \n",
       "60       2   1     2  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0,\n",
       "       1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0], dtype=int64)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0,\n",
       "       0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0], dtype=int64)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compare predictions to truth labels to evaluate the model\n",
    "y_preds = clf.predict(X_test)\n",
    "np.mean(y_preds == y_test) #Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracy_score(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make predictions with predict_proba() - use this if someone asks you \"what's the probability your model is assigning to each prediction?\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`predict` can be also used for regression models`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate and fit model\n",
    "model = RandomForestRegressor(n_estimators=100).fit(X_train, y_train)\n",
    "\n",
    "# Make predictions\n",
    "y_preds = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([23.081, 30.574, 16.759, 23.46 , 16.893, 21.644, 19.113, 15.334,\n",
       "       21.14 , 20.639])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_preds[:10] #Regression model has predict based on X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([23.6, 32.4, 13.6, 22.8, 16.1, 20. , 17.8, 14. , 19.6, 16.8])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(y_test[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.136382352941176"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compare the predictions to the truth\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "mean_absolute_error(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Evaluating a machine learning model\n",
    "Three ways to evaluate Scikit-Learn models/esitmators:\n",
    "\n",
    "1. Estimator score method\n",
    "2. The scoring parameter\n",
    "3. Problem-specific metric functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier()"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "clf = RandomForestClassifier()\n",
    "\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's do the same but for regression..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate and fit model\n",
    "model = RandomForestRegressor(n_estimators=100).fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8654448653350507"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluating a model using the scoring parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    " from sklearn.model_selection import cross_val_score\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "\n",
    "clf.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.81967213, 0.86885246, 0.81967213, 0.78333333, 0.76666667])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(clf, X, y, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.90322581, 0.80645161, 0.87096774, 0.9       , 0.86666667,\n",
       "       0.8       , 0.73333333, 0.86666667, 0.73333333, 0.8       ])"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(clf, X, y, cv=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9375    , 0.875     , 0.8125    , 0.8       , 0.8       ,\n",
       "       0.86666667, 0.86666667, 0.93333333, 0.93333333, 0.86666667,\n",
       "       0.86666667, 0.73333333, 0.8       , 0.66666667, 0.8       ,\n",
       "       0.86666667, 0.66666667, 0.73333333, 0.8       , 0.8       ])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(clf, X, y, cv=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8524590163934426, 0.8248087431693989)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "# Single training and test split score\n",
    "clf_single_score = clf.score(X_test, y_test)\n",
    "\n",
    "# Take the mean of 5-fold cross-validation score\n",
    "clf_cross_val_score = np.mean(cross_val_score(clf, X, y, cv=5))\n",
    "\n",
    "# Compare the two\n",
    "clf_single_score, clf_cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.78688525, 0.86885246, 0.80327869, 0.78333333, 0.76666667])"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Scoring parameter set to None by default\n",
    "cross_val_score(clf, X, y, cv=5, scoring=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Default scoring parameter of classifier = mean accuracy\n",
    "#clf.score()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification model evaluation metrics\n",
    "1. Accuracy\n",
    "2. Area under ROC curve \n",
    "3. Confusion matrix\n",
    "4. Classification Reports\n",
    "**Accuracy**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "cross_val_score = cross_val_score(clf, X, y, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.81967213, 0.90163934, 0.83606557, 0.78333333, 0.78333333])"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Heart Disease Classifier Cross-Validated Accuracy: 82.48%\n"
     ]
    }
   ],
   "source": [
    "print(f\"Heart Disease Classifier Cross-Validated Accuracy: {np.mean(cross_val_score) *100:.2f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Area uder the reciver operating characteritsitc curve (AUC/ROC)**\n",
    "* Area under the curve(AUC)\n",
    "* ROC curve\n",
    "\n",
    "ROC curves are a comparison of a model's true postive rate (tpr) versus a models false positive rate (fpr).\n",
    "\n",
    "* True positive = model predicts 1 when truth is 1\n",
    "* False positive = model predicts 1 when truth is 0\n",
    "* True negative = model predicts 0 when truth is 0\n",
    "* False negative = model predicts 0 when truth is 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create X_test... etc\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0.51, 0.49],\n",
       "        [0.17, 0.83],\n",
       "        [0.51, 0.49],\n",
       "        [0.72, 0.28],\n",
       "        [0.43, 0.57],\n",
       "        [0.12, 0.88],\n",
       "        [0.3 , 0.7 ],\n",
       "        [0.97, 0.03],\n",
       "        [0.15, 0.85],\n",
       "        [0.4 , 0.6 ]]),\n",
       " 61)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "# Fit the classifier\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions with probabilities\n",
    "y_probs = clf.predict_proba(X_test)\n",
    "\n",
    "y_probs[:10], len(y_probs) # In array there is probability of 1 and 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.49, 0.83, 0.49, 0.28, 0.57, 0.88, 0.7 , 0.03, 0.85, 0.6 ])"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_probs_positive = y_probs[:, 1]\n",
    "y_probs_positive[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.03448276, 0.03448276, 0.03448276, 0.03448276,\n",
       "       0.03448276, 0.03448276, 0.06896552, 0.06896552, 0.06896552,\n",
       "       0.10344828, 0.10344828, 0.13793103, 0.13793103, 0.13793103,\n",
       "       0.20689655, 0.20689655, 0.20689655, 0.27586207, 0.37931034,\n",
       "       0.37931034, 0.48275862, 0.48275862, 0.55172414, 0.55172414,\n",
       "       1.        ])"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Caculate fpr, tpr and thresholds\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_probs_positive)\n",
    "\n",
    "# Check the false positive rates\n",
    "fpr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/JklEQVR4nO3dd3gU5fbA8e+hhNA7Sm+CkCA1NEEEAQugIAIXRCmCiAL+FHtH5aqoVxApio2riFwRaYKAKEVApElvIhaiovSakHZ+f8wE17BJFshms7vn8zx5sjvz7syZLXOmnldUFWOMMeErV6ADMMYYE1iWCIwxJsxZIjDGmDBnicAYY8KcJQJjjAlzlgiMMSbMWSLIYURkm4i0DnQcOYWIPC4i7wRo3pNFZGQg5p3VRKS3iCy6wNde8HdSRFaKSIMLee2FEpF7ReSl7JxnsLNEkAER+VlE4kTkpIjsd1cMhfw5T1WNVtWl/pxHKhHJJyIvisiv7nL+ICIPiYhkx/y9xNNaRGI9h6nqC6o60E/zE3elsVVETolIrIhMF5Er/DG/CyUiI0RkysVMQ1U/UtVrfZjXOcnvQr+TInIjcEJVv3efjxCRRPf3dFREVolI8zSvKSYiE93f22kR2SIi/b1M+1YRWedO6w8R+UJEWrqjJwG3iUiZDGILis8+u1giyNyNqloIqA80AB4LbDjnT0TypDNqOtAW6AAUBm4HBgGv+yEGEZGc9n17Hfg/4F6gBFATmAV0zOoZZfAZ+F0A5z0Y+DDNsP+5v6dSwBKc7yAAIhIBLAYqA82BosBDwEsiMtyj3XBgDPACcAlQCZgAdAZQ1XjgC6BPBrFl2WcfyM82y6iq/aXzB/wMtPN4/jIwz+N5M2AVcBTYBLT2GFcCeB/4HTgCzPIY1wnY6L5uFVA37TyBckAcUMJjXAPgIJDXfX4HsMOd/kKgskdbBYYAPwA/eVm2tkA8UDHN8KZAMnCZ+3wp8CKwBjgGzE4TU0bvwVLg38BKd1kuA/q7MZ8A9gJ3uW0Lum1SgJPuXzlgBDDFbVPFXa6+wK/ue/GEx/zyA/91348dwMNAbDqfbQ13OZtk8PlPBsYD89x4vwOqe4x/HdgHHAfWA1d5jBsBfApMcccPBJoA37rv1R/AOCDC4zXRwJfAYeBP4HHgeiABSHTfk01u26LAu+50fgNGArndcf3c93y0O62R7rAV7nhxx/3lfqabgTo4GwGJ7vxOAnPT/g6A3G5cP7rvyXrSfIfcdhHu51khzXsyxeN5lPt5lnafD3BjKphmWv9y4yniLvdJoHsmv93ewJKL+OyXAgM9np99/7z9voA3gVfTTGM2MNx9XA6YARxw298b6PXbP2INdAA5+S/ND6ACsAV43X1eHjiEszWdC2jvPk/9Us8D/gcUB/ICV7vDG7pf9qbuj6qvO598Xub5NXCnRzyvAG+6j7sAe4DaQB7gSWBVmi/qlzgJKb+XZXsJWJbOcv/C3yvopTgrmjo4K+sZ/L1izuw9WIqzwo52Y8yLs8VVHWdldDVwGmjotm9NmhU33hPB2zgr/XrAGaC25zK573kFnBVceolgMPBLJp//ZJwVaRM3/o+AaR7jbwNKuuMeAPYDkR5xJ7qfUy433kY4iTOPuyw7gPvc9oVxVuoPAJHu86Zp3wOPec8C3nI/kzI4iTr1M+sHJAHD3Hnl55+J4DqcFXgx93OoDZT1WOaRGfwOHsL5HVzuvrYeUNLLexcNnMrgs4xwP6+DQB532DTgv16mlcddnutwEmNS6msy+OwaAocv4rNfSuaJ4OzvC2iFs1Eg7vjiOImwnPv5rweedpe7Gs5G0HWBXsel/uW0XfWcaJaInMD5kP8CnnGH3wbMV9X5qpqiql8C64AOIlIWuAEYrKpHVDVRVZe5r7sTeEtVv1PVZFX9L87KrJmXeU8FeoFzaAXo6Q4DuAt4UVV3qGoSzm5yfRGp7PH6F1X1sKrGeZl2KZwVjzd/uONTfaiqW1X1FPAU0ENEcmf0Hni8drKqblPVJPd9mKeqP6pjGbAIuCqdONLzrKrGqeomnL2Qeu7wHsAL7nseC4zNYBolM1h+T5+p6hr3Pf4I5xAhAKo6RVUPucv2HyAfzgoy1beqOst9b+JUdb2qrnbb/4yzIr/abdsJ2K+q/1HVeFU9oarfeQtIRC7B+X7dp6qnVPUvnC38nh7NflfVN9x5pf38E3ESTS2cFdcOVfXlvQBnz+ZJVd3lfoabVPWQl3bFcPYY0uohIkdxVpJ3At3c9xbS+U664w+640sCBz1ek54TOHsP3vj62WfG8/f1DU5ySP0ud8P5/H8HGuNsHD2nqgmquhdnY6an16kGgCWCzHVR1cI4W6u1+HsFWRno7p70Oup+uVsCZYGKOFsjR7xMrzLwQJrXVcTZckjrU6C5iJTD2eJQnC9c6nRe95jGYZwttPIer9+XwXIddGP1pqw73tt0fsHZsi9Fxu+B1xhE5AYRWS0ih932Hfhn0vHFfo/Hp4HUE/jl0swvo+U/RPrL78u8EJEHRGSHiBxzl6Uo/1yWtMteU0Q+d0+EHsdJ3qntK+IcbvFFZZzP4A+P9/0tnD0Dr/P2pKpf4xyWGg/8KSKTRKSIj/P2Nc4jOMkmrU9UtRjOsf2tOHtJqbx+J91j8KXc8YeAUj4cly+Mc9jLG18/+8ycfY/V2Q2YhrvhBtyKs+EAzudVLs3v5HGc9yBHsETgI3frdTLwqjtoH86WcjGPv4Kq+pI7roSIFPMyqX3Av9O8roCqfuxlnkdxtph74HyxPna/cKnTuSvNdPKr6irPSWSwSIuBpiJS0XOgiDTB+bF/7THYs00lnC3Kg5m8B+fEICL5cA4tvQpc4q4Q5uMksMzi9cUfOIeEvMWd1ldABRGJuZAZichVwCM4n01xd1mO8feywLnLMxHYCdRQ1SI4K4PU9vtwDpl5k3Y6+3D2Ikt5vO9FVDU6g9f8c4KqY1W1Ec4hnJo4h3wyfV0mcXr6AWdHtry3kap6EGevdoS7Bw3Od/IGESmYpvktOMu7GuccSzzOIbeM1MbZW/TGl8/+FFDA4/mlXtqkfa8+Brq5e+VNcb7r4LxnP6X5nRRW1Q7kEJYIzs8YoL2I1Mc5CXijiFwnIrlFJNK9/LGCu5v9BTBBRIqLSF4RaeVO421gsIg0da+kKSgiHUXE29YTOIeC+uD8GKZ6DH8TeExEogFEpKiIdPd1QVR1Mc4PYoaIRLvL0AxnK2aiqv7g0fw2EYkSkQLAc8Cnqpqc0XuQzmwjcA6fHACSROQGwPOSxj+BkiKS3i59Zj7BeU+Kuyugoek1dJdvAvCxG3OEG39PEXnUh3kVxjlWfQDIIyJP45zMzOw1x4GTIlILuNtj3OfApSJynziX9RYWkabuuD+BKqlXXbnfr0XAf0SkiIjkEpHqInI1PhCRxu73Ly/OCi8e5+Rp6ryqZfDyd4DnRaSG+/2tKyIl0zZS1UScFXu6ManqTpyLHB52B30IxALTRaSK+7u5DucQ3whVPaaqx3COtY8XkS4iUsBtd4OIvOwx+atxfoPe5uvLZ78R6OpO/zKcE9kZUucy2QPue7TQ3ZAD5/zNcRF5RETyu7+VOiLSOLNpZhdLBOdBVQ8AHwBPqeo+nMvVHsf58PfhbFWlvqe342w578Q5t3CfO411OMdGx+HsPu/BORGVnjk4Vzn86R4TT41lJjAKmOYeZtiKc9z4fNyCcwnfApwrMabgXIkyLE27D3H2hvbjnMi8140hs/fgH1T1hPvaT3CW/VZ3+VLH78TZqtrr7kJ7O1yWkedwViQ/4ayEPsXZkkzPvfx9iOQoziGPm4G5PsxrIc6KZjfO4bJ4Mj4UBfAgzjKfwNkg+F/qCPe9aQ/ciPM+/wC0cUenXmJ5SEQ2uI/74CTW7Tjv5af4frijiDv/I27sh/h7T/ddIMp9/2d5ee1rOJ/fIpyk9i7OyVJv3sL5HWTkFWCQiJRR1TM4V8ztw7lC67g7vydU9ZXUF6jqa8BwnAskUr93Q3FOoCMikTiHHP+bwXwz++xH41w99ac7nY/OnYRXH7vLcHajzd1ouhHn/NJPOHvT75D+OYxsl3qG2xivRGQpzpUeAbm792KIyN1AT1X1aUvZZD0RWQEMc7eWs2uew3AuaX0408YGcC7LMiYkuMeaq+EcR66BcynmuIAGFeZUtWXmrbJ8nm9k9zyDnSUCE0oicA5HVMXZ3Z+GcyzYGJMBOzRkjDFhzk4WG2NMmAu6Q0OlSpXSKlWqBDoMY4wJKuvXrz+oqqW9jQu6RFClShXWrVsX6DCMMSaoiMgv6Y2zQ0PGGBPmLBEYY0yYs0RgjDFhLujOEXiTmJhIbGws8fHxgQ4lqEVGRlKhQgXy5s0b6FCMMdkoJBJBbGwshQsXpkqVKkhgutsNeqrKoUOHiI2NpWrVqoEOxxiTjfx2aEhE3hORv0RkazrjRUTGisgeEdksIg0vdF7x8fGULFnSksBFEBFKlixpe1XGhCF/niOYjNOtXHpuwKkHUwOnr9SJFzMzSwIXz95DY8KT3w4NqepyEamSQZPOwAduRyurRaSYiJQ9jy7zjAkd8Qdgz1uQkhDoSEwOlJgIP/2Wi5qNr4Sy12b+gvMUyHME5fln/fZYd9g5iUBEBuHsNVCpUqVsCe585c6dmyuuuIKkpCSqVq3Khx9+SLFixQDYtm0bw4YNIzY2FlWlT58+PPnkk2e3wL/44gueeuopTp06harSqVMnXn311QzmZkJO7CzY/JT7xPbMzN++/7kcd0zqzl/HC7F73hIKhlgi8PZt91oBT1UnAZMAYmJicmSVvPz587Nx40YA+vbty/jx43niiSeIi4vjpptuYuLEiVx77bWcPn2aW265hQkTJjBkyBC2bt3K0KFDmTdvHrVq1SIpKYlJkyYFdmFM9lO3g7Cb/4D83npFNOEmPj6JZ59dxSuvrKVUqfxMeKcdBZvX9Mu8AnkfQSz/7FO2AvB7gGLJUs2bN+e3334DYOrUqbRo0YJrr3WyeIECBRg3bhwvveR06/vyyy/zxBNPUKtWLQDy5MnDPffcE5jAjTE5Rpcus3jppTX06RPNjh130LWrf5IABHaPYA4wVESm4XT0fCxLzg+svw+ObLzoyfxD8frQaIxPTZOTk/nqq68YMMDp4nTbtm00atToH22qV6/OyZMnOX78OFu3buWBBx7I2niNMUHpxIkE8ubNRWRkHh59tAkPPBBD+/ZV/D5ff14++jFOT1GXi0isiAwQkcEiMthtMh/Yi9Nn79tAUG8Gx8XFUb9+fUqWLMnhw4dp37494Fyfn97VOHaVjjEm1cKFP1Gnzvs8//y3ALRuXSlbkgD496qhXpmMV2BIls/Yxy33rJZ6juDYsWN06tSJ8ePHc++99xIdHc3y5cv/0Xbv3r0UKlSIwoULEx0dzfr166lXr15A4jYXIDke/liUtVf4HN6QeRsTkg4fjmP48KX897/bqFWrBB07Vsv2GELizuKcpGjRoowdO5bOnTtz991307t3b1544QUWL15Mu3btiIuL49577+Xhh51+tR966CG6du1Ky5YtqVmzJikpKYwZM4bhw4cHeElMun6dDt/2yfrp5oqAPAWzfromx/rqq1/o3Xsehw7F88QTzXjyyWZERmb/atkSgR80aNCAevXqMW3aNG6//XZmz57NsGHDGDJkCMnJydx+++0MHToUgLp16zJmzBh69erF6dOnERE6duwY4CUwGUqOc/5f8xVElsm66eYrCXkLZ930TI5XpkwBqlYtyoIF3ahfPwu/S+cp6PosjomJ0bQd0+zYsYPatWsHKKLQYu+lD/ZMgjV3QZffoEC5QEdjgoiq8t//bmPDhj8ZO7bt2WHZcb5QRNaraoy3cVaG2hhjssFPPx3luus+pX//BWzceIC4uEQgZ1w0YoeGjDHGj5KTUxg/fiOPPbacXLmECRPacddd9ciVK/AJIJUlAhOcEk9AXIDuP4zbH5j5mqB08GAcTz+9kquvrsibb7anUqUigQ7pHJYITHBa3Crrbxw8X7nzBXb+JsdKTEzmo4920KdPNJdcUpANG26natWiOeIwkDeWCExwOnMIyrSCywZn3tYf8l/qXOVjTBrr1+/njjsWsnnzAcqWLch111WlWrVigQ4rQ5YITPAqVB2qZHjfojHZJi4ukWef/ZZXX11LmTIFmDmzM9ddFxy9/dlVQ1nozz//5NZbb6VatWo0atSI5s2bM3PmTL/Oc926ddx7771+nYcxJnNdusxm1Kg19O9fh+3b+9OlS41Ah+Qz2yPIIqpKly5d6Nu3L1OnTgXgl19+Yc6cOX6db0xMDDExXi8NNsb42fHjZ4iIyE1kZB4ef7wpDz/cmLZtKwc6rPNmewRZ5OuvvyYiIoLBg/8+Zl25cmWGDRvG5MmTz95JDNCpUyeWLl0KwKJFi2jevDkNGzake/funDx5EoBHH32UqKgo6taty4MPPgjA9OnTqVOnDvXq1aNVq1YALF26lE6dOgEwYsQI7rjjDlq3bk21atUYO3bs2Xk+//zz1KpVi/bt29OrVy/r+MaYizR//l7q1JnMc885ReKuvrpiUCYBCNE9gtatp50zrEePy7nnngacPp1Ihw4zzhnfr18d+vWrw8GDp+nW7Z9b8UuX9sx0ntu2baNhw4bnFefBgwcZOXIkixcvpmDBgowaNYrXXnuNoUOHMnPmTHbu3ImIcPToUQCee+45Fi5cSPny5c8OS2vnzp0sWbKEEydOcPnll3P33XezadMmZsyYwffff09SUhINGzY8pzT2eVGFnaNhyzOB61oxJQHEtmNM9jt48DT337+UKVO2ExVVkptuqh7okC5aSCaCnGDIkCGsWLGCiIgIhgzxXmR19erVbN++nRYtWgCQkJBA8+bNKVKkCJGRkQwcOJCOHTue3eJv0aIF/fr1o0ePHnTt2tXrNDt27Ei+fPnIly8fZcqU4c8//2TFihV07tyZ/PnzA3DjjTde+IKlJMOG4bB7LJS93umrIVCq3Bq4eZuw9OWXP9O79zyOHDnD00835/HHm5IvX/CvRoN/CbzIaAu+QIG8GY4vVaqAT3sAaUVHRzNjxt97GuPHj+fgwYPExMSQJ08eUlJSzo6Lj48HnPMK7du35+OPPz5nemvWrOGrr75i2rRpjBs3jq+//po333yT7777jnnz5lG/fv2zXWN6ypfv72vbc+fOTVJSEllWTyopDr69DfZ9BrWGQ4NXbKvchJWyZQtSs2YJJk5sxxVXlA50OFnGfsVZ5JprriE+Pp6JEyeeHXb69GkAqlSpwsaNG0lJSWHfvn2sWbMGgGbNmrFy5Ur27Nlztv3u3bs5efIkx44do0OHDowZM+bsCv/HH3+kadOmPPfcc5QqVYp9+/b5FFvLli2ZO3cu8fHxnDx5knnz5p3/Ap45BF+3g30zoeFoaPgfSwIm5Kkq77yzmSFDFgNQp05pvvmmZ0glAQjRPYJAEBFmzZrF/fffz8svv0zp0qXPHvdv0aIFVatW5YorrqBOnTpnzyWULl2ayZMn06tXL86cOQPAyJEjKVy4MJ07dyY+Ph5VZfTo0YDTd8EPP/yAqtK2bVvq1avHsmXLMo2tcePG3HTTTdSrV4/KlSsTExND0aJFfV+4kz/Bkuvh1C/Q8hOo1O383yBjgszevUe5885FfP31r7RuXZG4uETy58+bY+8OvhhWhjpMnDx5kkKFCnH69GlatWrFpEmTvJ7cPue9PLQOlnWElERoNQfKtMzGqI3JfsnJKYwdu4EnnlhBnjy5ePXVqxk4sG6OKhJ3ITIqQ217BGFi0KBBbN++nfj4ePr27evbFU6/zYeVPSBfKWi7DIrW8n+gxgTYwYNxPPvst7RtW4mJE9tToULodxZkiSBMpN7k5rM978DawVCsHrSe59TWMSZEJSQkM2XKdvr1q8MllxRk48Y+VK5cJCQPA3kTMmf7gu0QV0509j3c/AysuRMubQ/tlloSMCFt7do/aNToQwYMWMjixb8AUKVKzq0U6g8hkQgiIyM5dOiQJYOLoKocOniQyLidsPU5qHYHXD3H+tA1Iev06UQefHApzZpN5ciReObMuZlrr60S6LACIiQODVWoUIHY2FgOHDgQ6FCCl6YQeWozFfYOgStGQJ2nIYy2iEz46dx5FosX/8KgQXV5+eWrKVo0fPuXCImrhsxFivsDlnaAo1ugySSofkegIzLGL44dO0O+fE6RuOXL95GcrLRpUynQYWUL67zepO/YDljYDE78AFd/bknAhKzPP/+R6Oj3efbZVQC0alUxbJJAZiwRhLO/voEvW0DKGWi3HMpdH+iIjMlyBw6c5tZbP+fGG2dSokQkXbsGTz8B2SUkzhGYC/DrdFh1OxSqAq0XOP+NCTGLFjlF4o4dO8Ozz17Jo482JSIid6DDynEsEYSjnaNhwwNQ+krnbuF8JQIdkTF+Ub58IWrXLsnEie2Iji4V6HByLDs0FE40Bdbf75SRrtgVrllsScCElJQUZdKkTdx995cAREeXYvnynpYEMmF7BOEiOd45FLTvU7j8PqseakLOnj1HuPPORSxduo82bf4uEmcyZ4kgHCSehKU3wIGV0PA1qHV/oCMyJsskJ6cwZsx6nnpqJXnz5uLtt69lwIArwurO4Ivl101CEbleRHaJyB4RedTL+KIiMldENonINhHp7894wtbv8+DACmj2niUBE3IOHoxj5MjVtG9fme3b+zNwYF1LAufJb4lARHID44EbgCigl4hEpWk2BNiuqvWA1sB/RCTCXzGFrZRE539pKyFtQsOZM0m8/fZmUlL0bJG4WbO6UL68lUS5EP7cI2gC7FHVvaqaAEwDOqdpo0BhcdJ3IeAwkOTHmIwxQe6775wicYMGLTpbJK5y5fAqEpfV/JkIygOefSnGusM8jQNqA78DW4D/U9WUNG0QkUEisk5E1lk9IWPC06lTCQwfvoTmzT/i2LEE5s3rGrZF4rKaPxOBt/SctrDRdcBGoBxQHxgnIkXOeZHqJFWNUdWY0qVDq69QY4xvunSZzejR6xk8uB7btvWjQ4dqgQ4pZPgzEcQCFT2eV8DZ8vfUH/hMHXuAnwDrBssYA8DRo/HExTnnuJ5+ujnLlv2LCRPaU6RI+FYK9Qd/JoK1QA0RqeqeAO4JzEnT5legLYCIXAJcDuz1Y0zGmCAxZ84eoqMn8+yz3wJw1VUVaNWqYiavMhfCb4lAVZOAocBCYAfwiapuE5HBIjLYbfY8cKWIbAG+Ah5R1YP+iskYk/P99dcpevacS+fOsyhVKj/dutUMdEghz+cbykSkIBCvqsm+vkZV5wPz0wx70+Px78C1vk7PGBPaFiz4id6953HyZCLPP9+CRx5pQt68ViTO39JNBCKSC+dwTm+gMXAGyCciB3BW7pNU9YdsidIYExYqVizMFVeUYsKEdkRFWX2g7JLRoaElQHXgMeBSVa2oqmWAq4DVwEsicls2xGiMCVEpKcrEiRu5665FgFMkbunSnpYEsllGh4baqWpi2oGqehiYAcwQEavoZIy5ILt3H2bgwEV8800s7dtXJj4+ichIK38WCOnuEahqoojkEpGtGbXxT1jGmFCVlJTCqFHfUbfuf9my5QDvv389Cxd2syQQQBm+86qa4haEq6Sqv2ZXUMaY0HXoUByjRq2lQ4dqjB/flrJlCwU6pLDnSwouC2wTkTXAqdSBqnqT36IyxoSUM2eSmDx5G3feWZdLLinIpk19qFjxnCICJkB8SQTP+j0KY0zI+vbb3xkwYAE7dhymevVitGtX2ZJADpPpDWWqugzYBRQFigC73GHGGJOukycTuO++r2nRYiqnTiWyYMEttGtXOdBhGS8yTQQiMhBYA3QFugGrReQOfwdmjAluXbrM4vXXNzBkSAO2bu3PdddVDXRIJh2imrYgaJoGIruAK1X1kPu8JLBKVS/PhvjOERMTo+vWrQvErLPH4e9heRdIic+6aSbFQdIJuHEPFK6eddM1Jo0jR+KJjMxN/vx5WbEiFoCWLSsEOCoDICLrVTXG2zhfzhHEAic8np/gn/0MmKx0fAec/hUq94S8xbJuupGloZBtkRn/+eyz3QwZ8hV9+kQxatTVlgCCiC+J4DfgOxGZjdOfQGdgjYgMB1DV1/wYX/i64lkoYsW2TM63f/8phg5dzIwZP1C/fhl69rRK8sHGl0Two/uXarb73zoHNSbMffHFXnr3ns/p04m88MJVPPhgjBWJC0K+JILtqjrdc4CIdE87zBgTfipXLkKDBmUYP74ttWqVDHQ45gL50h/BYz4OM8aEuJQUZdy4Ddx550IAoqJK8dVXPSwJBLmMylDfAHQAyovIWI9RRYAkfwdmjMlZdu06zIABC1m58jeuu66KFYkLIRl9ir8D64Gb3P+pTgD3+zMoY0zOkZiYzKuvruPZZ1dRoEBeJk++nj59ohGRQIdmski6iUBVNwGbROQjqzJqTPg6ciSeV15Zy403VueNN9py6aUFAx2SyWLpniMQkbkicmM646qJyHN2h7ExoSk+PokJE74nJUUpU6Ygmzf3Zfr0mywJhKiMDg3dCQwHxojIYeAAEAlUwbmcdJyqzk7/5caYYLRiRSwDBixk9+4j1KxZgnbtKlOhgl0tHsoyOjS0H3gYeFhEquCUo44Ddqvq6ewJzxiTXU6cSOCxx5YzfvxGqlQpwqJF3axIXJjw6ZS/qv4M/OzXSIwxAdWlyyyWLPmV//u/howc2ZJChSICHZLJJnbtlzFh7PDhOCIj81CgQF6ef74FIi1p3rxcoMMy2cyXG8qMMSHo0093Ubv2+4wYsQqAK68sb0kgTPmUCEQkv4gEpOy0MSZr/fHHSbp2nU337nOpWLEwvXvXDnRIJsB86ZjmRmAjsMB9Xl9E5vg5LmOMH8yb9yNRUe/zxRc/MWpUK1av7k29emUCHZYJMF/OEYwAmgBLAVR1o3sVkTEmyFSrVozGjS9l3Li21KxZItDhmBzCl0NDSap6zO+RGGOyXHJyCq+/vp4BAxYAULt2SRYt6m5JwPyDL4lgq4jcCuQWkRoi8gawys9xGWMu0vbtB7nqqmncd98S9u8/RXy81Yo03vmSCIYB0cAZYCpwDPg/fwZljLlwCQnJjBz5LQ0afMju3UeYMqUDn3/e1SqFmnT5kgg6quoTqtrY/XsSpyJppkTkehHZJSJ7ROTRdNq0FpGNIrJNRJadT/DGmHMdPRrP6NHrufnmy9i+vR+9e0dZpVCTIb91TCMiuYHxwA1AFNBLRKLStCkGTABuUtVooLsP8Rhj0oiLS2TcuA1ni8Rt2dKPadNupEwZKxJnMufPjmmaAHtUda87vWk4Hd9v92hzK/CZqv4KoKp/nV/4xpjly/cxcOAifvjhCLVrl6Rt28qUK1co0GGZIJJZxzTruPCOacoD+zyexwJN07SpCeQVkaVAYeB1Vf0g7YREZBAwCKBSpUo+zDoHOnMYTu7NvJ0vbYwBjh8/w6OPLmfixE1UrVqUxYu707atFYkz58+XjmmmXmDHNN4OSqqX+TcC2gL5gW9FZLWq7k4TyyRgEkBMTEzaaQSHpTfAoTW+t89TwH+xmJDQpcssli7dx/33N+L551tQsKAViTMXxpfLCKqIyIs4x/kjUweqarVMXhcLVPR4XgFnLyNtm4Oqego4JSLLgXrAbkJNwlEofRVEPZx524iSUKCC30MywefgwdMUKJCXAgXy8u9/X4UINGtm9YHMxfElEbwPPAOMBtoA/fG+tZ/WWqCGiFQFfgN64pwT8DQbGCcieYAInENHo30LPQgVKA/lOwU6ChOEVJX//W8Xw4Z9Rb9+0bzySmsrEGeyjC9XDeVX1a8AUdVfVHUEcE1mL1LVJGAosBDYAXyiqttEZLCIDHbb7MCpYbQZWAO8o6pbL2xRjAlNv/12gi5dZtGr1+dUrVqUPn2iAx2SCTG+7BHEi0gu4AcRGYqzde9TlSpVnQ/MTzPszTTPXwFe8S1cY8LL55//SO/e80hMTOHVV6/mvvsakTu3VY83WcuXRHAfUAC4F3ge5/BQXz/GZIxxXXZZMa68shxvvNGWyy4rHuhwTIjKcNPCvSmsh6qeVNVYVe2vqreo6upsii/4qcKvMyDud5C8gY7G5HDJySmMHr2Ofv2+AKBWrZJ88UU3SwLGrzJMBKqaDDQSuz/9whzbDl+3hxXdoFBV364YMmFr27aDtGjxMcOHL+XgwTgrEmeyjS+Hhr4HZovIdOBU6kBV/cxvUQW7hGOwZQTsfgPyFIZGb0CNwZDLin6ZcyUkJPPSS98xcuRqihbNx9SpHenZs5bVBzLZxpc1UwngEP+8UkgBSwRpaQrs/S9sehTiD8Bld0Ldf0NkqUBHZnKwo0fjGTv2e7p3v5wxY9pQurTdTGiyV6aJQFX7Z0cgQe/gGlg/zLl7uFRzaD0fSjQKdFQmhzp9OpG3397M0KEN3CJxfSlb1uoDmcCwYxUXK/4v2PgY7H0PIi+F5h9Ald4gdomf8W7Jkl8ZOHAhe/ceo06dUrRtW9mSgAkoSwQX48SPsKARJJ2C2g9Cnacgb5FAR2VyqGPHzvDww8uYNGkz1asXY8mSHrRuHaRFFE1IsURwMQ6vg8Rj0PZruKRNoKMxOVyXLrNYvjyWhx5qzIgRV1KggF1ObHKGTBOBiFwCvACUU9Ub3M5lmqvqu36PLlhEXhroCEwOdeDAaQoWdIrEvfjiVeTOLTRuXDbQYRnzD74cyJ6MUy8otcLVbpy7jY0x6VBVpk7dQe3a7/PMMysBp0qoJQGTE/mSCEqp6idACpwtJpfs16iMCWKxsSe46aaZ9O49j8suK0a/fnUCHZIxGfLlHMEpESmJ26mMiDQDjvk1KmOC1Jw5e7jttvluqYg2DBvWwIrEmRzPl0TwADAHqC4iK4HSQDe/RmVMkKpZszgtW5Zn3Li2VKtWLNDhGOMTX24oWy8iVwOX43RIs+sCu640JuQkJaUwZsx6Nm8+wAcfdKBWrZLMn39LoMMy5rxkus8qIpuAh4F4Vd1qScAYx+bNB2je/CMeemgZx48nWJE4E7R8OXh5E5AEfCIia0XkQRGxu2BM2DpzJolnnllJo0Yf8uuvJ/jkkxuZObMzkZF2W44JTpkmArd7ypdVtRFOn8N1gZ/8HpkxOdTx4wlMmLCRXr1qsX17f7p3v9wqhZqg5tMmjIhUAXoA/8K5dNQK65uwcupUApMmbebeextSunQBtm7txyWXFAx0WMZkCV/uLP4OyAtMB7qr6l6/R2VMDvLVV79w552L+OmnY9SrV4ZrrqlkScCEFF/2CPqq6k6/R2JMDnP0aDwPPriMd9/dQo0axVm27F+0alUx0GEZk+XSTQQicpuqTgE6iEiHtONV9TW/RmZMgN1882y++SaWRx5pwjPPNCd/fisSZ0JTRnsEqfu+hb2MUz/EYkzA/fnnKQoVykvBghG89FIr8uQRGjWyooImtKWbCFT1LffhYlVd6TlORFr4NSpjspmqMmXKdu67bwn9+9fh1Vdb07SpFYgz4cGX+wje8HGYMUHp11+P07HjZ/Tp8wWXX16CAQOuCHRIxmSrjM4RNAeuBEqLyHCPUUWA3P4OzJjsMHv2Hm67bR6qMHbsNdxzT30rEmfCTkbnCCKAQm4bz/MEx7GicybIqSoiQq1aJWjduiJvvNGWKlWKBjosYwIio3MEy4BlIjJZVX/JxpiM8ZukpBT+85+1bNlykClTOnL55SWYO7droMMyJqAyOjQ0RlXvA8aJyDlXCanqTf4MzJistmnTX9xxx0I2bPiTm2+uQXx8ktUHMoaMDw196P5/NTsCMcZf4uOTGDlyNaNGraFkyUg+/fQmbrmlZqDDMibHyOjQ0Hr3/7LUYSJSHKioqpuzITZjssSJEwm89dYmeveuzWuvtaZEifyBDsmYHMWX/giWikgRESkBbALeFxGf7ioWketFZJeI7BGRRzNo11hEkkXETkKbLHHyZAKvvrqW5OQUSpcuwPbt/Zk8+QZLAsZ44ct1ckVV9TjQFXjfLUfdLrMXiUhuYDxwAxAF9BKRqHTajQIWnk/gxqRn0aKfqVNnMg8/vIzly2MBKF26QICjMibn8iUR5BGRsjhlqD8/j2k3Afao6l5VTQCmAZ29tBsGzAD+Oo9pG3OOw4fj6N//C6677lMiI/PwzTe9aNPG+lAyJjO+XDLxHM7W+kpVXSsi1YAffHhdeWCfx/NYoKlnAxEpD9wMXAM0Tm9CIjIIGARQqZL9sI13N988m5Urf+Pxx5vy1FPN7YogY3zkS+f103H6Ikh9vhfwpXdub102pb0MdQzwiKomZ9TDk6pOAiYBxMTEWME7c9b+/acoXNgpEvfKK1cTEZGb+vXLBDosY4KKLyeLK4jITBH5S0T+FJEZIlLBh2nHAp7F2ysAv6dpEwNME5Gfce5WniAiXXwL3YQzVWXy5K1ERb3P0087NRGbNClrScCYC+DLOYL3gTlAOZzDPXPdYZlZC9QQkaoiEgH0dKdzlqpWVdUqqloF+BS4R1Vn+R6+CUc//3yM66+fQf/+C4iOLsmgQfUCHZIxQc2Xg6ilVdVzxT9ZRO7L7EWqmiQiQ3HOL+QG3lPVbSIy2B3/5oUEnC1W3Qb7Psu8nSY5/8WKlGWXmTN/4Pbb5yMC48a15e6765Mrl3Ucb8zF8CURHBSR24CP3ee9gEO+TFxV5wPz0wzzmgBUtZ8v08wWhzdAgYpQwdtFTmnkKwmFa/g/pjCXWiQuOrok7dpV5vXX21C5shWJMyYr+JII7gDGAaPd5yvdYaGteD1o8HKgowh7iYnJvPLKWrZuPcjUqZ2oWbMEs2Z1CXRYxoQUX64a+hWwAnMm223Y8CcDBixk48a/6NHjcs6cSSJfPrsk1Jis5stVQ9VEZK6IHHCvHJrt3ktgjF/ExSXy2GPLadJkCvv3n2LmzM787383WhIwxk98Ocs5FfgEKItz5dB0/j5fYEyWO3UqkXff3ULfvtFs396fLl3sHIwx/uRLIhBV/VBVk9y/KZx7Y5gxF+XEiQRefnkNyckplCrlFIl7993rKV48MtChGRPyfNnXXuJWDp2GkwD+Bcxzq5Giqof9GJ8JAwsW/MRddy1i374TNGlyKa1bV6JUKSsSZ0x28SUR/Mv9f1ea4XfgJAY7X2AuyKFDcQwfvoQPPthO7dolWLnyVpo3LxfosIwJO75cNVQ1OwIx4adr19msWvU7Tz3VjCeeaGYng40JEPvlmWz1xx8nKVw4gkKFInj1VadIXL16Vh/ImECy2ggmW6gq7723hdq1/y4S17hxWUsCxuQAtkdg/G7v3qPcddeXLF78C61aVWDwYCsSZ0xOkmkiEKejgN5ANVV9TkQqAZeq6hq/R2eC3mef7eb22+eTO3cuJk5sx6BB9axInDE5jC97BBOAFJxexJ4DTuB0LZluj2LGpBaJu+KK0lx/fVXGjGlDxYpFAh2WMcYLX84RNFXVIUA8gKoeASL8GpUJWgkJyYwc+S233joPVaVGjeLMmNHZkoAxOZgviSBRRHLj3k0sIqVx9hCM+Yd16/bTuPEUnnrKORmckJAc4IiMMb7wJRGMBWYCZUTk38AK4AW/RmWCSlxcIg8/vIymTT/i4ME4Zs/uwscfd7L7AowJEr7cUPaRiKwH2uJ0SN9FVXf4PTITNE6dSmTy5K0MGHAFL7/cimLFrD6QMcHEl6uGKgGncfoqPjvM7afAhKnjx88wYcJGHnqoMaVKFWDHjjsoWTJ/oMMyxlwAX/bd5+GcHxAgEqgK7AKi/RiXycHmzfuRwYMX8/vvJ2nWrCytW1eyJGBMEMv0HIGqXqGqdd3/NYAmOOcJTJg5cOA0vXvPo1OnmRQtGsGqVbfSunWlQIdljLlI5302T1U3iIjdQxCGbrllDqtX/86IEVfy2GNNiYjIHeiQjDFZwJdzBMM9nuYCGgIH/BaRyVF+++0ERYvmo1ChCEaPbk2+fLmpU6d0oMMyxmQhXy4fLezxlw/nnEFnfwZlAk9VefvtzURF/V0krlGjSy0JGBOCMtwjcG8kK6SqD2VTPCYH+PHHo9x550KWLNlHmzYVGTKkQaBDMsb4UbqJQETyqGqSiDTMzoBMYH366S769PmCvHlzMWnStQwceAVO3UFjTKjKaI9gDc75gI0iMgeYDpxKHamqn/k5NpONUovE1atXho4dqzF6dBsqVCgc6LCMMdnAl6uGSgCHcKqPpt5PoIAlghCQkJDMiy9+x/bth5g2rRM1ahRn+vSbAh2WMSYbZZQIyrhXDG3l7wSQSv0alckWa9b8wYABC9m69SC33lqbhIRkqw9kTBjK6FefGyjEPxNAKksEQez06USefnolo0evp2zZgsydezOdOlUPdFjGmADJKBH8oarPZVskJtvExSUxZcp2Bg2qy6hRrShSJF+gQzLGBFBG9xFc9KUiInK9iOwSkT0i8qiX8b1FZLP7t0pErDNbPzl27Az//vdqkpJSKFkyPzt23MHEie0tCRhjMkwEbS9mwu49COOBG4AooJeIRKVp9hNwtarWBZ4HJl3MPI13c+f+ePbGsBUrYgEoXtxKRRtjHOkmAlU9fJHTbgLsUdW9qpoATCPNHcmqusrt+hJgNVDhIudpPBw4cJpevT7npptmUrJkJN9919uKxBljzuHPS0TKA/s8nscCTTNoPwD4wtsIERkEDAKoVMlWZL5KLRL33HMteOSRJlYkzhjjlT8Tgc9XG4lIG5xE0NLbeFWdhHvYKCYmxq5YykBs7AmKFXOKxI0Z04Z8+XITHV0q0GEZY3IwX4rOXahYoKLH8wrA72kbiUhd4B2gs6oe8mM8IS0lRXnrrU1ERb1/tvP4hg0vsSRgjMmUP/cI1gI1RKQq8BvQE7jVs4HbDeZnwO2qutuPsYS0H344wp13LmTZsljatq3EsGFWJM4Y4zu/JQK3YN1QYCHOzWnvqeo2ERnsjn8TeBooCUxwC5slqWqMv2IKRdOnO0Xi8uXLzbvvXkf//nWsSJwx5rz4tZ6Aqs4H5qcZ9qbH44HAQH/GEKpSi8Q1aFCGzp2r89prbShXrlCgwzLGBCF/niMwfnDmTBJPP72CHj3moqpcdllxpk270ZKAMeaCWSIIIqtX/07Dhh/y/POryZ8/DwkJyYEOyRgTAiwRBIFTpxK4//4lXHnlVE6cSGD+/K588EEHqxRqjMkStiYJAvHxyUybtpN77qnPiy+2onDhiECHZIwJIZYIcqijR+N5443veeyxpm6RuP4UK2b1gYwxWc8ODeVAs2b9QFTU+zz77CpWrfoNwJKAMcZvLBHkIH/+eYoePeZw882zKVOmAN9915tWrSpm/kJjjLkIdmgoB+nWbQ5r1uxn5MiWPPxwY/LmtSJxxhj/s0QQYL/+epzixSMpXDiCsWOvIV++3ERFWX0gY0z2sUNDAZKSoowf/z3R0U6HMQANGlxiScAYk+1sjyAAdu06zMCBC1mx4jfat6/M//1fw0CHZIwJY5YIstknn+ykT58vyJ8/D++/fz19+0ZbkThjTEBZIsgmqUXiGjW6lK5da/Daa2249NKCgQ7LGGPsHIG/xccn8cQT39Ct2xxUlerVizF1aidLAsaYHMMSgR+tWvUbDRp8wAsvfEfhwhFWJM4YkyNZIvCDkycTuPfer2jZ8mNOn05kwYJbmDz5BisSZ4zJkWzN5AcJCcl8+uluhgxpwAsvXGVF4owxOZolgixy+HAcY8du4Mknm1OiRH527LiDokXzBTosY4zJlB0aygIzZuwmKup9Ro5cfbZInCUBY0ywsERwEf744yS33DKbbt3mUK5cIdatu92KxBljgo4dGroIPXrMZe3a/bz00lU88EBj8uSxvGqMCT6WCM7TL78co0SJ/BQuHMEbb7Qlf/48XH55iUCHZYwxF8w2YX2UkqK88cYGoqMn89RTKwCoX7+MJQFjTNCzPQIf7Nx5iIEDF7Fy5W9cf30V7r+/UaBDMsaYLGOJIBPTpu2kb98vKFQoLx98cAO33RZlReKMMSHFEkE6UlKc42aNG19K9+41+c9/WnPJJVYfyBgTeiwRpBEXl8izk2PY9VcFPlvmFImbMqVjoMMyxhi/sZPFHr75Jpb69T9g1Iy6lCySRGJiSqBDMsYYv7NEAJw4kcCQIYtp1WoaiYkpfPn8At559GciIqzzeGNM6LNEACQmJjNr1h7uu68RW7b0pV393wMdkjHGZJuwPUdw6FAcr7++nqefvpISJfKzc+cdViXUGBOW/LpHICLXi8guEdkjIo96GS8iMtYdv1lE/N6Lu6oyffouoqLe58UX1/Dtt87WvyUBY0y48lsiEJHcwHjgBiAK6CUiUWma3QDUcP8GARP9FQ/A77+fpGvX2fToMZeKFQuzbt1tXHVVBX/O0hhjcjx/HhpqAuxR1b0AIjIN6Axs92jTGfhAVRVYLSLFRKSsqv6R5dH8vpAe7Rezfk9JXu7/Pfd32Uae2Jcg1kvbkz9CsTpZHoIxxuRE/kwE5YF9Hs9jgaY+tCkP/CMRiMggnD0GKlWqdGHR5C3C+Af/In++/dSsmALUTr9tkSiodseFzccYY4KMPxOBtzoMegFtUNVJwCSAmJiYc8b7pHRz6vVvfkEvNcaYUObPk8WxgGcvLRWAtNdl+tLGGGOMH/kzEawFaohIVRGJAHoCc9K0mQP0ca8eagYc88v5AWOMMeny26EhVU0SkaHAQiA38J6qbhORwe74N4H5QAdgD3Aa6O+veIwxxnjn1xvKVHU+zsrec9ibHo8VGOLPGIwxxmTMSkwYY0yYs0RgjDFhzhKBMcaEOUsExhgT5sQ5Xxs8ROQA8MsFvrwUcDALwwkGtszhwZY5PFzMMldW1dLeRgRdIrgYIrJOVWMCHUd2smUOD7bM4cFfy2yHhowxJsxZIjDGmDAXbolgUqADCABb5vBgyxwe/LLMYXWOwBhjzLnCbY/AGGNMGpYIjDEmzIVkIhCR60Vkl4jsEZFHvYwXERnrjt8sIg0DEWdW8mGZe7vLullEVolIvUDEmZUyW2aPdo1FJFlEumVnfP7gyzKLSGsR2Sgi20RkWXbHmNV8+G4XFZG5IrLJXeagrmIsIu+JyF8isjWd8Vm//lLVkPrDKXn9I1ANiAA2AVFp2nQAvsDpIa0Z8F2g486GZb4SKO4+viEcltmj3dc4VXC7BTrubPici+H0C17JfV4m0HFnwzI/DoxyH5cGDgMRgY79Ipa5FdAQ2JrO+Cxff4XiHkETYI+q7lXVBGAa0DlNm87AB+pYDRQTkbLZHWgWynSZVXWVqh5xn67G6Q0umPnyOQMMA2YAf2VncH7iyzLfCnymqr8CqGqwL7cvy6xAYRERoBBOIkjK3jCzjqoux1mG9GT5+isUE0F5YJ/H81h32Pm2CSbnuzwDcLYoglmmyywi5YGbgTcJDb58zjWB4iKyVETWi0ifbIvOP3xZ5nFAbZxubrcA/6eqKdkTXkBk+frLrx3TBIh4GZb2Gllf2gQTn5dHRNrgJIKWfo3I/3xZ5jHAI6qa7GwsBj1fljkP0AhoC+QHvhWR1aq629/B+Ykvy3wdsBG4BqgOfCki36jqcT/HFihZvv4KxUQQC1T0eF4BZ0vhfNsEE5+WR0TqAu8AN6jqoWyKzV98WeYYYJqbBEoBHUQkSVVnZUuEWc/X7/ZBVT0FnBKR5UA9IFgTgS/L3B94SZ0D6HtE5CegFrAme0LMdlm+/grFQ0NrgRoiUlVEIoCewJw0beYAfdyz782AY6r6R3YHmoUyXWYRqQR8BtwexFuHnjJdZlWtqqpVVLUK8ClwTxAnAfDtuz0buEpE8ohIAaApsCOb48xKvizzrzh7QIjIJcDlwN5sjTJ7Zfn6K+T2CFQ1SUSGAgtxrjh4T1W3ichgd/ybOFeQdAD2AKdxtiiClo/L/DRQEpjgbiEnaRBXbvRxmUOKL8usqjtEZAGwGUgB3lFVr5chBgMfP+fngckisgXnsMkjqhq05alF5GOgNVBKRGKBZ4C84L/1l5WYMMaYMBeKh4aMMcacB0sExhgT5iwRGGNMmLNEYIwxYc4SgTHGhDlLBCbbuBVAN3r8Vcmg7clsDC1dIlJORD51H9cXkQ4e427KqOqpH2KpIiK3XsDr8ovIMhHJ7T5/xa3S+cp5TKO0e1mqCUF2+ajJNiJyUlULZXXb7CIi/YAYVR3qx3nkUVWvBdNEpDXwoKp2Os9pDgHyqOrr7vPjQGlVPXM+MYnI+zj3Jaw8n/mbnM/2CEzAiEghEflKRDaIyBYROad6qIiUFZHl7h7EVhG5yh1+rYh86752uoickzTcwmtjxOl/YauINHGHlxCRWW4t99Vu6Q1E5GqPvZXvRaSwuxW+1b2r9TngX+74f4lIPxEZJ049/J9FJJc7nQIisk9E8opIdRFZIE4BuG9EpJaXOEeIyCQRWQR84M7zG3fZNojIlW7Tl3DuGt4oIveLSG53636tuyx3pfNW98a54xgRmQMUBL5zl2GyiLzpzm+3iHRy2/Vz39e5wCJ3OrPcaZlQE+ja2/YXPn9AMk5xsI3ATJw724u440rh3CmZupd60v3/APCE+zg3UNhtuxwo6A5/BHjay/yWAm+7j1vh1ncH3gCecR9fA2x0H88FWriPC7nxVfF4XT9gnMf0zz7HWdG2cR//C2fLGeAroIb7uCnwtZc4RwDrgfzu8wJApPu4BrDOfdwa+NzjdYOAJ93H+YB1QNU0044A9qcZdtLj8WRgAc5GYQ2cOjaR7rLFAiU82pYHtgT6e2R/Wf8XciUmTI4Wp6r1U5+ISF7gBRFphVMOoTxwCbDf4zVrgffctrNUdaOIXA1EASvdchkRwLfpzPNjcGq8i0gRESmGU3n1Fnf41yJSUkSKAiuB10TkI5ya/rHie9XS/+EkgCU49XAmuHspVwLTPaaTL53Xz1HVOPdxXmCciNTHSZ4103nNtUBd+bvntaI4K/OfPNqUAo5mEvsn6pRt/kFE9uIUbAP4UlU96+L/BZTLZFomCFkiMIHUG6dHqUaqmigiP+NsjZ7lrsBbAR2BD90TnEdwVlK9fJhH2pNgSjplfFX1JRGZh1PHZbWItAPifVyWOcCLIlICpwz01ziHYI56Jr8MnPJ4fD/wJ07V0FwZxCDAMFVdmMF040jznnrh7T1KGxPudOIwIcfOEZhAKgr85SaBNkDltA1EpLLb5m3gXZwu/FYDLUTkMrdNARFJb6v5X26bljhVGo/hHFbq7Q5vjVO2+biIVFfVLao6CucwS9rj+SdwDk2dQ1VP4pQ9fh3n8E2yOvXwfxKR7u68RHzrK7oo8Ie7lX47ziExb/NfCNzt7i0hIjVFpGCauI4AuUUko2TQXURyiUh1nC4hd6XTriYQtAXsTPosEZhA+giIEZF1OCvmnV7atAY2isj3OIdzXlfVAzjHsD8Wkc04ieGck7CuIyKyCqeXsgHusBHufDfjnIDt6w6/zz0xvAlnyzdtL25LgKjUk8Ve5vU/4Db3f6rewAB3mtvw3p1mWhOAviKyGmflm7plvhlIEqeT9vtx+pbYDmwQp6Pzt/C+l7+IjDsi2gUsw1newaqa3h5IG2CeD/GbIGOXj5qQJSJLcS63XBfoWAJJRBoAw1X1di/jJuPswXzqw3SWA531776vTYiwPQJjQpyqfg8sEfeGsgshIqWB1ywJhCbbIzDGmDBnewTGGBPmLBEYY0yYs0RgjDFhzhKBMcaEOUsExhgT5v4fAZ97vV7h3agAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create a function for plotting ROC curves\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_roc_curve(fpr, tpr):\n",
    "    \"\"\"\n",
    "    Plots a ROC curve given the false positive rate (fpr)\n",
    "    and true positive rate (tpr) of a model.\n",
    "    \"\"\"\n",
    "    # Plot roc curve\n",
    "    plt.plot(fpr, tpr, color=\"orange\", label=\"ROC\")\n",
    "    # Plot line with no predictive power (baseline)\n",
    "    plt.plot([0, 1], [0, 1], color=\"darkblue\", linestyle=\"--\", label=\"Guessing\")\n",
    "    \n",
    "    # Customize the plot\n",
    "    plt.xlabel(\"False positive rate (fpr)\")\n",
    "    plt.ylabel(\"True positive rate (tpr)\")\n",
    "    plt.title(\"Receiver Operating Characteristic (ROC) Curve\")\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "plot_roc_curve(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8669181034482759"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(y_test, y_probs_positive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8ZUlEQVR4nO3dd3gU5fbA8e8hEEIJHZTepIOAdFEEAUGQYkOKBQSRK+hFrwX1p6JwraiIgIpX4SoiV1CaICBIERBpBgxVRIWoYOgtIe38/pgJLmGTLJDNZrPn8zx5sjvzzsx5Z3fnzLwz846oKsYYY0JXnkAHYIwxJrAsERhjTIizRGCMMSHOEoExxoQ4SwTGGBPiLBEYY0yIs0SQw4jIVhFpG+g4cgoReUpE/hOgZU8RkdGBWHZWE5F+IrL4Iqe96O+kiKwWkcYXM+3FEpGHROTl7FxmsLNEkAER+VVE4kTkpIjsdzcMhf25TFWtp6rL/bmMVCKSX0ReEpG9bj1/EpHHRESyY/le4mkrIjGew1T1RVUd5KflibvRiBaRUyISIyIzRKSBP5Z3sURkpIhMvZR5qOonqnqDD8s6L/ld7HdSRLoBJ1T1B/f9SBFJdH9PR0VkjYi0SjNNMRF5x/29nRaRH0VkgJd59xWRDe68/hSRr0TkGnf0JOBOESmTQWxB8dlnF0sEmeumqoWBRkBj4MnAhnPhRCRvOqNmAO2BLkAkcBcwGHjLDzGIiOS079tbwD+Bh4ASQE1gNtA1qxeUwWfgdwFc9hDg4zTD/uf+nkoBy3C+gwCISDiwBKgMtAKKAo8BL4vIIx7lHgHGAi8ClwGVgIlADwBVjQe+Au7OILYs++wD+dlmGVW1v3T+gF+BDh7vXwXme7xvCawBjgKbgbYe40oAk4E/gCPAbI9xNwFR7nRrgCvTLhMoB8QBJTzGNQYOAvnc9/cC2935LwIqe5RVYCjwE/CLl7q1B+KBimmGtwCSgSvc98uBl4B1wDFgTpqYMloHy4F/A6vdulwBDHBjPgHsAe53yxZyy6QAJ92/csBIYKpbpopbr3uAve66eNpjeQWA/7rrYzvwOBCTzmdbw61n8ww+/ynABGC+G+/3QHWP8W8B+4DjwEbgWo9xI4GZwFR3/CCgOfCdu67+BMYD4R7T1AO+Bg4DB4CngM5AApDorpPNbtmiwAfufH4HRgNh7rj+7jp/053XaHfYKne8uOP+cj/TLUB9nJ2ARHd5J4F5aX8HQJgb18/uOtlImu+QWy7c/TwrpFknUz3e13U/z9Lu+4FuTIXSzOsON54ibr1PArdn8tvtByy7hM9+OTDI4/3Z9eft9wW8C4xJM485wCPu63LA50CsW/6hQG/fzok10AHk5L80P4AKwI/AW+778sAhnL3pPEBH933ql3o+8D+gOJAPuM4dfpX7ZW/h/qjucZeT38syvwHu84jnNeBd93VPYDdQB8gL/B+wJs0X9WuchFTAS91eBlakU+/f+HsDvRxnQ1MfZ2P9OX9vmDNbB8txNtj13Bjz4exxVcfZGF0HnAaucsu3Jc2GG++J4H2cjX5D4AxQx7NO7jqvgLOBSy8RDAF+y+Tzn4KzIW3uxv8JMN1j/J1ASXfcv4D9QIRH3Inu55THjbcJTuLM69ZlOzDcLR+Js1H/FxDhvm+Rdh14LHs28J77mZTBSdSpn1l/IAl40F1WAc5NBJ1wNuDF3M+hDlDWo86jM/gdPIbzO6jlTtsQKOll3dUDTmXwWYa7n9dBIK87bDrwXy/zyuvWpxNOYkxKnSaDz+4q4PAlfPbLyTwRnP19AW1wdgrEHV8cJxGWcz//jcCzbr2r4ewEdQr0Ni71L6cdqudEs0XkBM6H/BfwnDv8TmCBqi5Q1RRV/RrYAHQRkbLAjcAQVT2iqomqusKd7j7gPVX9XlWTVfW/OBuzll6WPQ3oA07TCtDbHQZwP/CSqm5X1SScw+RGIlLZY/qXVPWwqsZ5mXcpnA2PN3+641N9rKrRqnoKeAboJSJhGa0Dj2mnqOpWVU1y18N8Vf1ZHSuAxcC16cSRnudVNU5VN+MchTR0h/cCXnTXeQwwLoN5lMyg/p6+UNV17jr+BKeJEABVnaqqh9y6vQ7kx9lApvpOVWe76yZOVTeq6lq3/K84G/Lr3LI3AftV9XVVjVfVE6r6vbeAROQynO/XcFU9pap/4ezh9/Yo9oeqvu0uK+3nn4iTaGrjbLi2q6ov6wKcI5v/U9Wd7me4WVUPeSlXDOeIIa1eInIUZyN5H3Cbu24hne+kO/6gO74kcNBjmvScwDl68MbXzz4znr+vb3GSQ+p3+Tacz/8PoBnOztELqpqgqntwdmZ6e51rAFgiyFxPVY3E2Vutzd8byMrA7e5Jr6Pul/saoCxQEWdv5IiX+VUG/pVmuoo4ew5pzQRaiUg5nD0OxfnCpc7nLY95HMbZQyvvMf2+DOp10I3Vm7LueG/z+Q1nz74UGa8DrzGIyI0islZEDrvlu3Bu0vHFfo/Xp4HUE/jl0iwvo/ofIv36+7IsRORfIrJdRI65dSnKuXVJW/eaIvKleyL0OE7yTi1fEae5xReVcT6DPz3W+3s4RwZel+1JVb/BaZaaABwQkUkiUsTHZfsa5xGcZJPWZ6paDKdtPxrnKCmV1++k2wZfyh1/CCjlQ7t8JE6zlze+fvaZObuO1TkMmI674wb0xdlxAOfzKpfmd/IUzjrIESwR+Mjde50CjHEH7cPZUy7m8VdIVV92x5UQkWJeZrUP+Hea6Qqq6qdelnkUZ4+5F84X61P3C5c6n/vTzKeAqq7xnEUGVVoCtBCRip4DRaQ5zo/9G4/BnmUq4exRHsxkHZwXg4jkx2laGgNc5m4QFuAksMzi9cWfOE1C3uJOaylQQUSaXsyCRORa4Amcz6a4W5dj/F0XOL8+7wA7gBqqWgRnY5Bafh9Ok5k3aeezD+cospTHei+iqvUymObcGaqOU9UmOE04NXGafDKdLpM4Pf2EcyBb3ttIVT2Ic1Q70j2CBuc7eaOIFEpT/Fac+q7FOccSj9PklpE6OEeL3vjy2Z8CCnq8v9xLmbTr6lPgNveovAXOdx2cdfZLmt9JpKp2IYewRHBhxgIdRaQRzknAbiLSSUTCRCTCvfyxgnuY/RUwUUSKi0g+EWnjzuN9YIiItHCvpCkkIl1FxNveEzhNQXfj/BimeQx/F3hSROoBiEhREbnd14qo6hKcH8TnIlLPrUNLnL2Yd1T1J4/id4pIXREpCLwAzFTV5IzWQTqLDcdpPokFkkTkRsDzksYDQEkRSe+QPjOf4ayT4u4GaFh6Bd36TQQ+dWMOd+PvLSIjfFhWJE5bdSyQV0SexTmZmdk0x4GTIlIb+IfHuC+By0VkuDiX9UaKSAt33AGgSupVV+73azHwuogUEZE8IlJdRK7DByLSzP3+5cPZ4MXjnDxNXVa1DCb/DzBKRGq4398rRaRk2kKqmoizYU83JlXdgXORw+PuoI+BGGCGiFRxfzedcJr4RqrqMVU9htPWPkFEeopIQbfcjSLyqsfsr8P5DXpbri+ffRRwizv/K3BOZGdInctkY911tMjdkQPn/M1xEXlCRAq4v5X6ItIss3lmF0sEF0BVY4GPgGdUdR/O5WpP4Xz4+3D2qlLX6V04e847cM4tDHfnsQGnbXQ8zuHzbpwTUemZi3OVwwG3TTw1llnAK8B0t5khGqfd+ELcinMJ30KcKzGm4lyJ8mCach/jHA3txzmR+ZAbQ2br4ByqesKd9jOcuvd165c6fgfOXtUe9xDaW3NZRl7A2ZD8grMRmomzJ5meh/i7ieQoTpPHzcA8H5a1CGdDswunuSyejJuiAB7FqfMJnB2C/6WOcNdNR6Abznr+CWjnjk69xPKQiGxyX9+Nk1i34azLmfje3FHEXf4RN/ZD/H2k+wFQ113/s71M+wbO57cYJ6l9gHOy1Jv3cH4HGXkNGCwiZVT1DM4Vc/twrtA67i7vaVV9LXUCVX0DeATnAonU790wnBPoiEgETpPjfzNYbmaf/Zs4V08dcOfzyfmz8OpTtw5nd9rcnaZuOOeXfsE5mv4P6Z/DyHapZ7iN8UpEluNc6RGQu3svhYj8A+itqj7tKZusJyKrgAfdveXsWuaDOJe0Pp5pYQM4l2UZkyu4bc3VcNqRa+Bcijk+oEGFOFW9JvNSWb7Mt7N7mcHOEoHJTcJxmiOq4hzuT8dpCzbGZMCahowxJsTZyWJjjAlxQdc0VKpUKa1SpUqgwzDGmKCycePGg6pa2tu4oEsEVapUYcOGDYEOwxhjgoqI/JbeOGsaMsaYEGeJwBhjQpwlAmOMCXGWCIwxJsRZIjDGmBDnt0QgIh+KyF8iEp3OeBGRcSKyW0S2iMhV/orFGGNM+vx5RDAF57Fy6bkRpz+YGjjPSn3Hj7EYY4xJh9/uI1DVlSJSJYMiPYCP3AetrBWRYiJS9gIemXdhdk+CX6dlXs4YY3KYxCThl/0FqdmgBjQZm+XzD+Q5gvKc2397DOc+ZvEsERksIhtEZENsbOzFLe3XaXAk6uKmNcaYAPlhdxGaD7uGdo+24lScZD7BRQjkncXeauS1BzxVnQRMAmjatOnF95JXvBF0WH7RkxtjTHaJj0/i+efX8Npr6ylVqgATJ3Wg0DU1/bKsQCaCGM59pmwF4I8AxWKMMTlKz56zWbToVwYMqM/rr7elePEIvy0rkE1Dc4G73auHWgLH/HZ+wBhjgsCJEwnExycBMGJEcxYvvo0PP+zs1yQAfjwiEJFPgbZAKRGJAZ4D8gGo6rvAApzniu4GTgMD/BWLMcbkdIsW/cLgwYu58866/Pvf19K2baVsW7Y/rxrqk8l4BYb6a/nGGBMMDh+O45FHlvPf/26ldu0SdO1aLdtjCLpuqI0xJrdYuvQ3+vWbz6FD8Tz9dEv+7/9aEhGR/ZtlSwTGGBMgZcoUpGrVoixceBuNGpUJWBzW15AxxmQTVWXKlGgeemgpAA0alGbNmr4BTQJgicAYY7LFL78cpVOnmQwYsJCoqFji4hIBEPHPTWIXwpqGjDHGj5KTU5gwIYonn1xJnjzCxIkduP/+huTJE/gEkMoSgTHG+NHBg3E8++xqrruuIu++25FKlYoEOqTzWCIwxpgslpiYzCefbOfuu+tx2WWF2LTpLqpWLZojmoG8sURgjDFZaOPG/dx77yK2bImlbNlCdOpUlWrVigU6rAzZyWJjjMkCcXGJjBixkhYtPiE29jSzZvWgU6eqgQ7LJ3ZEYIwxWaBnzzksXvwrgwY14LXXrqNYMf/2D5SVLBEYY8xFOn78DOHhYURE5OWpp1rw+OPNaN++cqDDumDWNGSMMRdhwYI91K8/hRde+A6A666rGJRJACwRGGPMBTl48DR33bWArl2/IDIynO7dqwc6pEtmTUPGGOOjr7/+lX795nPkyBmefbYVTz3Vgvz5g38zGvw1MMaYbFK2bCFq1izBO+90oEGD0oEOJ8tY05AxxqRDVfnPf7YwdOgSAOrXL8233/bOVUkALBEYY4xXe/YcpUOHGdx332K2bTuUozqJy2qWCIwxxkNycgpvvrmB+vWnsH79ft57ryNLl/aiQIF8gQ7Nb+wcgTHGeDh4MI7nn/+O9u0r8c47HalQITLQIfmdJQJjTMhLSEhm6tRt9O9fn8suK0RU1N1UrlwkVzYDeWOJwBgT0tav/5N7711EdPRBKlSI5IYbqlClStFAh5Wt7ByBMSYknT6dyKOPLqdly2kcORLP3Lk3c8MNVQIdVkDYEYExJiT16DGbJUt+Y/DgK3n11esoWjR/oEMKGEsExpiQcezYGfLndzqJe+aZljz1VAvatasU6LACzpqGjDEh4csvf6Zevck8//waANq0qWhJwGWJwBiTq8XGnqZv3y/p1m0WJUpEcMstNQIdUo5jTUPGmFxr8WKnk7hjx87w/PNXM2JEC8LDwwIdVo5jicAYk2uVL1+YOnVK8s47HahXr1Sgw8mxrGnIGJNrpKQokyZt5h//+BqAevVKsXJlb0sCmbBEYIzJFXbvPkL79p9x//1fs3Pn4bOdxJnMWSIwxgS15OQUXn99PVde+V82bTrA++/fkOs7ictqfk0EItJZRHaKyG4RGeFlfFERmScim0Vkq4gM8Gc8xpjc5+DBOEaPXkvHjpXZtm0AgwZdGTJ9BGUVvyUCEQkDJgA3AnWBPiJSN02xocA2VW0ItAVeF5Fwf8VkjMkdzpxJ4v33t5CSomc7iZs9uyfly+f+nkL9wZ9HBM2B3aq6R1UTgOlAjzRlFIgUJ30XBg4DSX6MyRgT5L7//k+aNPmYwYMXs2TJbwBUrlzUjgIugT8TQXlgn8f7GHeYp/FAHeAP4Efgn6qaknZGIjJYRDaIyIbY2Fh/xWuMycFOnUrgkUeW0arVJxw7lsD8+beEbCdxWc2ficBbetY07zsBUUA5oBEwXkSKnDeR6iRVbaqqTUuXzl3PCjXG+KZnzzm8+eZGhgxpyNat/enSpVqgQ8o1/JkIYoCKHu8r4Oz5exoAfKGO3cAvQG0/xmSMCSJHj8afvQz02WdbsWLFHUyc2JEiRUK3p1B/8GciWA/UEJGq7gng3sDcNGX2Au0BROQyoBawx48xGWOCxNy5u6lXbwrPP/8dANdeW4E2bSpmMpW5GH5LBKqaBAwDFgHbgc9UdauIDBGRIW6xUcDVIvIjsBR4QlUP+ismY0zO99dfp+jdex49esymVKkC3HZbzUCHlOv53NeQiBQC4lU12ddpVHUBsCDNsHc9Xv8B3ODr/IwxudvChb/Qr998Tp5MZNSo1jzxRHPy5bNO4vwt3UQgInlwmnP6Ac2AM0B+EYnF2bhPUtWfsiVKY0xIqFgxkgYNSjFxYgfq1rX+gbJLRk1Dy4DqwJPA5apaUVXLANcCa4GXReTObIjRGJNLpaQo77wTxf33LwacTuKWL+9tSSCbZdQ01EFVz+u1SVUPA58Dn4uIdeZhjLkou3YdZtCgxXz7bQwdO1YmPj6JiAjrGT8Q0j0iUNVEEckjItEZlfFPWMaY3CopKYVXXvmeK6/8Lz/+GMvkyZ1ZtOg2SwIBlOGaV9UUt0O4Sqq6N7uCMsbkXocOxfHKK+vp0qUaEya0p2zZwoEOKeT5koLLAltFZB1wKnWgqnb3W1TGmFzlzJkkpkzZyn33XclllxVi8+a7qVjxvE4ETID4kgie93sUxphc67vv/mDgwIVs336Y6tWL0aFDZUsCOUymN5Sp6gpgJ1AUKALsdIcZY0y6Tp5MYPjwb2jdehqnTiWycOGtdOhQOdBhGS8yTQQiMghYB9wC3AasFZF7/R2YMSa49ew5m7fe2sTQoY2Jjh5Ap05VAx2SSYcvTUOPAY1V9RCAiJQE1gAf+jMwY0zwOXIknoiIMAoUyMfIkVczcuTVXHNNhUCHZTLhS19DMcAJj/cnOPc5A8YYwxdf7KJu3cmMHLkGgGuuqWBJIEj4ckTwO/C9iMzBeZ5AD2CdiDwCoKpv+DE+Y0wOt3//KYYNW8Lnn/9Eo0Zl6N3bepIPNr4kgp/dv1Rz3P/2cFBjQtxXX+2hX78FnD6dyIsvXsujjza1TuKCkC+JYJuqzvAcICK3px1mjAk9lSsXoXHjMkyY0J7atUsGOhxzkXw5R/Ckj8OMMblcSooyfvwm7rtvEQB165Zi6dJelgSCXEbdUN8IdAHKi8g4j1FFgCR/B2aMyVl27jzMwIGLWL36dzp1qmKdxOUiGX2KfwAbge7u/1QngIf9GZQxJudITExmzJgNPP/8GgoWzMeUKZ25++56iEigQzNZJN1EoKqbgc0i8on1MmpM6DpyJJ7XXltPt27Vefvt9lx+eaFAh2SyWLrnCERknoh0S2dcNRF5we4wNiZ3io9PYuLEH0hJUcqUKcSWLfcwY0Z3SwK5VEZNQ/cBjwBjReQwEAtEAFVwLicdr6pz0p/cGBOMVq2KYeDARezadYSaNUvQoUNlKlSwq8Vzs4yahvYDjwOPi0gVnO6o44Bdqno6e8IzxmSXEycSePLJlUyYEEWVKkVYvPg26yQuRPh0yl9VfwV+9WskxpiA6tlzNsuW7eWf/7yK0aOvoXDh8ECHZLKJXftlTAg7fDiOiIi8FCyYj1GjWiNyDa1alQt0WCab+XJDmTEmF5o5cyd16vzdSdzVV5e3JBCifEoEIlJARGr5OxhjjP/9+edJbrllDrffPo+KFSPp169OoEMyAebLg2m6AVHAQvd9IxGZ6+e4jDF+MH/+z9StO5mvvvqFV15pw9q1/WjYsEygwzIB5ss5gpFAc2A5gKpGuVcRGWOCTLVqxWjW7HLGj29PzZolAh2OySF8aRpKUtVjfo/EGJPlkpNTeOutjQwcuBCAOnVKsnjx7ZYEzDl8SQTRItIXCBORGiLyNs6jKo0xOdi2bQe59trpDB++jP37TxEfb31FGu98SQQPAvWAM8A04BjwT38GZYy5eAkJyYwe/R2NG3/Mrl1HmDq1C19+eYv1FGrS5Usi6KqqT6tqM/fv/3B6JM2UiHQWkZ0isltERqRTpq2IRInIVhFZcSHBG2POd/RoPG++uZGbb76Cbdv6069fXesp1GTIbw+mEZEwYAJwI1AX6CMiddOUKQZMBLqraj3gdh/iMcakEReXyPjxm852Evfjj/2ZPr0bZcpYJ3Emc/58ME1zYLeq7nHnNx3nwffbPMr0Bb5Q1b0AqvrXhYVvjFm5ch+DBi3mp5+OUKdOSdq3r0y5coUDHZYJIhkdEfwBbADicR5Mk/o3F+jkw7zLA/s83se4wzzVBIqLyHIR2Sgid3ubkYgMFpENIrIhNjbWh0Ubk/sdP36GBx74muuu+x9JSSksWXI77dtbJ3HmwvnyYJppF/lgGm+Nkupl+U2A9kAB4DsRWauqu9LEMgmYBNC0adO08zAmJPXsOZvly/fx8MNNGDWqNYUKWSdx5uL4chlBFRF5CaedPyJ1oKpWy2S6GKCix/sKOEcZacscVNVTwCkRWQk0BHZhjDnPwYOnKVgwHwUL5uPf/74WEWjZ0voHMpfGl5PFk4F3cM4LtAM+Aj72Ybr1QA0RqSoi4UBvnGYlT3OAa0Ukr4gUBFoA230N3phQoapMn76DOnUm89xzqwFo1aqcJQGTJXxJBAVUdSkgqvqbqo4Ers9sIlVNAoYBi3A27p+p6lYRGSIiQ9wy23H6MNoCrAP+o6rRF1cVY3Kn338/Qc+es+nT50uqVi3K3XfXC3RIJpfxpWkoXkTyAD+JyDDgd8CnXqpUdQGwIM2wd9O8fw14zbdwjQktX375M/36zScxMYUxY65j+PAmhIVZ7/Ema/mSCIYDBYGHgFE4zUP3+DEmY4zriiuKcfXV5Xj77fZccUXxQIdjcqkME4F7U1gvVX0MOAkMyJaojAlRyckpjBu3ic2bY5ky5UZq1y7JV1/dFuiwTC6X4TGmqiYDTcTuTzfG77ZuPUjr1p/yyCPLOXgwzjqJM9nGl6ahH4A5IjIDOJU6UFW/8FtUxoSQhIRkXn75e0aPXkvRovmZNq0rvXvXtv6BTLbxJRGUAA5x7pVCClgiMCYLHD0az7hxP3D77bUYO7YdpUsXDHRIJsRkmghU1c4LGJPFTp9O5P33tzBsWGO3k7h7KFvW+gcygWEdlBuTzZYt28ugQYvYs+cY9euXon37ypYETEDZBcnGZJNjx85w//2Luf76zxARli3rZZ3EmRzBjgiMySY9e85m5coYHnusGSNHXk3BgvkCHZIxgA+JQEQuA14Eyqnqje7DZVqp6gd+j86YIBcbe5pChZxO4l566VrCwoRmzcoGOixjzuFL09AUnP6CUnu32oVzt7ExJh2qyrRp28/pJK5ly3KWBEyO5EsiKKWqnwEpcLYzuWS/RmVMEIuJOUH37rPo128+V1xRjP796wc6JGMy5Ms5glMiUhL3oTIi0hI45teojAlSc+fu5s47F5CcnMKbb7bjwQcbWydxJsfzJRH8C+c5AtVFZDVQGrDOT4zxombN4lxzTXnGj29PtWrFAh2OMT7x5YayjSJyHVAL5/GTOy/y0ZXG5DpJSSmMHbuRLVti+eijLtSuXZIFC24NdFjGXJBMj1lFZDPwOBCvqtGWBIxxbNkSS6tWn/DYYys4fjzBOokzQcuXxsvuOI+p/ExE1ovIoyJSyc9xGZNjnTmTxHPPraZJk4/Zu/cEn33WjVmzehARYbflmOCUaSJwH0/5qqo2AfoCVwK/+D0yY3Ko48cTmDgxij59arNt2wBuv72W9RRqgppPuzAiUgXoBdyBc+no436MyZgc59SpBCZN2sJDD11F6dIFiY7uz2WXFQp0WMZkCV/uLP4eyAfMAG5X1T1+j8qYHGTp0t+4777F/PLLMRo2LMP111eyJGByFV+OCO5R1R1+j8SYHObo0XgefXQFH3zwIzVqFGfFijto06ZioMMyJsulmwhE5E5VnQp0EZEuacer6ht+jcyYALv55jl8+20MTzzRnOeea0WBAtZJnMmdMjoiSD32jfQyTv0QizEBd+DAKQoXzkehQuG8/HIb8uYVmjS5PNBhGeNX6SYCVX3PfblEVVd7jhOR1n6NyphspqpMnbqN4cOXMWBAfcaMaUuLFtZBnAkNvtxH8LaPw4wJSnv3Hqdr1y+4++6vqFWrBAMHNgh0SMZkq4zOEbQCrgZKi8gjHqOKAGH+DsyY7DBnzm7uvHM+qjBu3PU88EAj6yTOhJyMzhGEA4XdMp7nCY5jnc6ZIKeqiAi1a5egbduKvP12e6pUKRrosIwJiIzOEawAVojIFFX9LRtjMsZvkpJSeP319fz440GmTu1KrVolmDfvlkCHZUxAZdQ0NFZVhwPjReS8q4RUtbs/AzMmq23e/Bf33ruITZsOcPPNNYiPT7L+gYwh46ahj93/Y7IjEGP8JT4+idGj1/LKK+soWTKCmTO7c+utNQMdljE5RkZNQxvd/ytSh4lIcaCiqm7JhtiMyRInTiTw3nub6devDm+80ZYSJQoEOiRjchRfnkewXESKiEgJYDMwWUR8uqtYRDqLyE4R2S0iIzIo10xEkkXETkKbLHHyZAJjxqwnOTmF0qULsm3bAKZMudGSgDFe+HKdXFFVPQ7cAkx2u6PukNlEIhIGTABuBOoCfUSkbjrlXgEWXUjgxqRn8eJfqV9/Co8/voKVK2MAKF26YICjMibn8iUR5BWRsjjdUH95AfNuDuxW1T2qmgBMB3p4Kfcg8Dnw1wXM25jzHD4cx4ABX9Gp00wiIvLy7bd9aNfOnqFkTGZ8uWTiBZy99dWqul5EqgE/+TBdeWCfx/sYoIVnAREpD9wMXA80S29GIjIYGAxQqZL9sI13N988h9Wrf+epp1rwzDOt7IogY3zky8PrZ+A8iyD1/R7Al6dze3tkU9rLUMcCT6hqckZPeFLVScAkgKZNm1qHd+as/ftPERnpdBL32mvXER4eRqNGZQIdljFBxZeTxRVEZJaI/CUiB0TkcxGp4MO8YwDPztsrAH+kKdMUmC4iv+LcrTxRRHr6FroJZarKlCnR1K07mWefdfpEbN68rCUBYy6CL+cIJgNzgXI4zT3z3GGZWQ/UEJGqIhIO9Hbnc5aqVlXVKqpaBZgJPKCqs30P34SiX389RufOnzNgwELq1SvJ4MENAx2SMUHNl0bU0qrqueGfIiLDM5tIVZNEZBjO+YUw4ENV3SoiQ9zx715MwCa0zZr1E3fdtQARGD++Pf/4RyPy5LEHxxtzKXxJBAdF5E7gU/d9H+CQLzNX1QXAgjTDvCYAVe3vyzxNaErtJK5evZJ06FCZt95qR+XK1kmcMVnBl6ahe3EuHd3v/t3mDjPG7xITk3nxxbX06zcfgJo1SzB7dk9LAsZkIV+uGtoLWAdzJttt2nSAgQMXERX1F7161eLMmSTy57dLQo3Jar5cNVRNROaJSKx75dAc914CY/wiLi6RJ59cSfPmU9m//xSzZvXgf//rZknAGD/xpWloGvAZUBbnyqEZ/H2+wJgsd+pUIh988CP33FOPbdsG0LNnjUCHZEyu5ksiEFX9WFWT3L+pnH9jmDGX5MSJBF59dR3JySmUKuV0EvfBB50pXjwi0KEZk+v5cqy9zO05dDpOArgDmO/2RoqqHvZjfCYELFz4C/ffv5h9+07QvPnltG1biVKlrJM4Y7KLL4ngDvf//WmG34uTGOx8gbkohw7F8cgjy/joo23UqVOC1av70qpVuUCHZUzI8eWqoarZEYgJPbfcMoc1a/7gmWda8vTTLe1ksDEBYr88k63+/PMkkZHhFC4czpgxTidxDRta/0DGBJIvJ4uNuWSqyocf/kidOn93EtesWVlLAsbkAHZEYPxuz56j3H//1yxZ8htt2lRgyBDrJM6YnCTTRCDOgwL6AdVU9QURqQRcrqrr/B6dCXpffLGLu+5aQFhYHt55pwODBze0TuKMyWF8OSKYCKTgPEXsBeAEzqMl032imDGpncQ1aFCazp2rMnZsOypWLBLosIwxXvhyjqCFqg4F4gFU9QgQ7teoTNBKSEhm9Ojv6Nt3PqpKjRrF+fzzHpYEjMnBfEkEiSIShns3sYiUxjlCMOYcGzbsp1mzqTzzjHMyOCEhOcARGWN84UsiGAfMAsqIyL+BVcCLfo3KBJW4uEQef3wFLVp8wsGDccyZ05NPP73J7gswJkj4ckPZJyKyEWiP80D6nqq63e+RmaBx6lQiU6ZEM3BgA159tQ3Filn/QMYEE1+uGqoEnMZ5VvHZYe5zCkyIOn78DBMnRvHYY80oVaog27ffS8mSBQIdljHmIvhy7D4f5/yAABFAVWAnUM+PcZkcbP78nxkyZAl//HGSli3L0rZtJUsCxgSxTM8RqGoDVb3S/V8DaI5znsCEmNjY0/TrN5+bbppF0aLhrFnTl7ZtKwU6LGPMJbrgs3mquklE7B6CEHTrrXNZu/YPRo68miefbEF4eFigQzLGZAFfzhE84vE2D3AVEOu3iEyO8vvvJyhaND+FC4fz5pttyZ8/jPr1Swc6LGNMFvLl8tFIj7/8OOcMevgzKBN4qsr772+hbt2/O4lr0uRySwLG5EIZHhG4N5IVVtXHsikekwP8/PNR7rtvEcuW7aNdu4oMHdo40CEZY/wo3UQgInlVNUlErsrOgExgzZy5k7vv/op8+fIwadINDBrUAKffQWNMbpXREcE6nPMBUSIyF5gBnEodqapf+Dk2k41SO4lr2LAMXbtW480321GhQmSgwzLGZANfrhoqARzC6X009X4CBSwR5AIJCcm89NL3bNt2iOnTb6JGjeLMmNE90GEZY7JRRomgjHvFUDR/J4BU6teoTLZYt+5PBg5cRHT0Qfr2rUNCQrL1D2RMCMroVx8GFObcBJDKEkEQO306kWefXc2bb26kbNlCzJt3MzfdVD3QYRljAiSjRPCnqr6QbZGYbBMXl8TUqdsYPPhKXnmlDUWK5A90SMaYAMroPoJLvlRERDqLyE4R2S0iI7yM7yciW9y/NSJiD7P1k2PHzvDvf68lKSmFkiULsH37vbzzTkdLAsaYDBNB+0uZsXsPwgTgRqAu0EdE6qYp9gtwnapeCYwCJl3KMo138+b9fPbGsFWrYgAoXty6ijbGONJNBKp6+BLn3RzYrap7VDUBmE6aO5JVdY376EuAtUCFS1ym8RAbe5o+fb6ke/dZlCwZwfff97NO4owx5/HnJSLlgX0e72OAFhmUHwh85W2EiAwGBgNUqmQbMl+ldhL3wguteeKJ5tZJnDHGK38mAp+vNhKRdjiJ4Bpv41V1Em6zUdOmTe2KpQzExJygWDGnk7ixY9uRP38Y9eqVCnRYxpgczJdO5y5WDFDR430F4I+0hUTkSuA/QA9VPeTHeHK1lBTlvfc2U7fu5LMPj7/qqsssCRhjMuXPI4L1QA0RqQr8DvQG+noWcB+D+QVwl6ru8mMsudpPPx3hvvsWsWJFDO3bV+LBB62TOGOM7/yWCNwO64YBi3BuTvtQVbeKyBB3/LvAs0BJYKLbsVmSqjb1V0y50YwZTidx+fOH8cEHnRgwoL51EmeMuSB+7U9AVRcAC9IMe9fj9SBgkD9jyK1SO4lr3LgMPXpU54032lGuXOFAh2WMCUL+PEdg/ODMmSSefXYVvXrNQ1W54oriTJ/ezZKAMeaiWSIIImvX/sFVV33MqFFrKVAgLwkJyYEOyRiTC1giCAKnTiXw8MPLuPrqaZw4kcCCBbfw0UddrKdQY0yWsC1JEIiPT2b69B088EAjXnqpDZGR4YEOyRiTi1giyKGOHo3n7bd/4MknW7idxA2gWDHrH8gYk/WsaSgHmj37J+rWnczzz69hzZrfASwJGGP8xo4IcpADB07x4INLmTFjFw0blmbevJtp0uTyQIdlTLZJTEwkJiaG+Pj4QIcStCIiIqhQoQL58uXzeRpLBDnIbbfNZd26/YwefQ2PP96MfPmskzgTWmJiYoiMjKRKlSp2Y+RFUFUOHTpETEwMVatW9Xk6SwQBtnfvcYoXjyAyMpxx464nf/4w6ta1/oFMaIqPj7ckcAlEhJIlSxIbG3tB09k5ggBJSVEmTPiBevWcB8YANG58mSUBE/IsCVyai1l/dkQQADt3HmbQoEWsWvU7HTtW5p//vCrQIRljQpgdEWSzzz7bQcOG/yU6+iCTJ3dm0aLbqFKlaKDDMsa4wsLCaNSoEfXr16dbt24cPXr07LitW7dy/fXXU7NmTWrUqMGoUaNQ/fsRKV999RVNmzalTp061K5dm0cffTQANbhwlgiySeqXpUmTy7nllhps334v/ftbT6HG5DQFChQgKiqK6OhoSpQowYQJEwCIi4uje/fujBgxgl27drF582bWrFnDxIkTAYiOjmbYsGFMnTqV7du3Ex0dTbVq1QJZFZ9Z05CfxccnMWrUd+zYcZiZM7tTvXoxpk27KdBhGZPzbRwOR6Kydp7FG0GTsT4Xb9WqFVu2bAFg2rRptG7dmhtuuAGAggULMn78eNq2bcvQoUN59dVXefrpp6lduzYAefPm5YEHHsja+P3Ejgj8aM2a32nc+CNefPF7IiPDrZM4Y4JIcnIyS5cupXv37oDTLNSkSZNzylSvXp2TJ09y/PhxoqOjzxsfLOyIwA9Onkzgqae+Zfz4H6hYMZKFC2+lUyffr+k1xnBBe+5ZKS4ujkaNGvHrr7/SpEkTOnbsCPz9DBBvgr2J144I/CAhIZmZM3cxdGhjoqMHWBIwJoikniP47bffSEhIOHuOoF69emzYsOGcsnv27KFw4cJERkZSr149Nm7cGIiQL5klgixy+HAcI0euJikphRIlCrB9+728/XZ76ynUmCBVtGhRxo0bx5gxY0hMTKRfv36sWrWKJUuWAM6Rw0MPPcTjjz8OwGOPPcaLL77Irl3O49dTUlJ44403Ahb/hbBEkAU+/3wXdetOZvTotWc7iStaNH+AozLGXKrGjRvTsGFDpk+fToECBZgzZw6jR4+mVq1aNGjQgGbNmjFs2DAArrzySsaOHUufPn2oU6cO9evX588//wxwDXxj5wguwZ9/nmTYsKV88cVPNG5choULb6NRozKBDssYcwlOnjx5zvt58+adfd2gQQOWL1+e7rQ33XQTN90UfFcFWiK4BL16zWP9+v28/PK1/Otfzcib1w6wjDHBxxLBBfrtt2OUKFGAyMhw3n67PQUK5KVWrRKBDssYYy6a7cL6KCVFefvtTdSrN4VnnlkFQKNGZSwJGGOCnh0R+GDHjkMMGrSY1at/p3PnKjz8cHDeNGKMMd5YIsjE9Ok7uOeeryhcOB8ffXQjd95ZN+hvHjHGGE+WCNKRkqLkySM0a3Y5t99ek9dfb8tllxUKdFjGGJPl7BxBGnFxiYwYsZJbb52DqlK9ejGmTu1qScCYEHHgwAH69u1LtWrVaNKkCa1atWLWrFl+XeaGDRt46KGH/LqMjFgi8PDttzE0avQRr7yyjpIlC5CYmBLokIwx2UhV6dmzJ23atGHPnj1s3LiR6dOnExMT49flNm3alHHjxvl1GRmxpiHgxIkERoxYycSJUVStWpSvv76dDh0qBzosY0Je27bTzxvWq1ctHnigMadPJ9Kly+fnje/fvz79+9fn4MHT3Hbb3HPGLV/eO8PlffPNN4SHhzNkyJCzwypXrsyDDz7IlClT2LBhA+PHjwecm8ceffRR2rZty+LFi3nuuec4c+YM1atXZ/LkyRQuXJgRI0Ywd+5c8ubNyw033MCYMWOYMWMGzz//PGFhYRQtWpSVK1eyfPlyxowZw5dffsnIkSPZu3cve/bsYe/evQwfPvzs0cKoUaP45JNPqFixIqVKlaJJkyZZ8vAbSwRAYmIys2fvZvjwJowe3ZpChax/IGNC0datW7nqqgt7dOzBgwcZPXo0S5YsoVChQrzyyiu88cYbDBs2jFmzZrFjxw5E5OyTzl544QUWLVpE+fLlz3n6macdO3awbNkyTpw4Qa1atfjHP/7B5s2b+fzzz/nhhx9ISkriqquuyrJur0M2ERw6FMdbb23k2WevpkSJAuzYca91EGdMDpPRHnzBgvkyHF+qVMFMjwAyM3ToUFatWkV4eDhDhw71Wmbt2rVs27aN1q1bA5CQkECrVq0oUqQIERERDBo0iK5du57teqJ169b079+fXr16ccstt3idZ9euXcmfPz/58+enTJkyHDhwgFWrVtGjRw8KFCgAQLdu3S6pbp78eo5ARDqLyE4R2S0iI7yMFxEZ547fIiJ+f4q7qjJjxk7q1p3MSy+t47vv/gCwJGCMoV69emzatOns+wkTJrB06VJiY2PJmzcvKSl/nzeMj48HnG1Kx44diYqKIioqim3btvHBBx+QN29e1q1bx6233srs2bPp3LkzAO+++y6jR49m3759NGrUiEOHDp0XR/78f3daGRYWRlJS0jnPRs5qfksEIhIGTABuBOoCfUSkbppiNwI13L/BwDv+igfgj4P5ueWWOfTqNY+KFSPZsOFOrr22gj8XaYwJItdffz3x8fG8887fm6LTp08DUKVKFaKiokhJSWHfvn2sW7cOgJYtW7J69Wp27959tvyuXbs4efIkx44do0uXLowdO5aoqCgAfv75Z1q0aMELL7xAqVKl2Ldvn0+xXXPNNcybN4/4+HhOnjzJ/Pnzs6ze/mwaag7sVtU9ACIyHegBbPMo0wP4SJ1Ut1ZEiolIWVX1S9+tvUY3YePPv/Lqq214+OGm1kmcMeYcIsLs2bN5+OGHefXVVylduvTZdv/WrVtTtWpVGjRoQP369c+eSyhdujRTpkyhT58+nDlzBoDRo0cTGRlJjx49iI+PR1V58803Aee5BT/99BOqSvv27WnYsCErVqzINLZmzZrRvXt3GjZsSOXKlWnatClFixbNmnr763BDRG4DOqvqIPf9XUALVR3mUeZL4GVVXeW+Xwo8oaob0sxrMM4RA5UqVWry22+/XXhAG4ezeVc+CjR5kpo1rX8gY3Ki7du3U6dOnUCHkWOdPHmSwoULc/r0adq0acOkSZO8ntz2th5FZKOqNvU2X38eEXjrhyFt1vGlDKo6CZgE0LRp04vLXE3G0tC6CDLGBLHBgwezbds24uPjueeeey74Cqf0+DMRxAAVPd5XAP64iDLGGGOAadOm+WW+/mwkXw/UEJGqIhIO9AbmpikzF7jbvXqoJXDMX+cHjDHBwZ9Xx4SCi1l/fjsiUNUkERkGLALCgA9VdauIDHHHvwssALoAu4HTwAB/xWOMyfkiIiI4dOgQJUuWtF5+L4KqcujQISIiIi5oOr+dLPaXpk2b6oYNGzIvaIwJOomJicTExJy9Rt9cuIiICCpUqEC+fPnOGR6ok8XGGHNB8uXLR9WqVQMdRsixC+mNMSbEWSIwxpgQZ4nAGGNCXNCdLBaRWOAibi0GoBRwMAvDCQZW59BgdQ4Nl1Lnyqpa2tuIoEsEl0JENqR31jy3sjqHBqtzaPBXna1pyBhjQpwlAmOMCXGhlggmBTqAALA6hwarc2jwS51D6hyBMcaY84XaEYExxpg0LBEYY0yIy5WJQEQ6i8hOEdktIiO8jBcRGeeO3yIiWfN0hwDyoc793LpuEZE1ItIwEHFmpczq7FGumYgku0/NC2q+1FlE2opIlIhsFZHMn4GYw/nw3S4qIvNEZLNb56DuxVhEPhSRv0QkOp3xWb/9UtVc9YfT5fXPQDUgHNgM1E1TpgvwFc4T0loC3wc67myo89VAcff1jaFQZ49y3+B0eX5boOPOhs+5GM5zwSu578sEOu5sqPNTwCvu69LAYSA80LFfQp3bAFcB0emMz/LtV248ImgO7FbVPaqaAEwHeqQp0wP4SB1rgWIiUja7A81CmdZZVdeo6hH37Vqcp8EFM18+Z4AHgc+Bv7IzOD/xpc59gS9UdS+AqgZ7vX2pswKR4jzAoDBOIkjK3jCzjqquxKlDerJ8+5UbE0F5YJ/H+xh32IWWCSYXWp+BOHsUwSzTOotIeeBm4N1sjMuffPmcawLFRWS5iGwUkbuzLTr/8KXO44E6OI+5/RH4p6qmZE94AZHl26/c+DwCb481SnuNrC9lgonP9RGRdjiJ4Bq/RuR/vtR5LPCEqibnkqdd+VLnvEAToD1QAPhORNaq6i5/B+cnvtS5ExAFXA9UB74WkW9V9bifYwuULN9+5cZEEANU9HhfAWdP4ULLBBOf6iMiVwL/AW5U1UPZFJu/+FLnpsB0NwmUArqISJKqzs6WCLOer9/tg6p6CjglIiuBhkCwJgJf6jwAeFmdBvTdIvILUBtYlz0hZrss337lxqah9UANEakqIuFAb2BumjJzgbvds+8tgWOq+md2B5qFMq2ziFQCvgDuCuK9Q0+Z1llVq6pqFVWtAswEHgjiJAC+fbfnANeKSF4RKQi0ALZnc5xZyZc678U5AkJELgNqAXuyNcrsleXbr1x3RKCqSSIyDFiEc8XBh6q6VUSGuOPfxbmCpAuwGziNs0cRtHys87NASWCiu4ecpEHcc6OPdc5VfKmzqm4XkYXAFiAF+I+qer0MMRj4+DmPAqaIyI84zSZPqGrQdk8tIp8CbYFSIhIDPAfkA/9tv6yLCWOMCXG5sWnIGGPMBbBEYIwxIc4SgTHGhDhLBMYYE+IsERhjTIizRGCyjdsDaJTHX5UMyp7MxtDSJSLlRGSm+7qRiHTxGNc9o15P/RBLFRHpexHTFRCRFSIS5r5/ze2l87ULmEdp97JUkwvZ5aMm24jISVUtnNVls4uI9AeaquowPy4jr6p67TBNRNoCj6rqTRc4z6FAXlV9y31/HCitqmcuJCYRmYxzX8LqC1m+yfnsiMAEjIgUFpGlIrJJRH4UkfN6DxWRsiKy0j2CiBaRa93hN4jId+60M0TkvKThdrw2VpznL0SLSHN3eAkRme325b7W7XoDEbnO42jlBxGJdPfCo927Wl8A7nDH3yEi/UVkvDj94f8qInnc+RQUkX0ikk9EqovIQnE6gPtWRGp7iXOkiEwSkcXAR+4yv3XrtklErnaLvoxz13CUiDwsImHu3v16ty73p7Oq++HccYyIzAUKAd+7dZgiIu+6y9slIje55fq763UesNidz2x3Xia3CXTf2/YXOn9AMk7nYFHALJw724u440rh3CmZepR60v3/L+Bp93UYEOmWXQkUcoc/ATzrZXnLgffd121w+3cH3gaec19fD0S5r+cBrd3Xhd34qnhM1x8Y7zH/s+9xNrTt3Nd34Ow5AywFarivWwDfeIlzJLARKOC+LwhEuK9rABvc122BLz2mGwz8n/s6P7ABqJpm3uHA/jTDTnq8ngIsxNkprIHTj02EW7cYoIRH2fLAj4H+Htlf1v/lui4mTI4Wp6qNUt+ISD7gRRFpg9MdQnngMmC/xzTrgQ/dsrNVNUpErgPqAqvd7jLCge/SWean4PTxLiJFRKQYTs+rt7rDvxGRkiJSFFgNvCEin+D06R8jvvda+j+cBLAMpz+cie5RytXADI/55E9n+rmqGue+zgeMF5FGOMmzZjrT3ABcKX8/ea0ozsb8F48ypYCjmcT+mTrdNv8kIntwOmwD+FpVPfvF/wsol8m8TBCyRGACqR/OE6WaqGqiiPyKszd6lrsBbwN0BT52T3AewdlI9fFhGWlPginpdOOrqi+LyHycflzWikgHIN7HuswFXhKREjjdQH+D0wRz1DP5ZeCUx+uHgQM4vYbmySAGAR5U1UUZzDeONOvUC2/rKG1MuPOJw+Q6do7ABFJR4C83CbQDKqctICKV3TLvAx/gPMJvLdBaRK5wyxQUkfT2mu9wy1yD00vjMZxmpX7u8LY43TYfF5Hqqvqjqr6C08yStj3/BE7T1HlU9SROt8dv4TTfJKvTH/4vInK7uywR354VXRT4091LvwunSczb8hcB/3CPlhCRmiJSKE1cR4AwEckoGdwuInlEpDrOIyF3plOuJhC0HdiZ9FkiMIH0CdBURDbgbJh3eCnTFogSkR9wmnPeUtVYnDbsT0VkC05iOO8krOuIiKzBeUrZQHfYSHe5W3BOwN7jDh/unhjejLPnm/YpbsuAuqkni70s63/Ane7/VP2Age48t+L9cZppTQTuEZG1OBvf1D3zLUCSOA9pfxjn2RLbgE3iPOj8Pbwf5S8m4wcR7QRW4NR3iKqmdwTSDpjvQ/wmyNjloybXEpHlOJdbbgh0LIEkIo2BR1T1Li/jpuAcwcz0YT4rgR7697OvTS5hRwTG5HKq+gOwTNwbyi6GiJQG3rAkkDvZEYExxoQ4OyIwxpgQZ4nAGGNCnCUCY4wJcZYIjDEmxFkiMMaYEPf/BzqIffhEam8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot perfect ROC curve and AUC score\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_test)\n",
    "plot_roc_curve(fpr, tpr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Perfect AUC score\n",
    "roc_auc_score(y_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Confusion Matrix**\n",
    "\n",
    "A confusion matrix is a quick way to compare the labels a model predicts and the actual labels it was supposed to predict.\n",
    "\n",
    "In essence, giving you an idea of where the model is getting confused."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[23,  6],\n",
       "       [ 6, 26]], dtype=int64)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "y_preds = clf.predict(X_test)\n",
    "\n",
    "confusion_matrix(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Predicted Labels</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Actual Labels</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Predicted Labels   0   1\n",
       "Actual Labels           \n",
       "0                 23   6\n",
       "1                  6  26"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Visualize confusion matrix with pd.crosstab()\n",
    "pd.crosstab(y_test,\n",
    "            y_preds,\n",
    "            rownames=[\"Actual Labels\"],\n",
    "            colnames=[\"Predicted Labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEACAYAAACatzzfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAASgUlEQVR4nO3dbWxUZZ/H8d+0QiHMDJRCYKX0gUqiQCBZSRVNDEuh3OMDpoqKViRATIu0kVJUNCRqgoYiBCyQIDuomCxJ0ygElUq3uBIgvrE1vqGAOI0tvdeHYszQWqTTOfviXojDYGem7XCu9nw/ZF70OjPnXInJz3/+13XOcVmWZQkAYKwUuycAAOgbQQ0AhiOoAcBwBDUAGI6gBgDDEdQAYLjb7J5A95Ftdk8BhvEs3WH3FGCo0NX2Af2+pyMQ93dHTJg2oGsNJtuDGgBumXCv3TPoF4IagHNYYbtn0C8ENQDnCBPUAGA0i4oaAAzXG0rKacPhsGpqanTw4EFdvHhRGRkZKigoUHl5udxutyRp0aJFam1tjfrt119/rfHjx/d5foIagHMkaTHR7/dr586dWr16tebNm6eWlhZVV1frwoUL2r9/v7q6utTW1qbKykrl5+dH/Nbr9cY8P0ENwDmS0PqwLEt+v19PPfWUKisrJUn33Xef0tPTVVFRoebmZnV3d8uyLBUUFCgvLy/haxDUAJwjCYuJXV1dWrJkiXw+X8T4tGn/2ofd2tqqjo4OpaWlKScnp1/XIKgBOEYyFhPdbrc2bdoUNd7Q0CBJuuOOO3T69GmNGzdO69ev1+nTp9Xb26v58+frtdde08SJE2Neg6AG4BwJVNTBYFDBYDBq3Ov1xuwrf/fdd9q3b58WLlyovLw8nT17Vh0dHZo+fbqWL1+uQCCg6upqPffcczp06JBGjRrV5/kIagDO0dsT91cPHDig3bt3R42XlZWpvLz8b3/X2Nio0tJSZWZmavPmzZKkTZs2ybIszZkzR5I0d+5c5eXl6ZlnntGRI0f05JNP9jkXghqAcyTQ+lixYoWKioqixvuqpo8ePaqNGzcqJydHfr9f6enpkqTZs2dHfffuu++Wx+PR2bNnY86FoAbgHAm0PuJpcfzVBx98oKqqKuXn52vPnj3yeDySpD/++EN1dXWaOXOm7rzzzuvftyxLPT0918O8LzzmFIBzWOH4Pwmora3Vli1b5PP55Pf7r4e0JKWlpamqqiqqjXL8+HFduXIlal/1zVBRA3COJGzPu3Tpkt566y1NmTJFxcXFOnPmTMTxrKwsrVmzRlu2bNHmzZu1YMECnT9/Xrt27VJBQYHuueeemNcgqAE4hhWOfzExXidPnlR3d7fa29tVXFwcdXzr1q1auXKl3G63PvroI9XW1mrs2LFatmxZn4uSf+WyLMsa7IknghcH4Ea8OAB/Z6AvDrjSdCTu74769yUDutZgoqIG4Bw8PQ8ADMcbXgDAcFTUAGA43vACAIZL0osDko2gBuAcVNQAYDbLYjERAMxGRQ0AhmPXBwAYjooaAAzHrg8AMBytDwAwHK0PADAcQQ0AhqP1AQCGYzERAAxH6wMADEfrAwAMR0UNAIYjqAHAcPa+y7vfCGoAzhFi1wcAmI3FRAAwHD1qADAcPWoAMBwVNQAYjqAGALNZvbzcFgDMRkUNAIZjex4AGC7Mrg8AMNtwb320t7erpaVFnZ2dSklJkcfjUW5uriZPnpzM+QHA4Bmui4n19fV69913FQgEZN2wWdzlcik7O1vr1q3TP/7xj6RNEgAGxXCsqA8fPqyNGzfK5/OpvLxc2dnZGjNmjCzLUldXl3788UcdO3ZMFRUV6unp0SOPPHKr5g0AiRuOPep9+/bp6aef1uuvv37T4zNmzJDP59Mbb7yh9957j6AGYLYhuusjpa+D7e3tWrhwYcyTFBQUqK2tbdAmBQBJEbbi/xikz6CeOnWqTp06FfMkX331FYuKAIxnhcNxf0zSZ+ujtLRUL730kn755RcVFhYqNzdXbrdbLpdLnZ2d13vUn332md58881bNWcA6J/huOvj4YcfVmpqqnbs2KHPP/9cLpcr4rhlWcrMzNTbb7+toqKipE4UAAbMsJZGvGJuz/P5fPL5fGpra1MgEFBnZ6csy7q+jzorK+tWzBMABs6wlka84r7hZerUqZo6dWoy5wIAyZWkijocDqumpkYHDx7UxYsXlZGRoYKCApWXl8vtdkuSTp06pR07dujChQvKyMjQs88+q1WrVsV1fm4hB+AcSdqe5/f7tXPnTq1evVrz5s1TS0uLqqurdeHCBe3fv19NTU0qLS2Vz+fTiy++qMbGRm3dulWWZWn16tUxz09QA3COJFTUlmXJ7/frqaeeUmVlpSTpvvvuU3p6uioqKtTc3Kzq6mrNmDFD77zzjiTpgQceUCgU0t69e7V8+XKNHDmyz2v0uT0PAIYTK9Qb9ydeXV1dWrJkiR5++OGI8WnTpkmSvv/+e33zzTcqLCyMOL548WIFg0E1NTXFvAYVNQDnSEJF7Xa7tWnTpqjxhoYGSf+6g7unp0e5ubkRx7OzsyVJLS0tuvfee/u8BkENwDkS6FEHg0EFg8Goca/XK6/X2+dvv/vuO+3bt08LFy7U5cuXJen6ouI1Y8aMkSR1dnbGnAtBDcA5EqioDxw4oN27d0eNl5WVqby8/G9/19jYqNLSUmVmZmrz5s1qaWmRpKj7UK5JSYndgSaoATiGlUBQr1ix4qY38vVVTR89elQbN25UTk6O/H6/0tPT1dHRISm6cr72t8fjiTkXghqAcySwSBhPi+OvPvjgA1VVVSk/P1979uy5HsBZWVlKTU1Va2trxPev/X1j7/pm2PUBwDmS9PS82tpabdmyRT6fT36/P6JKTktL09y5c1VfXx/x8pVjx47J4/Fo1qxZMc9PRQ3AOZKw6+PSpUt66623NGXKFBUXF+vMmTMRx7OysrRmzRqtXLlSFRUVKioq0rfffqv9+/ersrJSo0ePjnkNghqAY9z4OsHBcPLkSXV3d6u9vV3FxcVRx7du3apHH31Uu3btUnV1tdauXatJkybp5ZdfjvsWcpeVjJknoPvINjsvDwN5lu6wewowVOhq+4B+H3y+MPaX/p/3P+sHdK3BREUNwDmG62NOAWC4sELD/DGnADDkDc2cJqgBOEciN7yYhKAG4BwENQAYjtYHAJiN1gcAGM4KEdQAYDZaHwBgtiS92zbpCGoAzkFQA4DZqKgBwHBWyO4Z9A9BDcAxqKgBwHAENQCYzrr5m8BNR1ADcAwqagAwnBWmogYAo4V7CWoAMBqtDwAwHK0PADCcNTQfnkdQA3AOKmoAMByLiQBgOCpqADCcxZ2JAGA2tucBgOHCVNQAYDZaHwBgOHZ9AIDh2PUBAIajRw0AhqNHDQCG41kfAGA4Wh8AYLgwi4n941m6w+4pwDDd/zxp9xQwTFFRA4DhWEwEAMNRUQOA4Ybopg+CGoBz9IZT7J5CvxDUABxjiD7llKAG4ByWkt+jbm5u1tKlS3X8+HFNnjz5+viiRYvU2toa9f2vv/5a48eP7/OcBDUAxwgnuUkdCARUUlKiUCgUMd7V1aW2tjZVVlYqPz8/4pjX6415XoIagGOEk1RRh0Ih1dTUaPv27RoxYkTU8XPnzsmyLBUUFCgvLy/h8w/NzjoA9IMlV9yfRDQ2Nmrbtm1atWqVNmzYEHW8ublZaWlpysnJ6de8qagBOEZvAgEcDAYVDAajxr1eb1S7Ii8vTw0NDcrIyNAnn3wS9Ztz585p3LhxWr9+vU6fPq3e3l7Nnz9fr732miZOnBhzLgQ1AMdIZNfHgQMHtHv37qjxsrIylZeXR4xNmDChz3OdPXtWHR0dmj59upYvX65AIKDq6mo999xzOnTokEaNGtXn7wlqAI6RSFCvWLFCRUVFUePxLP7daNOmTbIsS3PmzJEkzZ07V3l5eXrmmWd05MgRPfnkk33+nqAG4BiJ9J5v1uLor9mzZ0eN3X333fJ4PDp79mzM37OYCMAxwq74P4Pljz/+0McffxwVyJZlqaenR+np6THPQVADcIywXHF/BktaWpqqqqqi+t3Hjx/XlStXovZV3wytDwCO0WvDNVNTU7VmzRpt2bJFmzdv1oIFC3T+/Hnt2rVLBQUFuueee2Keg6AG4Bhhlz2POV25cqXcbrc++ugj1dbWauzYsVq2bFnU7pG/47Ise1/3eNvIKXZeHgbiDS/4OyMmTBvQ72v/rTju7z7xv/81oGsNJipqAI7B0/MAwHBD9N22BDUA50jkFnKTENQAHIOKGgAMR48aAAzHy20BwHC0PgDAcLQ+AMBwvVTUAGA2KmoAMBxBDQCGY9cHABiOXR8AYDhaHwBgODteHDAYCGoAjkHrAwAMR+sDAAzHrg8AMFx4iEY1QQ3AMVhMBADD0aMGAMOx6wMADEePGgAMNzRjmqAG4CD0qAHAcL1DtKYmqAE4BhU1ABiOxUQAMNzQjGmCGoCD0PoAAMOxmAgAhqNHDQCGG5oxTVADcJBhW1H//PPPCZ1w0qRJ/Z4MACTTsF1MLCgoUG9v/E9xbW5uHtCEACBZrOFaUdfW1qqkpERXr15VZWWlbruNbgmAoWnY7vq466679OGHH+qJJ57Qr7/+qhdeeOFWzAsABt1QbX2kxPOladOmaf369fL7/frtt9+SPScASIqwZcX9MUncfYxly5Zp+vTpyZwLACSVWfEbv7iDOjU1Vfn5+cmcCwAk1VDdnhdX6wMAhgMrgX/91dzcrJkzZ+qnn36KGD916pQef/xxzZkzRwsWLND7778f9znZwgHAMUJJrqgDgYBKSkoUCoUixpuamlRaWiqfz6cXX3xRjY2N2rp1qyzL0urVq2Oel6AG4BjJ2kcdCoVUU1Oj7du3a8SIEVHHq6urNWPGDL3zzjuSpAceeEChUEh79+7V8uXLNXLkyD7PT+sDgGOEE/gkorGxUdu2bdOqVau0YcOGiGN//vmnvvnmGxUWFkaML168WMFgUE1NTTHPT1ADcAzLsuL+JCIvL08NDQ0qKytTampqxLG2tjb19PQoNzc3Yjw7O1uS1NLSEvP8tD4AOEYiuz6CwaCCwWDUuNfrldfrjRibMGHC357n8uXLkiS32x0xPmbMGElSZ2dnzLkQ1AAcI5FbyA8cOKDdu3dHjZeVlam8vDzu81yrzl0u102Pp6TEbmwQ1AAcI5GKesWKFSoqKooav7GajsXj8UiKrpyv/X3teF8IagCOkUjv+WYtjv7IyspSamqqWltbI8av/X1j7/pmWEwE4BjJ2vXRl7S0NM2dO1f19fUR/6M4duyYPB6PZs2aFfMcBDUAx7gVdybezJo1a9TU1KSKigqdOHFCO3fu1P79+1VSUqLRo0fH/D1BDcAxwrLi/gymefPmadeuXfrhhx+0du1affrpp3r55Zf1/PPPx/V7l5XohsFBdtvIKXZeHgbq/udJu6cAQ42YMG1Av/+PzEVxf/d/Lv73gK41mFhMBOAYw/ZVXAAwXJj2QoB4EdQAHGNoxjRBDcBBhuqLAwhqAI5BUAOA4XqtofkecoIagGOw6wMADGfzbSP9RlADcAx61ABgOCpqADBc76A+F+/WIagBOAZ3JgKA4dj1AQCGo6IGAMNRUQOA4aioAcBw3EIOAIaj9QEAhrOoqAHAbNxCDgCG4xZyADAcFTUAGK43TI8aAIzGrg8AMBw9agAwHD1qADAcFTUAGI7FRAAwHK0PADAcrQ8AMByPOQUAw7GPGgAMR0UNAIYL85hTADAbi4kAYLihGtQua6jOHAAcIsXuCQAA+kZQA4DhCGoAMBxBDQCGI6gBwHAENQAYjqAGAMMR1ABgOIIaAAxHUBvgs88+00MPPaTZs2fL5/Pp8OHDdk8JBmlubtbMmTP1008/2T0V2ISgtlldXZ02bNig+++/X3v27FF+fr5eeeUVffHFF3ZPDQYIBAIqKSlRKBSyeyqwEc/6sNmiRYs0a9Ys7dix4/rYunXrdO7cOdXV1dk4M9gpFAqppqZG27dv14gRI/T777/rxIkTmjx5st1Tgw2oqG3U1tam1tZWFRYWRowvXrxYgUBAbW1tNs0MdmtsbNS2bdu0atUqbdiwwe7pwGYEtY0CgYAkKTc3N2I8OztbktTS0nLL5wQz5OXlqaGhQWVlZUpNTbV7OrAZz6O20eXLlyVJbrc7YnzMmDGSpM7Ozls+J5hhwoQJdk8BBqGittG15QGXy3XT8ZQU/vMAIKht5fF4JEVXzl1dXRHHATgbQW2ja73p1tbWiPEff/wx4jgAZyOobZSdna3MzMyoPdP19fXKycnR7bffbtPMAJiExUSbrV27Vq+++qrGjh2r+fPn68svv1RdXV3EvmoAzkZQ2+yxxx7T1atX9f7776u2tlZTp05VVVWVHnzwQbunBsAQ3JkIAIajRw0AhiOoAcBwBDUAGI6gBgDDEdQAYDiCGgAMR1ADgOEIagAwHEENAIb7P+1M7Gjyk/YcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Make our confusion matrix more visual with Seaborn's heatmap()\n",
    "import seaborn as sns\n",
    "\n",
    "# Set the font scale \n",
    "sns.set(font_scale=1.5)\n",
    "\n",
    "# Create a confusion matrix\n",
    "conf_mat = confusion_matrix(y_test, y_preds)\n",
    "\n",
    "# Plot it using Seaborn\n",
    "sns.heatmap(conf_mat);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOQAAADfCAYAAADm6n/jAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAehElEQVR4nO3daVgUZ7738S+bKCAIiePKJoGocQcXokZRcd8V9RzEUeKuiLsy6pg8zmg0Jh6TaMbjkkQvFzAuRzFmjBp41LgkOUyMG0JQREBkE2Rtlnpe+NiZDqINNnQh/8915UVX3V39rw4/766qu+4yURRFQQihCqbGLkAI8TsJpBAqIoEUQkUkkEKoiARSCBUxN3YBVSX/2EZjl/DKqD92k7FLeKUUaxLLXSc9pBAqIoEUQkUkkEKoiARSCBWRQAqhIhJIIVREAimEikgghVARCaQQKiKBFEJFJJBCqIgEUggVkUAKoSISSCFURAIphIpIIIVQEQmkECoigRRCRSSQQqiIBFIIFZFACqEiEkghVEQCKYSKSCCFUBEJpBAqIoEUQkUkkEKoiARSCBWRQAqhIhJIIVREAimEikgghVCRV/aBrWrxQ3QC289EcfN+GiYmJrRz+hNzBnrRzrmRts2V2ES2/vNnbidnYG1pgW+7Fswd6IWVpYURK685Xn/dgb+tWc6wof2pV68uUVG/8pcV67h85X+NXVqFSQ9ZhX76LZk5O7/lcb6GOQM7M8O3Ewnp2bz7eTi/3nsIwI+xScz875MUlZQyb1Bnhnq6c+jSTWbvOElpqWLkPVA/Gxtrvj97GL+xw9j237tZ/d6HNG3ahO9OhfHWW28au7wKkx6yCn147CKN7WzYEzSSenWefNXDPN0Z9eHXfPbtT2ybPpiPwy/TuIE1O2cNpa7FkzaNG9iw7sgFfrh9nx4tHY25C6q3bOlc3vRwo2+/sZw7fxmAsIPHiIm+yOJFs5kSGGzkCitGesgqkp1XyO3kdHzbt9CGEeC1+lZ4tmjML3dTKCwqxt66LqO7ttSGEcCzRWMAYpLTq73umiZgoh/fnDyjDSNASkoqS5et4fy/LaspjN5DJiYmcufOHXJycjA1NaV+/fq4urrSuHFjY5f2UqzrWnB0yTidMD71KK8AczMTLC3M2TptUJn10UkZwJOeUpTPxcWR5s2bsPGjrdpl1tZW5Obm8Y9tXxmxssozWiBPnTrF5s2biYuLQ1F0j5VMTExwdnZm/vz5DBw40EgVvhwzU1OcG9qVWX47KZ1/3U3hbY/mZdYlZT7mx9hkPg6/xBuN7enTxqUaKq253N9wBeBhahrr161k6lR/7OxsiY29w+Il7xN+4jsjV1hxRgnk0aNHWb58OYMGDSIoKAhnZ2esra1RFIXc3Fzi4+P55z//yYIFCygqKmLYsGHGKNPg8gqLWHkgAoApPh101mXlFTB47QEA6lqYs2zk21haGP0HjKrZNXjyD977q5dSVFzEgoWrKSktYdHCmRz6eieDh/hz5uw5I1dZMSbKH7unajB48GC6du3K6tWrn9vuvffe46effiI8PLzCn5F/bGNly6sS+Zpi5u36lh9/SyawTwfmDeqssz47r5CLt+9TVFLKvvPXiE5K5wP/Pvi2a2Gkin9Xf+wmY5fwTP7+Y/jqi09ITU2n1Vs9efQoCwA7O1tu3/qBuDvxeL89xMhVllWsSSx3nVFO6iQmJtKvX78Xtuvbty8JCQnVUFHVys4vZNb2b/jxt2RGdvYgaKBXmTa2VpYM6ODGUE93ds0eRpMGNnx0/JIRqq05cnPzADhy9KQ2jABZWdkcDz+FZ6d2WFtbGau8SjFKIB0dHTl//vwL20VERNT4kzsZOflM+8cJ/nU3hTFdW7La7x1MTEye+566Fub0bO3Eg0e5ZOYWVFOlNU9S4gMAUlPTyqxLTU3D1NQUGxvr6i7rpRjlIGXmzJksWbKEhw8f0r9/f1xdXbGxscHExIScnBztMWR4eDjvv/++MUo0iNwCDbO2nyQ6KZ2JPduweLi3zvo7Dx8xe8dJJvduz/i3W+usyysswsQE6pjJlanyXLt+i4KCAlq39iizzsXFifz8fFJTa9alo3IDmZSUVKkNNm3a9IVthg4dipmZGZs2beLEiRNlegxFUWjevDlr165l1KhRlapDDdYduUB0Ujr/2aNsGAEcX7Mlp0DD1xdvMrrLm1iYmwFPzrae+fUOni2aYF23TnWXXWPk5eVzPPw7RgwfQOvWHty4cRt4cjlk2FBf/ufYPyktLTVylRVT7kmdli1bvvCn1bPcvHmzQu0TEhKIi4sjJycHRVG01yGdnJwq/Nn/ztgndeJSMhm98Wts6tZhyXBvzE3LfpdDPN058XMMKw5E0M7pTwzu9AZZeYUcuHCdopJSvpwzjDcaOxihel1qPakD4OzcnIsXTgDw6Wc70Wg0BM2dirV1Pbp0G8SdO/eMXGFZzzupU24POWfOnEoFsqIcHR1xdHz1hof9HJcMQE6BhtVhkc9sM8TTnSGe7pibm/Ll91f56Pgl6tWxoIt7U4IGeuHcsEE1Vlwzxcffp3vPYaxbu4JFC2diYmLC+fNXWBayRpVhfBGjXPaoDsbuIV8lau4ha6JK9ZDliY6OJiIigqSkJCZNmoSVlRW3b9+mV69eL1WkEKKCgVyzZg379u1DURRMTEwYOHAg2dnZBAcH07t3bzZv3oylpWVV1SrEK0/vc+q7d+9m7969TJ8+nbCwMO34U29vbyZPnkxERATbt2+vskKFqA30DuSBAwcYOHAgCxYs0DkJY2try/Llyxk+fHilhrgJIX6ndyATEhLo1q1bueu9vLxITk42SFFC1FZ6B9Le3p4HDx6Uuz4mJgY7u7K3Gwkh9Kd3IH19fdm3bx+xsbHaZU+vU0ZGRhIaGoqPj4/hKxSiFtH7OmR2djb+/v7Ex8fj7u7OjRs38PT0JDc3l1u3btGsWTPCwsJwcDD+yBKQ65CGJNchDcsgt1/Z2toSFhbGtGnT0Gg0WFpa8ssvv5Cfn8+UKVM4dOiQasIoRE0lI3XEC0kPaVgGHakTExNDREQEiYmJmJmZ4eTkRJ8+fV7J8ahCVDe9A1lcXMyqVas4evRomUmp1q9fz9SpU1m4cKHBCxSiNtE7kFu3buXIkSOMGjWKSZMmaXvEuLg4vvjiC7Zv307Dhg0JCAiosmKFeNXpfQzp4+NDhw4d2LTp2ccTM2fOJC4ujlOnThm0wMqSY0jDkWNIwzLIWdaMjAw6d+5c7vrevXuTkpJSscqEEDr0DmT79u05d678OS6vXr1Kq1atDFKUELWV3nPqTJs2jXnz5rFo0SLeffddXF1dMTExITExkbCwMLnbQwgDqNCcOk+bPm95RefUqSpyDGk4cgxpWKqeU0cI8btyAxkUFFSddQghqMRInezsbPLy8nTmuywpKSE3N5dLly4xefJkQ9YnRK2idyBTUlJYunQpV65ceW47CaQQlaf3ZY8NGzZw5coVBg8ezMiRI1EUhenTpzN27FhsbW2xtLRk//79VVmrEK88vQN58eJFRo4cyUcffcSKFSswMTGhZ8+erFmzhqNHj2JlZcV339W8B2QKoSZ6BzI7O5tOnToBYGNjQ9OmTbl27RoATZo0wc/Pj7Nnz1ZNlULUEnoH0s7Ojvz8fO1rJycnoqOjta8dHR2fO+eOEOLF9A5kp06dOHz4MI8fPwbAw8ODy5cvU1hYCMCvv/6KjY1N1VQpRC2hdyBnzZrFnTt36NWrF5mZmYwbN46UlBRGjx7NtGnTCAsLo3fv3lVYqhCvPr0D2bp1a8LCwhg+fDj29va4ubmxZcsWCgoKiIqKYtCgQSxdurQqaxXilSdz6ogXkrGshlWpsaxV+QRlIcSzlRvIPn36VMsTlIUQv5O7PYRQkVf2GNK8TjNjl/DKyE8qf6YIUXEWr7cod53eZ1mFEFVPAimEikgghVARCaQQKiKBFEJFKjyFR3R0NBERESQlJTFp0iSsrKy4ffs2vXr1qor6hKhVKhTINWvWsG/fPhRFwcTEhIEDB5KdnU1wcDC9e/dm8+bNWFpaVlWtQrzy9P7Junv3bvbu3cv06dMJCwvTzsXq7e3N5MmTZaJkIQxA70AeOHCAgQMHsmDBAp1nQdra2rJ8+XKGDx9OeHh4lRQpRG2hdyATEhLo1q1bueu9vLxITk42SFFC1FZ6B9Le3v65U3TExMRgZ2dnkKKEqK30DqSvry/79u0jNjZWu+zp4PPIyEhCQ0Px8fExfIVC1CJ6Dy7Pzs7G39+f+Ph43N3duXHjBp6enuTm5nLr1i2aNWtGWFgYDg4OVV2zXmRwueHI4HLDMsjgcltbW8LCwpg2bRoajQZLS0t++eUX8vPzmTJlCocOHVJNGIWoqeT2K/FC0kMa1vN6SL0HBug7pYdM4SFE5ekdSH2n9JApPISoPL0D+awpPUpKSkhLSyMyMhJra2t5pqQQL0nvQD4vbDk5OUyYMIH4+HiDFCVEbWWQ269sbGzw8/MjNDTUEJsTotYy2P2QRUVFZGZmGmpzQtRKL32WVaPRcPPmTXbt2kWrVq0MVpgQtZFBzrIqioKlpSWLFi0yWGFC1EZ6B3Lu3LnPXG5qakrDhg3p27evjNQR4iXpHcgmTZrg6emJi4tLFZYjRO2m90mdtWvXcuLEiaqsRYhaT+9A1qtXT+bLEaKK6f2T9b333mPlypUUFhbSo0cPHBwcMDMzK9NOxrIKUXl63+3Rrl07iouLKS0tfe6YVrWMZZW7PQxH7vYwLIPc7TFt2jR5PJ0QVazcHjIkJIQJEybQvn376q7JIKSHNBzpIQ2rUjMGHDlyhHv37lVJQUKIZ5NnewihIhJII3j9dQf+8fkGEhP+RUbaLc58d5CuXToZu6wa4cLln5k0azFefUbSud8opgaH8Ms13ROJGZmPWP3BZt4Z+h909R3N5LlLy7RRq3KPIVu2bMn48ePp2LFjhTY4cuRIQ9T10tR6DGljY83FH07QtEkjNn+ynczMLGbPmkKzZo3x7j6E69ejjV1iGWo5hvwx6iqBQct5w9WZUUP6U1xSQuiRcB6mpbN760batn6T3Nw8JkybT2paOgHjR2Fb34b9h47zMDWd/Tv+C/cWLsbejeceQz43kBU5q/r0ATxy2eP51vyfZSxbOpe+/cZy7vxlABo1akhM9EUOHT7BlMBgI1dYlloCOXbyHLKyczi2bxv16tYFIC0jk+H/OZ3Wb7qzY/NaNm/7kh17wvjis/V4dWj7pE16BgP9AvH16cG6VYuNuQvAS1z2GDduHB06dDB0PbVawEQ/vjl5RhtGgJSUVJYuW0NRUZERK1O3rOzHRMfe4c8TRmvDCPC6gz1eHdty8cr/oigKx06e5p23O2vDCPD6aw4snjsVc/OyA1nU5rmB9PLyYtiwYdVVyyvPxcWR5s2bsPGjrdpl1tZW5Obm8Y9tXxmxMvWzsbYifP92nTA+9ehRNmZmZiQmp5CSms4Ufz/gya+2/PwCrKzqMWH00OouuVLkpE41cn/DFYCHqWmsX7eS9NSbZGXGcOvGeYYO8TVydepmZmaGs2Mz/tTwNZ3l0bF3iPr1Bh3atiY+IREAB3s7Nn62A+8BY+niO5pB4wKJOH/JGGVXmASyGtk1ePIwovdXL2XQ4L4sWLiaP0+ZR15+Poe+3knfPj2NXGHNkpeXz1/WbATg3Yl+PM7JBeCz7Xv4vxevsDx4JmtXLaZuXUvmhazh4o9RxixXL+X+ZB01ahROTk5V9sEpKSkVat+oUaMqqqT6WFrWAaBBA1tavdWTR4+yAAgP/47bt37gb39bzpm31XECRe3yCwqYu+x9omPjmBowns4d23Hs2zMAPM7JIXz/Duxs6wPQu3tXBo0L5L/+8QXenSt21aC6lRvIdevWVekH9+3bl5KSEr3bq+Xs7cvIzc0D4MjRk9owAmRlZXM8/BSTAvy0x5SifNmPc5izdDVRV28wamh/gmf8GUB7fNmvV3dtGAFs69vg06Mb/3PyNHl5+VhZ1TNK3frQe3C5oR08eJAZM2ag0WhYtGgR5uZGK6XaJCU+eb5mampamXWpqWmYmppiY2MtgXyO9MxHzFiwglsxcfiNGMRflwRpL881+v/Hlw4NGpR5n4O9HYqikJcvgXymVq1a8eWXX+Ln50dqaiqzZ882VinV5tr1WxQUFNC6tUeZdS4uTuTn55Oamm6EymqG3Nw8bRgnjR/F0nnTdda/0cKFOnUsiL1TdsLuxOQULOvUwb6Buh8qbNSTOi1atGDhwoXs2LGDjIwMY5ZSLfLy8jke/h1DBvfTCaWLiyPDhvpy7PgpSktLjVihuv3t463cioljot+IMmEEsKpXF58e3Yj84Qqxcb+H8n7SA74/fwmfnt2eeVO9mhj9cXQlJSX8/PPPvPHGGwadtU6tI3WcnZtz8cKTuYk+/WwnGo2GoLlTsbauR5dug7hzR3132KhhpM5vd+8xwn8G9W2sWTZvBmbPuMg/bEAfEpNT+I+pT0Y7+Y8bgYW5BXsPHiW/oJDQnZ/g2KxJdZdeRqWGztV0ag0kgKurE+vWrqBf356YmJhw/vwVloWs4dat2Be/2QjUEMjQIydYs/Gz57a5duEkAAmJyWz6fBcXf4xCURQ6tW/Dojnv4uZSdVcNKkICKV6KGgL5KjHII82FEFVPAimEikgghVARCaQQKiKBFEJFJJBCqIgEUggVkUAKoSISSCFURAIphIpIIIVQEQmkECoigRRCRSSQQqiIBFIIFZFACqEiEkghVEQCKYSKSCCFUBEJpBAqIoEUQkUkkEKoiARSCBWRQAqhIhJIIVREAimEikgghVARCaQQKiKBFEJFJJBCqIgEUggVkUAKoSISSCFU5JV9grIQNZH0kEKoiARSCBWRQAqhIhJIIVREAimEikgghVARCaQQKiKBFEJFJJBCqIgEUggVkUAaSXh4OEOGDKFdu3YMGjSIo0ePGrukGu/mzZu89dZbPHjwwNilVJoE0ghOnjzJ4sWL6d69O1u2bKFLly4sW7aMb7/91til1VhxcXHMmDGD4uJiY5fyUmRwuRH4+vrSpk0bNm3apF02f/58oqOjOXnypBErq3mKi4sJDQ3lo48+wsLCgkePHhEZGUnjxo2NXVqlSA9ZzRISErh37x79+/fXWT5gwADi4uJISEgwUmU1088//8zGjRsJDAxk8eLFxi7npUkgq1lcXBwArq6uOsudnZ0BuHPnTrXXVJO5ublx+vRp5s6di5mZmbHLeWnmxi6gtnn8+DEANjY2Osutra0ByMnJqfaaarLXX3/d2CUYlPSQ1ezpIbuJickzl5uayv+S2kz+71ez+vXrA2V7wtzcXJ31onaSQFazp8eO9+7d01keHx+vs17UThLIaubs7Ezz5s3LXHM8deoULi4uNG3a1EiVCTWQkzpGMGfOHEJCQrCzs6N3796cPXuWkydP6lyXFLWTBNIIRo8ejUajYdeuXRw8eBBHR0fWr1/P4MGDjV2aMDIZqSOEisgxpBAqIoEUQkUkkEKoiARSCBWRQAqhIhJIIVREAvkSli9fzptvvqnzX6tWrejUqRN+fn4cOXKkWuro06cPAQEB2tcBAQH06dOnwtvJyckhIyPDYHU9/X5eto0h31dd26ssGRhgACEhIdjb2wNP7trIycnh2LFjLF++nMzMTAIDA6u1npkzZ5Kfn1+h91y7do1Zs2axceNGunbtWkWViReRQBpAv379aN68uc6ysWPHMnjwYLZs2cLEiROpU6dOtdXTvXv3Cr/n9u3bPHz4sAqqERUhP1mrSN26denTpw85OTnExMQYuxxRQ0ggq9DTm5BLSkqAJ8d6K1eu5C9/+Qtt27blnXfe0R6zRUVFMWXKFDp27EjHjh0JDAzk6tWrZbb5zTffMGLECNq1a8fQoUO5dOlSmTbPOob87bffCA4OpmvXrnh6ehIQEMBPP/0EwKeffkpISAgAkyZN0nnvgwcPWLp0Kd26daNt27aMHDmSY8eOlfnMa9euERgYSMeOHenZsye7d++uzFcGwMWLF5k6dSpdu3blrbfeomfPnvz1r38lOzu7TNuoqCjGjBlD27Zt6d+/P19++WWZNvrugxrIT9YqUlpaypUrV6hTpw5ubm7a5SdOnMDV1ZUVK1aQlpaGg4MDFy5cYMaMGbRs2ZLg4GA0Gg2HDx/G39+fL774Ai8vLwAOHz5MSEgIHTt2ZMmSJcTHxzNz5kxKS0tp1qxZubXcvXuXcePGYW5uzsSJE3FwcODAgQNMmTKFvXv34uvrS2pqKqGhocycOZO2bdsCkJKSgp+fH4qiEBAQgJ2dHWfOnGHJkiU8fPiQqVOnAhATE0NAQAC2trbMnj2boqIitmzZov2HqCLOnz/PtGnT6NSpE/PmzcPExIQLFy4QGhpKUVER69at02kfGBhIv379GD16NKdPn2bdunU8fvyYoKCgCu2Daiii0pYtW6Z4eHgo169fV9LT05X09HTl4cOHSlRUlBIcHKx4eHgoa9eu1bb38fFRWrZsqcTHx2uXlZSUKH379lUmTJigFBcXa5fn5uYqvr6+yogRIxRFUZTi4mLF29tbGTNmjKLRaLTtDh06pHh4eCgTJ07ULps4caLi4+OjfR0cHKy0a9dOuXv3rnZZRkaG4unpqcybN09nO5cuXdLZvy5duigpKSk6+71w4UKlTZs2SlpamqIoihIUFKR06NBBSUpK0raJjY1V2rRpo3h4eOj1HT717rvvKj4+PkphYaFOu3HjxikdO3Ys877169drl5WUlCiTJk1S2rRpo2RkZFRoH/5Yh7HIT1YDGDVqFN7e3nh7e9OjRw/Gjx/PmTNnCAgIYNGiRTptnZyccHJy0r6+ceMGCQkJ9OvXj6ysLDIyMsjIyKCgoAAfHx9u3rzJgwcPuH79Ounp6YwePRoLCwvt+0eMGIGdnV25tZWWlhIZGUmvXr20M9sB2Nvbs2/fPlauXFnu+06fPo2Xlxfm5ubaujIyMujfvz8ajYYLFy5QWlrKuXPn6NWrF02aNNG+383NjR49elT4u9y2bRuHDh3SOQmWmZmJjY0NeXl5Zdr/ew9namrKxIkT0Wg0/PDDD3rvg5rIT1YD+PDDD7Wzn5mammJra4ubmxuWlpZl2r722ms6r59O5bFhwwY2bNjwzO0nJydrp8f/9zADmJmZ6QTtjx49ekReXt4z23h4eJT7vszMTB4/fszp06c5ffp0uXU93f4f6wJo0aIFZ8+eLfcznsXMzIyEhAQ2b95MbGws9+7dIyUl5ZltGzRogIODg84yR0dHABITE/XeBzWRQBpAp06dylz2KM8f5w4tLS0FIDg4mA4dOjzzPS1atND+URYWFpZZ/3Qbz/L0OK6is9k9fd+AAQOYMGHCM9s8/eOvTF3lOXDgAKtXr8bV1RUvLy/69+9P+/bt2bNnD8ePH9dp+8eZ+0B39r6K7oMaSCCN7OnJGCsrK95++22ddVevXiUrK4u6detq/3Du3r2r00ZRFBITE3F3d3/m9u3t7albt652Eq1/t3PnTtLS0li2bFmZdQ4ODtSrV4/i4uIydSUlJXHjxg3q1auHvb09NjY2ZeoCuH//frn7/SyFhYV88MEHdO3alV27dmFu/vuf5+bNm8u0z8rKIicnR2eO26d1ODk56b0PaiLHkEbWpk0bGjZsyJ49e7RTQcKTYWzz588nJCQEMzMzWrduTbNmzdi/f7/OKJwTJ06QmZlZ7vbNzc3p3r07kZGROj/PsrKy2Llzp/Yn89Me9GmvZm5uzjvvvENkZCS3bt3S2eYHH3zAnDlzyMzMxMTEBF9fX86dO8ft27e1be7fv09ERESFvouCggLy8/NxcXHRCePNmze5cuUKgM7DdEpLS/n666+1r4uLi/nqq6+wsrLC29tb731QE+khjczCwoJVq1Yxf/58Ro8ezdixY7G0tOTgwYMkJSWxceNG7R/nqlWrmDNnDuPHj2fMmDGkpKSwd+9eGjRo8NzPWLRoEX5+fvj5+eHv74+NjQ1hYWHk5eUxf/58AO2x2P79+0lLS2PYsGEsXryYy5cv4+/vj7+/P02bNiUiIoLvv/+e8ePHa3vl4OBgIiIiCAgIYPLkyZiZmbFnzx6sra3RaDR6fxd2dna0b9+ew4cPY2Njg6urKzExMRw8eFD7D0Zubq72JFa9evX45JNPSE5OxsnJiW+++YaoqChWr16tnd9W331QCwmkCgwYMIBdu3bx+eefs3XrVkxNTXF3d+fzzz/Hx8dH287Hx4dt27bx6aef8vHHH9OoUSP+/ve/s3fv3udu383NjdDQUD7++GN27NiBqakp7dq1Y/369do/SG9vbwYNGsT333/PpUuX6N+/P05OToSFhfHJJ59oA+zo6EhISIjOYPYmTZqwf/9+NmzYwI4dO6hTpw5+fn7Ak7OmFbF582bWrVvHoUOH0Gg0NGvWjOnTp+Pm5kZQUBCXLl1iwIABANja2rJ+/XrWrl3L3r17cXZ25sMPP2T48OHa7em7D2ohk1wJoSJyDCmEikgghVARCaQQKiKBFEJFJJBCqIgEUggVkUAKoSISSCFURAIphIr8P+NBiQ+I2eXdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_conf_mat(conf_mat):\n",
    "    \"\"\"\n",
    "    Plots a confusion matrix using Seaborn's heatmap().\n",
    "    \"\"\"\n",
    "    fig, ax = plt.subplots(figsize=(3,3))\n",
    "    ax = sns.heatmap(conf_mat,\n",
    "                     annot=True, # Annotate the boxes with conf_mat info\n",
    "                     cbar=False)\n",
    "    plt.xlabel(\"Predicted label\")\n",
    "    plt.ylabel(\"True label\")\n",
    "    \n",
    "    # Fix the broken annotations (this happened in Matplotlib 3.1.1)\n",
    "#     bottom, top = ax.get_ylim()\n",
    "#     ax.set_ylim(bottom + 0.5, top-0.5);\n",
    "    \n",
    "plot_conf_mat(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x218043c5e80>"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVAAAAEWCAYAAAAw6c+oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuxklEQVR4nO3deVyU5d4/8M/MsK+CehAQEEktNQ0lDVdQAbU0RVFPigmKuJN7pC3PkyfDNbXMUqk0LDCXY5pllPqkJ/WncioLcQGRZDEFgQEEZub+/YGMjQPMMMJsfN7ndb/Oa677mnu+g/HlWu9bJAiCACIiajSxoQMgIjJVTKBERDpiAiUi0hETKBGRjphAiYh0xARKRKQjJlAianaC4p6hQ2gWInNYB/p51mKUyu4YOowmNbvTZ/jw6suGDqNZnA61N3QIzeLzrK2Y4jvH0GE0qTaernjv1KomuZbs7kRAkd9wJXE7WLRObpLP0wcLQwfQFEpld1BSXWDoMJqcOX4nACjIdjR0CM2mIPsvQ4dgtOTyXEB+q+FKEoVJJSVTipWITJjw4H8NEWk4b2yYQIlILxQQIEDRYB0mUCKiOsgEBRRCwwlUrOG8sWECJSK9kEOAQkMLU1MX39hwGRMR6YXiQQLVdOgqPT0d3bp1Q36+6kx/SEgIunTponYUFhYq6/z222+IjIyEv78/BgwYgA0bNqC6ulrjZ7IFSkR6oRAEyDWtmtRxVWVmZiZiY2Mhk8lUysvKypCTk4PFixejT58+KuecnJwAANnZ2Zg2bRr8/f3x3nvv4fr169i4cSOkUineeOONBj+XCZSI9ELx4GiIqJHXlMlkSE5Oxvr162Fpaal2PiMjA4IgYOjQofDz86vzGh9//DEcHR2xdetWWFlZYfDgwbCxscGqVasQGxsLNze3ej+fXXgi0gs5BK2Oxrhw4QLWrVuH6OhoLFmyRO18eno6rK2t0aFDh3qvcfr0aQQHB8PKykpZNnz4cMjlcpw6darBz2cCJSK9kAlAtYZD1sgevJ+fH1JTUzFv3jxIJBK18xkZGWjVqhUWLVqEgIAA+Pv7Y+HChfjrr5oNDxUVFcjLy4Ovr6/K+1xdXeHg4ICsrKwGP59deCLSCzlEkGvopIsenM/Ly4NcLlc55+TkpBy3rNWmTZsGr3f58mXcuXMHnTp1QmRkJDIzM7F582ZMnToVBw4cQGlpKQDAwcFB7b329vaQSqUNXp8JlIj0QiHUHJrqAMDkyZNx65bqts958+Zh/vz5jfrMlStXQhAE9OzZEwAQEBAAPz8/vPTSSzh06BAGDx4MABCJ1BO7IAgQixvupDOBEpFeKLRogYofnE9KSqqzBdpYPXr0UCvr3bs3HB0dcfnyZTz//PMAUGdLs7y8HI6ODd+3gQmUiPRCmy58bQJ1d3d/7M8rLy/H0aNH0a1bNzz55JPKckEQUF1dDRcXF9jb28PNzQ3Z2dkq77179y6kUqna2Kh6vEREeiATxKjWcMiEpktJ1tbWSEhIwPvvv69S/sMPP+D+/fvKdaH9+/fH8ePHUVVVpazz3XffQSKRqK0dfRRboESkF3KIIdfQZtN0vjEkEglmz56Nd999F6tWrcKQIUNw5coVbNmyBUOHDkXfvn0BADNmzMCRI0cwc+ZMvPzyy7hx4wY2bNiACRMmwMPDo8HPYAIlIr2omURquAuvaZKpsaKiouDg4IBdu3Zh7969cHZ2xqRJk1Qmo/z8/JCYmIg1a9ZgwYIFcHFxQVRUlFYTVkygRKQX2kwiKRq9F+mh8PBwhIeHq5VHREQgIiKiwfcGBAQgJSWl0Z/JBEpEeiGHGHINY5xN2YXXByZQItILBcRQaEiQms4bGyZQItKLakGMKkF9u+XfiZtwFl4fmECJSC8UEGkc43ycMVBDYAIlIr1QaLGMiV14IqI6yAUtJpHYhSciUsdJJCIiHSkEQK7nhfTNjQmUiPSiWrBAtdBwytF03tiYVrREZLI4iUREpCO5INLYhdd03tgwgRKRXtSsA9XUAmUCJSJSo9BiGZOCy5iIiNRVCxJUa9jKqem8sWECJSK9qLmdHbvwRESNpoBI8w2VmUCJiNTp+5Ee+sAESkR6IQhijZNEAieRiIjUafNYY03njQ0TKBHpRc1jjRueZW/KxxrrAxMoEemFQosuPNeBEhHVgfcDJSLSkaDFIz0EjoESEamruZmIphYoEygRkRqFoMVCeiZQIiJ1Mi32wsu4F540+eNNa5RnixHwaYVKeeFZCa6/bwVphgRprWbCbog1/BZUwsJO9f13T0uQ9ZEVSv6QQCQCnHvK4Te/Es49FXr8FlTL2VWGqPg8KG4HYn9GEa79ZovEd9xx+aK9oUMzKub4TCTTitYM3Npnidx9VmrlheckuBhjC6FahCcWVmLYlEG4tdcSabG2EP6WF4v+nwRps2xRXSqC34JK+M6uRHmOGOen2aH4N/5z6putvRzrDlzDoFH3ANt/YteadmjTrhoJe6/Dp0uFxve3JHKIlDdVrvcwsUkk/sbpiSAHMj+0Qvpb1nWev7rOGjbuAnp/Wg6vf1ZjxrtT0GlpJYrTLHD39MNuTUaCNWzaCeizpxw+U6vRIboazyaVQ2ILXN9U97Wp+Uycdxvt/Srxxsu+EDsuwMGdbbF47BMAgAlz/jJwdMZFEB6Og9Z3CCb2UDmDJ9DDhw/j+eefR48ePTBixAgcPHjQ0CE1OXklcDbCDpkfWMN9lAzWbgq181YuAjzHVUNi87DcJUAOAJBeqUmg1cWANEMMtzAZJLYP61m3EeASIMO9X0xr/Mj0CRgWUYRzPzjh0lkHZWnRX5bY/r8euHSWXfi/q11Ir+kwJQYdAz169CiWLFmCqVOnYuDAgUhNTcXy5cthY2OD4cOHGzK0JqWoBGRlIjy9rgJuw2U4Far6iyWxBvw/Uu/ulV6u+Y/Jxr0m4Vo4AP0Ol6kkz1rV90QQMX/qlZtXFdp6VGPvh20BAIIgwMZOjvvlEhz+rI2BozM+NVs5G06Q3MrZCBs2bMCIESPw2muvAQAGDhyI4uJibNq0yawSqIUD0O9IGcRa/rQrckX47txxXFltA/tOcrQdKgMAiCSAnY96H6c0Q4x7aRK07i9vyrBJA8+OVQCAe3csMOP1XAi3e+Pf16TIzbLCtrc8cPZ7ZwNHaFzMcSunwaLNycnBzZs3ERoaqlIeFhaGzMxM5OTkGCiypicSQ+vkWV0MnA51wLrorZBXAV3iKyFpYGhTVg78/lpNv7/D9KomiJa05eBU8wfr5aX56DO0BCLHlVgz3wv3K8R4M/EG/AeWGjhC46J4sBNJ02FKDJZAMzMzAQC+vr4q5T4+PgCArKwsvcdkLLqvrcCyz+bBoaMCaTG2KDhWd/aVVwC/zLOFNEOCDtOr4PIsW6D6ZGlVM7Ri7yzHohefgMguHD/sc8WS8CdQVixBVHyegSM0LnIBmmfhTWwSyWBd+NLSmr/ODg4OKuX29jXjg1KpVOtrze70WdMFpge/Wc6Bm31bLO/6P3VXCKz5v0Hjn0PM04uRv1GGDa9sU6kivVeGlaNWo+hcBoZHBWPRR7MhEpnGX+/lZrJcVbj/HYR789Gq/XgcKHobAPC9Yi8AQFH8KpxcD+CY7FOIxJxMAnhD5SYlPFiv8OgvfW25WKz9D/LDqy+jpLqg6YJrZsXV9qgsy0PCH/WP8y7v+i3ey3oRFoHWyEuywtunR8DKpeZnU3VXhIuxtpBelsAzogrViw5hTfohfYX/2FK7Oxo6hCbRxb8Mm48ASe8cx661EfhesRch4ggAwPQVuZgwV8BE98ko+svSwJHqzs2nLT7P2tok1zLHrZwGS/eOjjW/RI+2NMvKylTOtwRlmWKcCrVHzpfqv2iyMhEgEiC2Eh68hjJ5ek+twlNvVsJEGp5m58ZlG1TdF8GnS6XauXZeVaisEKH4Ljf71ZJBDJmg4TD8yspGMVi0tWOfN2/eVCnPzs5WOd8S2HorIJOKcCvZEorqh+UVuSLcTrWAS4AcFg96gZdX2UB6WQKvKVXovEz9F5f0p7JCgjPHnNB3WAl8Ot9Xlrt5VeK50BL8fMwZCgX/utUStFgDyi68lnx8fNC+fXt8++23CAkJUZYfO3YMHTp0gIeHh6FC0zuxBdAl/j5+j7fFhWl2aPdCNXZb7sW5zXYQiYAur9UkyrLrYuR/bQkLRwGOT8qR97X6P5/7KJm+w2/RdqzyQI9+ZVjz1XUI0g8xfvZtjJl+B5X3xfhkdTtDh2dUFILmLrqCk0jamzt3LuLj4+Hs7IygoCD8+OOPOHr0KDZu3GjIsAzCfZQMYssK3Ei0wpU11sh1OAKXZ+XwW1AJ+w41/1UVna9ZKS8rFeGPlXWspgfgPopLZ/Sp4E8rxL3wBKavyMOgtjvwUlwZLp21x/a3PZB/k1tr/06bZUqmtozJoAk0PDwcVVVVSExMxN69e+Hl5YWEhASMHDnSkGE1uwHHyuosdxsug9vwmhbk8q5fqU0ytZ9YjfYTq+t6KxlQ/k1r/Cu2A4Ji9iLcPcLQ4Rgtc5xEMvgI96RJkzBp0iRDh0FEzUzQIoEKTKBEROpkCjFkCg174TWcNzZMoESkFxwDJSLSEbvwREQ6UkCLZUz6CaXJMIESkV5wFp6ISEcKhRhyDZNECk4iERGp4yQSEZGO2IUnItKRIIg0zrKbzSx8bm6uThdsSTcBISLttagW6JAhQ3S6w3l6evpjBUREZkrQooVpLndjmjt3rsk8IoKIjJ9cEEGu4f6ocnNpgc6fP1+fcRCRmeMsPICMjAycOHECubm5mDp1Kuzs7HDlyhUMHjy4OeIjIjPRoiaR6vL2229jz549EAQBIpEIw4cPR0lJCeLi4hAUFIRNmzbB2po3kSUidea4F17rZf+7du1CUlISZs6ciZSUFOXTMwMDAzFt2jScOHEC27dvb7ZAici0CYJ2hynROoF++eWXGD58OBYuXAgvLy9luZOTE1599VWMHj0ahw8fbpYgicj01XbhNR26Sk9PR7du3ZCfn69SfurUKYwbNw49e/bEkCFDkJiYqPbe3377DZGRkfD398eAAQOwYcMGVFdrfvqD1gk0JycHzz33XL3nAwICkJeXp+3liKiFkT/YC6/p0EVmZiZiY2Mhk6k+VPHixYuYNWsWOnbsiC1btmDUqFFYs2YNdu7cqayTnZ2NadOmwdraGu+99x6io6PxySefYPXq1Ro/V+sxUBcXF7XM/ndXr16Fs7OztpcjohZGgOYuemN78DKZDMnJyVi/fj0sLS3Vzm/evBldu3bF2rVrAQCDBg2CTCbDtm3bEBkZCSsrK3z88cdwdHTE1q1bYWVlhcGDB8PGxgarVq1CbGws3Nzc6v18rdN9SEgI9uzZg2vXrinLateJnjx5EsnJyQgODtb6ixNRyyII2nTjG3fNCxcuYN26dYiOjsaSJUtUzlVWVuL8+fMIDQ1VKQ8LC0NJSQkuXrwIADh9+jSCg4NhZWWlrDN8+HDI5XKcOnWqwc/XugUaFxeHc+fOITw8HJ06dYJIJML777+PhIQEXL58GZ6enoiLi9P2ckTU0mgzxtnIMVA/Pz+kpqaidevW2L9/v8q5nJwcVFdXw9fXV6Xcx8cHAJCVlYWePXsiLy9PrY6rqyscHByQlZXV4OdrnUCdnJyQkpKCHTt24NixY7C2tsYvv/wCT09PREVFITY2ll14IqqXAM1d9NrzeXl5kMvlKuecnJzg5OSkUtamTZt6r1VaWgoAcHBwUCm3t7cHAEil0nrr1NaTSqUNxtuodaC2traYP38+dykRUaMJChEEDVs5a89PnjwZt27dUjk3b968RuWe2qWW9W1JF4vFDdYRBAFiccOjnI3eiXT16lWcOHECt27dgkQigbe3N4YMGaKytImI6FGN2YmUlJRUZwu0MRwdHQFArRVZ+9rR0VHZ8qyrpVleXq68Rn20TqAymQyvv/46Dh48qMzatRISEjBjxgwsWrRI28sRUQujzUL52vPu7u6P/Xne3t6QSCS4efOmSnnta19fX9jb28PNzQ3Z2dkqde7evQupVKo2NvoorWfht27digMHDmDMmDE4cOAAzp8/j/PnzyMlJQVhYWHYvn07du/ere3liKiFae6F9I+ytrZGQEAAjh07ptLo++677+Do6Iju3bsDAPr374/jx4+jqqpKpY5EIkGfPn0a/AytW6AHDhzAiBEj1BaX9ujRAxs3bkRFRQV2796NyMhIbS9JRC2KSItZ9qbdCz979mxERUVh4cKFGDt2LNLS0rBz504sXrwYtra2AIAZM2bgyJEjmDlzJl5++WXcuHEDGzZswIQJEzTeIF7rFmhhYSGeffbZes8HBQWhoKBA28sRUQtjiL3wgYGB2LJlC65fv465c+fi66+/xrJlyxATE6Os4+fnh8TERJSXl2PBggX45JNPEBUVhRUrVmi8vtYt0J49e+Knn37CSy+9VOf5X3/9FU899ZS2lyOiFqYxs/C6CA8PR3h4uFp5SEgIQkJCGnxvQEAAUlJSGv2ZWj8TKSYmBgsWLMDixYsxffp0+Pr6QiQS4datW0hJSeHdmIioYY1ZCGoiGvVMJEEQcOTIEXzzzTdq5QAwfvx4PhOJiOrWDDuRDI3PRCIi/WhJLVDuNiKipmdejbJG70QqKSlBeXk5FAqFskwul6OsrAxnzpzBtGnTmjI+IjIXAgCFFnVMiNYJtKCgAMuWLcO5c+carMcESkR1ErRYB2ouY6CPWrNmDc6dO4eRI0fCysoKBw4cQGxsLAoLC3Hs2DFUVlbi008/bcZQiciUNWYrp6nQeiH9zz//jDFjxmD9+vVYsWIFRCIRBg4ciLfffhsHDx6EnZ0dvv/+++aMlYhMmaDlYUK0TqAlJSXo1asXgJp753l4eODSpUsAajb+R0RE4Mcff2yeKInI9NV24TUdJkTrLryzszMqKiqUr729vZGRkaF87eXl1eAzk4ioZRMJNYemOqZE6xZor169sH//fuUdnDt37oyzZ8+isrISQM1jQeu6qzMREQBAIdLuMCFaJ9DZs2cjKysLgwcPRlFRESZMmICCggKEh4cjJiYGKSkpCAoKasZQicjkmdH4J9CIBNq1a1ekpKRg9OjRcHFxgZ+fHz744APcv38faWlpGDFiBJYtW9acsRKRKTPDSaRGLaTv0qUL3nrrLeXroKAgtjqJSDstaSvno3dj0pamG5ASUQvVkhbS13U3Jm3wbkxEVCctZuHNpgXKuzERUZNqSV14U7ob0+lQexRkN/z4UVOzXAGkdjev71Tru9z/GjqEZmN2303i2WSXMsd1oI2+GxMRkU5a0hgoEVGTM7EWpiZMoESkHy1pDJSIqCmJFDWHpjqmhAmUiPSDLVAgIyMDJ06cQG5uLqZOnQo7OztcuXIFgwcPbo74iMhMtPhZ+Lfffht79uyBIAgQiUQYPnw4SkpKEBcXh6CgIGzatAnW1tbNFSsRmTIznIXX+mYiu3btQlJSEmbOnImUlBTls+ADAwMxbdo0nDhxAtu3b2+2QInIxJnhzUS0TqBffvklhg8fjoULF8LLy0tZ7uTkhFdffRWjR4/G4cOHmyVIIjJ9Ijzsxtd7GDrIRtI6gebk5OC5556r93xAQADy8vKaJCgiMj+1s/CaDlOi9Rioi4tLg4/suHr1KpydnZskKCIyQ2Y4C691CzQkJAR79uzBtWvXlGW1Nxs5efIkkpOTERwc3PQREpF5MMMxUK1boHFxcTh37hzCw8PRqVMniEQivP/++0hISMDly5fh6emJuLi45oyViEyYOS5j0roF6uTkhJSUFMTExKCqqgrW1tb45ZdfUFFRgaioKOzbtw+urq7NGSsRkVFp1DpQW1tbzJ8/36RudUdERsIMx0C1TqDaPuKDj/QgorqIBC32wptrAtX2ER98pAcR1aklt0DresSHXC7HnTt3cPLkSdjb27NrT0T1a0nPRHpUQ8lRKpVi0qRJyM7ObpKgiMgMmWELVOtZ+IY4ODggIiICycnJTXE5IjJDGrdxatNCNTJNdj/Q6upqFBUVNdXliMjcKB4cmuqYkMeeha+qqkJ6ejoSExPx1FNPNVlgRGRezHEhfZPMwguCAGtrayxevLjJAiMiM2RiCVITrRPovHnz6iwXi8Vo27Ythg4dyp1IRFQ/M5xE0jqBuru7o3fv3ujQoUMzhkNE5socu/Baz8K/8847OHLkSHPGQkTmrCXfjcnW1pbPOyIinbXoxxq/9dZbWLlyJSorKzFgwAC4urpCIpGo1eNeeCKqU0seA120aBFkMhm2bNmC999/v9563AtPRHURQfMzj0ztmUhaJ9CYmBitbiZCRFSnltQCjY+Px6RJk9CzZ08ADe+FJyLSpPapnJrqmJJ6Z+EPHDiAmzdv6jMWIjJnLXkWnojocbToWXgiosfSksZAAeD8+fOQy+WNuuCYMWMeJx4iMlct7YbKKSkpSElJ0epCgiBAJBIxgRJR3VpaC3TChAl45pln9BQKEZkzc9wL32ACDQgIwKhRo/QVCxGZMwGab5hsTgmUiKiptLgWKBFRk2lJY6Bjx46Ft7e3PmMhIjMmEgSIhIYzpKbzj5LJZOjVqxcqKytVyu3s7JCWlgYAOHXqFDZu3Ihr166hdevWmDJlCqKjoxsXfD3qTaCrV69ukg8gIgLQLC3QrKwsVFZWIiEhQeVm72JxzSbLixcvYtasWRgxYgTi4uJw4cIFrFmzBoIgYPr06Y37sDqwC09EetEcY6CXL1+GWCxGWFgYbG1t1c5v3rwZXbt2xdq1awEAgwYNgkwmw7Zt2xAZGQkrK6vGfeAjmuS58EREmoiEh9s56z0amUDT09Ph7e1dZ/KsrKzE+fPnERoaqlIeFhaGkpISXLx48XG+DgAmUCLSl2a4mUhGRgasrKwwffp0+Pv749lnn8Ubb7wBqVSKnJwcVFdXw9fXV+U9Pj4+AGq6/4+LXXgj4+wqQ1R8HhS3A7E/owjXfrNF4jvuuHzR3tChtWgbl3ghN8saa/ddUymfP7ITrvy39t8mAsAzAIABz9/D69tvKOtdOmuPT951x9Vf7ODQSoZ+YcWIXJIP59aN2yptyhrThc/Ly1PbRu7k5AQnJyeVssuXL0MqlSIiIgKzZs3CpUuXsGXLFmRlZWHRokUAAAcHB5X32NvX/HtJpdLH+DY1mECNiK29HOsOXENrt2rAdg52rdiP0VF3kLD3OhaM7ITsDPVuCjW/b/e44ts9rdEjUPUXThCAnKs26Df8HgY8XwyR8zoIxUsAAP9oX6Ws98t/HLDipY6wd5Jj0oICiCUCDmxvi/+edsTGQ1fh2KqFJNFGTCJNnjwZt27dUjk1b948tfsSb9y4Ec7OzujSpQsA4Nlnn0Xr1q2xdOlSnD59GgDqvRF87UTT42ACNSIT591Ge79KLB3nh40/L8DBnT/h5KFW+PRMOibM+Qtr47isTJ/kcuCLTW74fH27Os8X5FihokyCwLASDB1XBHG7QVDkF6nV27rSE2IxsPHQVXh0qEms/YYXY/awLvhikxtmvpnbrN/DWDSmBZqUlFRnC/RRffr0USsLCgpSef1oS7P2taOjo4aINTOaMdD09HR069YN+fn5hg7FQAQMiyjCuR+ccOnswy5H0V+W2P6/Hrh0ll14faq6L8LcsC7Yvc4dQ8cXoY17lVqdGxk2AADvTvfrvU5+jhVuXLbF0PGFyuRZ855K9A0pQepe16YP3lgpBIg0HFDUZFB3d3e0b99e5Xg0gd69exd79+5FTk6OSvn9+zX/Hq1bt4ZEIlG7MXzt60fHRnVhFAk0MzMTsbGxkMlkhg7FYNy8qtDWoxoX/68meQqCABu7mr/Ahz9rg6N7WhsyvBanqlKM8lIxXtt2A0s33UQdD6BF9oME6vUggVaUqSfSu3mWAIAOT6qf8+hQieJCC9y+ZdmEkRuxJp5EEolEeOONN/D555+rlH/zzTeQSCTo168fAgICcOzYMQh/W6D/3XffwdHREd27d3/ML2TgLrxMJkNycjLWr18PS8sW8h9RPTw71rRO7t2xwIzXcyHc7o1/X5MiN8sK297ywNnvnQ0cYcti5yjHJ6fTIWngNyQ7wwZ2DnJ8/JYnTh5qhYqySLj7PIVpy/MQNOYeAMDGrubuGRVl6m2V0qKaixf9ZYF/eFY3+XcwNrXLmDTV0ZarqysmT56M3bt3w8HBAQEBAbhw4QK2bduGyZMnw8fHB7Nnz0ZUVBQWLlyIsWPHIi0tDTt37sTixYvrXPrUWAZtgV64cAHr1q1DdHQ0lixZYshQDM7Bqaa1+fLSfPQZWgKR40qsme+F+xVivJl4A/4DSw0cYcsiFqPB5AnUJNByqQTSEgmWbr6JxTvnwNZegdVzOiD1KxcAgHfn+7BzlOPUkVb4+y7FqvsinD9RMwZXVWkUHcHm1wzLmJYvX46FCxfiyJEjmDlzJg4ePIgFCxYgPj4eABAYGIgtW7bg+vXrmDt3Lr7++mssW7YMMTExTfKVDNoC9fPzQ2pqKlq3bo39+/cbMhSDs7Sq+dNs7yzH9AFP4kBROH7Y9wXOfO+MT/+Tjqj4PKT99PiD3tR0Rky5C4W8EKOj7gAAxO2CMTgoFrFDnsSOtz0QPLYIllYCxs28jd3r3fHuXB9Mml8AhVyET9e0w/2KmsQpkZjYHTR01Bw7kSwtLRETE9NgQgwJCUFISEjjLqwlg/7pa9OmDVq35tgeANwvr/mnOP2NM6TFD/+ulZVIcOZ7J3TqUaEcEyXj8MLUu8rkWcvaVsDQcUUo+ssSN6/UjJG+tLAAY2Nu4/8OtcKsoU9iTmgXiEXAhDm3AaAFLWMStDtMiFksY/o8a6uhQ3hsQtUvEAoj8PysSRi19BUAwPeKvQAARelaoGw7DhV/BJGkrQGjbMEkcwCrthC3O9pgNXG7q3Dp+B2AHai0+Qridl0gBjDnI+Cf/3sPt67moa1XG7j5tEXiij0QS/4N94CLENs83p5sU8CnchqpKb5zUJD9l6HDeCzWtnJ89bsI575JwtszTuN7xV6EiCMAACu23UDfEBHGWM+GQlH3omBT8l3ufw0dQuPJuwJVWVDkdwIA3MmzRPw/O2Lw6HuYsqgAQE3yVOR3ws2LngDa4h/2Y6DIl+H4gVZwdZOhZz8pnJ+ouZwiH/j1hyfQqYcIFve6abxRu8FIPCFue6JJLmWON1RuIaPXxq+yQoIzx5zQd1gJfDo/XPLi5lWJ50JL8PMxZ7NInuaijXs1ykslOJrUGmWlD3+Nbt+yxPcprujZvxSu/6hZlrd/e1t8sMIT8r+t0jub6oTfzzlg1LQ7j17ajGnTfTetDGoWLVBzsWOVB3r0K8Oar65DkH6I8bNvY8z0O6i8L8Ynq+veDUOGM3fVLfzPdF8sHN0JIybfxX3sw7+3dIbYQsC8d/5U1psw9zZWxfji9akdMWBkMfJzrLD/47boHVSCIeHqO5fMFVug1KwK/rRC3AtP4Nef7SGU7cBLcQXI/N0GC0c/gfyb1oYOjx7Rb0Qx3kzMhI2dAjv/5YGvNnyNp3qXYeO/r8K708M7pA98vhjxW2+g6LYltr3pgZP/boWI2bfxxo6sOhfom61mWMZkaGyBGpn8m9b4V2wHBMXsRbh7hKHDoQd2nfujzvJ+w0vQb3gJgIdjoHUJGnNPubi+pWILtBmFh4cjIyMD7dqxq0pkluSCdocJYQuUiPTCHFugTKBEpCfaLJQ3rQzKBEpE+qFFC9TE8icTKBHpSTM81tjQmECJSC9EckCkYZJIZGK3BWACJSK9EAkCRBrGQDWdNzZMoESkH+zCExHpirPwREQ64TpQIiJdaXPDZI6BEhGpE8kFLWbhmUCJiNRxEomISDdcxkREpDPOwhMR6Ubx4NBUx4QwgRKRXrALT0SkK4UAKDQ0MRVMoERE6tiFJyLSjQhadOE5iUREVAfuRCIi0hETKBGRjrR56ia3chIR1UGLZUxsgRIR1YVdeCIiHQnQvM7TtPInEygR6QlboEREOmICJSLSkVxRc2iqY0KYQIlIPwRFzaGpjglhAiUiPeH9QImIdKOA5ll402qAMoESkZ5wEomISEdMoEREOpLLaw5NdUwIEygR6QknkYiIdMMuPBGRjjgLT0SkI0EBgQvpiYh0wK2cREQ6EhSaH2vMFigRUR04iUREpBtBIUDQ0AIVNE0yGRkmUCLSD7ZAiYh0pBC0WMbEBEpEpEZQyCFo2KopKLiVk4hInSBocUNltkD1ro2nq6FDaBZuPm0NHULzkHgaOoLmY27fTdyuyS7V2sNF4yRRaw+XJvs8fRAJgomlfCIiIyE2dABERKaKCZSISEdMoEREOmICJSLSERMoEZGOmECJiHTEBEpEpCMmUCIiHTGBEhHpiAnUyBw+fBjPP/88evTogREjRuDgwYOGDom0lJ6ejm7duiE/P9/QoZCeMIEakaNHj2LJkiXo378/PvjgA/Tp0wfLly/Ht99+a+jQSIPMzEzExsZCJpMZOhTSI+6FNyIhISHo3r07Nm7cqCx75ZVXkJGRgaNHjxowMqqPTCZDcnIy1q9fD0tLS9y7dw8nT55Eu3ZNdxMOMl5sgRqJnJwc3Lx5E6GhoSrlYWFhyMzMRE5OjoEio4ZcuHAB69atQ3R0NJYsWWLocEjPmECNRGZmJgDA19dXpdzHxwcAkJWVpfeYSDM/Pz+kpqZi3rx5kEgkhg6H9Mws7gdqDkpLSwEADg4OKuX29vYAAKlUqveYSLM2bdoYOgQyILZAjUTtULRIJKqzXCzmPxWRseFvpZFwdHQEoN7SLCsrUzlPRMaDCdRI1I593rx5U6U8Oztb5TwRGQ8mUCPh4+OD9u3bq635PHbsGDp06AAPDw8DRUZE9eEkkhGZO3cu4uPj4ezsjKCgIPz44484evSoyrpQIjIeTKBGJDw8HFVVVUhMTMTevXvh5eWFhIQEjBw50tChEVEduBOJiEhHHAMlItIREygRkY6YQImIdMQESkSkIyZQIiIdMYESEemICdRIvPrqq+jSpYvK8dRTT6FXr16IiIjAgQMH9BLHkCFDEBkZqXwdGRmJIUOGNPo6UqkUhYWFTRZX7c/nces05fv0dT0yXlxIb2Ti4+Ph4uICoOZOTFKpFIcOHcKrr76KoqIiREdH6zWeWbNmoaKiolHvuXTpEmbPno1169ahb9++zRQZkeExgRqZYcOGoX379ipl48ePx8iRI/HBBx9gypQpsLKy0ls8/fv3b/R7rly5gtu3bzdDNETGhV14E2BjY4MhQ4ZAKpXi6tWrhg6HiB5gAjURtTdalsvlAGrGKleuXInXXnsNTz/9NAYNGqQcc0xLS0NUVBT8/f3h7++P6Oho/Prrr2rX/Oabb/Diiy+iR48eeOGFF3DmzBm1OnWNgV6/fh1xcXHo27cvevfujcjISJw/fx4AsGXLFsTHxwMApk6dqvLe/Px8LFu2DM899xyefvppjBkzBocOHVL7zEuXLiE6Ohr+/v4YOHAgdu3apcuPDADw888/Y8aMGejbty+6deuGgQMH4o033kBJSYla3bS0NIwbNw5PP/00QkND8emnn6rV0fY7UMvALrwJUCgUOHfuHKysrODn56csP3LkCHx9fbFixQrcuXMHrq6uOH36NGJjY/Hkk08iLi4OVVVV2L9/PyZPnoxPPvkEAQEBAID9+/cjPj4e/v7+WLp0KbKzszFr1iwoFAp4enrWG8uNGzcwYcIEWFhYYMqUKXB1dcWXX36JqKgoJCUlISQkBH/99ReSk5Mxa9YsPP300wCAgoICREREQBAEREZGwtnZGT/88AOWLl2K27dvY8aMGQCAq1evIjIyEk5OTpgzZw6qq6vxwQcfKP9wNMapU6cQExODXr16YcGCBRCJRDh9+jSSk5NRXV2N1atXq9SPjo7GsGHDEB4ejtTUVKxevRqlpaWYP39+o74DtSACGYXly5cLnTt3Fn7//Xfh7t27wt27d4Xbt28LaWlpQlxcnNC5c2fhnXfeUdYPDg4WnnzySSE7O1tZJpfLhaFDhwqTJk0SZDKZsrysrEwICQkRXnzxRUEQBEEmkwmBgYHCuHHjhKqqKmW9ffv2CZ07dxamTJmiLJsyZYoQHBysfB0XFyf06NFDuHHjhrKssLBQ6N27t7BgwQKV65w5c0bl+/Xp00coKChQ+d6LFi0SunfvLty5c0cQBEGYP3++8Mwzzwi5ubnKOteuXRO6d+8udO7cWaufYa3p06cLwcHBQmVlpUq9CRMmCP7+/mrvS0hIUJbJ5XJh6tSpQvfu3YXCwsJGfYdH4yDzxS68kRk7diwCAwMRGBiIAQMGYOLEifjhhx8QGRmJxYsXq9T19vaGt7e38vUff/yBnJwcDBs2DMXFxSgsLERhYSHu37+P4OBgpKenIz8/H7///jvu3r2L8PBwWFpaKt//4osvwtnZud7YFAoFTp48icGDByufFgoALi4u2LNnD1auXFnv+1JTUxEQEAALCwtlXIWFhQgNDUVVVRVOnz4NhUKBn376CYMHD4a7u7vy/X5+fhgwYECjf5YfffQR9u3bpzLpVlRUBAcHB5SXl6vV/3sLUiwWY8qUKaiqqsJ//vMfrb8DtSzswhuZtWvXKp/0KBaL4eTkBD8/P1hbW6vVbd26tcrr2seBrFmzBmvWrKnz+nl5ecjPzwcAleQLABKJRCUxPurevXsoLy+vs07nzp3rfV9RURFKS0uRmpqK1NTUeuOqvf6jcQFAx44d8eOPP9b7GXWRSCTIycnBpk2bcO3aNdy8eRMFBQV11m3VqhVcXV1Vyry8vAAAt27d0vo7UMvCBGpkevXqpbaMqT6PPodcoVAAAOLi4vDMM8/U+Z6OHTsqk0hlZaXa+dpr1KV2HLKxTwitfV9YWBgmTZpUZ53aZKVLXPX58ssv8eabb8LX1xcBAQEIDQ1Fz549sXv3bnz99dcqdR99Giqg+kTUxn4HahmYQM1I7eSPnZ0d+vXrp3Lu119/RXFxMWxsbJS/6Ddu3FCpIwgCbt26hU6dOtV5fRcXF9jY2CgfdPd3O3fuxJ07d7B8+XK1c66urrC1tYVMJlOLKzc3F3/88QdsbW3h4uICBwcHtbgA4M8//6z3e9elsrIS7777Lvr27YvExERYWDz8T33Tpk1q9YuLiyGVSuHg4KAsq43D29tb6+9ALQvHQM1I9+7d0bZtW+zevVv5OGSgZlvlK6+8gvj4eEgkEnTt2hWenp744osvVHYZHTlyBEVFRfVe38LCAv3798fJkydVuqvFxcXYuXOncgihtoVa22q0sLDAoEGDcPLkSVy+fFnlmu+++y7mzp2LoqIiiEQihISE4KeffsKVK1eUdf7880+cOHGiUT+L+/fvo6KiAh06dFBJnunp6Th37hwAQCaTKcsVCgW++uor5WuZTIbPPvsMdnZ2CAwM1Po7UMvCFqgZsbS0xOuvv45XXnkF4eHhGD9+PKytrbF3717k5uZi3bp1ymTy+uuvY+7cuZg4cSLGjRuHgoICJCUloVWrVg1+xuLFixEREYGIiAhMnjwZDg4OSElJQXl5OV555RUAUI4lfvHFF7hz5w5GjRqFJUuW4OzZs5g8eTImT54MDw8PnDhxAsePH8fEiROVrd64uDicOHECkZGRmDZtGiQSCXbv3g17e3tUVVVp/bNwdnZGz549sX//fjg4OMDX1xdXr17F3r17lQm+rKxMOWlma2uLzZs3Iy8vD97e3vjmm2+QlpaGN998E46OjgCg9XegloMJ1MyEhYUhMTERH374IbZu3QqxWIxOnTrhww8/RHBwsLJecHAwPvroI2zZsgUbNmyAm5sb/vWvfyEpKanB6/v5+SE5ORkbNmzAjh07IBaL0aNHDyQkJCgTSGBgIEaMGIHjx4/jzJkzCA0Nhbe3N1JSUrB582ZlwvXy8kJ8fLzKzUvc3d3xxRdfYM2aNdixYwesrKwQEREBoGZWvTE2bdqE1atXY9++faiqqoKnpydmzpwJPz8/zJ8/H2fOnEFYWBgAwMnJCQkJCXjnnXeQlJQEHx8frF27FqNHj1ZeT9vvQC0HHypHRKQjjoESEemICZSISEdMoEREOmICJSLSERMoEZGOmECJiHTEBEpEpCMmUCIiHTGBEhHpiAmUiEhH/x91RndRGuq1jQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import plot_confusion_matrix\n",
    "\n",
    "plot_confusion_matrix(clf, X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Classification Report**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.79      0.79        29\n",
      "           1       0.81      0.81      0.81        32\n",
      "\n",
      "    accuracy                           0.80        61\n",
      "   macro avg       0.80      0.80      0.80        61\n",
      "weighted avg       0.80      0.80      0.80        61\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(y_test, y_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\manik\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0.0</th>\n",
       "      <th>1.0</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>macro avg</th>\n",
       "      <th>weighted avg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.99990</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.499950</td>\n",
       "      <td>0.99980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.99990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.99995</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.499975</td>\n",
       "      <td>0.99985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>9999.00000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>10000.000000</td>\n",
       "      <td>10000.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0.0  1.0  accuracy     macro avg  weighted avg\n",
       "precision     0.99990  0.0    0.9999      0.499950       0.99980\n",
       "recall        1.00000  0.0    0.9999      0.500000       0.99990\n",
       "f1-score      0.99995  0.0    0.9999      0.499975       0.99985\n",
       "support    9999.00000  1.0    0.9999  10000.000000   10000.00000"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Where precision and recall become valuable\n",
    "disease_true = np.zeros(10000)\n",
    "disease_true[0] = 1 # only one positive case\n",
    "\n",
    "disease_preds = np.zeros(10000) # model predicts every case as 0\n",
    "\n",
    "pd.DataFrame(classification_report(disease_true,\n",
    "                                   disease_preds,\n",
    "                                   output_dict=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To summarize classification metrics:\n",
    "\n",
    "Accuracy is a good measure to start with if all classes are balanced (e.g. same amount of samples which are labelled with 0 or 1).\n",
    "* Precision and recall become more important when classes are imbalanced.\n",
    "* If false positive predictions are worse than false negatives, aim for higher precision.\n",
    "* If false negative predictions are worse than false positives, aim for higher recall.\n",
    "* F1-score is a combination of precision and recall."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Regression model evaluation metrics\n",
    "Model evaluation metrics documentation - https://scikit-learn.org/stable/modules/model_evaluation.html\n",
    "\n",
    "1. R^2 (pronounced r-squared) or coefficient of determination.\n",
    "2. Mean absolute error (MAE)\n",
    "3. Mean squared error (MSE)\n",
    "R^2\n",
    "\n",
    "What R-squared does: Compares your models predictions to the mean of the targets. Values can range from negative infinity (a very poor model) to 1. For example, if all your model does is predict the mean of the targets, it's R^2 value would be 0. And if your model perfectly predicts a range of numbers it's R^2 value would be 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=100)\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8654448653350507"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "# Fill an array with y_test mean\n",
    "y_test_mean = np.full(len(y_test), y_test.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21.488235294117654"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529, 21.48823529, 21.48823529, 21.48823529,\n",
       "       21.48823529, 21.48823529])"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.220446049250313e-16"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Model only predicting the mean gets an R^2 score of 0\n",
    "r2_score(y_test, y_test_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Model predicting perfectly the correct values gets an R^2 score of 1\n",
    "r2_score(y_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mean absolue error (MAE)**\n",
    "\n",
    "MAE is the average of the aboslute differences between predictions and actual values. It gives you an idea of how wrong your models predictions are."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.136382352941176"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean absolute error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "\n",
    "y_preds = model.predict(X_test)\n",
    "mae = mean_absolute_error(y_test, y_preds)\n",
    "mae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual values</th>\n",
       "      <th>predicted values</th>\n",
       "      <th>differences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>23.6</td>\n",
       "      <td>23.081</td>\n",
       "      <td>-0.519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>32.4</td>\n",
       "      <td>30.574</td>\n",
       "      <td>-1.826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>13.6</td>\n",
       "      <td>16.759</td>\n",
       "      <td>3.159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>22.8</td>\n",
       "      <td>23.460</td>\n",
       "      <td>0.660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>16.1</td>\n",
       "      <td>16.893</td>\n",
       "      <td>0.793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>17.9</td>\n",
       "      <td>13.159</td>\n",
       "      <td>-4.741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>9.6</td>\n",
       "      <td>12.476</td>\n",
       "      <td>2.876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>17.2</td>\n",
       "      <td>13.612</td>\n",
       "      <td>-3.588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>22.5</td>\n",
       "      <td>20.205</td>\n",
       "      <td>-2.295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>21.4</td>\n",
       "      <td>23.832</td>\n",
       "      <td>2.432</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual values  predicted values  differences\n",
       "173           23.6            23.081       -0.519\n",
       "274           32.4            30.574       -1.826\n",
       "491           13.6            16.759        3.159\n",
       "72            22.8            23.460        0.660\n",
       "452           16.1            16.893        0.793\n",
       "..             ...               ...          ...\n",
       "412           17.9            13.159       -4.741\n",
       "436            9.6            12.476        2.876\n",
       "411           17.2            13.612       -3.588\n",
       "86            22.5            20.205       -2.295\n",
       "75            21.4            23.832        2.432\n",
       "\n",
       "[102 rows x 3 columns]"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data={\"actual values\": y_test,\n",
    "                        \"predicted values\": y_preds})\n",
    "df[\"differences\"] = df[\"predicted values\"] - df[\"actual values\"]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual values</th>\n",
       "      <th>predicted values</th>\n",
       "      <th>differences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>23.6</td>\n",
       "      <td>23.081</td>\n",
       "      <td>-0.519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>32.4</td>\n",
       "      <td>30.574</td>\n",
       "      <td>-1.826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>13.6</td>\n",
       "      <td>16.759</td>\n",
       "      <td>3.159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>22.8</td>\n",
       "      <td>23.460</td>\n",
       "      <td>0.660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>16.1</td>\n",
       "      <td>16.893</td>\n",
       "      <td>0.793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>17.9</td>\n",
       "      <td>13.159</td>\n",
       "      <td>-4.741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>9.6</td>\n",
       "      <td>12.476</td>\n",
       "      <td>2.876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>17.2</td>\n",
       "      <td>13.612</td>\n",
       "      <td>-3.588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>22.5</td>\n",
       "      <td>20.205</td>\n",
       "      <td>-2.295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>21.4</td>\n",
       "      <td>23.832</td>\n",
       "      <td>2.432</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual values  predicted values  differences\n",
       "173           23.6            23.081       -0.519\n",
       "274           32.4            30.574       -1.826\n",
       "491           13.6            16.759        3.159\n",
       "72            22.8            23.460        0.660\n",
       "452           16.1            16.893        0.793\n",
       "..             ...               ...          ...\n",
       "412           17.9            13.159       -4.741\n",
       "436            9.6            12.476        2.876\n",
       "411           17.2            13.612       -3.588\n",
       "86            22.5            20.205       -2.295\n",
       "75            21.4            23.832        2.432\n",
       "\n",
       "[102 rows x 3 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data={\"actual values\": y_test,\n",
    "                        \"predicted values\": y_preds})\n",
    "df[\"differences\"] = df[\"predicted values\"] - df[\"actual values\"]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mean squared error (MSE)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.867437068627442"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean squared error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "y_preds = model.predict(X_test)\n",
    "mse = mean_squared_error(y_test, y_preds)\n",
    "mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.867437068627439"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate MSE by hand\n",
    "squared = np.square(df[\"differences\"])\n",
    "squared.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the scoring parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.81967213, 0.90163934, 0.83606557, 0.78333333, 0.78333333])"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "cv_acc = cross_val_score(clf, X, y, cv=5, scoring=None)\n",
    "cv_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy is: 82.48%\n"
     ]
    }
   ],
   "source": [
    "# Cross-validated accuracy\n",
    "print(f'The cross-validated accuracy is: {np.mean(cv_acc)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy is: 82.48%\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "cv_acc = cross_val_score(clf, X, y, cv=5, scoring=\"accuracy\")\n",
    "print(f'The cross-validated accuracy is: {np.mean(cv_acc)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8085601538512754"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Precision\n",
    "cv_precision = cross_val_score(clf, X, y, cv=5, scoring=\"precision\")\n",
    "np.mean(cv_precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8424242424242424"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Recall\n",
    "cv_recall = cross_val_score(clf, X, y, cv=5, scoring=\"recall\")\n",
    "np.mean(cv_recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.841476533416832"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_f1 = cross_val_score(clf, X, y, cv=5, scoring=\"f1\")\n",
    "np.mean(cv_f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How about our regression model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6243870737930857"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "cv_r2 = cross_val_score(model, X, y, cv=5, scoring=None)\n",
    "np.mean(cv_r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6243870737930857"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "cv_r2 = cross_val_score(model, X, y, cv=5, scoring=None)\n",
    "np.mean(cv_r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-2.13045098, -2.49771287, -3.45471287, -3.81509901, -3.11813861])"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean absolute error\n",
    "cv_mae = cross_val_score(model, X, y, cv=5, scoring=\"neg_mean_absolute_error\")\n",
    "cv_mae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-21.12863512415064"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean squared error\n",
    "cv_mse = cross_val_score(model, X, y, cv=5, scoring=\"neg_mean_squared_error\")\n",
    "np.mean(cv_mse)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using different evalution metrics as Scikit-Learn functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier metrics on the test set\n",
      "Accuracy: 85.25%\n",
      "Precision: 0.8484848484848485\n",
      "Recall: 0.875\n",
      "F1: 0.8615384615384615\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Make some predictions\n",
    "y_preds = clf.predict(X_test)\n",
    "\n",
    "# Evaluate the classifier\n",
    "print(\"Classifier metrics on the test set\")\n",
    "print(f\"Accuracy: {accuracy_score(y_test, y_preds)*100:.2f}%\")\n",
    "print(f\"Precision: {precision_score(y_test, y_preds)}\")\n",
    "print(f\"Recall: {recall_score(y_test, y_preds)}\")\n",
    "print(f\"F1: {f1_score(y_test, y_preds)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Regression evaluation functions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Regression model metrics on the test set\n",
      "R^2: 0.8654448653350507\n",
      "MAE: 2.136382352941176\n",
      "MSE: 9.867437068627442\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=100)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions using our regression model\n",
    "y_preds = model.predict(X_test)\n",
    "\n",
    "# Evaluate the regression model\n",
    "print(\"Regression model metrics on the test set\")\n",
    "print(f\"R^2: {r2_score(y_test, y_preds)}\")\n",
    "print(f\"MAE: {mean_absolute_error(y_test, y_preds)}\")\n",
    "print(f\"MSE: {mean_squared_error(y_test, y_preds)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Improving a model\n",
    "First predictions = baseline predictions. \n",
    "First model = baseline model.\n",
    "\n",
    "From a data perspective:\n",
    "* Could we collect more data? (generally, the more data, the better)\n",
    "* Could we improve our data?\n",
    "\n",
    "From a model perspective:\n",
    "* Is there a better model we could use?\n",
    "* Could we improve the current model?\n",
    "\n",
    "Hyperparameters vs. Parameters\n",
    "* Parameters = model find these patterns in data\n",
    "* Hyperparameters = settings on a model you can adjust to (potentially) improve its ability to find pattern . \n",
    "\n",
    "Three ways to adjust hyperparameters:\n",
    "1. By hand\n",
    "2. Randomly with RandomSearchCV\n",
    "3. Exhaustively with GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "clf = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'ccp_alpha': 0.0,\n",
       " 'class_weight': None,\n",
       " 'criterion': 'gini',\n",
       " 'max_depth': None,\n",
       " 'max_features': 'auto',\n",
       " 'max_leaf_nodes': None,\n",
       " 'max_samples': None,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 100,\n",
       " 'n_jobs': None,\n",
       " 'oob_score': False,\n",
       " 'random_state': None,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.get_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Tuning hyperparameters by hand\n",
    "Let's make 3 sets, training, validation and test."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'ccp_alpha': 0.0,\n",
       " 'class_weight': None,\n",
       " 'criterion': 'gini',\n",
       " 'max_depth': None,\n",
       " 'max_features': 'auto',\n",
       " 'max_leaf_nodes': None,\n",
       " 'max_samples': None,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 100,\n",
       " 'n_jobs': None,\n",
       " 'oob_score': False,\n",
       " 'random_state': None,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.get_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We're going to try and adjust:\n",
    "\n",
    "* `max_depth`\n",
    "* `max_features`\n",
    "* `min_samples_leaf`\n",
    "* `min_samples_split`\n",
    "* `n_estimators`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_preds(y_true, y_preds):\n",
    "    \"\"\"\n",
    "    Performs evaluation comparison on y_true labels vs. y_pred labels\n",
    "    on a classification.\n",
    "    \"\"\"\n",
    "    accuracy = accuracy_score(y_true, y_preds)\n",
    "    precision = precision_score(y_true, y_preds)\n",
    "    recall = recall_score(y_true, y_preds)\n",
    "    f1 = f1_score(y_true, y_preds)\n",
    "    metric_dict = {\"accuracy\": round(accuracy, 2),\n",
    "                   \"precision\": round(precision, 2),\n",
    "                   \"recall\": round(recall, 2),\n",
    "                   \"f1\": round(f1, 2)}\n",
    "    print(f\"Acc: {accuracy * 100:.2f}%\")\n",
    "    print(f\"Precision: {precision:.2f}\")\n",
    "    print(f\"Recall: {recall:.2f}\")\n",
    "    print(f\"F1 score: {f1:.2f}\")\n",
    "    \n",
    "    return metric_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 82.22%\n",
      "Precision: 0.81\n",
      "Recall: 0.88\n",
      "F1 score: 0.85\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.82, 'precision': 0.81, 'recall': 0.88, 'f1': 0.85}"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Shuffle the data\n",
    "heart_disease_shuffled = heart_disease.sample(frac=1)\n",
    "\n",
    "# Split into X & y\n",
    "X = heart_disease_shuffled.drop(\"target\", axis=1)\n",
    "y = heart_disease_shuffled[\"target\"]\n",
    "\n",
    "# Split the data into train, validation & test sets\n",
    "train_split = round(0.7 * len(heart_disease_shuffled)) # 70% of data\n",
    "valid_split = round(train_split + 0.15 * len(heart_disease_shuffled)) # 15% of data\n",
    "X_train, y_train = X[:train_split], y[:train_split]\n",
    "X_valid, y_valid = X[train_split:valid_split], y[train_split:valid_split]\n",
    "X_test, y_test = X[valid_split:], y[:valid_split]\n",
    "\n",
    "clf = RandomForestClassifier()\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Make baseline predictions\n",
    "y_preds = clf.predict(X_valid)\n",
    "\n",
    "# Evaluate the classifier on validation set\n",
    "baseline_metrics = evaluate_preds(y_valid, y_preds)\n",
    "baseline_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 82.22%\n",
      "Precision: 0.84\n",
      "Recall: 0.84\n",
      "F1 score: 0.84\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "# Create a second classifier with different hyperparameters\n",
    "clf_2 = RandomForestClassifier(n_estimators=100)\n",
    "clf_2.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions with different hyperparameters\n",
    "y_preds_2 = clf_2.predict(X_valid)\n",
    "\n",
    "# Evalute the 2nd classsifier\n",
    "clf_2_metrics = evaluate_preds(y_valid, y_preds_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter tuning with RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   2.0s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    1.9s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   1.9s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   2.1s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   1.9s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   2.7s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.3s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.3s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.5s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.4s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.2s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.9s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   1.0s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.9s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.5s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.5s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.6s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   2.2s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.7s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.5s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.9s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:   32.1s finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "grid = {\"n_estimators\": [10, 100, 200, 500, 1000, 1200],\n",
    "        \"max_depth\": [None, 5, 10, 20, 30],\n",
    "        \"max_features\": [\"auto\", \"sqrt\"],\n",
    "        \"min_samples_split\": [2, 4, 6],\n",
    "        \"min_samples_leaf\": [1, 2, 4]}\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Split into X & y\n",
    "X = heart_disease_shuffled.drop(\"target\", axis=1)\n",
    "y = heart_disease_shuffled[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate RandomForestClassifier\n",
    "clf = RandomForestClassifier(n_jobs=1)\n",
    "\n",
    "# Setup RandomizedSearchCV\n",
    "rs_clf = RandomizedSearchCV(estimator=clf,\n",
    "                            param_distributions=grid, \n",
    "                            n_iter=10, # number of models to try\n",
    "                            cv=5,\n",
    "                            verbose=2)\n",
    "\n",
    "# Fit the RandomizedSearchCV version of clf\n",
    "rs_clf.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 200,\n",
       " 'min_samples_split': 6,\n",
       " 'min_samples_leaf': 2,\n",
       " 'max_features': 'sqrt',\n",
       " 'max_depth': None}"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rs_clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 81.97%\n",
      "Precision: 0.77\n",
      "Recall: 0.86\n",
      "F1 score: 0.81\n"
     ]
    }
   ],
   "source": [
    "# Make predictions with the best hyperparameters\n",
    "rs_y_preds = rs_clf.predict(X_test)\n",
    "\n",
    "# Evaluate the predictions\n",
    "rs_metrics = evaluate_preds(y_test, rs_y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameter tuning with GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': [10, 100, 200, 500, 1000, 1200],\n",
       " 'max_depth': [None, 5, 10, 20, 30],\n",
       " 'max_features': ['auto', 'sqrt'],\n",
       " 'min_samples_split': [2, 4, 6],\n",
       " 'min_samples_leaf': [1, 2, 4]}"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_2 = {'n_estimators': [100, 200, 500],\n",
    "          'max_depth': [None],\n",
    "          'max_features': ['auto', 'sqrt'],\n",
    "          'min_samples_split': [6],\n",
    "          'min_samples_leaf': [1, 2]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 12 candidates, totalling 60 fits\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.4s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.3s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.1s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.1s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.8s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.8s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.8s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.5s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.1s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.5s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.8s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   1.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   1.1s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   1.1s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.3s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.8s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.7s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  60 out of  60 | elapsed:   26.9s finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Split into X & y\n",
    "X = heart_disease_shuffled.drop(\"target\", axis=1)\n",
    "y = heart_disease_shuffled[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate RandomForestClassifier\n",
    "clf = RandomForestClassifier(n_jobs=1)\n",
    "\n",
    "# Setup GridSearchCV\n",
    "gs_clf = GridSearchCV(estimator=clf,\n",
    "                      param_grid=grid_2, \n",
    "                      cv=5,\n",
    "                      verbose=2)\n",
    "\n",
    "# Fit the GridSearchCV version of clf\n",
    "gs_clf.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': None,\n",
       " 'max_features': 'sqrt',\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 6,\n",
       " 'n_estimators': 200}"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 78.69%\n",
      "Precision: 0.74\n",
      "Recall: 0.82\n",
      "F1 score: 0.78\n"
     ]
    }
   ],
   "source": [
    "\n",
    "gs_y_preds = gs_clf.predict(X_test)\n",
    "\n",
    "# evaluate the predictions\n",
    "gs_metrics = evaluate_preds(y_test, gs_y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's compare our different models metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlwAAAINCAYAAADrzqHiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABKn0lEQVR4nO3dd3RU1d7G8WcmIQSSEAIkdEIgSAlEKYI0KWIogmKQIiqiUo1KlSbKFRBFauiIUkWkKCAlgIgUUUSkWBBEgiQCSiSFFEibef/gMu8dE0gwOcwEvp+1XIvs035n5tzkufvss4/JarVaBQAAAMOYHV0AAADAnY7ABQAAYDACFwAAgMEIXAAAAAYjcAEAABiMwAUAAGAwAhcAAIDBXB1dQE7i4pJlsTBV2P8qWdJTly4lOboMFBBcL8gtrhXcCq4Xe2azST4+Hjdc7vSBy2KxEriywWeCW8H1gtziWsGt4HrJPW4pAgAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABjM6Z9SBADcHa5cSVZSUoIyM9MdXQpy4eJFsywWi6PLuC1cXArJ09NbRYrceNqHnBC4AAAOl56epsTEOBUvXkqFChWWyWRydEnIgaurWRkZd37gslqtSk9PVXz833J1LaRChdz+1X64pQgAcLjExHh5enrLzc2dsAWnYjKZ5ObmLg8PbyUlxf/r/RC4AAAOl5GRpsKFizi6DOCG3N2LKD097V9vT+ACADicxZIps9nF0WUAN2Q2u8hiyfz32+djLQAA/GvcSoQzy+v1SeACAAAwGE8pAgCcmlexInIv7Ng/V1dTM5R4+cotb/fEE53UoEFDjRr1ugFV5d6FC+fVteujev318WrbtoO2bt2kSZPe1KefbpGfX2mH1na3IHABAJyae2FXdRq20aE1bJr2mBIdWkH+aty4mRYsWCIfnxKOLuWuQeACAOAu4+PjIx8fH0eXcVchcAEAYKD09DRNnfq2Pv98m1xdXdWq1cN68cWXVbSohzIzM/XRR8u1Y0eEzp07J7PZpGrVqqtv34GqV6+BJCk19apmz56p/fv3Kj4+TmXLllPHjp3Vs+cztmMkJMRrwYI52rdvj1JSUlS9eg0NHPiygoPvy7amf95SfOut/+jSpUtq3fohffjhMv3115/y9w/QwIEvq1Gjxrbt/vzzgubNm6WDBw8oIyNdwcF19fLLQxQQUMXQz/BOQOACAMBAO3fuUJ069+qNNybqwoVzWrhwnuLjYzVx4ruaNy9cn322XgMGvKwqVaoqJiZGS5cu0htvjNK6dZvl7u6u8PBp+u67b/XSS4Pl41NCBw58rXnzwuXj46P27TsqNTVVgwa9qLi4WA0YEKYSJUppw4ZPNHjwi5o7d5Fq1gzKVZ3Hj/+oixf/VJ8+A+Th4an331+gsWNHaP36CHl6eio+Pl4DB76gIkWKaPjwUSpSxF0ffrhML77YR0uWrFSZMmUN/iQLNgIXAAAGKl68uKZNm6XChd0lSa6urpo2bbLOnInU33/HqH//MHXp0s22fuHCbnrttRE6c+a0atYM0tGjh9WgQSM99FCIJKlevQYqWrSovL2LS5K2b9+q06dPadGiZapRo5Yk6YEHmqhv32e1cOFczZw5L1d1JiUlafHilSpXrrwkqUiRInrppX46cuSQmjdvqdWrV+ry5QQtXLhEfn6l5epqVoMGjdS9++NatuwDjRw5Nr8+sjsSgQsAHCi/nsD7t0/RwXiNGzezhS1JataspaZOfUcnThzXm2++LUmKi4tTVNRZ/fFHlPbv3ydJSk+/9hLvevUaaMOGTxQT85caN26qxo2bqXfvPrb9ff/9Qfn6+ikw8B5lZGTY2ps0aaYVK5bY9pOTkiVL2cKWJPn6+kmSrly5+t/jfKfq1WuqRImS/z2OWS4urrr//kb67rtv/8Unc3chcAGAA+XXE3h32lN0d5J/PglYvHhxSdLff8foxInjmjbtHf3yy3G5u7srIKCKSpcuI0myWq+t/8orw+Tr66cdOyI0Y8YUzZgxRbVrB2vYsFGqVu0eJSQk6OLFv9Sy5QPZHj8hIT5Xdbq7u9v9bDab/1vHtRdUX76coD/+iM72OK6uxImc8AkBAGCgxET7KBwXFytJKlzYXcOGvazAwOpasWKN/P0ry2w265tvvtLu3bts67u5uenZZ1/Qs8++oD///FP79+/VsmUfaMKE17V8+Wp5enqqcuUAjR37ZrbH9/Yurr//jsnzeXh4eKp+/fs1cODLkiQXF7MyMy153u/dgpnmAQAw0KFDB5WZ+f/v4Pvyy52SpNq16yghIUHdu/dUQEAVW4/SgQNfS7rWs5SWlqaePbto1aoPJUllypRRly7d1KZNiC5e/EuSdN999fTnnxdUqpSvatSoZftv3749Wrv243zrfbrvvnqKijorf/8A1ahRSzVrXjvOxo3r9fnn2/PlGHcyergAADBQTMxfGjdutDp3fkKnTv2qRYvmq0OHTqpUqbI8PDy0dOn7MpmuvRx59+5d2rLl2i3mK1euyM3NTTVr1tKSJYtUqJCrqlatpqios9q6dbNatnxIktShw6Nat26NBg9+Uc8885x8ff20f/8+rV69Us891zff3lHZo8dT2r59i4YMCVP37j3l7e2tLVs2afv2rRo9+o18OcadjMAFAICBOnd+QomJlzV69DAVLuyurl17qG/fgXJ1ddXbb0/TvHmzNHbsSBUt6qFq1aprzpz3NHz4IP3ww1E1btxUw4ePkbd3ca1a9aFiYy/Jx6eEOnXqrD59BkiSihYtqnnzFmnBgjmaPXuGUlJSVK5ceQ0Z8qq6dOmeb+fh6+un+fMXa+HCOZo8+S1lZKSrUqXK+s9/3lKbNm3z7Th3KpPVen1YnnO6dClJFotTl3jb+fp6KSaG4bHIHa4X5+br65Vvg+bz+j078lr588+zKlPGP9tlBfldincyV1ezMjLurjFcN7tOzWaTSpb0vOG29HABAJxa4uUrPIGJAo9B8wAAAAajhwsAIElKy0yXr69XnvdzNT1NifGp+VARcOcgcAEAJEluLoXUbfXAPO9nTff5ShSBC/hf3FIEAAAwGIELAADAYAQuAAAAgzGGCwCAAsJsNslsztvM8RaLlfktHYDABQBAAWE2m/RbdHye9hFYsTiBywG4pQgAAGAwAhcAAIDBuKUIAHBqPt5ucnUr7NAaMtJSFZeQlq/7fOmlfnJxcVV4+DxJUmTkaU2cOE5nzpxWpUr+Wrbs41ztJyUlWRvWLdehg18pIT5Ovn5l1abto3oo5FGZTHkb74X8Q+ACnFR+/ZGxZOTvHwk4J0tGWr7MEu+MXN0KK/KtLg6tocprn0gy9n9Ly5a9rwsXzmvSpCny8SmZ6+3mzpyg07+dUJduvVW2fCX9/MP3WvbBbKWkJOvRx3tmWd9qtcrVNe83uKxWxoHdCgIX4KTy64/MtT8UzPp9pzO7uuX5erl2rcBREhISVLVqoBo3bpbrbc6e+U3HjhzUy0PfUKPGLSVJtevUU0pykjZvWJVt4DKZTEq9cDrP9RYuW1USoSu3GMMFAIBBrFarVq9eqZ49u6h166bq0SNU69ZlvVXYrFkDHTp0UEePHlazZg20deum3O1fVrVq01FBtevZtZctX0kpKclKTEzIl/NA3tHDBQCAQebNm6U1az5Sz569VK9eA/344zGFh0+T2exit96CBUsUHj5FmZmZGjp0lMqXr5Cr/VcOqKYX+g/N0v79d/vlXbyEPD2L5ct5IO8IXAAAGCAxMVFr1nyk7t17qn//MEnS/fc3UkzMRR09ethu3dq166hoUU9lZmaodu06eTruti2f6Jefj+rp3mEMmnciBC4AAAzw888/KjMzUw8+2MqufeTIsZKuPaWY33ZErNfKZfPUqHFLte0Qmu/7x79H4AIAwACXL18bP+XjU8LwY1ksFn384UJt3bRWTZo9pP4vjaJ3y8kQuAAAMICHh6ckKS4uzm5M1rlzf+jixb/ybVqFjIwMzQufqIMH9qpDp6568pkBhC0nROACAMAAtWrVlqurq/bv32s3Lmv58sU6cuR7+fmVzpfjLJr3rr77dp+e7v2i2j3yRL7sE/mPwJVLXsWKyL1w3j8uS3qazIXc8rSPtHya4PBqepoS45mfCUD+sqTd+u+oixfNdpNxWi1WZRbwFyz7+PioS5fuWrVqhVxdXXXvvXX1ww9HFRGxWSNGvKZt27bk+RhHvj+g/ft2ql6DJgqsVku//XrcbnnlgGpyLVQoz8dB3hG4csm9sKs6DduY5/1smvZYvkxO2G31wDzXsqb7fCUyISaAfGZ2c9P+x27t91yRIa8oKSXD9rNnYFXpv4ErIy3V4ZOyZqT9u9+VYWGD5OPjo88+W6+VK5epfPkKGj36DbVv3zFfAtd33+6VJB0+9LUOH/o6y/LwBatVsqRvno+DvCNwAQCc2rV3GBbMV1SZzWY9/XRvPf107yzL5sx5z+7n6+9UvBX9Xhyhfi+O+Lfl4TYicAEA4GQyMzNvMKjerMzMTEmSSZLZxSWbdeCMch24Nm/erPnz5ys6Olrly5dX//791blz5xuuHxsbqylTpmjfvn1KS0tT3bp1NXr0aFWuXDkfygYA4M41aNDALJOj/lMp39KaOW/VbaoIeZWrwBUREaHhw4erV69eat68uXbu3KmRI0fK3d1d7dq1y7K+1WpVWFiYoqKi9Oqrr6p48eKaNWuWevXqpU2bNsnb2zvfTwRwJvn1kAWAu9OIEWOUkpKSpd3FxazovxIlSYVcHTsY3mq12j3o8G9YrFZZMgv2wxG5lau/CNOnT1f79u01ZswYSVLz5s2VkJCg8PDwbAPX77//rsOHD2vy5Mm2XrCqVauqTZs22rVrlx5//PH8OwPACeXHQxabpj2WT9UAKGgqVaqcbburq1mFPONvay03YjKZdDo2Kk/7qFqikiy6OwJXjtE0OjpaUVFRCgkJsWtv27atIiMjFR0dnWWb1NRrT3N4eHjY2q73asXHx+elXgAAgAInx8AVGRkpSQoICLBr9/f3lySdOXMmyzY1atRQo0aNNHfuXJ0+fVqxsbGaOHGiihYtqjZt2uRH3QAAAAVGjrcUExOv3Sv29PS0a7/ee5WUlJTtdv/5z3/Up08fdejQQZLk5uamuXPnqmLFirdUYMmSnjmvhH8tPyZQhfPje0ZBlNfxQSgYCtL3bDab//Xv0xwD1/XHUv/5Xqbr7WZz1g/q9OnT6tGjhypVqqQxY8bI3d1da9as0SuvvKL3339fDRo0yHWBly4lyeIEsw3fqX+wYmISHV3CHcnZrhe+Z+flbNeKM8nIsDi6BKdTkMJJbhWk79lisdzw96nZbLppJ1GOgcvL69ovg3/2ZCUnJ9st/19Lly6VJC1evNg2dqtp06bq2bOnJk2apE8//TSnwwIAANwxcozK18duRUXZP4lw9uxZu+X/6/z586patard9A8mk0n169fXb7/9lqeCAQAACpoce7j8/f1VoUIFbdu2TQ8//LCtfceOHapcubLKlSuXZZuAgACtX79eCQkJdqHr2LFjKl++fD6VDgC4G3gVLyz3Qm4OreFqepoS4wvGu2eP/3xUk/4zVK+PD1f1mnUcXY7T2bp1kyZNelOffrpFfn6lb9txczUPV1hYmEaPHi1vb2+1bNlSu3btUkREhGbMmCHp2qzyUVFRCgwMlKenp3r37q3PPvtML7zwgvr16yd3d3dt3LhRBw8etG0Dx7OkpeV5/EjG1VTFJRbMd5wBKBjcC7mp2+qBDq1hTff5SlTBCFxwTrkKXKGhoUpLS9PixYu1du1aVaxYUZMnT7Y9gbh7926NHj1ay5cvV6NGjVShQgWtWrVKU6ZM0ahRo2Q2m3XPPfdoyZIlatKkiaEnhNwzu7lp/2Nd8rSPphs/kQhcAADcVK7fPdKjRw/16NEj22WhoaEKDQ21a6tataoWLFiQt+oAACjAnniik1q0aK1ffz2hkydPqFOnx/Tyy0N16tRJLV68SD/+eFSJiYkqUaKkWrZ8SAMGvKTChQtLkpo1a6Dhw0fr+PGftHfvbmVmZqpJk6YKfXKAvL19bMf44vNNiti0Vpf+vqiqgTX0YOv2Wer47dfjWvfxEkVGnpSL2aT6dWprwLNPq6yfnyQpYteXCl/0gd5+bbTmLlmms9F/qEK5sho2oJ9MJpNmvb9YkVFRKl+mjF5+4TnVD77xrcofDx/V2hUf6Y+zUXJxdVHN2kHq8VwvlatYwbbOd18f0IaP1+h81B/y8iqmkJD26tt3oNzc/v/W8e7dX2j16o/022+nlJGRrnLlyuuJJ3ro8cefkCQdPnxIr7wyQK++OkbLln2gzMwMjR8/Wffee5+++eYrLVu2WL/99qs8Pb3UsmVr9esXpqJFi/5/nT/+oDVrPtKpUyfl41NCXbv2UI8eT//Lbzpnd97zpQAAOJF16z5W9eo1NWHCO3rooRDFxFxUWFg/paWl6bXX/qOpU2epdeuHtXbtKq1b97HdtgsWzJYkTZjwtl588RV99dVefbRsvm35joj1WvLeDN1bt6GGjpygwHtqafHC6Xb7+OmH7zX+9Vfk4uqiAS+N0vDho3TqzBmFjRqr2P95+0taWrrenjVHTzzSQeNHDNPVq6kaN2Waxk+fqY4Pt9G4oUNksVg0ftpMpaZlf2fj4oU/NX3CJFWpFqhh415T31fCdP6Pc5oyboJtOqn9X+7RzInvqKK/vyZPnqZevZ7Txo2f6s03X7Pt56uv9mjs2JGqVStI77wzTRMnvquyZctp2rR3dPz4T3bHXLRongYNGqYBA15WzZq1tH//Po0YMUSlSvlqwoR31LfvQG3fHqG33x5vt92UKZMUEtJO7747U0FBdTRnzkwdOPB1Lr/VW8fbdQEAMJCfXxmFhQ2yzWd54MDXuuee6pow4R1bj8v99zfSoUPf6ujRw3rqqWdt2wYG3qMxY8b9dx3p5Mnj2vXlLknX5sPc8MmHeqBpKz3z3EuSpDr33q8rKcn64vNNtn2s/uh9la9QWcNGTZLZbFZgxeKqWdZHT4cN0uoNn2lg716SpEyLRc/16KZ2rVtKkqLOnde8pcs1ImygHmnT+r/rZOqNd6fp3IU/Vcy/ZpZzPf3rKaWlpumx7k/Ip2QJSVJJ31L6/sBBXb1yVe5F3PXxkuWq27CBBgwbpKolKun++xvLz6+0Ro8erh9+OKrg4Pv0++9n1KFDJ7388lDbvuvUCVaHDg/p6NHDqlWrtq398ce7qkWL1rafFy9+TzVq1NTEiZNtbVarVR9//KFSUpJtbWFhg9SpU2dJUu3awdq3b7cOH/5ODzxgzNAnAhcAAAYKCKhiN3n4Aw800QMPNFFGRobOnInUuXPROn36N8XFxcnHp4TdtnXq3Gv3s59faaVevSpJunA+WpcT4lT//qZ26zRq0tIWuK5evaLfI3/VE92fs5uovLSvr+rUqqEjPx+327ZmtWq2f/v8d5aBoOr/31bs+tycycnKTmCNe1TIzU2vDx6uRs2a6N4G9VUzuLaqVr9HknQ++g/F/n1Jj/fsrszMTKWnp8tkMqlJk6YqVKiQvv/+oOrVq6fevZ+XJKWkpCgq6ndFR0frxIlrtWZmZsjV1SwXl2vnU63aPXIxm5RpsSo19ap+/fWE+vV70a6ujh0fU8eOj9m13XvvfbZ/u7u7y8enhBITs397Tn4gcAEAYKASJexDlMVi0cKFc/Xpp2t15UqK/PxKq1atIBUuXFjWf7xY5fp4rutMJpOs1mszsyclXZYkeRUrbrdOcZ+Stn+nJCfJarXKu7h9DZJUwru4/rr4t11b0SJFsqz3zxpuxrd0ab0+eaI+W/upvty+U9s2blZRTw89/EgHde3VU4mXr83S/sGsefpg1rws21+IPK2k304rIfGyZix+T18fOiiZTKpQpqyC7qkhSUq9FKuk307ryrnzkqQiV67IZDZJFqsuX74sq9UqHx+fLPv+J3d3+3M1m822z9YIBC4AAG6jDz9cqjVrPtKrr47Rgw+2sr2ruG/fXre0Hy+vaz1Ql+Pj7NqTEi/b/l20qKdMJpMS4mOzbH8pLk7exfL/1VJVq9+jIWNHKSM9XSd//kVfRGzXxtVrVblqgMpWuDYX59P9nlf1WrVUwbuMUqL/sG3r/d8etElzwxV94bzeHT1OtardI7dChXQ1NVURu7+46bE9PK59lnFx8XbtKSnJ+vHHHxQU5Lh5yRg0DwDAbfTDD0dVtWo1dejQyRa2YmIu6vTp07fUw1KmbAWVLOmnb7/Zbdd++ND/D/x2L1JElavcowNf75bF8v/7vvj33/rpxEnVqVkjbyfzD9s/26xBvfsqPT1droUKKei+YPV55drtvUt//63yFSuomLe3/v7roqrcE6iaNWupepWq8vby0vsff6io8+ckST/9ekItGjXWfbWC5FaokCTp4LEjknTTz6ho0aIKDLxHX3+91659797dGjbs5SyvKbyd6OECAOA2qlkzSMuWfaCVK5epVq3aOncuWsuXL1F6epquXLmS6/2YTCZ1f7qv5oW/pQ8WTlfDB5rr1K/H9cWOTXbrdXvyeb371mhNe+c1tWn7qCJ/MWnRgtkqWqSIunXqmK/nFnRvsFZ9sEwzJrytkE4dZHZx0Rdbt6uQm5vqNrxfZhcXde3VU0vmLpTJZFa7Vu108dSvWv7JGiWlJCvQv7IkqXqVQO3cv1eB/gEq6VNCP586oVWfrZfJZNLV1JtPQNunT3+NHj1c48e/rrZtOygm5i8tWDBH7do9ojJlyuTr+d4KAhdwh0vLTM/zGwUK0mtNAGf3zDPPKSEhXmvWfKSkpCSVLl1Gbdt2kNls1ooVS5WcnGS7NZaTJs0ektlk1oZPVuirPdtVoVIVPd9/iObOnGhbp86992vk2Mn6ZM1SzZr2pooUKaIGwUHq/8zTKlki57FOt6KCfyUN/89YfbLyY82ZPF2WzEwFVAvUqInjVKZcWUlS6/ZtVaRoUW1et15fbN2mou7uCq5RS89366kSxa/VM3LAS5q97APNWvb+tf2WKashz/fXzv179ePJX25aQ7NmLfT229O0ZMkijR49TMWL+6hTp8fVu3effD3XW2WyWv85RM+5XLqUJIvF8SX6+nqp07CNed7PpmmPKfKtvM3uXuW1T/LlNRdrus/Pl5nmY2IS81zLnSY/rpf8uFak/Lle1nSfz/dsEH63XFNkyCvy9fj/P/6egVWVkXHt1hHvUvx/rq5m/RYdn6d9BFYsrtQLp/NcS+GyVXU6NipP+6haopKSfst7Lf97vRjpzz/PqkwZ/2yXmc0mlSx546BMDxcAwKklxqfyHkMUeAyaBwAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGDPNAwCcmo+Xm1zdCzu0hoyrqYpLTLttx7tw4by6dn1Ur78+Xm3bdrjheoNffFJBdeqr78Dht622/PbU4BdVL6iOhvXN+2ulnBmBCwDg1FzdC+f5va951XTjJ9JtDFwlS5bSggVLVKFCxdt2TBiLwAUgR5a0NPn6euV5P7e7lwAoqNzc3FS7dh1Hl4F8ROACkCOzm1u+9DDc7l4CwNHS09O1YMEc7dy5TcnJyWrcuJlq166j2bNn6KuvDkmSXnqpn8qUKauUlBQdOnRQDRs+oLCwQVluKf722ynNnTtTP/xwTJ5exdStZ58cj5+WmqqVy+fr8KGvlXg5Qb5+ZfREly56onVT2zoJlxO1cMVK7T/4nVKuXlH1KlXUv9fTqlOzhm2d+IQEfbBqtb49fESX4uJUxN1d9Rs0Uudne8i3tJ8kaeLI11TKz09XrlzRT0eOKbh+XQ0aM0IpKSlau+xDHdz/ja6kXFEF/0rq1usp1a577/9/TpkZWrBymXbu36erqVcVdE8NDXqur8r5lc6X78EZELgAADDI5MkT9eWXO9W370D5+wdo48ZPtHDh3Czrff75NrVpE6JJk6bIZDJlWR4Tc1EvvdRXlSr568VBryklJVkff/ieLifE3fT4K5bO1U/HDumpXgNVrLiPfjhyULNnz5Sn0tWudUulpqVp6Lg3FZdwWf2e6akSxYtr4/bPNXTceM16a7xqVguU1WrViAmTlHLlivo/87RK+BTX6d/P6oNVaxSXFK8R49+wHe/r3Xv1wIPNNOT1UTLJJEtmpiaPfVN/nj+vrs/0VJly5bQrYrumjJugN2e8q6olKkmSdu3fpwbB92nkgJcUFx+v+SuXatLccM15c1IevwHnQeACAMAA5879oe3bt2rw4FfVpUs3SVKjRo317LM9dOZMpN26rq6uGjHiNRUu7C7p2qD5/7VmzSplZlo0Y8ZsxVy+1la2XEX9Z0zYTWs4cfyYagfX1wNNW0mSagXdp3KlfeRd7NoQgR279+r02SgtePdt1Qiseq3GenU1YMRoLVr5kab/5w3FXIpVkSJF9Eqf51W7RnVJUt3aQfrz8hV99tl6u+O5uLqozysvyq3wtYccDn/7nX47cVKvvjlW993fQJJUM7i23hj8qo4f+1EP3d9SkuRXqpTGDxkhV9drseTcX39q5cZPdOXqVRVxd8/Fp+38CFwAABjg8OFDslqtatmyta3NbDarVas2OnPmPbt1y5evYAtb2Tl27Ijq1LlX3t7FFXM5XpIUWK2mSpbyu2kNtYLu0xefb1JsbIzurdtI99V7QM8/31epF05fq/HHH1WqRAkFBlRWRmambbvGDerrw08+VXp6uvxKlVT4hP/IarXqwsWL+uP8BUWdO68ffjiqjIwMu+P5lSljC1uS9OvxX1SoUCHd26C+rc3FxUVvzZ5ut13NqvfYwpYklfG7dl7JV1IIXAAA4Mbi46/d7ite3MeuvUSJklnW9fHJ2va/Ll++rIoVsz6xWDyH7Z5+LkwlSvpq/76dWr54tpYvnq06dYI1+LlnFBhQWQmJiYq5dEkPPdEj2+0TEhNVqkQJfb5nn977cKUu/n1JxTw9Va1KgNzd3WW1Wu3W9/Ypbvdz4uVEeXkXy/Y26f9yL2w/7Yf5v+tbLdbsVi+QCFwAABigVClfSVJcXJxKlSpla78exG5F8eLFFRsbm6U9KfHyTbcrVMhNj3V5Wo91eVp/x/ylI99/o03rV2rizFlaGj5dnkWLyr9CeY0Z9HK223t7eemH479o0qzZeqLjI+r2aEf5lrwW8t5bt0k//vjDTY9f1KOoEi8nZmk//espFSpUyDaG627ATPMAABggOPg+ubi46Kuvdtu179u355b3Vb/+/frhh6O6dOlvW9u56N918a8LN9wmPT1Nrw56Vls3rZEklfItrYfbdVZISFvF/H1JknRvUJD+ivlbpUr4qEZgVdt/+w9+p0+2bJWrq6t+OvmrLBarnuvRzRa2MjMz9d1332bp4fqn6rVqKj0tTT8ePmprs2Rmat67M7Rt46Zb/hwKMnq4AAAwQPnyFdS2bQfNnRuutLQ0+fsHaOvWTTp16mSOt9j+qVu3J7V580a98sqL6vj408rMyNSaVR/YjXv6p0KF3FSlanV9una5XF1dVbFSVV04H60tWzapRZMHJEntW7fUp1siNHTceD3dJVSlSpbQN4e+15rPNqt3964ymUyqWS1QkhS+6AO1bdVSiUlJWr91m06d+lVWq1Vpqal247b+V71G96vqPdU0f9pMdX3mKZXy89WX2z9X3KVLat/50Vv6DAo6AhcAAAYZNmykihQpomXLPlBqaqqaNWuhxx7rou3bt97Sfry9i2vevPc1e/Z0LZwzWYXdi6jjY9114OvdN93uuX5D5FXMW1s+W6OE+DgV8y6uRx/trGcfbSdJKlqkiGZPGq+FK1Zq7pJlSrl6ReVKl9agvs8rtEN7SdeeSBzcr4/WbNykXV99LZ/i3qpbO0jvvDNVI0cO04mfjyu4Xt1sj292cdHIieP08ZLlWrPsQ6WmpiogsKpGT3pTFSv739JnUNARuAAATi3jauq1SXMdXMOtunw5QQcOfKM+fQZqyJARtvbXXx+lChUq2H6eM+e9LNuWLVvONjHqdeXLV9DUqTP1W3S8ra19x643rcHdvYie7h2mp3v///QRgRWL255SlKQSxYtr9Ms3n17i8fZt9Xj7tnZthctW1cqtG2w/j538Vrbbenh66oWXX9QLL7+Y7fKVM+dlaWv7YCu1fbDVTWsqaAhcAACnFpeYViDfUFC4cGHNmPGuduyorS5duqtw4cI6ePCA9uzZpVGjXnd0ebjNCFwAABigcGF3zZgxR++9N18TJryh1NSr8vcP0NixbyokpL2jy8NtRuACAMAgNWrU0vTpsx1dBpwA00IAAAAYjMAFAABgMAIXAMDxLFZZdee8xgV3npwmec0JgQsA4HDW5CRl5rwa4DDp6Wlycfn3Q98JXAAAh0v7co/ik+KUIXq64FysVqvS0lIVHx8jT8/i/3o/PKUIAHA4y+lIXd20RbGtWsjk4anE82dlsVgcXZbTMZvNSrqckqd9nD+foIyEW3+B9j+5Ws8qKflS3mq5alVqct5ruWzw9eLi4iovLx8VKeLxr/dB4AIAOAXL6UhdPR0pSaq38RPFxCQ6uCLn4+vrpVHDNuZpH5umPabIt7rkuZYqr32ibqsH5mkfa7rP1/7H8l5LQbheuKUIAABgMAIXAACAwQhcAAAABiNwAQAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABiMwAUAAGAwAhcAAIDBCFwAAAAGI3ABAAAYjMAFAABgMAIXAACAwQhcAAAABiNwAQAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABiMwAUAAGAwAhcAAIDBCFwAAAAGI3ABAAAYjMAFAABgMAIXAACAwQhcAAAABiNwAQAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABiMwAUAAGAwAhcAAIDBCFwAAAAGI3ABAAAYjMAFAABgMAIXAACAwQhcAAAABst14Nq8ebMeeeQRBQcHq3379tqwYcNN17dYLJo/f74eeughBQcHq1OnTtqyZUte6wUAAChwXHOzUkREhIYPH65evXqpefPm2rlzp0aOHCl3d3e1a9cu220mTZqk1atXa+jQoapRo4a2bNmiYcOGydPTUy1atMjXkwAAAHBmuQpc06dPV/v27TVmzBhJUvPmzZWQkKDw8PBsA1dUVJRWrlyp8ePHq2vXrpKkxo0b6/fff9e+ffsIXAAA4K6SY+CKjo5WVFSUhg4datfetm1bRUREKDo6WhUrVrRbtnPnTrm7u6tz58527R9++GHeKwYAAChgchzDFRkZKUkKCAiwa/f395cknTlzJss2J0+eVEBAgL7++ms9+uijqlWrlkJCQrR169b8qBkAAKBAyTFwJSYmSpI8PT3t2j08PCRJSUlJWbaJjY3VhQsXNGbMGD399NN6//33FRQUpCFDhujAgQP5UTcAAECBkeMtRavVKkkymUzZtpvNWTNbenq6YmNjtWDBArVq1UrStTFckZGRmjNnjh544IFcF1iypGfOK8GhfH29HF0CChCuF+QW1wpuhbNfLzkGLi+vayfwz56s5ORku+X/y8PDQy4uLmratKmtzWQyqUmTJlq3bt0tFXjpUpIsFustbWMEZ/8iHSkmJtHRJTgdrpcb43qxx7VyY1wrWXG93Jijrxez2XTTTqIcbyleH7sVFRVl13727Fm75f/L399fFotFGRkZdu3p6elZesoAAADudDkGLn9/f1WoUEHbtm2za9+xY4cqV66scuXKZdmmefPmslqtioiIsLVlZGRo3759ql+/fj6UDQAAUHDkah6usLAwjR49Wt7e3mrZsqV27dqliIgIzZgxQ9K1QfJRUVEKDAyUp6enGjdurBYtWmjixIlKSUlR5cqV9dFHH+ncuXOaNm2aoScEAADgbHIVuEJDQ5WWlqbFixdr7dq1qlixoiZPnqwOHTpIknbv3q3Ro0dr+fLlatSokSRp1qxZCg8P13vvvaeEhATVqlVLixcvVu3atY07GwAAACeUq8AlST169FCPHj2yXRYaGqrQ0FC7Nnd3d40cOVIjR47MW4UAAAAFXK5fXg0AAIB/h8AFAABgMAIXAACAwQhcAAAABiNwAQAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABiMwAUAAGAwAhcAAIDBCFwAAAAGI3ABAAAYjMAFAABgMAIXAACAwQhcAAAABiNwAQAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABiMwAUAAGAwAhcAAIDBCFwAAAAGI3ABAAAYjMAFAABgMAIXAACAwQhcAAAABiNwAQAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABiMwAUAAGAwAhcAAIDBCFwAAAAGI3ABAAAYjMAFAABgMAIXAACAwQhcAAAABiNwAQAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABiMwAUAAGAwAhcAAIDBCFwAAAAGI3ABAAAYjMAFAABgMAIXAACAwQhcAAAABiNwAQAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABiMwAUAAGAwAhcAAIDBCFwAAAAGI3ABAAAYjMAFAABgMAIXAACAwQhcAAAABiNwAQAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABiMwAUAAGAwAhcAAIDBCFwAAAAGI3ABAAAYjMAFAABgMAIXAACAwQhcAAAABiNwAQAAGIzABQAAYDACFwAAgMEIXAAAAAYjcAEAABgs14Fr8+bNeuSRRxQcHKz27dtrw4YNuT7IhQsXVL9+fc2bN+/f1AgAAFCg5SpwRUREaPjw4WratKnmzp2rhg0bauTIkdq2bVuO21qtVo0ZM0ZJSUl5LhYAAKAgcs3NStOnT1f79u01ZswYSVLz5s2VkJCg8PBwtWvX7qbbfvTRR4qMjMx7pQAAAAVUjj1c0dHRioqKUkhIiF1727ZtFRkZqejo6JtuO3XqVE2YMCHvlQIAABRQOQau671TAQEBdu3+/v6SpDNnzmS7ncVi0ahRo9S+fXs9+OCDea0TAACgwMrxlmJiYqIkydPT067dw8NDkm44NmvZsmWKjo7WggUL8lojAABAgZZj4LJarZIkk8mUbbvZnLWTLDIyUjNnztSsWbPk5eWVpwJLlvTMeSU4lK9v3r5j3F24XpBbXCu4Fc5+veQYuK4Hpn/2ZCUnJ9stvy4zM1OjRo1Su3bt1LRpU2VkZNiWWSwWZWRkyNU1V2P1JUmXLiXJYrHmen2jOPsX6UgxMYmOLsHpcL3cGNeLPa6VG+NayYrr5cYcfb2YzaabdhLlOIbr+titqKgou/azZ8/aLb/uwoULOnbsmDZs2KCgoCDbf5I0e/Zs278BAADuFjl2Nfn7+6tChQratm2bHn74YVv7jh07VLlyZZUrV85ufT8/P61bty7Lfp544gk9+eST6tKlSz6UDQAAUHDk6t5eWFiYRo8eLW9vb7Vs2VK7du1SRESEZsyYIUmKjY1VVFSUAgMD5enpqTp16mS7Hz8/vxsuAwAAuFPlaqb50NBQvfnmm/rqq68UFhamgwcPavLkyerQoYMkaffu3erevbt+/vlnQ4sFAAAoiHI9er1Hjx7q0aNHtstCQ0MVGhp60+1Pnjx5a5UBAADcIXL98moAAAD8OwQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBguQ5cmzdv1iOPPKLg4GC1b99eGzZsuOn6MTExGjt2rFq1aqW6desqNDRUERERea0XAACgwHHNzUoREREaPny4evXqpebNm2vnzp0aOXKk3N3d1a5duyzrp6WlqU+fPkpMTNQrr7wiPz8/bd++XYMHD1ZmZqY6duyY7ycCAADgrHIVuKZPn6727dtrzJgxkqTmzZsrISFB4eHh2QauvXv36sSJE1q7dq2Cg4MlSU2bNtX58+e1aNEiAhcAALir5HhLMTo6WlFRUQoJCbFrb9u2rSIjIxUdHZ1lGw8PD3Xv3l116tSxa69SpYqioqLyWDIAAEDBkmMPV2RkpCQpICDArt3f31+SdObMGVWsWNFuWePGjdW4cWO7tvT0dO3Zs0fVqlXLU8EAAAAFTY49XImJiZIkT09Pu3YPDw9JUlJSUq4ONHXqVP3+++/q16/frdYIAABQoOXYw2W1WiVJJpMp23az+eaZzWq1asqUKVq6dKleeOEFtWnT5pYKLFnSM+eV4FC+vl6OLgEFCNcLcotrBbfC2a+XHAOXl9e1E/hnT1ZycrLd8uykpaVp1KhR2rJli1544QWNGDHilgu8dClJFov1lrfLb87+RTpSTEyio0twOlwvN8b1Yo9r5ca4VrLierkxR18vZrPppp1EOQau62O3oqKiVL16dVv72bNn7Zb/U1JSkvr376/Dhw9rzJgxevbZZ2+pcAAAgDtFjmO4/P39VaFCBW3bts2ufceOHapcubLKlSuXZZvMzEwNHDhQx44d0/Tp0wlbAADgrparebjCwsI0evRoeXt7q2XLltq1a5ciIiI0Y8YMSVJsbKyioqIUGBgoT09Pffzxxzp48KC6d++usmXL6ujRo7Z9mUwm3XvvvYacDAAAgDPKVeAKDQ1VWlqaFi9erLVr16pixYqaPHmyOnToIEnavXu3Ro8ereXLl6tRo0bavn27JGn16tVavXq13b5cXFx0/PjxfD4NAAAA55WrwCVJPXr0UI8ePbJdFhoaqtDQUNvPy5cvz3tlAAAAd4hcv7waAAAA/w6BCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwGIELAADAYAQuAAAAgxG4AAAADEbgAgAAMBiBCwAAwGAELgAAAIMRuAAAAAxG4AIAADAYgQsAAMBgBC4AAACDEbgAAAAMRuACAAAwWK4D1+bNm/XII48oODhY7du314YNG266fnJyst588001bdpUdevWVd++ffX777/nsVwAAICCJ1eBKyIiQsOHD1fTpk01d+5cNWzYUCNHjtS2bdtuuM2QIUO0bds2DR8+XJMnT9Zff/2lXr16KTExMd+KBwAAKAhcc7PS9OnT1b59e40ZM0aS1Lx5cyUkJCg8PFzt2rXLsv6hQ4e0Z88eLVq0SA8++KAkqUGDBnrooYe0atUq9evXLx9PAQAAwLnl2MMVHR2tqKgohYSE2LW3bdtWkZGRio6OzrLN/v375eHhoaZNm9raSpQoofvvv1979+7Nh7IBAAAKjhwDV2RkpCQpICDArt3f31+SdObMmWy38ff3l4uLi117pUqVsl0fAADgTpbjLcXrY648PT3t2j08PCRJSUlJWbZJSkrKsv71bbJb/2bMZtMtrW8kP58i+bIfV2/fPO/Dt2iJfKhEKuyX91qc6TtyJvlxveTHtSLlz/WSH9eKxPWSHX63ZI9rJXv8bsmeo6+XnI5vslqt1putsGnTJg0fPly7du1S+fLlbe2///672rZtm+04rueff17p6elasWKFXfuMGTO0fPlyHTly5FbPAwAAoMDK8Zail5eXpKw9WcnJyXbL/5enp6dt+T+3ya7nCwAA4E6WY+C6PnYrKirKrv3s2bN2y/+5TXR0tP7ZeXb27Nls1wcAALiT5Ri4/P39VaFChSxzbu3YsUOVK1dWuXLlsmzTrFkzXb58WV9//bWtLTY2VocOHVKTJk3yoWwAAICCI1fzcIWFhWn06NHy9vZWy5YttWvXLkVERGjGjBmSroWpqKgoBQYGytPTU/fff78aNmyooUOHavjw4SpevLhmz54tLy8vPfnkk4aeEAAAgLPJcdD8dR9//LEWL16sCxcuqGLFiurXr586d+4sSfr00081evRoLV++XI0aNZIkJSQk6J133tHOnTtlsVhUv359jRo1SlWqVDHsZAAAAJxRrgMXAAAA/p1cv7waAAAA/w6BCwAAwGAELgAAAIMRuAAAAAxG4HJyPNMAAEDBR+Bycg8++KCmTp2q06dPO7oUAADwLzEthJObOnWqNm/erL/++ku1a9dWly5d9Mgjj2T7DkvgupMnT+rKlSuyWCxZltWrV88BFcFZLFiwINfrmkwm9e/f38BqgLsHgasAsFqt+uabb7R+/XrbRLKtW7fW448/rubNm8tkMjm6RDiJn376SYMGDdL58+ezLLNarTKZTPrll18cUBmcRY0aNXK9LtfL3eu5557L9d8Wk8mkDz74wOCKCj4CVwGTkpKi3bt3a9WqVTp06JB8fX3VpUsXPfnkk/Lz83N0eXCwJ598Un///bfCwsJUpkwZmc1ZRw00bNjQAZUBKEgmT56sJUuWqFixYqpevXqO669YseI2VFWwEbgKkJiYGG3evFkRERH68ccfVb58ebVo0UJfffWVLl68qLffflvt2rVzdJlwoODgYE2fPl1t2rRxdCkACrgPP/xQb7/9tpYsWcL/UcsHBC4nd+XKFe3YsUOfffaZDhw4oEKFCikkJERdunSxvbfSarWqT58+OnnypL766isHVwxHatmypcaNG6dWrVo5uhQ4qZCQkFsahrB9+3YDq4GzGzZsmE6cOKEtW7Y4upQCz9XRBeDmmjRpoqtXryo4OFjjxo1Thw4d5OnpabeOyWRS3bp1dfLkSQdVCWfRs2dPvffee3rggQdUpEgRR5cDJ1SvXj3GfSJHaWlpcnNz04gRIzRz5kydPn1aVatWdXRZBRo9XE5u8uTJeuKJJ3K80JOSkuTu7i5XVzL03eyNN95QRESELBaL7rnnniyhi8GtAHKjWbNmmjNnju677z7NmTNHXbt2VenSpR1dVoHGX2cnN3LkSJ04cUJLly5V7969JV175H/FihV67rnnbEHsn71euDudOXPG7im09PR0B1aDgiA1NVWnTp1Senq6baJli8WiK1eu6NChQxoyZIiDK4QjXL58WX/99Zckae7cuXrwwQcJXHlED5eT++abb9SvXz9Vq1ZNn376qSTpxx9/1PDhw3Xx4kUtW7ZMwcHBDq4SQEF08OBBDR48WHFxcdku9/Dw0KFDh25zVXAGzzzzjI4cOSI/Pz+dP39evr6+cnNzy3Zdk8mknTt33uYKCx4Cl5Pr3r27ypYtq2nTpsnFxcXWbrFYNHToUF26dInHcZHFb7/9poMHDyopKUk+Pj6qX7++qlSp4uiy4GR69uypuLg4DR06VJ999pnMZrNCQ0O1d+9erVq1SitXrlTdunUdXSYc4OLFi1qxYoXi4+O1bt06tWrVSiVKlLjh+hMnTryN1RVM3FJ0cr/++qsGDx5sF7YkyWw2q1u3bnrppZccVBmckcVi0RtvvKFPPvnE7j2cJpNJnTt31qRJkxgwDZtffvlFEydO1MMPP6zExER9/PHHatGihVq0aKH09HTNnz9f7733nqPLhAP4+flp2LBhkqT9+/frlVdeuaVJc5EV71J0cp6enoqKisp22blz53gSDXbee+89bdiwQcOGDdOePXv0888/a/fu3Ro6dKg2b96s999/39ElwolYLBbbuBx/f3+dOnXKtiwkJETHjx93VGlwIrt27SJs5QMCl5MLCQnRzJkztW/fPrv2b775RuHh4UxwCTvr1q3TgAED1KdPH5UuXVouLi4qU6aM+vbtq/79+2vdunWOLhFOpFKlSraQFRAQoCtXrigyMlKSlJmZqeTkZEeWB9xRuKXo5IYMGaKffvpJffv2VeHChVWiRAnFxcUpNTVVderU0auvvuroEuFEYmJiVL9+/WyX1atXj9tDsNOxY0dNmTJFFotFTz31lGrXrq233npLvXr10vz58xUYGOjoEoE7BoHLyXl6emrVqlXas2ePvv/+eyUkJMjT01P169dX69ats31XHu5eFStW1JEjR9S4ceMsy44cOSJfX18HVAVn1bdvX8XGxurw4cN66qmnNG7cOFtvqKenp+bPn+/oEoE7Bk8pAneQpUuXavr06Ro6dKg6dOigUqVK6e+//9aWLVs0Y8YM9e/fX2FhYY4uE04sKSlJkZGRqlKlCvP7AfmIwFUAbNu2Td999122ExMeOXJEX375pYMrhLPIzMzUmDFjtHHjRrunEa1Wqx599FG988479IrCzp49e3TgwAGNHDlSkvTDDz/YwvkDDzzg4OqAOwe3FJ3c3LlzNXv2bHl5eSkjI0OFChWSq6urYmNjZTab1bVrV0eXCCfi4uKiyZMnq0+fPvruu+90+fJlFStWTA0bNmQ8DrLYunWrhg0bpubNm9vaihQpIovFohdeeEHz58/Xgw8+6MAKgTsHPVxOrk2bNmrQoIHefvtthYeH68KFC5o8ebJ++ukn9evXTy+++KKefvppR5cJoAB69NFH1bBhQ40dOzbLsgkTJujYsWM82QrkE3q4nNyff/6pTp06yWQyKSgoSFu3bpUk1a5dWwMGDNDatWsJXHe5tm3bKjw8XDVq1FBISEiOE5tu3779NlUGZxcVFaUxY8Zku6xNmza214kByDsCl5MrWrSobcxNpUqV9Mcff+jq1atyd3dXzZo19ccffzi4QjhavXr15OHhYfs3M8kjt0qWLKmff/4527FaJ0+elLe3twOqAu5MBC4nV6dOHW3cuFGNGzdWQECAXFxcdODAAbVs2VJnzpy54ctEcfd4++23bf9+5513HFgJCppOnTppzpw58vDwUJs2bVSyZEnFxsZq165dmj17tnr27OnoEoE7BmO4nNy3336rF154Qc2bN9f8+fP12muvaefOnWrcuLH27NmjNm3aaMqUKY4uE04kOTlZycnJ8vPzU3p6ulauXKkLFy4oJCTkhpOi4u6Unp6uYcOGaceOHVmeag0JCdG0adNUqFAhB1YI3DkIXAXA8ePH9euvv6pz585KTU3VxIkTdfjwYQUHB2vUqFF0+8Pm2LFj6tu3r7p3765hw4Zp3LhxWr16tYoVK6bk5GTNnj1brVu3dnSZcDK//vqrbWJlLy8v1a9fn3fnAfmMwOXkli5dqhYtWiggIMDRpaAAeO6553TlyhVNmTJFpUqVUuPGjRUaGqo33nhDb7zxhn755RetXbvW0WXCCWVkZCguLk4+Pj5ydWW0CZDfmAHRyc2dO1dnz551dBkoII4dO6aBAweqYsWK2r9/v1JTU/XYY49Jkjp06GB7UTFw3U8//aQXXnhBdevWVYsWLXTy5EmNGjVKc+fOdXRpwB2FwOXkKlasqDNnzji6DBQQZrNZhQsXliTt27dPxYoVU3BwsKRrr2xxd3d3ZHlwMocPH1bPnj0VHx+vfv362d5kUaZMGc2ZM0cfffSRgysE7hz0Gzu5Nm3aaNq0adq9e7eqVaumUqVK2S03mUzq37+/g6qDs6ldu7bWrl0rd3d3bdu2TS1btpTJZNKlS5e0aNEi1a5d29ElwolMnTpVTZo00YIFC5SRkWHr1Ro8eLCuXr2qVatW8aQikE8Yw+Xkchq4ajKZ9Msvv9ymauDsfv75Z/Xp00dxcXEqUaKEPvroI1WuXFmNGzeW1WrV4sWLVatWLUeXCSdx7733atasWWrRooUyMzMVFBSkTz75REFBQfr222/Vv39/HT161NFlAncEeric3IkTJxxdAgqQoKAgff755zp9+rSqVaumokWLSrr2mpZ69eqpRIkSDq4QzsTDw0OXLl3Kdtlff/1lm1AXQN4xhgu4w3h6euree++1hS3p2q1pwhb+qXXr1po5c6aOHz9uazOZTIqJidHChQvVokULB1YH3Fm4pejknn/++RzXWbx48W2oBM6Kdyni34qPj9ezzz6rU6dOqXTp0rpw4YICAwN17tw5+fn5adWqVQR1IJ9wS9HJpaenZ2lLSUnR6dOnVbRoUYWEhDigKjgT3qWIf2vhwoV6/fXXFRkZqQMHDiggIECenp7q0aOHQkND7XpJAeQNPVwFVEJCgvr27auOHTuqV69eji4HTsxqtRLCkK369etr1qxZatq0qaNLAe54jOEqoLy9vdWvXz8tXbrU0aXAyaxatUpDhw61/Xzo0CGFhIRo/fr1DqwKzigoKEj79+93dBnAXYFbigXcjZ4wwt3pww8/1FtvvaWuXbva2sqUKaMGDRpo7NixMpvNtpnngaCgIC1fvlyff/65AgMDs53nb/z48Q6qDrizcEvRyR0+fDhLm8Vi0YULFzR79myVKlWK2aBh07ZtWz366KMKCwvLsmzOnDnavn27Nm3a5IDK4IxyepG5yWTSF198cZuqAe5s9HA5uZ49e8pkMtmNw7mekcuWLasxY8Y4sjw4mT///FP16tXLdln9+vW1aNGi21wRnNmuXbscXQJw1yBwObnly5dnaTOZTPL09FT16tVlNjMMD/+vXLly+vbbb9W4ceMsy77//nuVLl3aAVUBAAhcTq5hw4a6evWqfv75Z9WvX1/StV6MgwcPKiAggJcRw0737t01depUZWZm2iY7jYuL065du/TBBx9o0KBBji4RAO5KjOFyctHR0erdu7esVqut+3///v3q06ePAgMD9f7779NrATuTJ0/WihUrlJmZaWtzcXHRM888o5EjRzqwMgC4exG4nFxYWJj++OMPhYeHq3Llyrb2c+fO6aWXXlLVqlU1depUxxUIp5SYmKijR48qPj5eXl5eCg4OZsZwAHAgBgA5uUOHDmnIkCF2YUuSypcvr7CwMH399deOKQxOzcvLS1WrVlXFihXVsGFDFSlSxNElAcBdjTFcTs5qtSotLe2Gy69evXobq0FBsGvXLr377rs6e/asTCaT1q5dq3nz5snb21sTJkyQi4uLo0sEgLsOPVxO7v7779e8efMUHx9v13758mUtXLhQDRs2dExhcEq7du1SWFiYAgMDNXHiRFksFklSkyZN9Nlnn2nhwoUOrhAA7k6M4XJyZ86cUbdu3ZSZmal69eqpZMmSio2N1eHDh+Xq6qqPPvpIVatWdXSZcBKPP/64atasqUmTJikzM1NBQUH65JNPFBQUpPnz52vDhg3avn27o8sEgLsOPVxOLiAgQJs3b1b37t1tA6FjY2PVpUsXbdiwgbAFO6dPn1aHDh2yXVa/fn1duHDhNlcEAJAYw1UglC5dWi+//LKKFi0qSUpKSlJSUpLKlCnj4MrgbHx8fPT777+rWbNmWZb9/vvv8vHxcUBVAAB6uJxcSkqKhgwZom7dutnajh49qlatWmn06NE3HVCPu0+HDh0UHh6unTt3Kj09XdK1NxOcOHFC8+bNU7t27RxcIQDcnRjD5eQmTpyoTZs2acSIEerSpYskKTk5WZ9//rkmT56s7t27a/DgwY4tEk4jNTVVYWFh+uqrr+Tq6qqMjAwVK1ZMiYmJqlu3rt5//31bTykA4PYhcDm5Bx98UEOGDNHjjz+eZdnatWs1f/58XkCLLPbv368DBw4oPj5enp6eatiwoVq2bGl7AToA4PZiDJeTS0xMVMmSJbNdVrZsWV26dOk2VwRnNnjwYD355JNq2rSpmjZt6uhyAAD/xRguJ1e9enWtX78+22UbN25UtWrVbnNFcGb79u0TndYA4Hzo4XJyAwcO1IABA3Tu3Dm1adPGNg/Xl19+qaNHj2revHmOLhFOpEmTJlq/fr3q1asnNzc3R5cDAPgvxnAVAF9++aVmz56tX375xdZ7UaNGDb3yyitq3bq1g6uDMxk1apQ2b96sQoUKqWLFiipVqpTdcpPJpA8++MBB1QHA3YvAVUDExcUpLS1NmZmZcnFxkdVqVUpKir7//nt17drV0eXBSTzzzDM5rrNixYrbUAkA4H8RuJzcyZMnNXz4cP3222/ZLjeZTDp+/PhtrgrO7vTp0zp06JASEhJUsmRJNWrUSBUqVHB0WQBw12IMl5N79913FR8fr5EjR+rLL7+Um5ubWrVqpb1792rPnj1avny5o0uEE0lNTdWrr76qzz//3G7wvNls1hNPPKE333yTqSEAwAF4StHJHT16VIMGDVLv3r3VoUMHXblyRT179tSCBQv08MMPc3sIdt555x3t27dPY8eO1b59+/Tzzz9r7969GjVqlDZt2qS5c+c6ukQAuCsRuJxcWlqaKleuLEmqXLmyTpw4YVsWGhqqo0ePOqYwOKVt27Zp6NCheuqpp+Tr6ysXFxf5+fmpV69eGjRokNauXevoEgHgrkTgcnLlypXTH3/8Iela4EpKStK5c+ckSYULF1ZCQoIjy4OTSU9PV8WKFbNdVrVqVSUmJt7migAAEoHL6bVp00ZTp07V559/rtKlS6tKlSoKDw/X6dOntXTp0hv+ccXdqXPnzlq0aJGuXr1q126xWLRq1Sp17NjRQZUBwN2NpxSd3JUrV/Tqq68qNTVVixYt0r59+/TSSy8pLS1NLi4umj59ukJCQhxdJpzE7NmztWLFCrm6uqp169by8/NTfHy89u3bp3PnzqlTp062CVFNJpPGjx/v4IoB4O5A4Cog0tLSbH8oo6Oj9dNPPykoKEiVKlVycGVwJrcyEa7JZNIXX3xhYDUAgOsIXAAAAAZjDBcAAIDBCFwAAAAGI3ABAAAYjMAFAABgMAIXAACAwf4PB5RjYLQmurIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "compare_metrics = pd.DataFrame({\"baseline\": baseline_metrics,\n",
    "                                \"clf_2\": clf_2_metrics,\n",
    "                                \"random search\": rs_metrics,\n",
    "                                \"grid search\": gs_metrics})\n",
    "\n",
    "compare_metrics.plot.bar(figsize=(10, 8));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving and loading trained machine learning models\n",
    "Two ways to save and load machine learning models:\n",
    "\n",
    "1. With Python's pickle module\n",
    "2. With the joblib module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "# Save an extisting model to file\n",
    "pickle.dump(gs_clf, open(\"gs_random_random_forest_model_1.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load a saved model\n",
    "loaded_pickle_model = pickle.load(open(\"gs_random_random_forest_model_1.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 78.69%\n",
      "Precision: 0.74\n",
      "Recall: 0.82\n",
      "F1 score: 0.78\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.79, 'precision': 0.74, 'recall': 0.82, 'f1': 0.78}"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make some predictions\n",
    "pickle_y_preds = loaded_pickle_model.predict(X_test)\n",
    "evaluate_preds(y_test, pickle_y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Joblib**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gs_random_forest_model_1.joblib']"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from joblib import dump, load\n",
    "\n",
    "# Save model to file\n",
    "dump(gs_clf, filename=\"gs_random_forest_model_1.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import a saved joblib model\n",
    "loaded_joblib_model = load(filename=\"gs_random_forest_model_1.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 78.69%\n",
      "Precision: 0.74\n",
      "Recall: 0.82\n",
      "F1 score: 0.78\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.79, 'precision': 0.74, 'recall': 0.82, 'f1': 0.78}"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make and evaluate joblib predictions\n",
    "joblib_y_preds = loaded_joblib_model.predict(X_test)\n",
    "evaluate_preds(y_test, joblib_y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Putting it all together!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>Black</td>\n",
       "      <td>35820.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>32042.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>NaN</td>\n",
       "      <td>White</td>\n",
       "      <td>155144.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5716.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>66604.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>31570.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>215883.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>Blue</td>\n",
       "      <td>248360.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12732.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Make Colour  Odometer (KM)  Doors    Price\n",
       "0     Honda  White        35431.0    4.0  15323.0\n",
       "1       BMW   Blue       192714.0    5.0  19943.0\n",
       "2     Honda  White        84714.0    4.0  28343.0\n",
       "3    Toyota  White       154365.0    4.0  13434.0\n",
       "4    Nissan   Blue       181577.0    3.0  14043.0\n",
       "..      ...    ...            ...    ...      ...\n",
       "995  Toyota  Black        35820.0    4.0  32042.0\n",
       "996     NaN  White       155144.0    3.0   5716.0\n",
       "997  Nissan   Blue        66604.0    4.0  31570.0\n",
       "998   Honda  White       215883.0    4.0   4001.0\n",
       "999  Toyota   Blue       248360.0    4.0  12732.0\n",
       "\n",
       "[1000 rows x 5 columns]"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make              object\n",
       "Colour            object\n",
       "Odometer (KM)    float64\n",
       "Doors            float64\n",
       "Price            float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Steps we want to do (all in one cell):\n",
    "\n",
    "1. Fill missing data\n",
    "2. Convert data to numbers\n",
    "3. Build a model on the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22188417408787875"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Getting data ready #Sk-Learn Pipeline\n",
    "import pandas as pd\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "# Modelling\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "\n",
    "# Setup random seed\n",
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "\n",
    "# Import data and drop rows with missing labels\n",
    "data = pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "data.dropna(subset=[\"Price\"], inplace=True)\n",
    "\n",
    "# Define different features and transformer pipeline\n",
    "categorical_features = [\"Make\", \"Colour\"]\n",
    "categorical_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"missing\")),\n",
    "    (\"onehot\", OneHotEncoder(handle_unknown=\"ignore\"))])\n",
    "\n",
    "door_feature = [\"Doors\"]\n",
    "door_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=4))\n",
    "])\n",
    "\n",
    "numeric_features = [\"Odometer (KM)\"]\n",
    "numeric_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"mean\"))\n",
    "])\n",
    "\n",
    "# Setup preprocessing steps (fill missing values, then convert to numbers)\n",
    "preprocessor = ColumnTransformer(\n",
    "                    transformers=[\n",
    "                        (\"cat\", categorical_transformer, categorical_features),\n",
    "                        (\"door\", door_transformer, door_feature),\n",
    "                        (\"num\", numeric_transformer, numeric_features)\n",
    "                    ])\n",
    "\n",
    "# Creating a preprocessing and modelling pipeline\n",
    "model = Pipeline(steps=[(\"preprocessor\", preprocessor),\n",
    "                        (\"model\", RandomForestRegressor())])\n",
    "\n",
    "# Split data\n",
    "X = data.drop(\"Price\", axis=1)\n",
    "y = data[\"Price\"]\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Fit and score the model\n",
    "model.fit(X_train, y_train)\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 16 candidates, totalling 80 fits\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.4s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.4s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.4s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.4s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   3.8s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.8s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.7s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   3.1s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   3.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   3.7s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   3.4s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.7s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.5s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.8s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.5s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.7s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.4s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.4s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   4.5s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   3.7s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.7s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   3.0s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   3.1s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.4s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.5s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.5s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   3.6s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.1s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.1s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.6s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.8s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.1s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.6s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.6s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.1s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.7s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.9s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.8s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  80 out of  80 | elapsed:  1.9min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5,\n",
       "             estimator=Pipeline(steps=[('preprocessor',\n",
       "                                        ColumnTransformer(transformers=[('cat',\n",
       "                                                                         Pipeline(steps=[('imputer',\n",
       "                                                                                          SimpleImputer(fill_value='missing',\n",
       "                                                                                                        strategy='constant')),\n",
       "                                                                                         ('onehot',\n",
       "                                                                                          OneHotEncoder(handle_unknown='ignore'))]),\n",
       "                                                                         ['Make',\n",
       "                                                                          'Colour']),\n",
       "                                                                        ('door',\n",
       "                                                                         Pipeline(steps=[('imputer',\n",
       "                                                                                          SimpleImputer(fill_value=4,\n",
       "                                                                                                        strategy='constant'))]),\n",
       "                                                                         ['Doors']),\n",
       "                                                                        ('num',\n",
       "                                                                         Pipeline(steps=[('imputer',\n",
       "                                                                                          SimpleImputer())]),\n",
       "                                                                         ['Odometer '\n",
       "                                                                          '(KM)'])])),\n",
       "                                       ('model', RandomForestRegressor())]),\n",
       "             param_grid={'model__max_depth': [None, 5],\n",
       "                         'model__max_features': ['auto'],\n",
       "                         'model__min_samples_split': [2, 4],\n",
       "                         'model__n_estimators': [100, 1000],\n",
       "                         'preprocessor__num__imputer__strategy': ['mean',\n",
       "                                                                  'median']},\n",
       "             verbose=2)"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use GridSearchCV with our regression Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "pipe_grid = {\n",
    "    \"preprocessor__num__imputer__strategy\": [\"mean\", \"median\"],\n",
    "    \"model__n_estimators\": [100, 1000],\n",
    "    \"model__max_depth\": [None, 5],\n",
    "    \"model__max_features\": [\"auto\"],\n",
    "    \"model__min_samples_split\": [2, 4]    \n",
    "}\n",
    "\n",
    "gs_model = GridSearchCV(model, pipe_grid, cv=5, verbose=2)\n",
    "gs_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3339554263158365"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}