2022-50-007-ml-project.ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","cell_id":"ac98df94-54a7-4e1a-8138-7f5c7571be69","deepnote_cell_height":369,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":14322146,"execution_start":1658682577993,"source_hash":"f2620bd0","trusted":true},"outputs":[],"source":["# # This Python 3 environment comes with many helpful analytics libraries installed\n","# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n","# # For example, here's several helpful packages to load\n","\n","# import numpy as np # linear algebra\n","# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n","\n","# # Input data files are available in the read-only \"../input/\" directory\n","# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n","\n","# import os\n","# for dirname, _, filenames in os.walk('/kaggle/input'):\n","#     for filename in filenames:\n","#         print(os.path.join(dirname, filename))\n","\n","# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n","# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"]},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[],"source":["import time\n","start_time = time.time()"]},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Process Initialised\n"]}],"source":["print(\"Process Initialised\")"]},{"cell_type":"markdown","metadata":{"cell_id":"00001-86291d65-f0ea-4c22-8d29-15f5f0273874","deepnote_cell_height":82,"deepnote_cell_type":"markdown"},"source":["# Load Training Dataset"]},{"cell_type":"code","execution_count":4,"metadata":{"cell_id":"eef26f675db14448bd91c5b241573c9a","deepnote_cell_height":99,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":11860273,"execution_start":1658682577995,"source_hash":"b7dcc9c8","tags":[],"trusted":true},"outputs":[],"source":["import numpy as np\n","import pandas as pd"]},{"cell_type":"code","execution_count":5,"metadata":{"cell_id":"00002-a0331af6-c3fd-496c-b8c9-2e9fbcf41caa","deepnote_cell_height":99,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":1,"execution_start":1658682578049,"source_hash":"3f76624c","trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>17179</th>\n","      <td>17180</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17180</th>\n","      <td>17181</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17181</th>\n","      <td>17182</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17182</th>\n","      <td>17183</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17183</th>\n","      <td>17184</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>17184 rows × 5002 columns</p>\n","</div>"],"text/plain":["          id  label    0    1    2    3    4    5    6    7  ...  4990  4991  \\\n","0          1      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1          2      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2          3      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3          4      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4          5      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...      ...    ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","17179  17180      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17180  17181      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17181  17182      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17182  17183      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17183  17184      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","       4992  4993  4994  4995  4996  4997  4998  4999  \n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...     ...   ...   ...   ...   ...   ...   ...   ...  \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[17184 rows x 5002 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 16.2 s\n","Wall time: 16.3 s\n"]}],"source":["%%time\n","df_train = pd.read_csv(r\"./source/train_tfidf_features.csv\")\n","display(df_train)"]},{"cell_type":"code","execution_count":6,"metadata":{"cell_id":"00003-d747b669-d32c-43a4-b111-dea206d21537","deepnote_cell_height":534.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":16711,"execution_start":1658682597472,"source_hash":"849ce5a8","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 10.8 s\n","Wall time: 11 s\n"]},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>...</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>8592.500000</td>\n","      <td>0.381227</td>\n","      <td>0.000150</td>\n","      <td>0.001066</td>\n","      <td>0.001532</td>\n","      <td>0.000369</td>\n","      <td>0.000140</td>\n","      <td>0.000066</td>\n","      <td>0.000270</td>\n","      <td>0.000483</td>\n","      <td>...</td>\n","      <td>0.000202</td>\n","      <td>0.000429</td>\n","      <td>0.000286</td>\n","      <td>0.000075</td>\n","      <td>0.000260</td>\n","      <td>0.000709</td>\n","      <td>0.000257</td>\n","      <td>0.000121</td>\n","      <td>0.000308</td>\n","      <td>0.000159</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>4960.737848</td>\n","      <td>0.485702</td>\n","      <td>0.008297</td>\n","      <td>0.019532</td>\n","      <td>0.024741</td>\n","      <td>0.012334</td>\n","      <td>0.008276</td>\n","      <td>0.005065</td>\n","      <td>0.009907</td>\n","      <td>0.013106</td>\n","      <td>...</td>\n","      <td>0.010215</td>\n","      <td>0.013178</td>\n","      <td>0.011378</td>\n","      <td>0.005866</td>\n","      <td>0.010864</td>\n","      <td>0.017641</td>\n","      <td>0.010246</td>\n","      <td>0.006529</td>\n","      <td>0.010526</td>\n","      <td>0.008536</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>1.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>4296.750000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>8592.500000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>12888.250000</td>\n","      <td>1.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>17184.000000</td>\n","      <td>1.000000</td>\n","      <td>0.676327</td>\n","      <td>0.560830</td>\n","      <td>0.958430</td>\n","      <td>0.646740</td>\n","      <td>0.532789</td>\n","      <td>0.437760</td>\n","      <td>0.435835</td>\n","      <td>0.536746</td>\n","      <td>...</td>\n","      <td>0.611122</td>\n","      <td>0.540809</td>\n","      <td>0.566613</td>\n","      <td>0.592170</td>\n","      <td>0.617341</td>\n","      <td>0.850605</td>\n","      <td>0.484908</td>\n","      <td>0.398105</td>\n","      <td>0.430031</td>\n","      <td>0.528556</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8 rows × 5002 columns</p>\n","</div>"],"text/plain":["                 id         label             0             1             2  \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean    8592.500000      0.381227      0.000150      0.001066      0.001532   \n","std     4960.737848      0.485702      0.008297      0.019532      0.024741   \n","min        1.000000      0.000000      0.000000      0.000000      0.000000   \n","25%     4296.750000      0.000000      0.000000      0.000000      0.000000   \n","50%     8592.500000      0.000000      0.000000      0.000000      0.000000   \n","75%    12888.250000      1.000000      0.000000      0.000000      0.000000   \n","max    17184.000000      1.000000      0.676327      0.560830      0.958430   \n","\n","                  3             4             5             6             7  \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean       0.000369      0.000140      0.000066      0.000270      0.000483   \n","std        0.012334      0.008276      0.005065      0.009907      0.013106   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.646740      0.532789      0.437760      0.435835      0.536746   \n","\n","       ...          4990          4991          4992          4993  \\\n","count  ...  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean   ...      0.000202      0.000429      0.000286      0.000075   \n","std    ...      0.010215      0.013178      0.011378      0.005866   \n","min    ...      0.000000      0.000000      0.000000      0.000000   \n","25%    ...      0.000000      0.000000      0.000000      0.000000   \n","50%    ...      0.000000      0.000000      0.000000      0.000000   \n","75%    ...      0.000000      0.000000      0.000000      0.000000   \n","max    ...      0.611122      0.540809      0.566613      0.592170   \n","\n","               4994          4995          4996          4997          4998  \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean       0.000260      0.000709      0.000257      0.000121      0.000308   \n","std        0.010864      0.017641      0.010246      0.006529      0.010526   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.617341      0.850605      0.484908      0.398105      0.430031   \n","\n","               4999  \n","count  17184.000000  \n","mean       0.000159  \n","std        0.008536  \n","min        0.000000  \n","25%        0.000000  \n","50%        0.000000  \n","75%        0.000000  \n","max        0.528556  \n","\n","[8 rows x 5002 columns]"]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","df_train.describe()"]},{"cell_type":"markdown","metadata":{},"source":["# Task 1: Implement Logistic Regression\n","Recalled that you have learned about Logistic Regression in your earlier class. Your task is to implement a Logistic Regression model from scratch. \\\n","Note that you are NOT TO USE the sklearn logistic regression package or any other pre-defined logistic regression package for this task! \\\n","Usage of any logistic regression packages will result in 0 marks for this task.\n","\n","## Key Task Deliverables\n","1a. Code implementation of the Logistic Regression model. \\\n","1b. Prediction made by your Logistic Regression on the Test set. Note that you are welcome to submit your predicted labels to Kaggle but you will need to submit the final prediction output in the final project submission. Please label the file as \"LogRed_Prediction.csv\"."]},{"cell_type":"markdown","metadata":{"cell_id":"00005-49b9c0d6-4381-409b-82b2-2a298f466e01","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown"},"source":["-- `sigmoid(z)`: A function that takes in a Real Number input and returns an output value between 0 and 1."]},{"cell_type":"code","execution_count":7,"metadata":{"cell_id":"00006-2e3c692b-00a7-4160-acbf-0da222258054","deepnote_cell_height":135,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":0,"execution_start":1658687327448,"source_hash":"f9e00733","trusted":true},"outputs":[],"source":["def sigmoid(z):\n","    result = 1/(1 + np.exp(-z))\n","#     print(f\"sigmoid: {result}\")\n","    return result"]},{"cell_type":"markdown","metadata":{"cell_id":"00007-968b9856-b6f3-40bd-9301-356dee468412","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown"},"source":["-- `loss(y, y_hat)`: A loss function that allows us to minimize and determine the optimal parameters. The function takes in the actual labels y and the predicted labels yhat, and returns the overall training loss. Note that you should be using the Log Loss function taught in class."]},{"cell_type":"markdown","metadata":{"cell_id":"3bd8cc27f03146c58682dabe9bdefae8","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown","tags":[]},"source":["Note: We have decided the add a regulariser (denoted by the `lmb` term) to observe whether there is an improvement in utilising L2 regularisation in our Logisitic Regression Model. As such, we have decided to change the arguments of the function  to accomodate for regularisation."]},{"cell_type":"code","execution_count":8,"metadata":{"cell_id":"00008-79c60bbd-81a5-460c-91a3-d3d8b7cbfd6b","deepnote_cell_height":333,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":0,"execution_start":1658687327449,"source_hash":"296019a7","trusted":true},"outputs":[],"source":["def loss(y, X, w, b, lmb):\n","    y_hat = sigmoid(np.dot(X, w) + b)\n","    m = np.shape(y)[0]\n","    \n","    loss = -1 * np.where(y == 1, np.log(y_hat), np.log(1 - y_hat)).mean()\n","    reg = lmb * np.sum(w**2) / (2 * m)\n","    error = loss + reg\n","    \n","#     print(f\"training loss = {loss}, regularisation term = {reg}, training error = {error}\")\n","    return error"]},{"cell_type":"markdown","metadata":{"cell_id":"00009-9fef5aa0-38be-400f-a766-b0fbdaa5d8d6","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown"},"source":["-- `gradients(X, y, y_hat)`: The Gradient Descent Algorithm to find the optimal values of our parameters. The function takes in the training feature X, actual labels y and the predicted labels yhat, and returns the partial derivative of the Loss function with respect to weights (dw) and bias (db)."]},{"cell_type":"markdown","metadata":{"cell_id":"5d4d961b83aa42039d5853a59939b89a","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Likewise, the arguments of the `gradients` function has been altered to accommodate for L2 regularisation."]},{"cell_type":"code","execution_count":9,"metadata":{"cell_id":"00010-20d9c8e4-9d38-4ed5-95f4-3c2fd6c26fb1","deepnote_cell_height":243,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":2,"execution_start":1658687327454,"source_hash":"8919acd2","trusted":true},"outputs":[],"source":["def gradients(y, X, w, b, lmb):\n","    # m - number of training examples\n","    m = np.shape(X)[0]\n","    y_hat = sigmoid(np.dot(X, w) + b)\n","    \n","    dw = (1 / m) * (np.dot(X.T, (y_hat - y)) + lmb * w)\n","    db = (1 / m) * np.sum((y_hat - y))\n","    \n","#     print(f\"dw: {dw}, db: {db}\")\n","    return dw, db"]},{"cell_type":"markdown","metadata":{"cell_id":"00011-2af85165-6958-44ec-a0d5-2fa2a2a13a7c","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown"},"source":["-- `train(X, y, bs, epochs, lr)`: The training function for your model."]},{"cell_type":"markdown","metadata":{"cell_id":"59627bc57f214e5aab30f859f0436502","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["We added the `C` term to represent the penalty term `lmb`. The relationship is that `lmb = 1/C` if `C != 0`. Otherwise,`lmb = 0`, where we do not apply regularisation."]},{"cell_type":"code","execution_count":10,"metadata":{"cell_id":"00012-1ef81ca6-ff76-4273-8db1-e073ae3b5b60","deepnote_cell_height":1377,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":42,"execution_start":1658685590567,"source_hash":"44826cb7","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train(X, y, bs, epochs, lr, C):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = np.zeros((d, 1))\n","    b = 0\n","#     w = rng.uniform(size=(d,1))\n","#     b = rng.random()\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            w_new = w.copy() - lr * dw\n","            b_new = b - lr * db\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            if (loss_new < loss_old):\n","#                 print(w == w_new, b == b_new);\n","#                 print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","\n","                w = w_new\n","                b = b_new\n","                old_w.append(w_new)\n","                old_b.append(b_new)\n","                old_losses.append(loss_new)\n","    \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"00019-ced096de-42a1-4c68-ade7-88807850e19b","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown"},"source":["-- `predict(X, w, b)`: The prediction function where you can apply your validation and test sets."]},{"cell_type":"code","execution_count":11,"metadata":{"cell_id":"00020-e9cb6e0c-03aa-42db-943c-b6bb4a9913f0","deepnote_cell_height":135,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":6,"execution_start":1658687691735,"source_hash":"7d794aee","trusted":true},"outputs":[],"source":["def predict(X, w, b):\n","    y_pred = sigmoid(np.dot(X, w) + b)\n","    pred_labels = np.array([1 if i >= 0.5 else 0 for i in y_pred])\n","    return pred_labels"]},{"cell_type":"markdown","metadata":{"cell_id":"00021-c62a3fc4-3c62-432f-9f13-64ea72f7e3da","deepnote_cell_height":212,"deepnote_cell_type":"markdown"},"source":["## Performance Evaluation\n","\n","As per the grading rubric - \"Perfect Implementation of the Logistics Regression algorithm. Successfully trained the implemented model with the train set and achieved comparative performance compared to SKLearn Logistic Regression package\", we shall compare the performance of our model with the SKLearn LogisticRegression and SGDClassifier package.\n","\n","We shall first implement a function to evaluate the accuracy of our model. The goal is to achieve a Macro-F1 score that is within 0.05 of the Macro-F1 score of the SKLearn Package"]},{"cell_type":"code","execution_count":12,"metadata":{"cell_id":"00024-700a0602-b8f9-47d5-85c9-dfb5b28bf780","deepnote_cell_height":117,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":3,"execution_start":1658687562415,"source_hash":"72d19504","trusted":true},"outputs":[],"source":["from sklearn.linear_model import LogisticRegression, SGDClassifier\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import f1_score"]},{"cell_type":"code","execution_count":13,"metadata":{"cell_id":"00022-a17b0e65-fcbb-44d1-8ded-a7923d6d255c","deepnote_cell_height":225,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":1,"execution_start":1658687514001,"source_hash":"754349d7","trusted":true},"outputs":[],"source":["def score(y, y_hat):\n","    accuracy = np.sum(y == y_hat) / np.shape(y)[0]\n","    f1score = f1_score(y, y_hat, average='macro')\n","\n","    print(f\"Accuracy: {accuracy}\")\n","    print(f\"Macro-F1 score: {f1score}\")\n","\n","    # Return Macro-F1 score of the model\n","    return f1score"]},{"cell_type":"code","execution_count":14,"metadata":{"cell_id":"81cc9876cd5a44c9bf079ddd3cc51847","deepnote_cell_height":315,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":2,"execution_start":1658685600957,"source_hash":"c6dbbe00","tags":[],"trusted":true},"outputs":[],"source":["def perform(LogReg, SGD, **scores):\n","    models = []\n","\n","    for model, score in scores.items():\n","        result = max(abs(LogReg - score), abs(SGD - score))\n","        print(f\"Model: {model}, Macro-F1 Score: {score}, Difference: {result}\")\n","\n","        if result <= 0.05:\n","            models.append(model)\n","    \n","    quality = \"Success\" if len(models) > 0 else \"Failed\"\n","    print(f\"Model {quality}\")\n","\n","    return models"]},{"cell_type":"code","execution_count":15,"metadata":{"cell_id":"00025-7a497f62-9e30-42a1-bb11-ac7f470daeb6","deepnote_cell_height":148,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":267,"execution_start":1658682615773,"source_hash":"87db3624","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X: (17184, 5000), y: (17184,)\n","CPU times: total: 219 ms\n","Wall time: 234 ms\n"]}],"source":["%%time\n","X = df_train.iloc[:, 2:5002].to_numpy()\n","y = df_train.iloc[:,1].to_numpy()\n","print(f\"X: {X.shape}, y: {y.shape}\")"]},{"cell_type":"markdown","metadata":{"cell_id":"c46261c92089447cacdad83b77304580","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Splitting of Training Data Set to Perform Internal Validation of ML Model"]},{"cell_type":"code","execution_count":16,"metadata":{"cell_id":"00026-6f193c9e-3618-4484-8208-a051961372e4","deepnote_cell_height":168,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":1079,"execution_start":1658682616031,"source_hash":"2253195f","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 5000), y_train: (12888,)\n","X_test: (4296, 5000), y_test: (4296,)\n","CPU times: total: 2.09 s\n","Wall time: 2.21 s\n"]}],"source":["%%time\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=100)\n","print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n","print(f\"X_test: {X_test.shape}, y_test: {y_test.shape}\")"]},{"cell_type":"markdown","metadata":{"cell_id":"7c27a48a55ce4ae78c6976019a616738","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Initialisation of Hyperparameters for Logistic Regression Model"]},{"cell_type":"code","execution_count":17,"metadata":{"cell_id":"00027-ee4a0808-cb79-407d-9897-e37bd6ea5fe3","deepnote_cell_height":184,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":5,"execution_start":1658682617112,"source_hash":"13936e3c","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["128 16 0.1 0\n"]}],"source":["bs = 128\n","epochs = 16\n","lr = 0.1\n","C = 0\n","print(bs, epochs, lr, C)"]},{"cell_type":"markdown","metadata":{"cell_id":"565f4ff1d2444030a1be477b5bfbce67","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Evaluation of Logisitic Regression Model"]},{"cell_type":"code","execution_count":18,"metadata":{"cell_id":"5209c131890d49f7b7e89ee7958f4402","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":187364,"execution_start":1658682617138,"source_hash":"5b601220","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-0.00172004]\n"," [-0.02492764]\n"," [ 0.00550406]\n"," ...\n"," [ 0.00015613]\n"," [ 0.01175477]\n"," [ 0.00490417]]\n","-0.5020697905473851\n","0.6493706267318076\n","1404\n","Accuracy: 0.6259310986964618\n","Macro-F1 score: 0.3867287807981557\n","CPU times: total: 10min 23s\n","Wall time: 1min 36s\n"]},{"data":{"text/plain":["0.3867287807981557"]},"execution_count":18,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train(X_train, y_train, bs, epochs, lr, C)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model = score(y_test, predict(X_test, w, b))\n","Model"]},{"cell_type":"markdown","metadata":{"cell_id":"9fbda5801d0e4375a6c0ed673aa4581d","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Evaluation of SKLearn Logistic Regression Model (LogisticRegression and SGDClassifier)"]},{"cell_type":"code","execution_count":19,"metadata":{"cell_id":"1c2de54ac3904efeaeace8e6bf90af98","deepnote_cell_height":316.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":5404,"execution_start":1658682804499,"source_hash":"3125af87","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-0.58187735 -1.02280517 -0.39681143 ... -0.04059766  0.49543838\n","   0.40047111]]\n","[-0.84683029]\n","Accuracy: 0.7302141527001862\n","Macro-F1 score: 0.6906115962801043\n","CPU times: total: 20.7 s\n","Wall time: 2.85 s\n"]},{"data":{"text/plain":["0.6906115962801043"]},"execution_count":19,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","clf1 = LogisticRegression(random_state = 100).fit(X_train, y_train)\n","print(clf1.coef_)\n","print(clf1.intercept_)\n","SKLearnLogReg = score(y_test, clf1.predict(X_test))\n","SKLearnLogReg"]},{"cell_type":"code","execution_count":20,"metadata":{"cell_id":"c4a0d020717347f8be43aaf0b5405db5","deepnote_cell_height":334.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":3477,"execution_start":1658682809910,"source_hash":"f7a528bd","tags":[],"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["c:\\Users\\issac\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:173: FutureWarning: The loss 'log' was deprecated in v1.1 and will be removed in version 1.3. Use `loss='log_loss'` which is equivalent.\n","  warnings.warn(\n"]},{"name":"stdout","output_type":"stream","text":["[[-0.47493728 -0.87421955 -0.31040593 ... -0.0494175   0.42273107\n","   0.31604817]]\n","[-0.75282034]\n","Accuracy: 0.728584729981378\n","Macro-F1 score: 0.6895107183766978\n","CPU times: total: 2.84 s\n","Wall time: 2.89 s\n"]},{"data":{"text/plain":["0.6895107183766978"]},"execution_count":20,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","clf2 = SGDClassifier(loss=\"log\", random_state=100).fit(X_train, y_train)\n","# clf2 = SGDClassifier(loss=\"log_loss\", random_state=100).fit(X_train, y_train)\n","print(clf2.coef_)\n","print(clf2.intercept_)\n","SKLearnSGD = score(y_test, clf2.predict(X_test))\n","SKLearnSGD"]},{"cell_type":"code","execution_count":21,"metadata":{"cell_id":"a0a7f0b441634a08bb9ffa3969fbe745","deepnote_cell_height":184.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":16,"execution_start":1658682813394,"source_hash":"5dbd2ce0","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Model: Model, Macro-F1 Score: 0.3867287807981557, Difference: 0.30388281548194856\n","Model Failed\n"]},{"data":{"text/plain":["[]"]},"execution_count":21,"metadata":{},"output_type":"execute_result"}],"source":["perform(SKLearnLogReg, SKLearnSGD, Model=Model)"]},{"cell_type":"markdown","metadata":{"cell_id":"38e4ab11c51d42afbcb9b3ce1379977a","deepnote_cell_height":214.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on the above results, we can observe that our Logistic Regression Model is severely underperforming as compared to the SKLearn Packages. We believe that this can be due to 2 possible reasons - The initialisation of the parameters `w` and `b` and the greedy approach adopted in navigating the gradient descent algorithm. \n","\n","We shall try a random initialisation and try 2 additional variations in developing our gradient descent algorithm (One where we choose the parameters that ensures minimum training error after every epoch and another where we choose the the parameters that ensure minimum training error at the end of the algorithm).\n","\n","We will evaluate these 3 models and compare their scores with those of the SKLearn Packages."]},{"cell_type":"markdown","metadata":{"cell_id":"1a044d9873f446408ad2aa101bb7d9af","deepnote_cell_height":70,"deepnote_cell_type":"markdown","tags":[]},"source":["## Tuning the Logistic Regression Model"]},{"cell_type":"markdown","metadata":{"cell_id":"a38debc715c64e2fa2dcb9a4f93a9479","deepnote_cell_height":62,"deepnote_cell_type":"markdown","tags":[]},"source":["### Tuning the Gradient Descent Algorithm"]},{"cell_type":"markdown","metadata":{"cell_id":"9349f4342add4e35a19655934163f24a","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train1` function, we will implement a random initialisation of parameters `w` and `b` whose values are very close to 0 using the uniform distribution [0, 1)."]},{"cell_type":"code","execution_count":22,"metadata":{"cell_id":"00014-6addd038-1b3b-427b-8ffe-cbfefb160449","deepnote_cell_height":1377,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":11,"execution_start":1658682813420,"source_hash":"7a9d88a4","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train1(X, y, bs, epochs, lr, C):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","#     w = np.zeros((d, 1))\n","#     b = 0\n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            w_new = w.copy() - lr * dw\n","            b_new = b - lr * db\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            if (loss_new < loss_old):\n","#                 print(w == w_new, b == b_new);\n","#                 print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","\n","                w = w_new\n","                b = b_new\n","                old_w.append(w_new)\n","                old_b.append(b_new)\n","                old_losses.append(loss_new)\n","    \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"6071a755307649bdb4eecc970033e4b6","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train2` function, we will implement a random initialisation of parameters `w` and `b` whose values are very close to 0 using the uniform distribution [0, 1). Also, we will only choose the parameters that ensure minimum training error only at the end of the algorithm."]},{"cell_type":"code","execution_count":23,"metadata":{"cell_id":"00013-87646db4-96ee-4ef0-8640-5664dccf1c4f","deepnote_cell_height":1305,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":7,"execution_start":1658682813446,"source_hash":"af40613f","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train2(X, y, bs, epochs, lr, C):\n","    lmb = 0 if C == 0 else 1/C\n","\n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            w_new = w.copy() - lr * dw\n","            b_new = b - lr * db\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","#             print(w == w_new, b == b_new);\n","#             print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","\n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","\n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"0eb861086da24e4798148ff9390189c8","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train3` function, we will implement a random initialisation of parameters `w` and `b` whose values are very close to 0 using the uniform distribution [0, 1). Also, we will only choose the parameters that ensure minimum training error after every epoch."]},{"cell_type":"code","execution_count":24,"metadata":{"cell_id":"00015-9fce8db2-a072-46fa-8931-9c1bcf1e8ae3","deepnote_cell_height":1449,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":8,"execution_start":1658682813469,"source_hash":"f961edc1","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train3(X, y, bs, epochs, lr, C):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","#     w = np.zeros((d, 1))\n","#     b = 0\n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            w_new = w.copy() - lr * dw\n","            b_new = b - lr * db\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","#             print(w == w_new, b == b_new);\n","#             print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","        \n","        min_loss = min(old_losses)\n","        min_index = old_losses.index(min_loss)\n","        w = old_w[min_index]\n","        b = old_b[min_index]\n","        \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"code","execution_count":25,"metadata":{"cell_id":"fa96d2cb0d7544ad8f6577e60c12b238","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":188007,"execution_start":1658682813480,"source_hash":"142cfe6d","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[0.82953088]\n"," [0.56112497]\n"," [0.28441661]\n"," ...\n"," [0.54737681]\n"," [0.20428492]\n"," [0.14764031]]\n","-1.7553462188993563\n","0.6582263563878277\n","1477\n","Accuracy: 0.6240689013035382\n","Macro-F1 score: 0.48659922937049865\n","CPU times: total: 10min 20s\n","Wall time: 1min 39s\n"]},{"data":{"text/plain":["0.48659922937049865"]},"execution_count":25,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train1(X_train, y_train, bs, epochs, lr, C)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model1 = score(y_test, predict(X_test, w, b))\n","Model1"]},{"cell_type":"code","execution_count":26,"metadata":{"cell_id":"a22fce9348614d9892f976e6e9fdf754","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":196553,"execution_start":1658683000816,"source_hash":"8629bebf","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[0.82917508]\n"," [0.55758287]\n"," [0.28212895]\n"," ...\n"," [0.54710171]\n"," [0.2049865 ]\n"," [0.14875909]]\n","-1.7440156421330273\n","0.6568636717097833\n","1600\n","Accuracy: 0.6243016759776536\n","Macro-F1 score: 0.4920613897314961\n","CPU times: total: 10min 9s\n","Wall time: 1min 39s\n"]},{"data":{"text/plain":["0.4920613897314961"]},"execution_count":26,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train2(X_train, y_train, bs, epochs, lr, C)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model2 = score(y_test, predict(X_test, w, b))\n","Model2"]},{"cell_type":"code","execution_count":27,"metadata":{"cell_id":"59dce1de6aff487aa46898e2e7156489","deepnote_cell_height":492,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":31925,"execution_start":1658683197315,"source_hash":"d6b6ea82","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[0.82917524]\n"," [0.55793472]\n"," [0.28220347]\n"," ...\n"," [0.5470061 ]\n"," [0.20511076]\n"," [0.14884337]]\n","-1.7438202768654791\n","0.656952130767089\n","1600\n","Accuracy: 0.6243016759776536\n","Macro-F1 score: 0.4920613897314961\n","CPU times: total: 10min 6s\n","Wall time: 1min 21s\n"]},{"data":{"text/plain":["0.4920613897314961"]},"execution_count":27,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train3(X_train, y_train, bs, epochs, lr, C)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model3 = score(y_test, predict(X_test, w, b))\n","Model3"]},{"cell_type":"code","execution_count":28,"metadata":{"cell_id":"6aa2a0555ce94cafb6a7fedc78c72891","deepnote_cell_height":262.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":2,"execution_start":1658671969346,"source_hash":"c1d0bc52","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Model: Model, Macro-F1 Score: 0.3867287807981557, Difference: 0.30388281548194856\n","Model: Model1, Macro-F1 Score: 0.48659922937049865, Difference: 0.2040123669096056\n","Model: Model2, Macro-F1 Score: 0.4920613897314961, Difference: 0.19855020654860817\n","Model: Model3, Macro-F1 Score: 0.4920613897314961, Difference: 0.19855020654860817\n","Model Failed\n"]},{"data":{"text/plain":["[]"]},"execution_count":28,"metadata":{},"output_type":"execute_result"}],"source":["perform(SKLearnLogReg, SKLearnSGD, Model=Model, \\\n","Model1=Model1, Model2=Model2, Model3=Model3)"]},{"cell_type":"markdown","metadata":{"cell_id":"68ffe1f2501642fdba5f5ca7c6f9c718","deepnote_cell_height":489.1875,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on the above results, we can see that the random initialisation of `w` and `b` has improved the performance of the gradient descent algorithm. However, we have noted that varying the design of the gradient descent algorithm at this stage has minimal effect on the model performance.\n","\n","Upon inspection of the training dataset, we have noted that the training data is extremely sparse. Hence, we need to incorporate the idea of momentum and adaptive learning rate into our gradient descent algorithm. The introduction of momentum is to ensure that the gradient descent algorithm moves in the direction of the trend (weighted average of gradients) even in the presence of anomalous gradient values or zero gradient values. This would help accelerate the training process.\n","\n","Additionally, as a result of sparse data, we require an adaptive learning rate that is able to boost the respective parameters of `w_i` and `b` appropriately such that the algorithm is more sensitive to valuable data that are present within the training set. This would help improve the correction and accelerate the training process.\n","\n","Based on the 2 requirements, we have decided to use the `AdamW` optimiser, which uses a combination of momentum and RMSProp. Moreover, `AdamW` is an improvement over the `Adam` optimiser, in that it supports regularisation by introducing a Weight Decay component to the gradient descent algorithm. This is necessary due to the use of adaptive learning rate that skews the regularisation of parameters in the `Adam` optimiser.\n","\n","Likewise, we will introduce 3 different variants of the `AdamW` optimiser. \\\n","    1. One where we choose the parameters that ensure a smaller training error after every update (Greedy Approach) \\\n","    2. One where we choose the parameters that ensure minimum training error at the end of the algorithm \\\n","    3. One where we choose the parameters that ensures minimum training error after every epoch"]},{"cell_type":"markdown","metadata":{"cell_id":"80fa9c837cfb4b0d8459d540dd17e727","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train4` function, it is an improvement of the `train1` function. We will use the `AdamW` optimiser instead of the conventional gradient descent algorithm. We will maintain the random initialisation in this algorithm."]},{"cell_type":"code","execution_count":29,"metadata":{"cell_id":"00017-1ee80d48-ede9-4bb5-8887-503569bd3183","deepnote_cell_height":1719,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":39,"execution_start":1658671969347,"source_hash":"cd1189da","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train4(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","#     w = np.zeros((d, 1))\n","#     b = 0\n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            if (loss_new < loss_old):\n","#                 print(w == w_new, b == b_new);\n","#                 print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","\n","                w = w_new\n","                b = b_new\n","                old_w.append(w_new)\n","                old_b.append(b_new)\n","                old_losses.append(loss_new)\n","    \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"120b7bb1dfb344c58acb38d783fc6401","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train5` function, it is an improvement of the `train2` function. We will use the `AdamW` optimiser instead of the conventional gradient descent algorithm. We will maintain the random initialisation in this algorithm."]},{"cell_type":"code","execution_count":30,"metadata":{"cell_id":"00016-e5039b6e-272f-46e1-bc5a-11cafd259faf","deepnote_cell_height":1665,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":0,"execution_start":1658671969386,"source_hash":"48058778","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train5(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","#     w = np.zeros((d, 1))\n","#     b = 0\n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","#             print(w == w_new, b == b_new);\n","#             print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","\n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"eae2c6e325b646bc88b45a58a813eb3f","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["For the `train6` function, it is an improvement of the `train3` function. We will use the `AdamW` optimiser instead of the conventional gradient descent algorithm. We will maintain the random initialisation in this algorithm."]},{"cell_type":"code","execution_count":31,"metadata":{"cell_id":"00018-45469d69-b680-4896-aa5a-a35ec13eeae1","deepnote_cell_height":1791,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":0,"execution_start":1658687341941,"source_hash":"cee706ad","trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train6(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","#     w = np.zeros((d, 1))\n","#     b = 0\n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","#         print(f\"limit: {limit}\")\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","#             print(f\"epoch: {epoch}, start: {start}, end: {end}\")\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","#             print(f\"choice: {choice}\")\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","#             print(w == w_new, b == b_new);\n","#             print(f\"loss_new: {loss_new}, loss_old: {loss_old}\")\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","        \n","        min_loss = min(old_losses)\n","        min_index = old_losses.index(min_loss)\n","        w = old_w[min_index]\n","        b = old_b[min_index]\n","        \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","#     print(f\"old_w: {old_w}\")\n","#     print(f\"old_b: {old_b}\")\n","#     print(f\"old_losses: {old_losses}\")\n","#     print(f\"min_loss: {min_loss}\")\n","#     print(f\"min_index:\", min_index)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"markdown","metadata":{"cell_id":"1a7db9c26cea4c4eb02e5d4db1fb3ece","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on our research, the recommended values of the hyperparameters for the AdamW optimizer are `beta_m = 0.9`, `beta_v = 0.999` and `err = 1e-8`."]},{"cell_type":"code","execution_count":32,"metadata":{"cell_id":"c2ea795b5d2f47ff96be49f0fd2ca797","deepnote_cell_height":238,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":4,"execution_start":1658671969407,"source_hash":"a6484cd3","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["128 16 0.1 0 0.9 0.999 1e-08\n"]}],"source":["bs = 128\n","epochs = 16\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","print(bs, epochs, lr, C, beta_m, beta_v, err)"]},{"cell_type":"code","execution_count":33,"metadata":{"cell_id":"00032-4e0a19ee-70b6-47ee-bc12-542dbcd9c1d1","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":202294,"execution_start":1658671969431,"source_hash":"aafebce6","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-8.47741643]\n"," [-8.10057621]\n"," [-4.33031263]\n"," ...\n"," [ 1.54335685]\n"," [ 3.65972176]\n"," [ 3.82466584]]\n","-1.1785629010376373\n","0.2810261570260205\n","1462\n","Accuracy: 0.6722532588454376\n","Macro-F1 score: 0.6494362017804154\n","CPU times: total: 10min 15s\n","Wall time: 1min 19s\n"]},{"data":{"text/plain":["0.6494362017804154"]},"execution_count":33,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train4(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model4 = score(y_test, predict(X_test, w, b))\n","Model4"]},{"cell_type":"code","execution_count":34,"metadata":{"cell_id":"00033-823f1f81-4f98-42b9-b244-7aa6caad1dba","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":211254,"execution_start":1658672171550,"source_hash":"d445cb11","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-9.15852804]\n"," [-8.63754918]\n"," [-4.72636096]\n"," ...\n"," [ 1.62198725]\n"," [ 3.88039144]\n"," [ 3.71492837]]\n","-1.2682779368539099\n","0.28378523874847067\n","1598\n","Accuracy: 0.6715549348230913\n","Macro-F1 score: 0.6496150778344336\n","CPU times: total: 10min 2s\n","Wall time: 1min 20s\n"]},{"data":{"text/plain":["0.6496150778344336"]},"execution_count":34,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train5(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model5 = score(y_test, predict(X_test, w, b))\n","Model5"]},{"cell_type":"code","execution_count":35,"metadata":{"cell_id":"00034-67b8b9dc-2b70-457d-8257-6d3ac7a14ac0","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":203230,"execution_start":1658672382798,"source_hash":"253fb0c6","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-8.22759819]\n"," [-8.33745542]\n"," [-3.9632901 ]\n"," ...\n"," [ 1.95402751]\n"," [ 3.52822577]\n"," [ 3.67153536]]\n","-1.1590263536783283\n","0.2840447771365454\n","1532\n","Accuracy: 0.672951582867784\n","Macro-F1 score: 0.6507565038781207\n","CPU times: total: 10min 12s\n","Wall time: 1min 19s\n"]},{"data":{"text/plain":["0.6507565038781207"]},"execution_count":35,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model6 = score(y_test, predict(X_test, w, b))\n","Model6"]},{"cell_type":"code","execution_count":36,"metadata":{"cell_id":"bed05c58066f43559152cfe6d491fda7","deepnote_cell_height":340.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":285,"execution_start":1658672585746,"source_hash":"47f11e70","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Model: Model, Macro-F1 Score: 0.3867287807981557, Difference: 0.30388281548194856\n","Model: Model1, Macro-F1 Score: 0.48659922937049865, Difference: 0.2040123669096056\n","Model: Model2, Macro-F1 Score: 0.4920613897314961, Difference: 0.19855020654860817\n","Model: Model3, Macro-F1 Score: 0.4920613897314961, Difference: 0.19855020654860817\n","Model: Model4, Macro-F1 Score: 0.6494362017804154, Difference: 0.04117539449968888\n","Model: Model5, Macro-F1 Score: 0.6496150778344336, Difference: 0.04099651844567065\n","Model: Model6, Macro-F1 Score: 0.6507565038781207, Difference: 0.039855092401983594\n","Model Success\n"]},{"data":{"text/plain":["['Model4', 'Model5', 'Model6']"]},"execution_count":36,"metadata":{},"output_type":"execute_result"}],"source":["perform(SKLearnLogReg, SKLearnSGD, Model=Model, \\\n","Model1=Model1, Model2=Model2, Model3=Model3, \\\n","Model4=Model4, Model5=Model5, Model6=Model6)"]},{"cell_type":"markdown","metadata":{"cell_id":"00052-0bd975ab-19ef-4aff-86d1-65a85656d013","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown"},"source":["Based on the above score, we can deem that the performance of our Logistic Regression Model is comparable to that of SKLearn Logistic Regression Package. Moreover, our best performing model is Model6. Hence, we shall refine our Logistic Regression Model by tuning the hyperparameters."]},{"cell_type":"markdown","metadata":{"cell_id":"ae11d50cb29e4b888746694fde2f6fe7","deepnote_cell_height":62,"deepnote_cell_type":"markdown","tags":[]},"source":["### Hyperparameter Tuning"]},{"cell_type":"markdown","metadata":{"cell_id":"1b5f0d3cd5a84c7a8139b176069da8fe","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown","tags":[]},"source":["To further improve our model performance, we will first find the best learning rate, regularisation coefficient, momentum coefficient, RMSProp coefficient and the Error Term. Once we have chosen the optimal hyperparameters, we will then decide on the Batch Size and Epoch Size to train our model."]},{"cell_type":"code","execution_count":37,"metadata":{"cell_id":"00035-fc897196-e51a-4ee4-9bd3-f9063c3d47c6","deepnote_cell_height":372,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":233,"execution_start":1658677091895,"source_hash":"8b69acce","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Learning Rates: [1, 0.1, 0.01, 0.001]\n","Regularisation Coefficients: [0.1, 1, 10, 100, 1000, 0]\n","Momentum Coefficients: [0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0.0]\n","RMSProp Coefficients: [0.999, 0.997, 0.995, 0.993, 0.991, 0.989]\n","Error Terms: [1, 0.01, 0.0001, 1e-06, 1e-08]\n"]}],"source":["Ls = [10**(-i) for i in range(4)]\n","Cs = [0.1, 1, 10, 100, 1000, 0]\n","Bm = [(9 - i)/10 for i in range(10)]\n","Bv = [(999 - 2*i)/1000 for i in range(6)]\n","Errs = [10**(-2*i) for i in range(5)]\n","\n","print(f\"Learning Rates: {Ls}\")\n","print(f\"Regularisation Coefficients: {Cs}\")\n","print(f\"Momentum Coefficients: {Bm}\")\n","print(f\"RMSProp Coefficients: {Bv}\")\n","print(f\"Error Terms: {Errs}\")"]},{"cell_type":"code","execution_count":38,"metadata":{"cell_id":"00037-9aee6fca-0854-4294-bc7f-d7ec6603904e","deepnote_cell_height":848,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":189719,"execution_start":1658675006802,"source_hash":"7c0af1ce","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6538640595903166\n","Macro-F1 score: 0.60183066275978\n","303\n","1 0 0.9 0.999 1e-08 0.60183066275978\n","Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.6662472846585037\n","400\n","0.1 0 0.9 0.999 1e-08 0.6662472846585037\n","Accuracy: 0.6778398510242085\n","Macro-F1 score: 0.6537110228158275\n","400\n","0.01 0 0.9 0.999 1e-08 0.6537110228158275\n","Accuracy: 0.3743016759776536\n","Macro-F1 score: 0.2723577235772358\n","400\n","0.001 0 0.9 0.999 1e-08 0.2723577235772358\n","{0.60183066275978: 1, 0.6662472846585037: 0.1, 0.6537110228158275: 0.01, 0.2723577235772358: 0.001}\n","[0.1] 0.6662472846585037\n","CPU times: total: 10min 18s\n","Wall time: 1min 19s\n"]}],"source":["%%time\n","vals = {}\n","bs = 128\n","epochs = 4\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","\n","for lr in Ls:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    print(l.index(min(l)))\n","    print(lr, C, beta_m, beta_v, err, result)\n","    vals[result] = lr\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"2bd880bb5f8d45159419ada1dbf55c62","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["We shall choose a learning rate of `lr = 0.1`."]},{"cell_type":"code","execution_count":39,"metadata":{"cell_id":"49aed85a49d6433bb21c9f510a9c61ef","deepnote_cell_height":1012.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":301417,"execution_start":1658673145534,"source_hash":"48c55687","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","320\n","0.1 0.1 0.9 0.999 1e-08 0.384791636832307\n","Accuracy: 0.6261638733705773\n","Macro-F1 score: 0.3879851144518121\n","323\n","0.1 1 0.9 0.999 1e-08 0.3879851144518121\n","Accuracy: 0.680633147113594\n","Macro-F1 score: 0.5902111978259219\n","338\n","0.1 10 0.9 0.999 1e-08 0.5902111978259219\n","Accuracy: 0.7178770949720671\n","Macro-F1 score: 0.6847822296614012\n","315\n","0.1 100 0.9 0.999 1e-08 0.6847822296614012\n","Accuracy: 0.699487895716946\n","Macro-F1 score: 0.6741020618444631\n","400\n","0.1 1000 0.9 0.999 1e-08 0.6741020618444631\n","Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.6662472846585037\n","400\n","0.1 0 0.9 0.999 1e-08 0.6662472846585037\n","{0.384791636832307: 0.1, 0.3879851144518121: 1, 0.5902111978259219: 10, 0.6847822296614012: 100, 0.6741020618444631: 1000, 0.6662472846585037: 0}\n","[100] 0.6847822296614012\n","CPU times: total: 15min 3s\n","Wall time: 1min 59s\n"]}],"source":["%%time\n","vals = {}\n","bs = 128\n","epochs = 4\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","\n","for C in Cs:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    print(l.index(min(l)))\n","    print(lr, C, beta_m, beta_v, err, result)\n","    vals[result] = C\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"654fc227f0a3475496364d46011a61fd","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["We shall choose a regularisation coefficient of `C = 100`."]},{"cell_type":"code","execution_count":40,"metadata":{"cell_id":"ef8f9fd43d2347328baa1bc48f2c59d9","deepnote_cell_height":1097,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":532213,"execution_start":1658678203178,"source_hash":"994d17cb","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.6662472846585037\n","400\n","0.1 0 0.9 0.999 1e-08 0.6662472846585037\n","Accuracy: 0.6904096834264432\n","Macro-F1 score: 0.6673977263492802\n","400\n","0.1 0 0.8 0.999 1e-08 0.6673977263492802\n","Accuracy: 0.6901769087523277\n","Macro-F1 score: 0.6668398896756464\n","400\n","0.1 0 0.7 0.999 1e-08 0.6668398896756464\n","Accuracy: 0.6908752327746741\n","Macro-F1 score: 0.6674584588869584\n","400\n","0.1 0 0.6 0.999 1e-08 0.6674584588869584\n","Accuracy: 0.6901769087523277\n","Macro-F1 score: 0.6664851322495824\n","400\n","0.1 0 0.5 0.999 1e-08 0.6664851322495824\n","Accuracy: 0.6911080074487895\n","Macro-F1 score: 0.6672200359410805\n","400\n","0.1 0 0.4 0.999 1e-08 0.6672200359410805\n","Accuracy: 0.6901769087523277\n","Macro-F1 score: 0.6673659689525142\n","398\n","0.1 0 0.3 0.999 1e-08 0.6673659689525142\n","Accuracy: 0.6901769087523277\n","Macro-F1 score: 0.6673659689525142\n","398\n","0.1 0 0.2 0.999 1e-08 0.6673659689525142\n","Accuracy: 0.6894785847299814\n","Macro-F1 score: 0.6666598418821515\n","398\n","0.1 0 0.1 0.999 1e-08 0.6666598418821515\n","Accuracy: 0.688780260707635\n","Macro-F1 score: 0.6660410286078096\n","398\n","0.1 0 0.0 0.999 1e-08 0.6660410286078096\n","{0.6662472846585037: 0.9, 0.6673977263492802: 0.8, 0.6668398896756464: 0.7, 0.6674584588869584: 0.6, 0.6664851322495824: 0.5, 0.6672200359410805: 0.4, 0.6673659689525142: 0.2, 0.6666598418821515: 0.1, 0.6660410286078096: 0.0}\n","[0.6] 0.6674584588869584\n","CPU times: total: 25min 12s\n","Wall time: 3min 14s\n"]}],"source":["%%time\n","vals = {}\n","bs = 128\n","epochs = 4\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","\n","for beta_m in Bm:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    print(l.index(min(l)))\n","    print(lr, C, beta_m, beta_v, err, result)\n","    vals[result] = beta_m\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"c7c1bbe3b3134a8a9ec62f6210a7a5d1","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["We shall choose a Momentum coefficient of `beta_m = 0.6`."]},{"cell_type":"code","execution_count":41,"metadata":{"cell_id":"da16fd14ed2246e18f62329dfc8047b6","deepnote_cell_height":1012.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":305369,"execution_start":1658678735406,"source_hash":"2f130d35","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.6662472846585037\n","400\n","0.1 0 0.9 0.999 1e-08 0.6662472846585037\n","Accuracy: 0.6885474860335196\n","Macro-F1 score: 0.6656603211681549\n","400\n","0.1 0 0.9 0.997 1e-08 0.6656603211681549\n","Accuracy: 0.6894785847299814\n","Macro-F1 score: 0.6661331817080218\n","400\n","0.1 0 0.9 0.995 1e-08 0.6661331817080218\n","Accuracy: 0.6885474860335196\n","Macro-F1 score: 0.6651320818030984\n","400\n","0.1 0 0.9 0.993 1e-08 0.6651320818030984\n","Accuracy: 0.6883147113594041\n","Macro-F1 score: 0.6646593875910989\n","400\n","0.1 0 0.9 0.991 1e-08 0.6646593875910989\n","Accuracy: 0.688780260707635\n","Macro-F1 score: 0.6650710789626108\n","400\n","0.1 0 0.9 0.989 1e-08 0.6650710789626108\n","{0.6662472846585037: 0.999, 0.6656603211681549: 0.997, 0.6661331817080218: 0.995, 0.6651320818030984: 0.993, 0.6646593875910989: 0.991, 0.6650710789626108: 0.989}\n","[0.999] 0.6662472846585037\n","CPU times: total: 15min 6s\n","Wall time: 1min 54s\n"]}],"source":["%%time\n","vals = {}\n","bs = 128\n","epochs = 4\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","\n","for beta_v in Bv:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    print(l.index(min(l)))\n","    print(lr, C, beta_m, beta_v, err, result)\n","    vals[result] = beta_v\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"8d44fa87a5b741daaf184e4b1c9cefa3","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["We shall choose a RMSProp coefficient of `beta_v = 0.999`."]},{"cell_type":"code","execution_count":42,"metadata":{"cell_id":"9a061a8962f545b487bd86565bd3326b","deepnote_cell_height":928,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":240096,"execution_start":1658673935844,"source_hash":"61d62e58","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.41783054003724396\n","Macro-F1 score: 0.3849706750029923\n","400\n","0.1 0 0.9 0.999 1 0.3849706750029923\n","Accuracy: 0.7178770949720671\n","Macro-F1 score: 0.677383393357214\n","400\n","0.1 0 0.9 0.999 0.01 0.677383393357214\n","Accuracy: 0.6978584729981379\n","Macro-F1 score: 0.6746235436795763\n","400\n","0.1 0 0.9 0.999 0.0001 0.6746235436795763\n","Accuracy: 0.6892458100558659\n","Macro-F1 score: 0.6664535557239946\n","400\n","0.1 0 0.9 0.999 1e-06 0.6664535557239946\n","Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.6662472846585037\n","400\n","0.1 0 0.9 0.999 1e-08 0.6662472846585037\n","{0.3849706750029923: 1, 0.677383393357214: 0.01, 0.6746235436795763: 0.0001, 0.6664535557239946: 1e-06, 0.6662472846585037: 1e-08}\n","[0.01] 0.677383393357214\n","CPU times: total: 12min 8s\n","Wall time: 1min 51s\n"]}],"source":["%%time\n","vals = {}\n","bs = 128\n","epochs = 4\n","lr = 0.1\n","C = 0\n","beta_m = 0.9\n","beta_v = 0.999\n","err = 1e-8\n","\n","for err in Errs:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    print(l.index(min(l)))\n","    print(lr, C, beta_m, beta_v, err, result)\n","    vals[result] = err\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"399c9be926c8452a849f1915e51d7eda","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["We shall choose an Error Term of `err = 0.01`."]},{"cell_type":"markdown","metadata":{"cell_id":"e180cf97009646918155b28b3ed4f16e","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["Assuming that the hyperparameters are independent of one another, we will now decide on the Batch Size and Epoch Size to train our model."]},{"cell_type":"code","execution_count":43,"metadata":{"cell_id":"1d54624481ed4d52a55a467afccd3320","deepnote_cell_height":222,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":2,"execution_start":1658681589053,"source_hash":"f873e1e6","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Batch Sizes: [4096, 2048, 1024, 512, 256, 128, 64]\n","Epoch Sizes: [1, 2, 4, 8, 16, 32, 64, 128]\n"]}],"source":["Bs = [64*(2**i) for i in range(7)]\n","Bs.reverse()\n","Es = [2**(i) for i in range(8)]\n","\n","print(f\"Batch Sizes: {Bs}\")\n","print(f\"Epoch Sizes: {Es}\")"]},{"cell_type":"code","execution_count":44,"metadata":{"cell_id":"6299f80d74ff43aa91fac42121dc9712","deepnote_cell_height":934.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":372691,"execution_start":1658677441479,"source_hash":"6153f8ec","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.3961824953445065\n","Macro-F1 score: 0.32737814572174595\n","4096 8 0.1 100 0.6 0.999 0.01 0.32737814572174595\n","Accuracy: 0.4590316573556797\n","Macro-F1 score: 0.45136005106620936\n","2048 8 0.1 100 0.6 0.999 0.01 0.45136005106620936\n","Accuracy: 0.5833333333333334\n","Macro-F1 score: 0.5710142883665134\n","1024 8 0.1 100 0.6 0.999 0.01 0.5710142883665134\n","Accuracy: 0.6820297951582868\n","Macro-F1 score: 0.6424082898028336\n","512 8 0.1 100 0.6 0.999 0.01 0.6424082898028336\n","Accuracy: 0.7176443202979516\n","Macro-F1 score: 0.6713070335546949\n","256 8 0.1 100 0.6 0.999 0.01 0.6713070335546949\n","Accuracy: 0.7225325884543762\n","Macro-F1 score: 0.6757364652101494\n","128 8 0.1 100 0.6 0.999 0.01 0.6757364652101494\n","Accuracy: 0.7155493482309124\n","Macro-F1 score: 0.6692042924893535\n","64 8 0.1 100 0.6 0.999 0.01 0.6692042924893535\n","{0.32737814572174595: (4096, 8), 0.45136005106620936: (2048, 8), 0.5710142883665134: (1024, 8), 0.6424082898028336: (512, 8), 0.6713070335546949: (256, 8), 0.6757364652101494: (128, 8), 0.6692042924893535: (64, 8)}\n","[(128, 8)] 0.6757364652101494\n","CPU times: total: 19min 33s\n","Wall time: 2min 39s\n"]}],"source":["%%time\n","vals = {}\n","epochs = 8\n","lr = 0.1\n","C = 100\n","beta_m = 0.6\n","beta_v = 0.999\n","err = 0.01\n","\n","for bs in Bs:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    # print(l.index(min(l)))\n","    print(bs, epochs, lr, C, beta_m, beta_v, err, result)\n","    vals[result] = (bs, epochs)\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"d1d28c1877104b2688bed296c91551dc","deepnote_cell_height":97.1875,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on the above, we can observe that for fixed epoch, the general trend is that the smaller the batch size, the better the performance. However, we have noted that the performance drops when the batch size decreases from 128 to 64, implying that the model may have overfitted with the training dataset."]},{"cell_type":"code","execution_count":45,"metadata":{"cell_id":"00036-3b9435e2-6c4c-4e47-b4db-98d47bc454b7","deepnote_cell_height":934.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":852126,"execution_start":1658679040793,"source_hash":"74277725","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.728584729981378\n","Macro-F1 score: 0.6950824941075637\n","4096 256 0.1 100 0.6 0.999 0.01 0.6950824941075637\n","Accuracy: 0.7316108007448789\n","Macro-F1 score: 0.6966186243774475\n","2048 128 0.1 100 0.6 0.999 0.01 0.6966186243774475\n","Accuracy: 0.729050279329609\n","Macro-F1 score: 0.6937789760740024\n","1024 64 0.1 100 0.6 0.999 0.01 0.6937789760740024\n","Accuracy: 0.7309124767225326\n","Macro-F1 score: 0.6957748632444369\n","512 32 0.1 100 0.6 0.999 0.01 0.6957748632444369\n","Accuracy: 0.728584729981378\n","Macro-F1 score: 0.6903260458584444\n","256 16 0.1 100 0.6 0.999 0.01 0.6903260458584444\n","Accuracy: 0.7225325884543762\n","Macro-F1 score: 0.6757364652101494\n","128 8 0.1 100 0.6 0.999 0.01 0.6757364652101494\n","Accuracy: 0.7167132216014898\n","Macro-F1 score: 0.6630075908232557\n","64 4 0.1 100 0.6 0.999 0.01 0.6630075908232557\n","{0.6950824941075637: (4096, 256), 0.6966186243774475: (2048, 128), 0.6937789760740024: (1024, 64), 0.6957748632444369: (512, 32), 0.6903260458584444: (256, 16), 0.6757364652101494: (128, 8), 0.6630075908232557: (64, 4)}\n","[(2048, 128)] 0.6966186243774475\n","CPU times: total: 39min 17s\n","Wall time: 6min 12s\n"]}],"source":["%%time\n","vals = {}\n","lr = 0.1\n","C = 100\n","beta_m = 0.6\n","beta_v = 0.999\n","err = 0.01\n","\n","for bs in Bs:\n","    epochs = bs // 16\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    # print(l.index(min(l)))\n","    print(bs, epochs, lr, C, beta_m, beta_v, err, result)\n","    vals[result] = (bs, epochs)\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"aa8adfffda094951a754372820042fa1","deepnote_cell_height":52.390625,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on the above, a batch size of `bs = 2048` will result in the best model performance."]},{"cell_type":"code","execution_count":46,"metadata":{"cell_id":"949382c344aa4e0b81afbfc266a29986","deepnote_cell_height":994.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":255644,"execution_start":1658681600341,"source_hash":"b875ab39","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.3745344506517691\n","Macro-F1 score: 0.2724809483488569\n","2048 1 0.1 100 0.6 0.999 0.01 0.2724809483488569\n","Accuracy: 0.37756052141527\n","Macro-F1 score: 0.28019622016520107\n","2048 2 0.1 100 0.6 0.999 0.01 0.28019622016520107\n","Accuracy: 0.3957169459962756\n","Macro-F1 score: 0.32663601058793984\n","2048 4 0.1 100 0.6 0.999 0.01 0.32663601058793984\n","Accuracy: 0.4590316573556797\n","Macro-F1 score: 0.45136005106620936\n","2048 8 0.1 100 0.6 0.999 0.01 0.45136005106620936\n","Accuracy: 0.5833333333333334\n","Macro-F1 score: 0.5703066677276047\n","2048 16 0.1 100 0.6 0.999 0.01 0.5703066677276047\n","Accuracy: 0.675512104283054\n","Macro-F1 score: 0.6373252069053643\n","2048 32 0.1 100 0.6 0.999 0.01 0.6373252069053643\n","Accuracy: 0.7206703910614525\n","Macro-F1 score: 0.6803361332031991\n","2048 64 0.1 100 0.6 0.999 0.01 0.6803361332031991\n","Accuracy: 0.7316108007448789\n","Macro-F1 score: 0.6966186243774475\n","2048 128 0.1 100 0.6 0.999 0.01 0.6966186243774475\n","{0.2724809483488569: (2048, 1), 0.28019622016520107: (2048, 2), 0.32663601058793984: (2048, 4), 0.45136005106620936: (2048, 8), 0.5703066677276047: (2048, 16), 0.6373252069053643: (2048, 32), 0.6803361332031991: (2048, 64), 0.6966186243774475: (2048, 128)}\n","[(2048, 128)] 0.6966186243774475\n","CPU times: total: 12min 38s\n","Wall time: 2min 38s\n"]}],"source":["%%time\n","vals = {}\n","bs = 2048\n","lr = 0.1\n","C = 100\n","beta_m = 0.6\n","beta_v = 0.999\n","err = 0.01\n","\n","for epochs in Es:\n","    w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","    result = score(y_test, predict(X_test, w, b))\n","    # print(w)\n","    # print(b)\n","    # print(min(l))\n","    # print(l.index(min(l)))\n","    print(bs, epochs, lr, C, beta_m, beta_v, err, result)\n","    vals[result] = (bs, epochs)\n","\n","print(vals)\n","maxval = max(vals.keys())\n","res = [v for k, v in vals.items() if k==maxval]\n","print(res, maxval)"]},{"cell_type":"markdown","metadata":{"cell_id":"483816e99eca4b4296d6a329b5f2d5dd","deepnote_cell_height":74.796875,"deepnote_cell_type":"markdown","tags":[]},"source":["Based on the above, we can clearly see that a larger epoch size will improve the model performance. However, there is a tradeoff in terms of the time spent to train the model. As such, we shall choose an epoch size of `epochs = 256` to train our model."]},{"cell_type":"code","execution_count":47,"metadata":{"cell_id":"77b57ac8664f47079777bdb4f59f0959","deepnote_cell_height":189,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":2,"execution_start":1658683807833,"source_hash":"1d06537f","tags":[],"trusted":true},"outputs":[],"source":["bs = 2048\n","epochs = 256\n","lr = 0.1 #best: 0.1 (btw 0.10 and 0.16)\n","C = 100 #best: 100\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)"]},{"cell_type":"code","execution_count":48,"metadata":{"cell_id":"1030069b017a421c99dfc46ae7cbfde7","deepnote_cell_height":492.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":275476,"execution_start":1658683811716,"source_hash":"7abb0f5","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-0.15609145]\n"," [-1.1398634 ]\n"," [-0.47299625]\n"," ...\n"," [ 0.31361622]\n"," [ 0.59016236]\n"," [ 0.52905051]]\n","-1.3197810454686933\n","0.43972440359598974\n","1536\n","Accuracy: 0.7358007448789572\n","Macro-F1 score: 0.7062293681825118\n","CPU times: total: 11min 25s\n","Wall time: 2min 29s\n"]},{"data":{"text/plain":["0.7062293681825118"]},"execution_count":48,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model7 = score(y_test, predict(X_test, w, b))\n","Model7"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, we have achieved a macro f1-score of `0.70623`, which has surpassed the performance of the default setup of the SKLearn Logistic Regression Package. We shall now try to implement robust scaling to increase the separation of the training data. We assume that our model will perform better when the data points are scaled as the model would be able to find a better decision boundary that can clearly separate the labels."]},{"cell_type":"code","execution_count":49,"metadata":{"trusted":true},"outputs":[],"source":["from sklearn.preprocessing import RobustScaler as Scaler"]},{"cell_type":"code","execution_count":56,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>9</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>...</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>0.000150</td>\n","      <td>0.001066</td>\n","      <td>0.001532</td>\n","      <td>0.000369</td>\n","      <td>0.000140</td>\n","      <td>0.000066</td>\n","      <td>0.000270</td>\n","      <td>0.000483</td>\n","      <td>0.000406</td>\n","      <td>0.000418</td>\n","      <td>...</td>\n","      <td>0.000202</td>\n","      <td>0.000429</td>\n","      <td>0.000286</td>\n","      <td>0.000075</td>\n","      <td>0.000260</td>\n","      <td>0.000709</td>\n","      <td>0.000257</td>\n","      <td>0.000121</td>\n","      <td>0.000308</td>\n","      <td>0.000159</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>0.008297</td>\n","      <td>0.019532</td>\n","      <td>0.024741</td>\n","      <td>0.012334</td>\n","      <td>0.008276</td>\n","      <td>0.005065</td>\n","      <td>0.009907</td>\n","      <td>0.013106</td>\n","      <td>0.012402</td>\n","      <td>0.013211</td>\n","      <td>...</td>\n","      <td>0.010215</td>\n","      <td>0.013178</td>\n","      <td>0.011378</td>\n","      <td>0.005866</td>\n","      <td>0.010864</td>\n","      <td>0.017641</td>\n","      <td>0.010246</td>\n","      <td>0.006529</td>\n","      <td>0.010526</td>\n","      <td>0.008536</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>0.676327</td>\n","      <td>0.560830</td>\n","      <td>0.958430</td>\n","      <td>0.646740</td>\n","      <td>0.532789</td>\n","      <td>0.437760</td>\n","      <td>0.435835</td>\n","      <td>0.536746</td>\n","      <td>0.546247</td>\n","      <td>0.550237</td>\n","      <td>...</td>\n","      <td>0.611122</td>\n","      <td>0.540809</td>\n","      <td>0.566613</td>\n","      <td>0.592170</td>\n","      <td>0.617341</td>\n","      <td>0.850605</td>\n","      <td>0.484908</td>\n","      <td>0.398105</td>\n","      <td>0.430031</td>\n","      <td>0.528556</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8 rows × 5000 columns</p>\n","</div>"],"text/plain":["                  0             1             2             3             4  \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean       0.000150      0.001066      0.001532      0.000369      0.000140   \n","std        0.008297      0.019532      0.024741      0.012334      0.008276   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.676327      0.560830      0.958430      0.646740      0.532789   \n","\n","                  5             6             7             8             9  \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean       0.000066      0.000270      0.000483      0.000406      0.000418   \n","std        0.005065      0.009907      0.013106      0.012402      0.013211   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.437760      0.435835      0.536746      0.546247      0.550237   \n","\n","       ...          4990          4991          4992          4993  \\\n","count  ...  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean   ...      0.000202      0.000429      0.000286      0.000075   \n","std    ...      0.010215      0.013178      0.011378      0.005866   \n","min    ...      0.000000      0.000000      0.000000      0.000000   \n","25%    ...      0.000000      0.000000      0.000000      0.000000   \n","50%    ...      0.000000      0.000000      0.000000      0.000000   \n","75%    ...      0.000000      0.000000      0.000000      0.000000   \n","max    ...      0.611122      0.540809      0.566613      0.592170   \n","\n","               4994          4995          4996          4997          4998  \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean       0.000260      0.000709      0.000257      0.000121      0.000308   \n","std        0.010864      0.017641      0.010246      0.006529      0.010526   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.617341      0.850605      0.484908      0.398105      0.430031   \n","\n","               4999  \n","count  17184.000000  \n","mean       0.000159  \n","std        0.008536  \n","min        0.000000  \n","25%        0.000000  \n","50%        0.000000  \n","75%        0.000000  \n","max        0.528556  \n","\n","[8 rows x 5000 columns]"]},"execution_count":56,"metadata":{},"output_type":"execute_result"}],"source":["df_train.iloc[:, 2:5002].describe()"]},{"cell_type":"code","execution_count":58,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X: (17184, 5000), y: (17184,)\n"]},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>9</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>...</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","      <td>17184.000000</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>0.000202</td>\n","      <td>0.001437</td>\n","      <td>0.002067</td>\n","      <td>0.000498</td>\n","      <td>0.000189</td>\n","      <td>0.000089</td>\n","      <td>0.000364</td>\n","      <td>0.000652</td>\n","      <td>0.000548</td>\n","      <td>0.000564</td>\n","      <td>...</td>\n","      <td>0.000272</td>\n","      <td>0.000579</td>\n","      <td>0.000386</td>\n","      <td>0.000101</td>\n","      <td>0.000350</td>\n","      <td>0.000957</td>\n","      <td>0.000347</td>\n","      <td>0.000163</td>\n","      <td>0.000415</td>\n","      <td>0.000214</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>0.011192</td>\n","      <td>0.026349</td>\n","      <td>0.033375</td>\n","      <td>0.016639</td>\n","      <td>0.011164</td>\n","      <td>0.006832</td>\n","      <td>0.013364</td>\n","      <td>0.017679</td>\n","      <td>0.016730</td>\n","      <td>0.017821</td>\n","      <td>...</td>\n","      <td>0.013780</td>\n","      <td>0.017776</td>\n","      <td>0.015349</td>\n","      <td>0.007914</td>\n","      <td>0.014656</td>\n","      <td>0.023798</td>\n","      <td>0.013821</td>\n","      <td>0.008808</td>\n","      <td>0.014199</td>\n","      <td>0.011514</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>0.912352</td>\n","      <td>0.756548</td>\n","      <td>1.292902</td>\n","      <td>0.872439</td>\n","      <td>0.718721</td>\n","      <td>0.590529</td>\n","      <td>0.587933</td>\n","      <td>0.724059</td>\n","      <td>0.736876</td>\n","      <td>0.742259</td>\n","      <td>...</td>\n","      <td>0.824391</td>\n","      <td>0.729541</td>\n","      <td>0.764349</td>\n","      <td>0.798826</td>\n","      <td>0.832780</td>\n","      <td>1.147449</td>\n","      <td>0.654131</td>\n","      <td>0.537036</td>\n","      <td>0.580103</td>\n","      <td>0.713011</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8 rows × 5000 columns</p>\n","</div>"],"text/plain":["               0             1             2             3             4     \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean       0.000202      0.001437      0.002067      0.000498      0.000189   \n","std        0.011192      0.026349      0.033375      0.016639      0.011164   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.912352      0.756548      1.292902      0.872439      0.718721   \n","\n","               5             6             7             8             9     \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean       0.000089      0.000364      0.000652      0.000548      0.000564   \n","std        0.006832      0.013364      0.017679      0.016730      0.017821   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.590529      0.587933      0.724059      0.736876      0.742259   \n","\n","       ...          4990          4991          4992          4993  \\\n","count  ...  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean   ...      0.000272      0.000579      0.000386      0.000101   \n","std    ...      0.013780      0.017776      0.015349      0.007914   \n","min    ...      0.000000      0.000000      0.000000      0.000000   \n","25%    ...      0.000000      0.000000      0.000000      0.000000   \n","50%    ...      0.000000      0.000000      0.000000      0.000000   \n","75%    ...      0.000000      0.000000      0.000000      0.000000   \n","max    ...      0.824391      0.729541      0.764349      0.798826   \n","\n","               4994          4995          4996          4997          4998  \\\n","count  17184.000000  17184.000000  17184.000000  17184.000000  17184.000000   \n","mean       0.000350      0.000957      0.000347      0.000163      0.000415   \n","std        0.014656      0.023798      0.013821      0.008808      0.014199   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.832780      1.147449      0.654131      0.537036      0.580103   \n","\n","               4999  \n","count  17184.000000  \n","mean       0.000214  \n","std        0.011514  \n","min        0.000000  \n","25%        0.000000  \n","50%        0.000000  \n","75%        0.000000  \n","max        0.713011  \n","\n","[8 rows x 5000 columns]"]},"execution_count":58,"metadata":{},"output_type":"execute_result"}],"source":["X = df_train.iloc[:, 2:5002].to_numpy()\n","y = df_train.iloc[:,1].to_numpy()\n","print(f\"X: {X.shape}, y: {y.shape}\")\n","\n","scaler = Scaler(with_centering=False, unit_variance=True).fit(X)\n","X_scaler = scaler.transform(X)\n","scaled = pd.DataFrame(X_scaler)\n","scaled.describe()"]},{"cell_type":"code","execution_count":54,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 5000), y_train: (12888,)\n","X_test: (4296, 5000), y_test: (4296,)\n","CPU times: total: 2.06 s\n","Wall time: 2.06 s\n"]}],"source":["%%time\n","X_train, X_test, y_train, y_test = train_test_split(X_scaler, y, test_size=0.25, random_state=100)\n","print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n","print(f\"X_test: {X_test.shape}, y_test: {y_test.shape}\")"]},{"cell_type":"code","execution_count":55,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["[[-0.40477117]\n"," [-1.21947652]\n"," [-0.52880814]\n"," ...\n"," [ 0.26783656]\n"," [ 0.56554289]\n"," [ 0.60708187]]\n","-1.4062877696318705\n","0.40790460884152074\n","1535\n","Accuracy: 0.7267225325884544\n","Macro-F1 score: 0.7004623928115308\n","CPU times: total: 11min 59s\n","Wall time: 2min 35s\n"]},{"data":{"text/plain":["0.7004623928115308"]},"execution_count":55,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1 #best: 0.1 (btw 0.10 and 0.16)\n","C = 100 #best: 100\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","w, b, l = train6(X_train, y_train, bs, epochs, lr, C, beta_m, beta_v, err)\n","print(w)\n","print(b)\n","print(min(l))\n","print(l.index(min(l)))\n","Model8 = score(y_test, predict(X_test, w, b))\n","Model8"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, we have achieved a score of `0.70046`, which is a slight decrease in model performance. Hence, it would seem that scaling results in a negligible degradation in model performance."]},{"cell_type":"markdown","metadata":{"cell_id":"00068-89a646ec-2454-4320-b8c4-f5548f796b36","deepnote_cell_height":153.1875,"deepnote_cell_type":"markdown"},"source":["## Exporting Prediction\n","Prediction made by your Logistic Regression on the Test set. Note that you are welcome to submit your predicted labels to Kaggle but you will need to submit the final prediction output in the final project submission. Please label the file as \"LogRed_Prediction.csv\"."]},{"cell_type":"code","execution_count":59,"metadata":{"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 5001 columns</p>\n","</div>"],"text/plain":["         id    0    1    2    3    4    5    6    7    8  ...  4990  4991  \\\n","0     17185  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1     17186  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2     17187  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3     17188  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4     17189  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...     ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","4291  21476  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4292  21477  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4293  21478  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4294  21479  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4295  21480  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","      4992  4993  4994  4995  4996  4997  4998  4999  \n","0      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...    ...   ...   ...   ...   ...   ...   ...   ...  \n","4291   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4292   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4293   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4294   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4295   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[4296 rows x 5001 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 5.91 s\n","Wall time: 5.99 s\n"]}],"source":["%%time\n","df_test = pd.read_csv(r\"./source/test_tfidf_features.csv\")\n","display(df_test)"]},{"cell_type":"code","execution_count":61,"metadata":{"cell_id":"b98d59a7657446668ec90741a9492258","deepnote_cell_height":172.1875,"deepnote_cell_type":"code","deepnote_output_heights":[21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":174,"execution_start":1658684807590,"source_hash":"341533dd","tags":[],"trusted":true},"outputs":[{"data":{"text/plain":["array([0, 0, 1, ..., 1, 0, 0])"]},"metadata":{},"output_type":"display_data"}],"source":["features = df_test.iloc[:,1:]\n","\n","results = predict(features, w, b)\n","display(results)"]},{"cell_type":"code","execution_count":62,"metadata":{"cell_id":"ebef1c3a31934cdda95c464e20cd5788","deepnote_cell_height":148,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":false,"execution_millis":97,"execution_start":1658684807669,"source_hash":"774f40c6","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["(4296, 1) (4296, 1)\n"]}],"source":["df_ids = df_test.iloc[:, 0].to_frame()\n","df_results = pd.DataFrame(results)\n","print(df_results.shape, df_ids.shape)"]},{"cell_type":"code","execution_count":63,"metadata":{"cell_id":"5e35eb9faffc4389a0b402260b3b6d95","deepnote_cell_height":636,"deepnote_cell_type":"code","deepnote_output_heights":[21.1875],"deepnote_table_invalid":false,"deepnote_table_loading":false,"deepnote_table_state":{"filters":[],"pageIndex":0,"pageSize":10,"sortBy":[]},"deepnote_to_be_reexecuted":false,"execution_millis":65,"execution_start":1658684807702,"source_hash":"510702b8","tags":[],"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 2 columns</p>\n","</div>"],"text/plain":["         id  label\n","0     17185      0\n","1     17186      0\n","2     17187      1\n","3     17188      0\n","4     17189      0\n","...     ...    ...\n","4291  21476      0\n","4292  21477      0\n","4293  21478      1\n","4294  21479      0\n","4295  21480      0\n","\n","[4296 rows x 2 columns]"]},"metadata":{},"output_type":"display_data"}],"source":["df_submission = pd.concat([df_ids, df_results], axis =1)\n","df_submission = df_submission.rename(columns={0: 'label'})\n","display(df_submission)"]},{"cell_type":"code","execution_count":64,"metadata":{"cell_id":"e3da92dbbe6c42f28d0802f31f076964","deepnote_cell_height":99,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":23,"execution_start":1658684807735,"source_hash":"21d4263a","tags":[],"trusted":true},"outputs":[],"source":["# Write to csv\n","# df_submission.to_csv(\"LogRed_Prediction.csv\", index=False)"]},{"cell_type":"markdown","metadata":{"cell_id":"00053-672d396b-4f1d-48be-b852-56817a09cd49","deepnote_cell_height":399.1875,"deepnote_cell_type":"markdown"},"source":["# Task 2: Apply Dimension Reduction Techniques\n","\n","Dimension reduction is the transformation of data from a high-dimensional space into a low-dimensional space so that the low-dimensional representation retains some meaningful properties of the original data. \\\n","The train dataset contains 5000 TD-IDF features. In this task, you are to apply PCA to reduce the dimension of features.\n","\n","## Key Task Deliverables\n","\n","2a. Code implementation of PCA on the train and test sets. Note that you are allowed to use the sklearn package for this task. \\\n","2b. Report the Macro F1 scores for applying 2000, 1000, 500, and 100 components on the test set. Note that you will have to submit your predicted labels to Kaggle to retrieve the Macro F1 scores for the test set and report the results in your final report. \\\n","Use KNN as the machine learning model for your training and prediction (You are also allowed to use the sklearn package for KNN implementation) (set n_neighbors=2)."]},{"cell_type":"markdown","metadata":{},"source":["## Initial Setup\n","\n","Import Relevant Packages for Task 2"]},{"cell_type":"code","execution_count":107,"metadata":{"cell_id":"00054-b8072742-0be5-4e46-b68e-f2ede4fc6333","deepnote_cell_height":171,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":false,"execution_millis":0,"execution_start":1658689638099,"source_hash":"b9067867","trusted":true},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","import time\n","\n","from sklearn.decomposition import PCA, IncrementalPCA, KernelPCA, TruncatedSVD\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import f1_score\n","from sklearn.neighbors import KNeighborsClassifier as KNN"]},{"cell_type":"markdown","metadata":{},"source":["Define Functions to Evaluate the Model Performance from adopting Dimensionality Reduction. This will be done simultaneously to observe any improvement in Model Performance along with KNN from using Dimensionality Reduction Techniques"]},{"cell_type":"code","execution_count":108,"metadata":{"trusted":true},"outputs":[],"source":["def sigmoid(z):\n","    result = 1/(1 + np.exp(-z))\n","#     print(f\"sigmoid: {result}\")\n","    return result"]},{"cell_type":"code","execution_count":109,"metadata":{"trusted":true},"outputs":[],"source":["def loss(y, X, w, b, lmb):\n","    y_hat = sigmoid(np.dot(X, w) + b)\n","    m = np.shape(y)[0]\n","    \n","    loss = -1 * np.where(y == 1, np.log(y_hat), np.log(1 - y_hat)).mean()\n","    reg = lmb * np.sum(w**2) / (2 * m)\n","    error = loss + reg\n","    \n","#     print(f\"training loss = {loss}, regularisation term = {reg}, training error = {error}\")\n","    return error"]},{"cell_type":"code","execution_count":110,"metadata":{"trusted":true},"outputs":[],"source":["def gradients(y, X, w, b, lmb):\n","    # m - number of training examples\n","    m = np.shape(X)[0]\n","    y_hat = sigmoid(np.dot(X, w) + b)\n","    \n","    dw = (1 / m) * (np.dot(X.T, (y_hat - y)) + lmb * w)\n","    db = (1 / m) * np.sum((y_hat - y))\n","    \n","#     print(f\"dw: {dw}, db: {db}\")\n","    return dw, db"]},{"cell_type":"code","execution_count":111,"metadata":{"trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train3(X, y, bs, epochs, lr, C):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            w_new = w.copy() - lr * dw\n","            b_new = b - lr * db\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","        \n","        min_loss = min(old_losses)\n","        min_index = old_losses.index(min_loss)\n","        w = old_w[min_index]\n","        b = old_b[min_index]\n","        \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"code","execution_count":112,"metadata":{"trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train4(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            if (loss_new < loss_old):\n","                w = w_new\n","                b = b_new\n","                old_w.append(w_new)\n","                old_b.append(b_new)\n","                old_losses.append(loss_new)\n","    \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"code","execution_count":113,"metadata":{"trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train5(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","\n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"code","execution_count":114,"metadata":{"trusted":true},"outputs":[],"source":["# @param X - features\n","# @param y - labels\n","# @param bs - batch size\n","# @param epochs - number of iterations through dataset\n","# @param lr - learning rate\n","\n","def train6(X, y, bs, epochs, lr, C, beta_m, beta_v, err):\n","    lmb = 0 if C == 0 else 1/C\n","    \n","    # n - number of training examples, d - number of features\n","    n, d = np.shape(X)\n","    \n","    randomize = np.arange(n)\n","    rng = np.random.default_rng(100)\n","    \n","    w = rng.uniform(size=(d,1))\n","    b = rng.random()\n","    \n","    m_w = np.zeros((d, 1))\n","    m_b = 0\n","    v_w = np.zeros((d, 1))\n","    v_b = 0\n","    \n","    y = y.reshape(n, 1)\n","    \n","    old_losses = []\n","    old_w = []\n","    old_b = []\n","    \n","    old_w.append(w.copy())\n","    old_b.append(b)\n","    l = loss(y, X, w, b, lmb)\n","    old_losses.append(l)\n","    \n","    for epoch in range(epochs):\n","        limit = n // bs\n","\n","        for i in range(limit):\n","            start = i * bs\n","            end = start + bs\n","            \n","            rng.shuffle(randomize)\n","            choice = randomize[start:end]\n","            X_batch = X[choice]\n","            y_batch = y[choice]\n","            \n","            loss_old = loss(y, X, w, b, lmb)\n","            \n","            dw, db = gradients(y_batch, X_batch, w, b, lmb)\n","            \n","            #AdamW\n","            m_w = beta_m * m_w + (1 - beta_m) * dw\n","            m_b = beta_m * m_b + (1 - beta_m) * db\n","            v_w = beta_v * v_w + (1 - beta_v) * np.square(dw)\n","            v_b = beta_v * v_b + (1 - beta_v) * (db**2)\n","            \n","            #bias correction\n","            t = len(old_losses)\n","            m_what = m_w /(1 - beta_m**t)\n","            m_bhat = m_b /(1 - beta_m**t)\n","            v_what = v_w /(1 - beta_v**t)\n","            v_bhat = v_b /(1 - beta_v**t)\n","            \n","            w_new = w.copy() - lr * (m_what/(np.sqrt(v_what) + err) + lmb * w.copy() / bs)\n","            b_new = b - lr * lr * (m_bhat/(np.sqrt(v_bhat) + err))\n","            loss_new = loss(y, X, w_new, b_new, lmb)\n","            \n","            w = w_new\n","            b = b_new\n","            old_w.append(w_new)\n","            old_b.append(b_new)\n","            old_losses.append(loss_new)\n","        \n","        min_loss = min(old_losses)\n","        min_index = old_losses.index(min_loss)\n","        w = old_w[min_index]\n","        b = old_b[min_index]\n","        \n","    min_loss = min(old_losses)\n","    min_index = old_losses.index(min_loss)\n","    \n","    return old_w[min_index], old_b[min_index], old_losses"]},{"cell_type":"code","execution_count":115,"metadata":{"trusted":true},"outputs":[],"source":["def predict(X, w, b):\n","    y_pred = sigmoid(np.dot(X, w) + b)\n","    pred_labels = np.array([1 if i >= 0.5 else 0 for i in y_pred])\n","    return pred_labels"]},{"cell_type":"code","execution_count":116,"metadata":{"trusted":true},"outputs":[],"source":["def score(y, y_hat):\n","    accuracy = np.sum(y == y_hat) / np.shape(y)[0]\n","    f1score = f1_score(y, y_hat, average='macro')\n","\n","    print(f\"Accuracy: {accuracy}\")\n","    print(f\"Macro-F1 score: {f1score}\")\n","\n","    # Return Macro-F1 score of the model\n","    return f1score"]},{"cell_type":"markdown","metadata":{},"source":["Initialise Training Data for Task 2."]},{"cell_type":"code","execution_count":117,"metadata":{"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>17179</th>\n","      <td>17180</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17180</th>\n","      <td>17181</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17181</th>\n","      <td>17182</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17182</th>\n","      <td>17183</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17183</th>\n","      <td>17184</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>17184 rows × 5002 columns</p>\n","</div>"],"text/plain":["          id  label    0    1    2    3    4    5    6    7  ...  4990  4991  \\\n","0          1      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1          2      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2          3      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3          4      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4          5      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...      ...    ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","17179  17180      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17180  17181      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17181  17182      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17182  17183      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17183  17184      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","       4992  4993  4994  4995  4996  4997  4998  4999  \n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...     ...   ...   ...   ...   ...   ...   ...   ...  \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[17184 rows x 5002 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 15.1 s\n","Wall time: 15.6 s\n"]}],"source":["%%time\n","df_train = pd.read_csv(r\"./source/train_tfidf_features.csv\")\n","display(df_train)"]},{"cell_type":"code","execution_count":118,"metadata":{"cell_id":"9c9a773b88434f6db5fcea744c70d847","deepnote_cell_height":117,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":6,"execution_start":1658689631971,"source_hash":"db08c607","tags":[],"trusted":true},"outputs":[],"source":["# Define features and label\n","df_features = df_train.iloc[:,2:]\n","df_label = df_train.iloc[:, 1]"]},{"cell_type":"code","execution_count":119,"metadata":{"cell_id":"2e1a6b992de1448c8ab6e1b63c8d16af","deepnote_cell_height":616.796875,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":2024,"execution_start":1658688637833,"source_hash":"55cf74dd","tags":[],"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>9</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>17179</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17180</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17181</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17182</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17183</th>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>17184 rows × 5000 columns</p>\n","</div>"],"text/plain":["         0    1    2    3    4    5    6    7    8    9  ...  4990  4991  \\\n","0      0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1      0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2      0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3      0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4      0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...    ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","17179  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17180  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17181  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17182  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17183  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","       4992  4993  4994  4995  4996  4997  4998  4999  \n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...     ...   ...   ...   ...   ...   ...   ...   ...  \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[17184 rows x 5000 columns]"]},"metadata":{},"output_type":"display_data"}],"source":["display(df_features)"]},{"cell_type":"code","execution_count":120,"metadata":{"cell_id":"969eabaf5a124d86887172291e438e29","deepnote_cell_height":600,"deepnote_cell_type":"code","deepnote_output_heights":[232.390625],"deepnote_to_be_reexecuted":true,"execution_millis":26,"execution_start":1658688639858,"source_hash":"ef276ad2","tags":[],"trusted":true},"outputs":[{"data":{"text/plain":["0        1\n","1        0\n","2        1\n","3        0\n","4        1\n","        ..\n","17179    0\n","17180    0\n","17181    1\n","17182    1\n","17183    0\n","Name: label, Length: 17184, dtype: int64"]},"metadata":{},"output_type":"display_data"}],"source":["display(df_label)"]},{"cell_type":"markdown","metadata":{},"source":["## Initializing the Control Model for Evaluation"]},{"cell_type":"markdown","metadata":{},"source":["We will first train the model without dimensionality reduction. We will use this as a control model for evaluation."]},{"cell_type":"code","execution_count":17,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 5000), y_train: (12888,)\n","X_test: (4296, 5000), y_test: (4296,)\n","CPU times: total: 609 ms\n","Wall time: 629 ms\n"]}],"source":["%%time\n","X_train, X_test, y_train, y_test = train_test_split(df_features, df_label, \\\n","test_size=0.25, random_state=100)\n","print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n","print(f\"X_test: {X_test.shape}, y_test: {y_test.shape}\")"]},{"cell_type":"code","execution_count":18,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.45763500931098694\n","Macro-F1 score: 0.44500777124850965\n","CPU times: total: 43.5 s\n","Wall time: 7.14 s\n"]},{"data":{"text/plain":["0.44500777124850965"]},"execution_count":18,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","knn = KNN(n_neighbors=2)\n","knn.fit(X_train, y_train)\n","score(y_test, knn.predict(X_test))"]},{"cell_type":"markdown","metadata":{},"source":["We see that the KNearestNeighbor Model has resulted in a score of `0.44500` which indicates that KNN is not a good algorithm without dimensionality reduction."]},{"cell_type":"markdown","metadata":{},"source":["## Applying Principal Component Analysis (PCA)"]},{"cell_type":"markdown","metadata":{},"source":["We shall first do a simple evaluation of PCA by using 1000 components after dimensionality reduction. We started with 1000 components as we believed that using 100 components is too small and using 2000 components may result in overfitting. Hence, 1000 components is a good litmus test to evaluate the utility of dimensionality reduction."]},{"cell_type":"code","execution_count":19,"metadata":{"trusted":true},"outputs":[],"source":["n_components = 1000\n","random_state = 100"]},{"cell_type":"code","execution_count":20,"metadata":{"cell_id":"00056-8acc8aa1-45e7-4e0b-8a3c-4562d28d4d7d","deepnote_cell_height":292,"deepnote_cell_type":"code","deepnote_output_heights":[194],"deepnote_to_be_reexecuted":true,"execution_millis":147893,"execution_start":1658687140783,"source_hash":"9be852a0","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 3min 23s\n","Wall time: 27.8 s\n"]}],"source":["%%time\n","pca = PCA(n_components=n_components, random_state=random_state)\n","pca.fit(df_features.to_numpy())\n","# print(pca.explained_variance_ratio_)\n","X_pca = pca.transform(df_features.to_numpy())\n","\n","# plt.figure(figsize=(8,6))\n","# plt.scatter(x_pca[:,0], x_3d[:,1], c=y_label) # show plotting of labels visually\n","# plt.show()"]},{"cell_type":"code","execution_count":21,"metadata":{"cell_id":"00057-9ee82dfa-b05d-4471-91b1-da119e05dea0","deepnote_cell_height":484,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":1,"execution_start":1658687288706,"source_hash":"e6cf271d","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Total Variance Explained: 0.647878600154715\n","(17184, 1000)\n","(17184,)\n"]}],"source":["# check before running\n","print(\"Total Variance Explained:\", np.sum(pca.explained_variance_ratio_))\n","# print(X_pca)\n","print(X_pca.shape)\n","print(df_label.shape)"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, we can see that the top 1000 principal components explain `64.788%` of the variance of the dataset."]},{"cell_type":"code","execution_count":22,"metadata":{"cell_id":"00058-60ad6fff-42a2-4fe9-bba2-6e4d2a9e6a93","deepnote_cell_height":186,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":625,"execution_start":1658688674821,"source_hash":"11567c9b","trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","CPU times: total: 62.5 ms\n","Wall time: 64 ms\n"]}],"source":["%%time\n","X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","test_size=0.25, random_state=100)\n","print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")"]},{"cell_type":"code","execution_count":23,"metadata":{"cell_id":"60773b9298624ba5b5517c858c3670aa","deepnote_cell_height":251,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":6962,"execution_start":1658689289370,"source_hash":"d310c324","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6338454376163873\n","Macro-F1 score: 0.5734082986431286\n","CPU times: total: 10.2 s\n","Wall time: 1.19 s\n"]},{"data":{"text/plain":["0.5734082986431286"]},"execution_count":23,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","knn = KNN(n_neighbors=2)\n","knn.fit(X_pcatrain, y_pcatrain)\n","score(y_pcatest, knn.predict(X_pcatest))"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, the KNN model has observed a significant improvement in performance from `0.44500` to `0.57341`. Hence, there is a 29% improvement in model performance from using Principal Component Analysis. Moreover, the computation of PCA and KNN algorithm occurred less than a minute. Hence, PCA may be a viable technique to employ to improve our model performance."]},{"cell_type":"markdown","metadata":{},"source":["In preparation of our submission, we shall now observe how our model performance and computation time changes as we increase the number of principal components used in our model."]},{"cell_type":"code","execution_count":30,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100\n","0.20904054096604335\n","X_train: (12888, 100), y_train: (12888,)\n","X_test: (4296, 100), y_test: (4296,)\n","Accuracy: 0.6268621973929237\n","Macro-F1 score: 0.449384679410837\n","Model3: 0.449384679410837\n","\n","Accuracy: 0.686219739292365\n","Macro-F1 score: 0.62909110473182\n","Model4: 0.62909110473182\n","\n","Accuracy: 0.6859869646182495\n","Macro-F1 score: 0.629215754080892\n","Model5: 0.629215754080892\n","\n","Accuracy: 0.6859869646182495\n","Macro-F1 score: 0.628896074643623\n","Model6: 0.628896074643623\n","\n","Accuracy: 0.6559590316573557\n","Macro-F1 score: 0.5602725837280811\n","KNN: 0.5602725837280811 \n","\n","Time taken for 100 Principal Components: 40.77728796005249s \n","\n","n_components = 500\n","0.483136414608792\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.6273277467411545\n","Macro-F1 score: 0.4933758890271646\n","Model3: 0.4933758890271646\n","\n","Accuracy: 0.7164804469273743\n","Macro-F1 score: 0.6767566245232456\n","Model4: 0.6767566245232456\n","\n","Accuracy: 0.7160148975791434\n","Macro-F1 score: 0.6759843704522317\n","Model5: 0.6759843704522317\n","\n","Accuracy: 0.7160148975791434\n","Macro-F1 score: 0.6759843704522317\n","Model6: 0.6759843704522317\n","\n","Accuracy: 0.6531657355679702\n","Macro-F1 score: 0.5511605103373378\n","KNN: 0.5511605103373378 \n","\n","Time taken for 500 Principal Components: 91.41721653938293s \n","\n","n_components = 1000\n","0.647878600154715\n","X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","Accuracy: 0.616852886405959\n","Macro-F1 score: 0.4968542790485735\n","Model3: 0.4968542790485735\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6834221098977449\n","Model4: 0.6834221098977449\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6834221098977449\n","Model5: 0.6834221098977449\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6834221098977449\n","Model6: 0.6834221098977449\n","\n","Accuracy: 0.6338454376163873\n","Macro-F1 score: 0.5734082986431286\n","KNN: 0.5734082986431286 \n","\n","Time taken for 1000 Principal Components: 165.50782799720764s \n","\n","n_components = 1500\n","0.7515722940280163\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.6101024208566108\n","Macro-F1 score: 0.4958661769137143\n","Model3: 0.4958661769137143\n","\n","Accuracy: 0.7213687150837989\n","Macro-F1 score: 0.6869214722857382\n","Model4: 0.6869214722857382\n","\n","Accuracy: 0.7213687150837989\n","Macro-F1 score: 0.6869214722857382\n","Model5: 0.6869214722857382\n","\n","Accuracy: 0.7213687150837989\n","Macro-F1 score: 0.6869214722857382\n","Model6: 0.6869214722857382\n","\n","Accuracy: 0.5793761638733705\n","Macro-F1 score: 0.5626968579620144\n","KNN: 0.5626968579620144 \n","\n","Time taken for 1500 Principal Components: 235.85887789726257s \n","\n","n_components = 2000\n","0.8247068485234131\n","X_train: (12888, 2000), y_train: (12888,)\n","X_test: (4296, 2000), y_test: (4296,)\n","Accuracy: 0.6110335195530726\n","Macro-F1 score: 0.5013253894944569\n","Model3: 0.5013253894944569\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6914394503956895\n","Model4: 0.6914394503956895\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6914394503956895\n","Model5: 0.6914394503956895\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6914394503956895\n","Model6: 0.6914394503956895\n","\n","Accuracy: 0.5174581005586593\n","Macro-F1 score: 0.5169077523878887\n","KNN: 0.5169077523878887 \n","\n","Time taken for 2000 Principal Components: 310.6528036594391s \n","\n","n_components = 2500\n","0.8790757792783456\n","X_train: (12888, 2500), y_train: (12888,)\n","X_test: (4296, 2500), y_test: (4296,)\n","Accuracy: 0.6108007448789572\n","Macro-F1 score: 0.5084915741162457\n","Model3: 0.5084915741162457\n","\n","Accuracy: 0.7269553072625698\n","Macro-F1 score: 0.6942523327996672\n","Model4: 0.6942523327996672\n","\n","Accuracy: 0.7269553072625698\n","Macro-F1 score: 0.6942523327996672\n","Model5: 0.6942523327996672\n","\n","Accuracy: 0.7269553072625698\n","Macro-F1 score: 0.6942523327996672\n","Model6: 0.6942523327996672\n","\n","Accuracy: 0.48463687150837986\n","Macro-F1 score: 0.4829689803479872\n","KNN: 0.4829689803479872 \n","\n","Time taken for 2500 Principal Components: 401.5583200454712s \n","\n","n_components = 3000\n","0.9203410112108971\n","X_train: (12888, 3000), y_train: (12888,)\n","X_test: (4296, 3000), y_test: (4296,)\n","Accuracy: 0.6019553072625698\n","Macro-F1 score: 0.49785914661154584\n","Model3: 0.49785914661154584\n","\n","Accuracy: 0.722998137802607\n","Macro-F1 score: 0.689768295953948\n","Model4: 0.689768295953948\n","\n","Accuracy: 0.722998137802607\n","Macro-F1 score: 0.689768295953948\n","Model5: 0.689768295953948\n","\n","Accuracy: 0.722998137802607\n","Macro-F1 score: 0.689768295953948\n","Model6: 0.689768295953948\n","\n","Accuracy: 0.46997206703910616\n","Macro-F1 score: 0.4635234886906304\n","KNN: 0.4635234886906304 \n","\n","Time taken for 3000 Principal Components: 500.49055099487305s \n","\n","CPU times: total: 2h 37min 26s\n","Wall time: 29min 6s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","comps = [100, 500, 1000, 1500, 2000, 2500, 3000] #added 2 more n_values\n","random_state = 100\n","\n","for n_components in comps:\n","    start = time.time()\n","    pca = PCA(n_components=n_components, random_state=random_state)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}\")\n","    print(\"Total Variance Explained:\", np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","    Model3 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model3: {Model3}\\n\")\n","    \n","    w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model4 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model4: {Model4}\\n\")\n","    \n","    w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model5 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model5: {Model5}\\n\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore} \\n\")\n","    elapsed = time.time() - start\n","    print(f\"Time taken for {n_components} Principal Components: {elapsed}s \\n\")"]},{"cell_type":"markdown","metadata":{},"source":["## Test Set Predictions using PCA\n","\n","As per the grading rubric - \"Perfectly implemented the PCA and KNN. Implemented model is able to run on test sets with reduced components and performed detail analysis of the reduced components\", we shall predict the labels of the Test dataset.\n","\n","We shall first implement a simple pipeline to generate our predictions using 100, 500, 1000 and 2000 components respectively. We will then submit the results to Kaggle and report the scores."]},{"cell_type":"code","execution_count":25,"metadata":{"trusted":true},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 5001 columns</p>\n","</div>"],"text/plain":["         id    0    1    2    3    4    5    6    7    8  ...  4990  4991  \\\n","0     17185  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1     17186  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2     17187  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3     17188  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4     17189  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...     ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","4291  21476  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4292  21477  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4293  21478  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4294  21479  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4295  21480  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","      4992  4993  4994  4995  4996  4997  4998  4999  \n","0      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...    ...   ...   ...   ...   ...   ...   ...   ...  \n","4291   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4292   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4293   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4294   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4295   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[4296 rows x 5001 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 4.95 s\n","Wall time: 5.07 s\n"]}],"source":["%%time\n","df_test = pd.read_csv(r\"./source/test_tfidf_features.csv\")\n","display(df_test)"]},{"cell_type":"code","execution_count":31,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100\n","0.20904054096604335\n","(4296, 1) (4296, 1)\n","Time taken for 100 Principal Components: 12.13409686088562s \n","\n","n_components = 500\n","0.483136414608792\n","(4296, 1) (4296, 1)\n","Time taken for 500 Principal Components: 24.379785776138306s \n","\n","n_components = 1000\n","0.647878600154715\n","(4296, 1) (4296, 1)\n","Time taken for 1000 Principal Components: 39.55708885192871s \n","\n","n_components = 2000\n","0.8247068485234131\n","(4296, 1) (4296, 1)\n","Time taken for 2000 Principal Components: 70.41422748565674s \n","\n","CPU times: total: 16min 34s\n","Wall time: 2min 26s\n"]}],"source":["%%time\n","comps = [100, 500, 1000, 2000]\n","random_state = 100\n","\n","for n_components in comps:\n","    start = time.time()\n","    pca = PCA(n_components=n_components, random_state=random_state)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}\")\n","    print(\"Total Variance Explained:\", np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pca, df_label)\n","    \n","    features = df_test.iloc[:,1:]\n","    \n","    X_test = pca.transform(features.to_numpy())\n","    results = knn.predict(X_test)\n","#     display(results)\n","    \n","    df_ids = df_test.iloc[:, 0].to_frame()\n","    df_results = pd.DataFrame(results)\n","    print(df_results.shape, df_ids.shape)\n","    \n","    df_submission = pd.concat([df_ids, df_results], axis =1)\n","    df_submission = df_submission.rename(columns={0: 'label'})\n","    elapsed = time.time() - start\n","    print(f\"Time taken for {n_components} Principal Components: {elapsed}s \\n\")\n","#     display(df_submission)\n","    \n","#     df_submission.to_csv(f\"KNN_Prediction_PCA_{n_components}.csv\", index=False)"]},{"cell_type":"markdown","metadata":{},"source":["Note: Compare Computation Time, F1-Score and Cross Validation Improvement e.g. x% increase in score vs y% increase in computational time (To be removed)"]},{"cell_type":"markdown","metadata":{},"source":["Based on our submissions to Kaggle, we have attained the following scores for the test set:\n","\n","***Macro-F1 Score based on n principal components -***\n","* n = 100: public score = 0.54753\n","* n = 500: public score = 0.57102\n","* n = 1000: public score = 0.57573\n","* n = 2000: public score = 0.49821\n","\n","Based on the above, we can clearly observe that there is an optimal number of principal components to use to obtain the best performing model from PCA. Nonetheless, we feel that PCA is insufficient to boost our model performance. Hence, we shall explore other dimensionality reduction techniques such as TruncatedSVD and KernelPCA. However, we will first explore the use IncrementalPCA to evaluate the boost in computational time from using IncrementalPCA compared to regular PCA and the model performance attained from using it."]},{"cell_type":"markdown","metadata":{},"source":["## Trying other Dimensionality Reduction Methods"]},{"cell_type":"markdown","metadata":{},"source":["### Initializing the PCA Control Model for Evaluation\n","\n","We shall first initialise a control model to evaluate our model from using other Dimensionality Reduction Techniques"]},{"cell_type":"code","execution_count":15,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100\n","Total Variance Explained: 0.20904054096604335\n","X_train: (12888, 100), y_train: (12888,)\n","X_test: (4296, 100), y_test: (4296,)\n","Accuracy: 0.6268621973929237\n","Macro-F1 score: 0.449384679410837\n","Model3: 0.449384679410837\n","\n","Accuracy: 0.686219739292365\n","Macro-F1 score: 0.62909110473182\n","Model4: 0.62909110473182\n","\n","Accuracy: 0.6859869646182495\n","Macro-F1 score: 0.629215754080892\n","Model5: 0.629215754080892\n","\n","Accuracy: 0.6859869646182495\n","Macro-F1 score: 0.628896074643623\n","Model6: 0.628896074643623\n","\n","Accuracy: 0.6559590316573557\n","Macro-F1 score: 0.5602725837280811\n","KNN: 0.5602725837280811 \n","\n","Time taken for 100 Principal Components: 40.18019723892212s \n","\n","n_components = 500\n","Total Variance Explained: 0.483136414608792\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.6273277467411545\n","Macro-F1 score: 0.4933758890271646\n","Model3: 0.4933758890271646\n","\n","Accuracy: 0.7164804469273743\n","Macro-F1 score: 0.6767566245232456\n","Model4: 0.6767566245232456\n","\n","Accuracy: 0.7160148975791434\n","Macro-F1 score: 0.6759843704522317\n","Model5: 0.6759843704522317\n","\n","Accuracy: 0.7160148975791434\n","Macro-F1 score: 0.6759843704522317\n","Model6: 0.6759843704522317\n","\n","Accuracy: 0.6531657355679702\n","Macro-F1 score: 0.5511605103373378\n","KNN: 0.5511605103373378 \n","\n","Time taken for 500 Principal Components: 103.14000916481018s \n","\n","n_components = 1000\n","Total Variance Explained: 0.647878600154715\n","X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","Accuracy: 0.616852886405959\n","Macro-F1 score: 0.4968542790485735\n","Model3: 0.4968542790485735\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6834221098977449\n","Model4: 0.6834221098977449\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6834221098977449\n","Model5: 0.6834221098977449\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6834221098977449\n","Model6: 0.6834221098977449\n","\n","Accuracy: 0.6338454376163873\n","Macro-F1 score: 0.5734082986431286\n","KNN: 0.5734082986431286 \n","\n","Time taken for 1000 Principal Components: 166.6883625984192s \n","\n","n_components = 1500\n","Total Variance Explained: 0.7515722940280163\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.6101024208566108\n","Macro-F1 score: 0.4958661769137143\n","Model3: 0.4958661769137143\n","\n","Accuracy: 0.7213687150837989\n","Macro-F1 score: 0.6869214722857382\n","Model4: 0.6869214722857382\n","\n","Accuracy: 0.7213687150837989\n","Macro-F1 score: 0.6869214722857382\n","Model5: 0.6869214722857382\n","\n","Accuracy: 0.7213687150837989\n","Macro-F1 score: 0.6869214722857382\n","Model6: 0.6869214722857382\n","\n","Accuracy: 0.5793761638733705\n","Macro-F1 score: 0.5626968579620144\n","KNN: 0.5626968579620144 \n","\n","Time taken for 1500 Principal Components: 236.03642630577087s \n","\n","n_components = 2000\n","Total Variance Explained: 0.8247068485234131\n","X_train: (12888, 2000), y_train: (12888,)\n","X_test: (4296, 2000), y_test: (4296,)\n","Accuracy: 0.6110335195530726\n","Macro-F1 score: 0.5013253894944569\n","Model3: 0.5013253894944569\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6914394503956895\n","Model4: 0.6914394503956895\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6914394503956895\n","Model5: 0.6914394503956895\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6914394503956895\n","Model6: 0.6914394503956895\n","\n","Accuracy: 0.5174581005586593\n","Macro-F1 score: 0.5169077523878887\n","KNN: 0.5169077523878887 \n","\n","Time taken for 2000 Principal Components: 323.81342601776123s \n","\n","CPU times: total: 1h 13min 13s\n","Wall time: 14min 29s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","comps = [100, 500, 1000, 1500, 2000] #added 2 more n_values\n","random_state = 100\n","\n","for n_components in comps:\n","    start = time.time()\n","    pca = PCA(n_components=n_components, random_state=random_state)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}\")\n","    print(\"Total Variance Explained:\", np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","    Model3 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model3: {Model3}\\n\")\n","    \n","    w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model4 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model4: {Model4}\\n\")\n","    \n","    w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model5 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model5: {Model5}\\n\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore} \\n\")\n","    elapsed = time.time() - start\n","    print(f\"Time taken for {n_components} Principal Components: {elapsed}s \\n\")"]},{"cell_type":"markdown","metadata":{},"source":["### IncrementalPCA\n","\n","We will first implement IncrementalPCA to compare the computational time compared to regular PCA. We will evaluate the performance in terms of computational time and model performance. We shall first test the function to observe the computational time to generate 1000 principal components using IncrementalPCA."]},{"cell_type":"code","execution_count":16,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components=1000, batch_size=2000\n"]}],"source":["n_components = 1000\n","batch_size = 2*n_components # must be greater than n_components\n","\n","print(f\"n_components={n_components}, batch_size={batch_size}\")"]},{"cell_type":"code","execution_count":18,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 24min 47s\n","Wall time: 4min 3s\n"]}],"source":["%%time\n","increment = IncrementalPCA(n_components=n_components, batch_size=batch_size)\n","increment.fit(df_features.to_numpy())\n","X_increment = increment.transform(df_features.to_numpy())"]},{"cell_type":"code","execution_count":19,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Total Variance Explained: 0.8247068485234131\n","(17184, 1000)\n","(17184,)\n"]}],"source":["# check before running\n","print(\"Total Variance Explained:\", np.sum(pca.explained_variance_ratio_))\n","# print(X_increment)\n","print(X_increment.shape)\n","print(df_label.shape)"]},{"cell_type":"code","execution_count":20,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","CPU times: total: 46.9 ms\n","Wall time: 52 ms\n"]}],"source":["%%time\n","X_incrementtrain, X_incrementtest, y_incrementtrain, y_incrementtest = train_test_split(X_increment, df_label, test_size=0.25, random_state=100)\n","print(f\"X_train: {X_incrementtrain.shape}, y_train: {y_incrementtrain.shape}\")\n","print(f\"X_test: {X_incrementtest.shape}, y_test: {y_incrementtest.shape}\")"]},{"cell_type":"code","execution_count":21,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7195065176908753\n","Macro-F1 score: 0.6847195618177574\n","CPU times: total: 2min 32s\n","Wall time: 32.6 s\n"]},{"data":{"text/plain":["0.6847195618177574"]},"execution_count":21,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","w, b, l = train6(X_incrementtrain, y_incrementtrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","score(y_incrementtest.values,  predict(X_incrementtest, w, b))"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, we were able to achieve a model performance of `0.68472` from IncrementalPCA (n=1000), which is comparable to a model performance of `0.68342` from PCA (n=1000). We shall now compare computational time from running the IncrementalPCA. We will set a batch size of 4000 to standardise the comparison and reduce computational time."]},{"cell_type":"code","execution_count":22,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100, batch_size = 4000\n","Total Variance Explained: 0.20455528952164242\n","X_train: (12888, 100), y_train: (12888,)\n","X_test: (4296, 100), y_test: (4296,)\n","Accuracy: 0.6343109869646183\n","Macro-F1 score: 0.4624685846725459\n","Model3: 0.4624685846725459\n","\n","Accuracy: 0.6908752327746741\n","Macro-F1 score: 0.6369257086999023\n","Model4: 0.6369257086999023\n","\n","Accuracy: 0.6911080074487895\n","Macro-F1 score: 0.6380355527962599\n","Model5: 0.6380355527962599\n","\n","Accuracy: 0.6913407821229051\n","Macro-F1 score: 0.6377775313432503\n","Model6: 0.6377775313432503\n","\n","Accuracy: 0.6613128491620112\n","Macro-F1 score: 0.5686668113488322\n","KNN: 0.5686668113488322 \n","\n","Time taken for 100 Principal Components: 239.91583132743835s \n","\n","n_components = 500, batch_size = 4000\n","Total Variance Explained: 0.4750607106451144\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.6252327746741154\n","Macro-F1 score: 0.4903620199836186\n","Model3: 0.4903620199836186\n","\n","Accuracy: 0.714851024208566\n","Macro-F1 score: 0.6759182424233281\n","Model4: 0.6759182424233281\n","\n","Accuracy: 0.715316573556797\n","Macro-F1 score: 0.6762095149269713\n","Model5: 0.6762095149269713\n","\n","Accuracy: 0.715316573556797\n","Macro-F1 score: 0.6762095149269713\n","Model6: 0.6762095149269713\n","\n","Accuracy: 0.654096834264432\n","Macro-F1 score: 0.5528786189352892\n","KNN: 0.5528786189352892 \n","\n","Time taken for 500 Principal Components: 320.3189721107483s \n","\n","n_components = 1000, batch_size = 4000\n","Total Variance Explained: 0.6365485970168069\n","X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","Accuracy: 0.6212756052141527\n","Macro-F1 score: 0.5031069005987306\n","Model3: 0.5031069005987306\n","\n","Accuracy: 0.7188081936685289\n","Macro-F1 score: 0.6849681226762188\n","Model4: 0.6849681226762188\n","\n","Accuracy: 0.7188081936685289\n","Macro-F1 score: 0.6849681226762188\n","Model5: 0.6849681226762188\n","\n","Accuracy: 0.7188081936685289\n","Macro-F1 score: 0.6849681226762188\n","Model6: 0.6849681226762188\n","\n","Accuracy: 0.6291899441340782\n","Macro-F1 score: 0.5672758840014516\n","KNN: 0.5672758840014516 \n","\n","Time taken for 1000 Principal Components: 412.80407547950745s \n","\n","n_components = 1500, batch_size = 4000\n","Total Variance Explained: 0.7401017867402803\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.6145251396648045\n","Macro-F1 score: 0.5053861104954276\n","Model3: 0.5053861104954276\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6907014118132113\n","Model4: 0.6907014118132113\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6907014118132113\n","Model5: 0.6907014118132113\n","\n","Accuracy: 0.7241620111731844\n","Macro-F1 score: 0.6907014118132113\n","Model6: 0.6907014118132113\n","\n","Accuracy: 0.5735567970204841\n","Macro-F1 score: 0.5548794834020465\n","KNN: 0.5548794834020465 \n","\n","Time taken for 1500 Principal Components: 505.742333650589s \n","\n","n_components = 2000, batch_size = 4000\n","Total Variance Explained: 0.8148522103653635\n","X_train: (12888, 2000), y_train: (12888,)\n","X_test: (4296, 2000), y_test: (4296,)\n","Accuracy: 0.6087057728119181\n","Macro-F1 score: 0.5011204723869305\n","Model3: 0.5011204723869305\n","\n","Accuracy: 0.7248603351955307\n","Macro-F1 score: 0.6925846497192871\n","Model4: 0.6925846497192871\n","\n","Accuracy: 0.7248603351955307\n","Macro-F1 score: 0.6925846497192871\n","Model5: 0.6925846497192871\n","\n","Accuracy: 0.7248603351955307\n","Macro-F1 score: 0.6925846497192871\n","Model6: 0.6925846497192871\n","\n","Accuracy: 0.5235102420856611\n","Macro-F1 score: 0.5228566382542694\n","KNN: 0.5228566382542694 \n","\n","Time taken for 2000 Principal Components: 596.8671686649323s \n","\n","CPU times: total: 3h 22min 46s\n","Wall time: 34min 35s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","comps = [100, 500, 1000, 1500, 2000]\n","\n","for n_components in comps:\n","    start = time.time()\n","    batch_size = 4000\n","    pca = IncrementalPCA(n_components=n_components, batch_size=batch_size)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}, batch_size = {batch_size}\")\n","    print(\"Total Variance Explained:\", np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","    Model3 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model3: {Model3}\\n\")\n","    \n","    w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model4 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model4: {Model4}\\n\")\n","    \n","    w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model5 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model5: {Model5}\\n\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore} \\n\")\n","    elapsed = time.time() - start\n","    print(f\"Time taken for {n_components} Principal Components: {elapsed}s \\n\")"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, we observe that the IncrementalPCA offers comparable performance to PCA. However, it is significantly slower than PCA (e.g. IncrementalPCA (n=1000) took a total of `412.80s` for dimensionality reduction and training 4 models and enabled Model 6 to achieve a Macro-F1 score of `0.68497` while PCA (n=1000) took a total of `166.69s` and enabled Model 6 to achieve a Macro-F1 score of  `0.68342`). Hence, using PCA is sufficient in improving our model performance compared to IncrementalPCA while reducing computational time. We assume that this is due to the relatively small batch size. Hence, we could consider trying a large batch size as a future implementation."]},{"cell_type":"markdown","metadata":{},"source":["### TruncatedSVD\n","\n","We will now implement TruncatedSVD to compare the computational time compared to regular PCA. We will evaluate the performance in terms of computational time and model performance. We shall first test the function to observe the computational time to generate 1000 principal components using TruncatedSVD."]},{"cell_type":"code","execution_count":23,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components=1000, random_state=100\n"]}],"source":["n_components = 1000\n","random_state = 100\n","\n","print(f\"n_components={n_components}, random_state={random_state}\")"]},{"cell_type":"code","execution_count":24,"metadata":{"cell_id":"65b25c6d510f4ecfa805b96d6de5ffd0","deepnote_cell_height":166,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":187050,"execution_start":1658687820350,"source_hash":"db4df06f","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 4min 57s\n","Wall time: 45.3 s\n"]}],"source":["%%time\n","svd = TruncatedSVD(n_components=n_components, random_state=random_state)\n","svd.fit(df_features.to_numpy())\n","X_svd = svd.transform(df_features.to_numpy())"]},{"cell_type":"code","execution_count":25,"metadata":{"cell_id":"9d2894b2c7f540b1b173e4b8f26a79cf","deepnote_cell_height":484,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":30,"execution_start":1658688007408,"source_hash":"fbc03bf3","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Total Variance Explained: 0.650073164448684\n","(17184, 1000)\n","(17184,)\n"]}],"source":["# check before running\n","print(\"Total Variance Explained:\", np.sum(svd.explained_variance_ratio_))\n","# print(X_svd)\n","print(X_svd.shape)\n","print(df_label.shape)"]},{"cell_type":"code","execution_count":26,"metadata":{"cell_id":"bb4aac5d0fc7440f8216a9ac85999898","deepnote_cell_height":186,"deepnote_cell_type":"code","deepnote_to_be_reexecuted":true,"execution_millis":122,"execution_start":1658689301898,"source_hash":"8dafaa66","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","CPU times: total: 62.5 ms\n","Wall time: 60 ms\n"]}],"source":["%%time\n","X_svdtrain, X_svdtest, y_svdtrain, y_svdtest = train_test_split(X_svd, df_label, test_size=0.25, random_state=100)\n","print(f\"X_train: {X_svdtrain.shape}, y_train: {y_svdtrain.shape}\")\n","print(f\"X_test: {X_svdtest.shape}, y_test: {y_svdtest.shape}\")"]},{"cell_type":"code","execution_count":27,"metadata":{"cell_id":"063c67f915ab44e49b397ab3e1bb8e5c","deepnote_cell_height":418.1875,"deepnote_cell_type":"code","deepnote_output_heights":[null,21.1875],"deepnote_to_be_reexecuted":true,"execution_millis":98527,"execution_start":1658689410287,"source_hash":"c8d685df","tags":[],"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6836444589405164\n","CPU times: total: 2min 31s\n","Wall time: 40.6 s\n"]},{"data":{"text/plain":["0.6836444589405164"]},"execution_count":27,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","w, b, l = train6(X_svdtrain, y_svdtrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","score(y_svdtest.values,  predict(X_svdtest, w, b))"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, we were able to achieve a model performance of `0.68364` from TruncatedSVD (n=1000), which is comparable to a model performance of `0.68342` from PCA (n=1000). We shall now compare computational time from running the TruncatedSVD."]},{"cell_type":"code","execution_count":28,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100, batch_size = 4000\n","Total Variance Explained: 0.20770761680685512\n","X_train: (12888, 100), y_train: (12888,)\n","X_test: (4296, 100), y_test: (4296,)\n","Accuracy: 0.633147113594041\n","Macro-F1 score: 0.4597114735469538\n","Model3: 0.4597114735469538\n","\n","Accuracy: 0.688780260707635\n","Macro-F1 score: 0.631559639336222\n","Model4: 0.631559639336222\n","\n","Accuracy: 0.6890130353817505\n","Macro-F1 score: 0.633025514433183\n","Model5: 0.633025514433183\n","\n","Accuracy: 0.6894785847299814\n","Macro-F1 score: 0.6329432742672462\n","Model6: 0.6329432742672462\n","\n","Accuracy: 0.6585195530726257\n","Macro-F1 score: 0.565109424225936\n","KNN: 0.565109424225936 \n","\n","Time taken for 100 Principal Components: 38.85954546928406s \n","\n","n_components = 500, batch_size = 4000\n","Total Variance Explained: 0.48528147090419305\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.6135940409683427\n","Macro-F1 score: 0.4738487009566307\n","Model3: 0.4738487009566307\n","\n","Accuracy: 0.7171787709497207\n","Macro-F1 score: 0.6779712517078109\n","Model4: 0.6779712517078109\n","\n","Accuracy: 0.7167132216014898\n","Macro-F1 score: 0.6777980209862013\n","Model5: 0.6777980209862013\n","\n","Accuracy: 0.7169459962756052\n","Macro-F1 score: 0.6780034986112806\n","Model6: 0.6780034986112806\n","\n","Accuracy: 0.6538640595903166\n","Macro-F1 score: 0.5514192419011086\n","KNN: 0.5514192419011086 \n","\n","Time taken for 500 Principal Components: 111.06299638748169s \n","\n","n_components = 1000, batch_size = 4000\n","Total Variance Explained: 0.650073164448684\n","X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","Accuracy: 0.6054469273743017\n","Macro-F1 score: 0.48659276617913666\n","Model3: 0.48659276617913666\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6836444589405164\n","Model4: 0.6836444589405164\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6836444589405164\n","Model5: 0.6836444589405164\n","\n","Accuracy: 0.7190409683426443\n","Macro-F1 score: 0.6836444589405164\n","Model6: 0.6836444589405164\n","\n","Accuracy: 0.6368715083798883\n","Macro-F1 score: 0.5716805652843039\n","KNN: 0.5716805652843039 \n","\n","Time taken for 1000 Principal Components: 208.36183071136475s \n","\n","n_components = 1500, batch_size = 4000\n","Total Variance Explained: 0.7535549687544596\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.5998603351955307\n","Macro-F1 score: 0.48584701881984005\n","Model3: 0.48584701881984005\n","\n","Accuracy: 0.720903165735568\n","Macro-F1 score: 0.6870472512776712\n","Model4: 0.6870472512776712\n","\n","Accuracy: 0.720903165735568\n","Macro-F1 score: 0.6870472512776712\n","Model5: 0.6870472512776712\n","\n","Accuracy: 0.720903165735568\n","Macro-F1 score: 0.6870472512776712\n","Model6: 0.6870472512776712\n","\n","Accuracy: 0.5835661080074488\n","Macro-F1 score: 0.5648477343817014\n","KNN: 0.5648477343817014 \n","\n","Time taken for 1500 Principal Components: 296.55450105667114s \n","\n","n_components = 2000, batch_size = 4000\n","Total Variance Explained: 0.8263684695508671\n","X_train: (12888, 2000), y_train: (12888,)\n","X_test: (4296, 2000), y_test: (4296,)\n","Accuracy: 0.5947392923649907\n","Macro-F1 score: 0.4886192293368272\n","Model3: 0.4886192293368272\n","\n","Accuracy: 0.7216014897579144\n","Macro-F1 score: 0.6889435203589405\n","Model4: 0.6889435203589405\n","\n","Accuracy: 0.7216014897579144\n","Macro-F1 score: 0.6889435203589405\n","Model5: 0.6889435203589405\n","\n","Accuracy: 0.7216014897579144\n","Macro-F1 score: 0.6889435203589405\n","Model6: 0.6889435203589405\n","\n","Accuracy: 0.5223463687150838\n","Macro-F1 score: 0.5213295619146662\n","KNN: 0.5213295619146662 \n","\n","Time taken for 2000 Principal Components: 354.7220196723938s \n","\n","CPU times: total: 1h 16min 53s\n","Wall time: 16min 49s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","comps = [100, 500, 1000, 1500, 2000]\n","\n","for n_components in comps:\n","    start = time.time()\n","    pca = TruncatedSVD(n_components=n_components, random_state=random_state)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}, batch_size = {batch_size}\")\n","    print(\"Total Variance Explained:\", np.sum(pca.explained_variance_ratio_))\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train3(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C)\n","    Model3 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model3: {Model3}\\n\")\n","    \n","    w, b, l = train4(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model4 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model4: {Model4}\\n\")\n","    \n","    w, b, l = train5(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model5 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model5: {Model5}\\n\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore} \\n\")\n","    elapsed = time.time() - start\n","    print(f\"Time taken for {n_components} Principal Components: {elapsed}s \\n\")"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, we observe that the TruncatedSVD offers comparable performance to PCA. Also, it has a similar computational time compared to PCA (e.g. TruncatedSVD (n=1000) took a total of `208.36s` for dimensionality reduction and training 4 models and enabled Model 6 to achieve a Macro-F1 score of `0.68364` while PCA (n=1000) took a total of `166.69s` and enabled Model 6 to achieve a Macro-F1 score of  `0.68342`). This increase in computation is becomes less significant as we increase the number of components to generate. Hence, it would be better to use TruncatedSVD instead of PCA to improve our model performance as the additional computation time is acceptable for a slight increase in model performance."]},{"cell_type":"markdown","metadata":{},"source":["### KernelPCA\n","\n","We will now implement KernelPCA to compare the model performance compared to regular PCA. We are aware that KernelPCA is computationally expensive. Hence, we shall first test the function to observe the computational time to generate 1000 principal components using KernelPCA."]},{"cell_type":"code","execution_count":29,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components=1000, random_state=100, kernel=linear, n_jobs=-1\n"]}],"source":["n_components=1000\n","kernel=\"linear\"\n","n_jobs=-1\n","random_state=100\n","print(f\"n_components={n_components}, random_state={random_state}, kernel={kernel}, n_jobs={n_jobs}\")"]},{"cell_type":"code","execution_count":30,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: total: 1h 11min 38s\n","Wall time: 13min 54s\n"]}],"source":["%%time\n","kern = KernelPCA(n_components=n_components, kernel=kernel, n_jobs=n_jobs, random_state=random_state)\n","kern.fit(df_features.to_numpy())\n","X_kernel = kern.transform(df_features.to_numpy())"]},{"cell_type":"code","execution_count":35,"metadata":{},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAY4AAAD4CAYAAAD7CAEUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAihklEQVR4nO3df5RdZX3v8feHRAIoP2OulYTbiZLojVIVpwjC6nU1DQaxxAqWoK1UabGrUBHrraFeS8H+gFvbiAv0kgpKqZpAtDYVhRsDtaXVyAS6hESTjEmURK0j4ZcIgYTP/WPvCSeTMzP7TM6ZOXPO57XWrDnn2c/e8905MN959vNLtomIiKjqoIkOICIiJpckjoiIaEgSR0RENCSJIyIiGpLEERERDZk60QGMhxe+8IXu6emZ6DAiIiaVdevW/dT2jKHlXZE4enp66Ovrm+gwIiImFUnfr1eeR1UREdGQliYOSQslbZTUL2lJnePTJK0oj6+V1FOWT5d0l6SfSbp2yDkHS1omaZOk70o6u5X3EBER+2rZoypJU4DrgAXAduAeSatsb6ipdgHwsO3jJS0GrgbOBZ4CPgy8svyq9SHgJ7bnSjoIOKZV9xAREftrZYvjJKDf9hbbTwPLgUVD6iwCbipfrwTmS5LtJ2zfTZFAhno38FcAtp+1/dPWhB8REfW0MnHMBB6seb+9LKtbx/Zu4FFg+nAXlHRU+fIjku6VdKukFw1T90JJfZL6BgYGxngLEREx1GTrHJ8KzAL+w/aJwDeAj9araHuZ7V7bvTNm7DearLKlqzeN+dyIiE7UysSxAziu5v2ssqxuHUlTgSOBh0a45kPAz4Evlu9vBU5sRrDDuWbN5lZePiJi0mll4rgHmCNptqSDgcXAqiF1VgHnl6/PAe70COu8l8f+GXhDWTQf2DBc/YiIaL6WjaqyvVvSxcAdwBTgRtvrJV0J9NleBdwA3CypH9hJkVwAkLQNOAI4WNJbgNPLEVkfLM/5GDAAvKvZsS9dvWmflkbPktsAuGT+HC5dMLfZPy4iYlJRN2zk1Nvb67HOHO9ZchvbrjqzyRFFRLQ/Sets9w4tn2yd4xERMcGSOEZxyfw5Ex1CRERbSeIYRfo0IiL2lcQRERENSeKIiIiGJHFERERDkjgiIqIhSRwREdGQJI6IiGhIEkdERDQkiSMiIhqSxBEREQ1J4oiIiIYkcUREREOSOCIioiFJHBER0ZCWJg5JCyVtlNQvaUmd49MkrSiPr5XUU5ZPl3SXpJ9JunaYa6+S9EAr44+IiP21LHFImgJcB5wBzAPOkzRvSLULgIdtHw8sBa4uy58CPgx8YJhrvxX4WSvijoiIkbWyxXES0G97i+2ngeXAoiF1FgE3la9XAvMlyfYTtu+mSCD7kPQC4P3An7cu9IiIGE4rE8dM4MGa99vLsrp1bO8GHgWmj3LdjwB/A/y8OWFGREQjJlXnuKRXAy+1/Y8V6l4oqU9S38DAQOuDi4joEq1MHDuA42rezyrL6taRNBU4EnhohGueAvRK2gbcDcyV9C/1KtpeZrvXdu+MGTPGdAMREbG/ViaOe4A5kmZLOhhYDKwaUmcVcH75+hzgTtse7oK2P2n7WNs9wGnAJttvaHrkERExrKmturDt3ZIuBu4ApgA32l4v6Uqgz/Yq4AbgZkn9wE6K5AJA2ao4AjhY0luA021vaFW8ERFRjUb4A79j9Pb2uq+vb6LDiIiYVCSts907tHxSdY5HRMTES+KIiIiGVOrjkHQW8Cvl26/b/ufWhRQREe1s1BaHpL8CLgE2lF/vlfSXrQ4sIiLaU5UWx5nAq20/CyDpJuA+4E9aGVhERLSnqn0cR9W8PrIFcURExCRRpcXxV8B9ku4CRNHXsd8S6RER0R1GTRy2P18u6/HLZdEHbf+4pVFFRETbGvZRlaSXl99PBF5MsbrtduDYsiwiIrrQSC2O9wMXUixhPpSBX21JRBER0daGTRy2LyxfnmF7nw2VJB3S0qgiIqJtVRlV9R8VyyIiogsM2+KQ9AsUO/QdKuk1FCOqoFix9rBxiC0iItrQSH0cbwR+h2IDpr+tKX+cTP6LiOhaI/Vx3ATcJOls218Yx5giIqKNVZnH8QVJZwKvAA6pKb+ylYFFRER7qrLI4f8FzgX+kKKf423AL7Y4roiIaFNVRlW93vY7gYdtXwGcAsytcnFJCyVtlNQvab9lSiRNk7SiPL5WUk9ZPl3SXZJ+JunamvqHSbpN0nclrZd0VaW7jIiIpqmSOAbncPxc0rHAMxQzyUckaQpwHXAGMA84T9K8IdUuoEhIxwNLgatrfuaHgQ/UufRHbb8ceA1wqqQzKtxDREQ0SZXE8c+SjgL+GrgX2AZ8rsJ5JwH9trfYfhpYDiwaUmcRcFP5eiUwX5JsP2H7bp5LWgDY/rntu8rXT5fxzKoQS0RENMmIneOSDgLW2H4E+IKkLwOH2H60wrVnAg/WvN8OvG64OrZ3S3oUmA78dLSLl8ns14FrKsQSERFNMmKLo9y86bqa97sqJo2WkjQV+DzwcdtbhqlzoaQ+SX0DAwPjG2BERAer8qhqjaSzJWn0qvvYARxX835WWVa3TpkMjgQeqnDtZcBm2x8broLtZbZ7bffOmDGjkbgjImIEVRLHe4BbgV2SHpP0uKTHKpx3DzBH0mxJBwOLgVVD6qwCzi9fnwPcadsjXVTSn1MkmPdViCEiIpqsygTAw8dy4bLP4mLgDmAKcKPt9ZKuBPpsrwJuAG6W1A/spEguAEjaRrEu1sGS3gKcDjwGfAj4LnBv2Qi61vanxhJjREQ0rsrWsWNm+yvAV4aU/WnN66coJhTWO7dnmMs2+sgsIiKaqMqjqoiIiL2SOCIioiGVEoek0yS9q3w9Q9Ls1oYVERHtqsoih5cDHwQuK4ueB/xDK4OKiIj2VaXF8RvAWcATALZ/CIxppFVEREx+VRLH0+XcCgNIen5rQ4qIiHZWJXHcIul64ChJvwd8Dfi71oYVERHtqsoEwI9KWkAx+e5lwJ/aXt3yyCIioi2NmjjKEVT/NpgsJB0qqcf2tlYHFxER7afKo6pbgWdr3u8pyyIiogtVSRxTy02TgL0bKB3cupAiIqKdVUkcA5LOGnwjaREVNlqKiIjOVGWRw98HPivpWooFBh8E3tnSqCIiom2N2uKw/T3bJwPzgP9h+/W2+1sfWntYunrTRIcQEdFWqoyqmgacDfQAUwc3ArR9ZUsjaxPXrNnMpQvmTnQYERFto8qjqn8CHgXWAbtaG05ERLS7Koljlu2FLY+kjSxdvYlr1mze+75nyW0AXDJ/TlofEdH1qoyq+g9JJ4zl4pIWStooqV/SkjrHp0laUR5fK6mnLJ8u6S5JPys75WvPea2k+8tzPq7BZ2dNdOmCuWy76ky2XXUmwN7XSRoREdUSx2nAujIBfLv8pf3t0U6SNAW4DjiDomP9PEnzhlS7AHjY9vHAUuDqsvwp4MPAB+pc+pPA7wFzyq+uag1FREy0Ko+qzhjjtU8C+m1vAZC0HFgEbKipswj4s/L1SuBaSbL9BHC3pONrLyjpxcARtr9Zvv974C3AV8cY46gumT+nVZeOiJiUqgzH/b7t7wNPUiytvneJ9VHMpJjzMWh7WVa3ju3dFJ3w00e55vZRrgmApAsl9UnqGxgYqBBufXk8FRGxryo7AJ4laTOwFfg6sI0W/oXfLLaX2e613TtjxoyJDiciomNU6eP4CHAysMn2bGA+8M0K5+0Ajqt5P6ssq1tH0lTgSOChUa45a5RrRkREC1VJHM/Yfgg4SNJBtu8Ceiucdw8wR9JsSQcDi4FVQ+qsAs4vX58D3FnuNliX7R8Bj0k6uRxN9U6KeSYRETFOqnSOPyLpBcC/UqxZ9RPK/cdHYnu3pIuBO4ApwI2210u6EuizvQq4AbhZUj+wkyK5ACBpG3AEcLCktwCn294A/AHwGeBQikdmbf/YLCKik2iEP/CLCsUe409RLHD4DorHSZ8tWyGTQm9vr/v6+iY6jIiISUXSOtv7PWGqMqrqCdt7bO+2fZPtj0+mpNEMWegwIuI5wyYOSXeX3x+X9FjN1+OSHhu/ECde7fIjERHdbtg+Dtunld8PH79wIiKi3Y3YOV4uG7Le9svHKZ62kYUOIyLqGzFx2N5TrlH1323/YLyCageXLpi7N0H0LLlt74KHERHdrspw3KOB9ZK+Rc0wXNtnDX9KRER0qiqJ48Mtj6LNZaHDiIjnjJo4bH99PAJpZ+nTiIh4TpVFDk+WdE+5qdLTkvZ023DciIh4TpW1qq4FzgM2Uyzz8bsUGzRFREQXqpI4sN0PTClnkH+a7LoXEdG1qnSO/7xc3fY/Jf0f4EdUTDgREdF5qiSA3y7rXUwxHPc44OxWBhUREe2rSovjtcBtth8DrmhxPBER0eaqtDh+Hdgk6WZJby536ouIiC5VZVn1dwHHA7dSjK76nqRPtTqwiIhoT5VaD7afkfRVwBRDct9CMSw3IiK6TJUJgGdI+gzFPI6zgU8Bv1Dl4pIWlosk9ktaUuf4NEkryuNrJfXUHLusLN8o6Y015ZdKWi/pAUmfl3RIlVgiIqI5qvRxvBP4EvAy279j+yu2d492Urkk+3XAGcA84DxJ84ZUuwB42PbxwFLg6vLceRT7j7+CYs7IJyRNkTQTeC/Qa/uVFHuZLyYiIsZNlT6O82x/yfauBq99EtBve4vtp4HlwKIhdRYBN5WvVwLzJaksX257l+2tQH95PSgerx1adtIfBvywwbgiIuIAtHIi30zgwZr328uyunXKVsyjwPThzrW9A/go8AOKiYiP2v5/9X64pAsl9UnqGxgYaMLtREQETLIZ4JKOpmiNzAaOBZ4v6bfq1bW9zHav7d4ZM2aMZ5gRER2tlYljB8Us80GzyrK6dcpHT0cCD41w7q8BW20P2H4G+CLw+pZEHxERdQ07HFfS/RTDb+uy/UujXPseYI6k2RS/9BcDbx9SZxVwPvAN4BzgTtuWtAr4nKS/pWhZzAG+BTwLnCzpMOBJYD7QN0ocERHRRCPN43hz+f2i8vvN5fd3VLmw7d2SLgbuoBj9dKPt9ZKuBPpsrwJuAG6W1A/spBwhVda7BdgA7AYusr0HWCtpJXBvWX4fsKzarUZERDPIHrZRUVSQ7rP9miFl99o+saWRNVFvb6/7+tIwiYhohKR1tnuHllfp45CkU2vevL7ieRER0YGqJIALKCbgbZO0DfgE8O6WRtVmlq7eNNEhRES0jSoTANfZfhXwKuBVtl9t+97Wh9Y+rlmzeaJDiIhoG1XWqnqRpBsoZnI/KmmepAvGIbaIiGhDVVbH/QzwaeBD5ftNwAqKEVEda+nqTfu0NHqW3AbAJfPncOmCuRMVVkTEhKuSOF5o+xZJl8HeYbZ7WhzXhLt0wdy9CaJnyW1su+rMCY4oIqI9VOkcf0LSdMrJgJJOplhTKiIiulCVFsf7KWZ4v1TSvwMzKGZ5R0REFxo1cdi+V9L/BF4GCNhYrhMVERFdqNLWsRR7YfSU9U+UhO2/b1lUbSCd4xER9Y2aOCTdDLwU+E9gsFPcQEcnjnSOR0TUV6XF0QvM82iLWkVERFeoMqrqAeAXWh1IO7tk/pyJDiEiom1UmscBbJD0LWDvvuO2z2pZVG0mfRoREc+pkjj+rNVBtLulqzcleURElKoMx/36eATSzq5ZszmJIyKiNNLWsXfbPk3S4+y7hawA2z6i5dFFRETbGbZz3PZp5ffDbR9R83V41aQhaaGkjZL6JS2pc3yapBXl8bWSemqOXVaWb5T0xpryoyStlPRdSd+RdEpDd1zR0tWb6Fly2975G4OvszdHRHS7qhMAkfTfgEMG39v+wSj1pwDXAQuA7cA9klbZ3lBT7QLgYdvHS1oMXA2cK2kexf7jrwCOBb4maW657/g1wO22z5F0MHBY1XtoROZxRETUV2U/jrMkbQa2Al8HtgFfrXDtk4B+21tsPw0sBxYNqbMIuKl8vRKYL0ll+XLbu2xvBfqBkyQdCfwK5ZLutp+2/UiFWCIiokmqzOP4CHAysMn2bGA+8M0K580EHqx5v70sq1vH9m6KVXenj3DubGAA+LSk+yR9StLz6/1wSRdK6pPUNzAwUCHc4V0yf04eUUVElKokjmdsPwQcJOkg23dRzCafCFOBE4FP2n4N8ASwX98JgO1ltntt986YMeOAfuilC+Zm+9iIiFKVxPGIpBcA/wp8VtI1FL+wR7MDOK7m/ayyrG4dSVOBI4GHRjh3O7Dd9tqyfCVFIomIiHFSJXEsAp4ELgVuB74H/HqF8+4B5kiaXXZiL6bY16PWKuD88vU5wJ3lmlirgMXlqKvZwBzgW7Z/DDwo6WXlOfOBDbRIRlZFROyvygTA2tbFTcNW3P+83ZIuBu4ApgA32l4v6Uqgz/Yqik7umyX1AzspkgtlvVsoksJu4KJyRBXAH1K0fA4GtgDvqhpTozKyKiJifxpu0dvhJv4xCScA9vb2uq+vr+Hzhu7JMSh7ckREN5C0zvZ+fdrDJo5OMtbEUSstjojoNsMljkoTACWdCJxG0eK42/Z9TY6vLWUXwIiI/Y3a4pD0p8DbgC+WRW8BbrX9560NrXma1eJIwoiIbjJci6PKqKp3AL9s+3Lbl1NMBvztZgc4GWQuR0REtcTxQ2rWqAKmsf98jI6XXQAjIgpV+jgeBdZLWk3Rx7EA+JakjwPYfm8L45tw6eeIiNhXlT6O80c6brvy3I6J0ow+DsjIqojoLgcyquqrtn8y5GIvs72xadFFRMSkUSVx/JukD9u+BUDSH1HsozGvpZG1iTyqiojYV5XE8QZgmaS3AS8CvkOx10ZERHShKmtV/UjS7cBlwLPAEts/a3lkbWLoelVA+jkioquNmjgkfY1iSO4rKZY6v0HSv9r+QKuDi4iI9lNlHse1tt9p+xHb9wOnUAzR7RqDy6sPyvLqEdHNRk0ctr8k6TRJg8uXHw38Q2vDioiIdjVq4pB0OfBBij4OgINJ4oiI6FpVHlX9BnAW5Xaxtn8IHN7KoCIion1VGY77tG1LMoCk57c4praTkVUREc+p0uK4RdL1wFGSfg/4GvB3VS4uaaGkjZL6JS2pc3yapBXl8bWSemqOXVaWb5T0xiHnTZF0n6QvV4mjGdJBHhFRqLQDoKQFwOkU28beYXt1hXOmAJsoFkXcDtwDnGd7Q02dPwB+yfbvS1oM/IbtcyXNAz5PMdHwWIpkNXdw33FJ7wd6gSNsv3m0WJq1VhU8N5M8LY6I6HQHsh8Htlfb/l+2P1AlaZROAvptb7H9NLAcWDSkziJgcJHElcB8SSrLl9veZXsr0F9eD0mzgDOBT1WMo6myzEhEdLtKiWOMZgIP1rzfXpbVrWN7N8X8kOmjnPsx4I8pZrEPS9KFkvok9Q0MDIzxFiIiYqhWJo6mk/Rm4Ce2141W1/Yy2722e2fMmDEO0UVEdIcqo6r2Iek4YLHtvx6l6g6KJUoGzWL/nQMH62yXNBU4EnhohHPPAs6S9CaKXQmPkPQPtn+r0ftoVFbJjYgoVO0cnwG8DTiPorP6H0dbq6pMBJuA+RS/9O8B3m57fU2di4ATajrH32r7NyW9Avgcz3WOrwHmDHaOl+e+AfjAeHeOQ4bkRkR3aHgjJ0mHA28F3g7MBb4IzLY9q8oPtL1b0sXAHcAU4Ebb6yVdCfTZXgXcANwsqR/YCSwuz10v6RZgA7AbuKg2aUyEoS0OKBJIWhwR0W2GbXFIehL4FvC/gbvLSYBbbL9kPANshma1OM69/hus3bpzv/LXzT6GFe855YCvHxHRTsYyHPcyYBrwCeAySS9tVXCTxckvmd5QeUREJxo2cdj+mO2TeW7uxZeAYyV9UFJXPpu5dMFcXjf7mP3Kr1mzmXOv/8YERBQRMf6qLKu+xfZf2j6BcrY28JWWR9amhmtdrN26M8kjIrrCsIlD0vGSTq0ts/0A8FVgYasDa1eXLpjLJfPnTHQYERETZqQWx8eAx+qUPwosbUk0k9zarTuz6GFEdLyREseLyq1i91GW9bQsoklguL4OgJXrHqxbHhHRKUZKHEeNcOzQJscx6ax4zynMPOqQ/cp3PPJUlluPiI42UuLoK/ff2Iek3wVGXSuqG/z7kvkcPm1K3WM33r1lnKOJiBgfIyWO9wHvkvQvkv6m/Po6cAFwybhENwncf8XCui2Px3ftSasjIjrSSPM4/sv264ErgG3l1xW2T7H94/EJb3KYdfRhdcszvyMiOtFIw3EPkfQ+4GzgaeCTtu8cr8AmkxXvOWXYIboZaRURnWakR1U3UUz4ux84A/jouEQ0SY00v+ObWx4a52giIlpnpMQxz/Zv2b4eOAf4lXGKqeOs3bqTEy6/faLDiIhoipESxzODL8ptXWMUI83veHzXHuZ+6Ct5bBURk95IieNVkh4rvx4HfmnwtaR6M8qDor9juOTx9B5zzZrNnHrVmnGOKiKieUYaVTXF9hHl1+G2p9a8PmI8g5xshpscOGjHI09xwuW3p/UREZNSpa1jJ7tmbx1b1alXrWHHI0+NWGcwwZzz2uOyk2BEtJWxbOTUjB+6UNJGSf2SltQ5Pk3SivL4Wkk9NccuK8s3SnpjWXacpLskbZC0XlJbT0T89yXzh31sNWjHI0+x45Gn9j7CyryPiGh3LUsckqYA11EM5Z0HnCdp3pBqFwAP2z6eYsXdq8tz51HsP/4KiiXcP1FebzfwR7bnAScDF9W5ZlsZqc9jqB2PPLV3BFYSSES0q1a2OE4C+suNoJ4GlvPcboKDFlHMFwFYCcyXpLJ8ue1dtrcC/cBJtn9k+14A248D3wFmtvAemmJwguBw61oN9fiuPazdujMtkIhoS61MHDOB2jXGt7P/L/m9dcohv48C06ucWz7Weg2wtt4Pl3ShpD5JfQMDA2O/iya5dMFc7r9iIa+bfcyIHee1alsgp161Jp3pEdEWpk50AGMh6QXAF4D32a47NNj2MmAZFJ3j4xjeiFa85xQATrj8dh7ftafSOY/v2sPju/ZwzZrN3Hj3Fo449HnMOvqwvdeKiBhPrUwcO4Djat7PKsvq1dkuaSpwJPDQSOdKeh5F0vis7S+2JvTWu/+KhZx7/TfY/vDPAUYdfTVoMInseOQpTr1qDY89+Qzzjj0ySSQixk3LhuOWiWATMJ/il/49wNttr6+pcxFwgu3fl7QYeKvt35T0CuBzFP0kxwJrgDnAsxR9Ijttv69qLBM1HLcRg5MCqyaQoWYedQiPPfkMRxz6PKAY0RURcSCGG47b0nkckt5EsXf5FOBG238h6Uqgz/YqSYcAN1P0VewEFtveUp77IeDdFCOp3mf7q5JOA/6NYuHFZ8sf8ye2vzJSHJMhcQxaunoTN969pfJjrOEMTSSZJxIRjZqQxNEuJlPiGLR09SZWrntwzC2Qemo75R978hnefdpLkkwiYlhJHJMscQwabIEccejzmppEBg0mk9rWSTreIwKSOCZt4qh17vXfYMMPH21ZEqk1tHWSpBLRfZI4OiBx1Bp8lPXYk88ccH9Io4ZLKhnhFdFZkjg6LHHUqk0ig7/EW90iGUm9x19ptURMPkkcHZw46hmc49EOiaSew6dNqZtUhr7PsOKIiZPE0WWJY6ihm0dNxCOusahNMDBykslIsYjmSuLo8sRRT+3M9XZunTRqtEdl9Y7df8XC8Q80os0lcSRxVFabUKCzkspwRurwHynh1B6DPFqLzpLEkcTRFCMllcny+KuVmpGA6h3LzP+YCEkcSRzjYnCEFwz/y7FTWy2tNnQZmbEmo0bqpgXV3ZI4kjjaRu1seBj5F1eSzMQaaXDCgbaiGj0GWXNtvCVxJHFMSidcfntDfzl3+6OyblBv8MPQ9xOR1DpxMEYSRxJHVxhupFgSTrSTVvWF1Tt2II8bh0sck3IHwIjhjHVGeu08l7H+z5rHalHV0P9Wav9wGfpHTDOONVsSRwTN6QSuHRgAzf1LMkkp2kkeVUVMAoOP4Mb7eXwSVueYedQhDf+BlEdVEZPYRC0KOdLghInogE4ia9y2q85s+jVbmjgkLQSuodg69lO2rxpyfBrw98BrgYeAc21vK49dBlwA7AHea/uOKteMiOZpt9E/w/VFDX3f7qOqJvtgjJYlDklTgOuABcB24B5Jq2xvqKl2AfCw7eMlLQauBs6VNA9YDLwCOBb4mqTBwdujXTMiOlSnTEhsxmCMRpJas7WyxXES0G97C4Ck5cAioPaX/CLgz8rXK4FrJaksX257F7BVUn95PSpcMyKirU32BHhQC689E3iw5v32sqxuHdu7gUeB6SOcW+WaAEi6UFKfpL6BgYEDuI2IiKjVysQxoWwvs91ru3fGjBkTHU5ERMdoZeLYARxX835WWVa3jqSpwJEUneTDnVvlmhER0UKtTBz3AHMkzZZ0MEVn96ohdVYB55evzwHudDGxZBWwWNI0SbOBOcC3Kl4zIiJaqGWd47Z3S7oYuINi6OyNttdLuhLos70KuAG4uez83kmRCCjr3ULR6b0buMj2HoB612zVPURExP66Yua4pAHg+2M8/YXAT5sYzmTQbffcbfcLueducaD3/Iu29+sk7orEcSAk9dWbct/Juu2eu+1+IffcLVp1zx07qioiIlojiSMiIhqSxDG6ZRMdwATotnvutvuF3HO3aMk9p48jIiIakhZHREQ0JIkjIiIaksQxDEkLJW2U1C9pyUTH0yySjpN0l6QNktZLuqQsP0bSakmby+9Hl+WS9PHy3+Hbkk6c2DsYO0lTJN0n6cvl+9mS1pb3tqJcjYByxYIVZflaST0TGvgYSTpK0kpJ35X0HUmndPrnLOnS8r/rByR9XtIhnfY5S7pR0k8kPVBT1vDnKun8sv5mSefX+1nDSeKoo2YvkTOAecB55R4hnWA38Ee25wEnAxeV97YEWGN7DrCmfA/Fv8Gc8utC4JPjH3LTXAJ8p+b91cBS28cDD1PsDwM1+8QAS8t6k9E1wO22Xw68iuLeO/ZzljQTeC/Qa/uVFKtLDO7z00mf82eAoTtsNfS5SjoGuBx4HcWWFZcPJptKbOdryBdwCnBHzfvLgMsmOq4W3es/UWyMtRF4cVn2YmBj+fp64Lya+nvrTaYvigUx1wC/CnwZEMWM2qlDP3OKJW1OKV9PLetpou+hwfs9Etg6NO5O/px5btuFY8rP7cvAGzvxcwZ6gAfG+rkC5wHX15TvU2+0r7Q46qu878dkVjbNXwOsBV5k+0floR8DLypfd8q/xceAPwaeLd9PBx5xsQ8M7Htfw+0TM5nMBgaAT5eP5z4l6fl08OdsewfwUeAHwI8oPrd1dPbnPKjRz/WAPu8kji4l6QXAF4D32X6s9piLP0E6Zpy2pDcDP7G9bqJjGUdTgROBT9p+DfAEzz2+ADrycz6aYkfQ2RRbTj+f/R/pdLzx+FyTOOrr6H0/JD2PIml81vYXy+L/kvTi8viLgZ+U5Z3wb3EqcJakbcByisdV1wBHqdgHBva9r+H2iZlMtgPbba8t36+kSCSd/Dn/GrDV9oDtZ4AvUnz2nfw5D2r0cz2gzzuJo76O3fdDkiiWs/+O7b+tOVS7N8r5FH0fg+XvLEdnnAw8WtMknhRsX2Z7lu0eis/yTtvvAO6i2AcG9r/nevvETBq2fww8KOllZdF8im0KOvZzpnhEdbKkw8r/zgfvuWM/5xqNfq53AKdLOrpsqZ1ellUz0Z087foFvAnYBHwP+NBEx9PE+zqNohn7beA/y683UTzbXQNsBr4GHFPWF8UIs+8B91OMWJnw+ziA+38D8OXy9UsoNgjrB24FppXlh5Tv+8vjL5nouMd4r68G+srP+kvA0Z3+OQNXAN8FHgBuBqZ12ucMfJ6iD+cZipblBWP5XIF3l/feD7yrkRiy5EhERDQkj6oiIqIhSRwREdGQJI6IiGhIEkdERDQkiSMiIhqSxBEREQ1J4oiIiIb8f70UjGVrq8WBAAAAAElFTkSuQmCC","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"}],"source":["import matplotlib.pyplot as plt\n","\n","var_values = kern.eigenvalues_ / sum(kern.eigenvalues_)\n","plt.plot(np.arange(1, kern.n_components + 1), var_values, \"+\", linewidth=2)\n","plt.ylabel(\"PCA explained variance ratio\")\n","plt.show()"]},{"cell_type":"code","execution_count":32,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["(17184, 1000)\n","(17184,)\n"]}],"source":["# check before running\n","# print(X_kernel)\n","print(X_kernel.shape)\n","print(df_label.shape)"]},{"cell_type":"code","execution_count":33,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","CPU times: total: 62.5 ms\n","Wall time: 67 ms\n"]}],"source":["%%time\n","X_kerneltrain, X_kerneltest, y_kerneltrain, y_kerneltest = train_test_split(X_kernel, df_label, test_size=0.25, random_state=100)\n","print(f\"X_train: {X_kerneltrain.shape}, y_train: {y_kerneltrain.shape}\")\n","print(f\"X_test: {X_kerneltest.shape}, y_test: {y_kerneltest.shape}\")"]},{"cell_type":"code","execution_count":34,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7199720670391061\n","Macro-F1 score: 0.686003205410374\n","CPU times: total: 2min 46s\n","Wall time: 34.4 s\n"]},{"data":{"text/plain":["0.686003205410374"]},"execution_count":34,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","w, b, l = train6(X_kerneltrain, y_kerneltrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","score(y_kerneltest.values,  predict(X_kerneltest, w, b))"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, we were able to achieve a model performance of `0.68600` from KernelPCA - Linear (n=1000), which is comparable to a model performance of `0.68342` from PCA (n=1000). However, the computational time to perform KernelPCA is extremely large. We shall now compare computational time from running the KernelPCA."]},{"cell_type":"code","execution_count":37,"metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components = 100\n","X_train: (12888, 100), y_train: (12888,)\n","X_test: (4296, 100), y_test: (4296,)\n","Accuracy: 0.6848230912476723\n","Macro-F1 score: 0.629190550988352\n","Model6: 0.629190550988352\n","\n","Accuracy: 0.6552607076350093\n","Macro-F1 score: 0.5604755619304248\n","KNN: 0.5604755619304248 \n","\n","Time taken for 100 Principal Components: 524.6863536834717s \n","\n"]},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYgAAAD5CAYAAAA9SqL2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbA0lEQVR4nO3de7SddX3n8feHpFy8gYYUlUBPlIALmXrpEdFSR82kBbFEC5ZQq2ix6GpdplhnjK4WLa1T6ThGumA6UlDRugTES2MRXWmgeKkiJ2CLQEliwBK8EBEC6HCTz/zxPCc8bJ99znPO2c/Z++z9ea111n5ue+/vs56s/c3vLttERER02qPfAURExGBKgoiIiFpJEBERUSsJIiIiaiVBRERErSSIiIiotbjND5d0DHA2sAg43/b7O87vBXwc+DXgTuAk27dKGgNuAm4uL/2m7bdM9V3777+/x8bGensDERFDbvPmzT+2vbTuXGsJQtIi4FxgFbADuEbSBts3Vi47FbjL9iGS1gBnASeV575r+7lNv29sbIyJiYneBB8RMSIkfa/buTarmI4EttnebvtB4CJgdcc1q4ELy+1LgZWS1GJMERHRUJsJ4kDgtsr+jvJY7TW2HwZ2AUvKc8slXSfpKkm/0WKcERFRo9U2iDn4AXCw7Tsl/RrweUnPtn1P9SJJpwGnARx88MF9CDMiYni1WYK4HTiosr+sPFZ7jaTFwL7AnbYfsH0ngO3NwHeBQzu/wPZ5tsdtjy9dWtvGEhERs9RmgrgGWCFpuaQ9gTXAho5rNgCnlNsnAlfYtqSlZSM3kp4BrAC2txhrRER0aC1BlG0KbwW+TNFl9RLbN0g6U9Lx5WUXAEskbQPeDqwrj78E+HdJ36ZovH6L7Z+0FSvA+o1b2vz4iIgFR8My3ff4+Ljn0s11bN1l3Pr+43oYUUTE4JO02fZ43bmMpI6IiFqD2otpXqzfuIWzN23dvT+27jIA1q5cwemrfqFNPCJipKSKqZQqpogYRaliioiIGUuCKK1duaLfIUREDJQkiFLaHCIiHisJIiIiaiVBRERErSSIiIiolQQRERG1kiAiIqJWEkRERNRKgoiIiFpJEBERUSsJIiIiaiVBRERErSSIiIiolQQRERG1kiAiIqJWEkRERNRKgoiIiFpJEBERUSsJIiIiaiVBRERErSSIiIiolQQRERG1kiAiIqJWEkRERNRKgoiIiFqLm1wk6XjgJeXuVba/0F5IERExCKYtQUj6a2AtcGP59zZJ/7PtwCIior+alCCOA55r+xEASRcC1wHvbjOwiIjor6ZtEPtVtvdtIY6IiBgwTUoQfw1cJ+lKQBRtEetajSoiIvpu2gRh+1OS/gV4QXnonbZ/2GpUERHRd12rmCQ9q3x9PvA0YEf59/TyWEREDLGpShBvB04D/nfNOQMvbyWiiIgYCF1LELZPKzePtf2y6h/wiiYfLukYSTdL2ibpF9otJO0l6eLy/NWSxjrOHyzpPknvmME9RUREDzTpxfSvDY89hqRFwLnAscDhwMmSDu+47FTgLtuHAOuBszrOfxC4vEGMERHRY12rmCQ9FTgQ2EfS8yh6MAE8CXhcg88+Ethme3v5eRcBqykG201aDby33L4UOEeSbFvSq4BbgJ82vpuIiOiZqdogfgt4A7CM4n/yk+6l2SC5A4HbKvs7gBd2u8b2w5J2AUsk3Q+8E1gFpHopIqIPuiYI2xcCF0o6wfZn5jEmKEoV623fJ6nrRZJOo2hI5+CDD56fyCIiRkSTcRCfkXQc8Gxg78rxM6d56+3AQZX9ZeWxumt2SFpMMUr7ToqSxomS/oZiFPcjku63fU5HbOcB5wGMj497unuJiIjmpk0Qkv4vRZvDy4DzgROBbzX47GuAFZKWUySCNcDvdVyzATgF+Eb5uVfYNvAble9/L3BfZ3KIiIh2NenF9GLbr6fobfQXwIuAQ6d7k+2HgbcCXwZuAi6xfYOkM8vpwwEuoGhz2EYx7iJTeEREDIgmczHdX77+TNLTKaqAntbkw21/Efhix7EzKtv3A6+Z5jPe2+S7IiKit5okiC9I2g/4X8C1FKOo/77NoCIiov+mTBCS9gA22b4b+IykfwL2tr1rPoKLiIj+mbINolwk6NzK/gNJDhERo6FJI/UmSSdoqgEJERExdJokiDcDnwYekHSPpHsl3dNyXBER0WdNBso9cT4CiYiIwdJ0TeqIiBgxSRAREVErCSIiImo1ShCSjpb0xnJ7aTm/UkREDLFpE4Sk91CszfCu8tAvAf/QZlAREdF/TUoQrwaOp1zZzfb3gfRsiogYck0SxIPlFNwGkPT4dkOKiIhB0CRBXCLpw8B+kv4Q+GcyWV9ExNBrMlDuA5JWAfcAhwFn2N7YemQREdFXTVaUWw58dTIpSNpH0pjtW9sOLiIi+qdJFdOngUcq+z8vj0VExBBrkiAW235wcqfc3rO9kCIiYhA0SRA7K2tII2k18OP2QoqIiEHQZMnRtwCflHQOIOA24PWtRhUREX3XpBfTd4GjJD2h3L+v9agiIqLvmvRi2gs4ARgDFk8uLGf7zFYji4iIvmpSxfSPwC5gM/BAu+FERMSgaJIgltk+pvVIIiJioDTpxfSvkv5L65FERMRAaVKCOBp4g6RbKKqYBNj2r7YaWURE9FWTBHFs61FERMTAadLN9XsAkn4Z2Lv1iCIiYiA0WVHueElbgVuAq4BbgctbjisiIvqsSSP1XwJHAVtsLwdWAt9sNao+W79xS79DiIjouyYJ4iHbdwJ7SNrD9pXAeMtx9dXZm7b2O4SIiL5r0kh9dznNxlco5mS6g3J96oiIGF5NEsRq4H7gdOC1wL7A0E2zsX7jlseUHMbWXQbA2pUrOH3Vof0KKyKib2S73zH0xPj4uCcmJnryWWPrLuPW9x/Xk8+KiBhkkjbbrm026FqCkPQ120dLuheoZpHJgXJP6nGcERExQLomCNtHl69PnL9wBsPalSv6HUJERN9N2YtJ0iJJ/zFfwQyKtDlEREyTIGz/HLhZ0sHzFE9ERAyIJuMgngzcIGmTpA2Tf00+XNIxkm6WtE3Suprze0m6uDx/taSx8viRkr5d/v2bpFfP6K4iImLOmnRz/fPZfLCkRcC5wCpgB3CNpA22b6xcdipwl+1DJK0BzgJOAr4DjNt+WNLTgH+T9AXbD88mloiImLkmk/VdNcvPPhLYZns7gKSLKMZUVBPEauC95falwDmSZPtnlWv25rG9qCIiYh40mazvKEnXSLpP0oOSfi7pngaffSBwW2V/R3ms9pqydLALWFJ+7wsl3QBcD7ylrvQg6TRJE5Imdu7c2SCkiIhoqkkbxDnAycBWYB/gTRRVR62yfbXtZwMvAN4l6RemGrd9nu1x2+NLly5tO6SIiJHSJEFgexuwyPbPbX8UaLJG9e3AQZX9ZeWx2mskLaaYxuPOju++CbgPOKJJrL2WmV0jYlQ1SRA/k7Qn8G1JfyPp9IbvuwZYIWl5+f41QGfvpw3AKeX2icAVtl2+ZzGApF8BnkWxDsW8y8yuETGqmvzQv6687q0Us7geBJww3ZvKNoO3Al8GbgIusX2DpDMlHV9edgGwRNI24O3AZFfYoyl6Ln0b+BzwR7Z/3PiuIiJizqadrE/S7wCX2X5gfkKanV5O1tc5s+ukzOwaEcNmqsn6miSIjwIvp1gP4mLgS4M4HqGXCaIqM7tGxDCbKkFMW8Vk+43AIcCnKXozfVfS+b0NMSIiBk2TkdTYfkjS5RQD1vYBXkXR3XXoZWbXiBhVTQbKHSvpYxTjIE4Azgee2nJcAyNtDhExqpqUIF5P0fbw5kFvqI6IiN5pMhfTyfMRSEREDJZGI6kjImL0JEFEREStJIiIiKjVtQ1C0vVMsQ6D7V9tJaKIiBgIUzVSv7J8/ePy9RPl62vbCyciIgZF1wRh+3sAklbZfl7l1DpJ1/LoxHoRETGEmrRBSNKvV3Ze3PB9ERGxgDUZKHcq8BFJ+5b7dwN/0FpEERExEJoMlNsMPGcyQdje1XpUERHRd03mYjpA0gXARbZ3STpc0qnzEFtERPRRk7aEj1GsCvf0cn8L8CctxRMREQOiSYLY3/YlwCOweynRn7caVURE9F2TBPFTSUsoB81JOgpIO0RExJBr0ovp7cAG4JmSvg4sBU5sNaqIiOi7Jr2YrpX0X4HDAAE3236o9cgiIqKvGi05ChwJjJXXP18Stj/eWlQREdF30yYISZ8Angl8m0cbpw0kQUREDLEmJYhx4HDbXWd2jYiI4dOkF9N3gKe2HUhERAyWJiWI/YEbJX0LeGDyoO3jW4sqIiL6rkmCeG/bQSwU6zdu4fRVh/Y7jIiIedGkm+tV8xHIQnD2pq1JEBExMqZacvRrto+WdC+PXXpUgG0/qfXoIiKib6ZaUe7o8vWJ8xfO4Fm/cQtnb9q6e39s3WUArF25IqWJiBhqatp7VdIvA3tP7tv+z7aCmo3x8XFPTEy0+h1j6y7j1vcf1+p3RETMJ0mbbY/XnWuyHsTxkrYCtwBXAbcCl/c0woiIGDhNxkH8JXAUsMX2cmAl8M1WoxpQa1eu2L29fuOWPkYSEdG+JgniIdt3AntI2sP2lRSjq0dOtc2h2i4RETGMmoyDuFvSE4CvAJ+UdAfw03bDioiIfmuSIFYD9wOnA68F9gXObDOoQZUeTRExShr3Yhp089GLqSo9miJiGMyqF5OkeyXdU/m7t/ra8IuPkXSzpG2S1tWc30vSxeX5qyWNlcdXSdos6fry9eUN7zUiInpkqoFycxogJ2kRcC6wCtgBXCNpg+0bK5edCtxl+xBJa4CzgJOAHwO/bfv7ko4AvgwcOJd4eq3aoykiYhg1WlFO0vOBoymm3Pia7esavO1IYJvt7eVnXETRnlFNEKt5dDLAS4FzJKnj828A9pG0l+0HGBBpc4iIYddkoNwZwIXAEoqpvz8m6c8afPaBwG2V/R38Yilg9zW2HwZ2ld9TdQJw7SAlh04ZExERw6jJOIjXAi+w/R7b76EYNPe6dsMqSHo2RbXTm7ucP03ShKSJnTt3zkdItTImIiKGUZME8X0qczABewG3N3jf7cBBlf1lNe/bfY2kxRRdaO8s95cBnwNeb/u7dV9g+zzb47bHly5d2iCkiIhoqkkbxC7gBkkbKdogVgHfkvS3ALbf1uV91wArJC2nSARrgN/ruGYDcArwDeBE4ArblrQfcBmwzvbXZ3ZL8yNjIiJi2E07DkLSKVOdt33hFO99BfAhYBHwEdvvk3QmMGF7g6S9gU8AzwN+Aqyxvb1s43gXUK27+U3bd3T7rvkeB1GVMRERsVBNNQ6iSQni8s4fZkmH2b55ujfa/iLwxY5jZ1S27wdeU/O+vwL+qkFsERHRkiZtEF+V9LuTO5L+lKJtIEoZExERw6hJgngp8DpJn5b0FeBQijEOUaq2OaTLa0QMi2kThO0fAF8CXgSMARfavq/luBasdHmNiGExbRuEpH+m6Op6BEWX1AskfcX2O9oOLiIi+qdJI/U5tj9fbt8t6UXAu9sLaeHp1uX1hcufwsVvflG/woqImJNpE4Ttz0s6Glhh+6PAk4F/aD2yBeT0VYfuboeodnmdTBQREQtRk7mY3gO8k2JcAsCeJEHMWBqvI2KhaVLF9GqKgWzXApRTcM9pKvBh9sLlT3lMyaG6nRHWEbGQNEkQD5bTXxhA0uNbjmlBq7Y5pLopIhayJgniEkkfBvaT9IfAHwB/325Yw6OuNJH5miJiIWjSSP0BSauAe4DDgDNsb2w9siFQTQTV0kTaIyJiIWi0olyZEJIUZqhbKeHsTVtTgoiIgddkqo3ogczXFBELTaMSRPRGXXtEBtNFxKCacQlC0kGS/nsbwQyz01cdyq3vP253O8Tk9tW3/GT3NWmbiIhB0ihBSFoq6Y8kfRX4F+CAVqMaUZnoLyIGSdcqpnIw3O9QLBN6KPBZYLntZfMU29CaajBdRMSg6LrkqKT/B3wL+DPga+Vgue22nzGfATbVzyVH56JbckjbRETMh9kuOfouYA3wf4BPSbq4jeCCx4y2zsjriBgUXdsgbH/I9lHA6vLQ54GnS3qnpHTi75Gm3V+rDdhpzI6I+dC1iqn2YukI4GTgJNuHtBbVLCzUKqaqkz78jcf0apq0duUKzt60tbakERExF7OqYpJ0CHCA7a9PHrP9HUmXAx/tfZjRbaI/6N7Daf3GLRmVHRGtmKoN4kM8ugZE1S5gPfDbbQQUj+q2Ul3ndhJERLRhqnEQB9i+vvNgeWystYgCeHSiv7rBdZ3bk9I2ERG9NFUJYr8pzu3T4ziiw3SlgpQmIqJtU5UgJsr1Hx5D0puAze2FFHWqvZ3WrlwxbWkiImKuphoodwDwOeBBHk0I4xRrUr/a9g/nJcKGhqEX02x0Gy+RRYkioolZ9WKy/SPgxZJeBhxRHr7M9hUtxBiz1G1RooiIuZqqm+vewFuAQ4DrgQtsPzxfgUUzKSVERFumaoO4kKJK6XrgWOAD8xJRzFq1nSI9miJirqZKEIfb/n3bHwZOBF4yTzHFLFVLE5k6PCLmaqoE8dDkRqqWFraUJiJiNqYaB/EcSfeU2wL2KfcF2PaTWo8uZmSqkdeTpYvq1ByZpiMipjLVbK6LbD+p/Hui7cWV7SSHATTVyOtJ1QRS3U4pIyI6zXhN6lg4xtZdtrsUUd2u0y1ZJHFEjK4kiCHVOdq6arrE0aSUkcQRMfySIIZUZ9tCk6k5ZlLKSPVUxPBrNUFIOkbSzZK2SVpXc34vSReX56+WNFYeXyLpSkn3STqnzRhHwVSr1s22lFHV2aU2JY2I4dBagpC0CDiXYpDd4cDJkg7vuOxU4K5ydbr1wFnl8fuBPwfe0VZ8o6Ramuic9K9qpqWMbokjVVQRw6HNEsSRwDbb220/CFzEo+tbT1pNMWIb4FJgpSTZ/qntr1EkiuiharLoljg6zSRxTPXDn8QRsbC0mSAOBG6r7O8oj9VeUw7G2wUsaTGm6KJpKaOqmiwmr5v84Z9tFVWTxJEkEjE/FnQjtaTTJE1Imti5c2e/wxkaTUoZnYmjyRgM6E3iSBKJmB9tJojbgYMq+8vKY7XXSFoM7Avc2fQLbJ9ne9z2+NKlS+cYbkxnNtVT0Lu2jTozTSIR0VybCeIaYIWk5ZL2BNYAGzqu2QCcUm6fCFzhbisYxcDq7FI7myqqmSaO2SYRSKkjoqnWEkTZpvBW4MvATcAltm+QdKak48vLLgCWSNoGvB3Y3RVW0q3AB4E3SNpR0wMqBtRsqqiquiWOXiWRVF1FNNNqG4TtL9o+1PYzbb+vPHaG7Q3l9v22X2P7ENtH2t5eee+Y7afYfoLtZbZvbDPWaN9cE0fVbJNI0x/4VF1FTD2ba8S8mGnimEkSgSI5rF25grM3bX1ML6tJ3banc/amrZkZN4ZaEkQMrG6JYzZJ5PRVh9au3d25XTWTJNItWSRxxEK2oLu5Rsw0iUxnLu0fk+bSrpFqqxgkSRAx9Jr2sppN+0dVL8Z1pMdVDJIkiBg5vay66vW4jk696nGVhBKzkQQRUWM2VVe96pLbRo+ruSSUJJrRlQQRMUtzneyw23bTea1mWzKZaUKZaxVYksrClQQR0QNzHdfR+VltDBacS1VXN22XXqK/kiAiemwu7Rq97nHVq4Qylyqwqn5WhyXxzFwSRMQ8adKu0eseV1UzTSi9rAIbhOqwNhLPsCedJIiIAdarHlezSShTxTSfpZc29Crx9CrpTHeuX5IgIha4mZZMZppQepFo+l0dNh+Jp5clnEFpt8lUGxEjZqYJZa5VYLOtDoOpp0WZyfbkflWTqVRmut2G6jQuTbZ7KQkiImat7dJLr/U68fQy6Ux3rh+SICJi3vSzOqytxNPLEk4vEs/alSt6VppIgoiIgdXL6rA2Es8glnZ6KY3UETFyepV4epV0pjvXLxqWJaDHx8c9MTHR7zAiInqq2/oivVp3RNJm2+O155IgIiJG11QJIlVMERFRKwkiIiJqJUFEREStJIiIiKiVBBEREbWGpheTpJ3A92bwlv2BH7cUziAbxfsexXuG0bzvUbxnmNt9/4rtpXUnhiZBzJSkiW5du4bZKN73KN4zjOZ9j+I9Q3v3nSqmiIiolQQRERG1RjlBnNfvAPpkFO97FO8ZRvO+R/GeoaX7Htk2iIiImNoolyAiImIKI5kgJB0j6WZJ2ySt63c8bZB0kKQrJd0o6QZJa8vjT5G0UdLW8vXJ/Y61DZIWSbpO0j+V+8slXV0+84sl7dnvGHtJ0n6SLpX0H5JukvSiUXjWkk4v/31/R9KnJO09jM9a0kck3SHpO5Vjtc9Xhb8t7//fJT1/tt87cglC0iLgXOBY4HDgZEmH9zeqVjwM/Kntw4GjgD8u73MdsMn2CmBTuT+M1gI3VfbPAtbbPgS4Czi1L1G152zgS7afBTyH4t6H+llLOhB4GzBu+whgEbCG4XzWHwOO6TjW7fkeC6wo/04D/m62XzpyCQI4Ethme7vtB4GLgNV9jqnnbP/A9rXl9r0UPxgHUtzrheVlFwKv6kuALZK0DDgOOL/cF/By4NLykqG6b0n7Ai8BLgCw/aDtuxmBZ02xKuY+khYDjwN+wBA+a9tfAX7Scbjb810NfNyFbwL7SXrabL53FBPEgcBtlf0d5bGhJWkMeB5wNXCA7R+Up34IHNCvuFr0IeB/AI+U+0uAu20/XO4P2zNfDuwEPlpWq50v6fEM+bO2fTvwAeA/KRLDLmAzw/2sq7o93579xo1ighgpkp4AfAb4E9v3VM+56MI2VN3YJL0SuMP25n7HMo8WA88H/s7284Cf0lGdNKTP+skU/1teDjwdeDy/WA0zEtp6vqOYIG4HDqrsLyuPDR1Jv0SRHD5p+7Pl4R9NFjfL1zv6FV9Lfh04XtKtFNWHL6eon9+vrIaA4XvmO4Adtq8u9y+lSBjD/qz/G3CL7Z22HwI+S/H8h/lZV3V7vj37jRvFBHENsKLs6bAnRaPWhj7H1HNlvfsFwE22P1g5tQE4pdw+BfjH+Y6tTbbfZXuZ7TGKZ3uF7dcCVwInlpcN1X3b/iFwm6TDykMrgRsZ8mdNUbV0lKTHlf/eJ+97aJ91h27PdwPw+rI301HArkpV1IyM5EA5Sa+gqKdeBHzE9vv6G1HvSToa+CpwPY/Wxb+boh3iEuBgitlvf9d2Z+PXUJD0UuAdtl8p6RkUJYqnANcBv2/7gT6G11OSnkvRKL8nsB14I8V/AIf6WUv6C+Akil571wFvoqhvH6pnLelTwEspZm39EfAe4PPUPN8yWZ5DUd32M+CNtidm9b2jmCAiImJ6o1jFFBERDSRBRERErSSIiIiolQQRERG1kiAiIqJWEkRERNRKgoiIiFpJEBERUev/AxemOmVS6fTmAAAAAElFTkSuQmCC","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["n_components = 500\n","X_train: (12888, 500), y_train: (12888,)\n","X_test: (4296, 500), y_test: (4296,)\n","Accuracy: 0.7169459962756052\n","Macro-F1 score: 0.6780034986112806\n","Model6: 0.6780034986112806\n","\n","Accuracy: 0.6524674115456238\n","Macro-F1 score: 0.5456504896637868\n","KNN: 0.5456504896637868 \n","\n","Time taken for 500 Principal Components: 639.6375532150269s \n","\n"]},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAZUAAAD4CAYAAAAkRnsLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAj4ElEQVR4nO3df5hdVX3v8feHxASr/AyBIgEnloneqPzQaQiF21tJo0GUwAMtibbEFg0+khqp3mtoi1Xsj+BVIxa1RKFGakkQah1FTWOgViwEJkiBgEmGEC6JaGICSaBNIPi9f+x1kp2Tc87smZw9kzPn83qe85y91157zdpxnC/rx15LEYGZmVkzHDLUFTAzs+HDQcXMzJrGQcXMzJrGQcXMzJrGQcXMzJpm5FBXYCgdc8wx0dHRMdTVMDNrKStXrvxlRIytda2tg0pHRwc9PT1DXQ0zs5Yi6cl619z9ZWZmTeOgYmZmTeOgYmZmTeOgYmZmTeOgYmZmTeOgMkALlq0Z6iqYmR10HFQG6Lrla4e6CmZmBx0HFTMza5q2fvmxvxYsW7NPC6Vj3h0AzJ3SyZVTJwxVtczMDhpq5026urq6YqBv1HfMu4P1889rco3MzA5+klZGRFeta+7+MjOzpnFQGaC5UzqHugpmZgcdB5UB8hiKmdn+HFTMzKxpSg0qkqZJWi2pV9K8GtdHS1qSrq+Q1JHSp0paKenh9H1O7p43p/ReSZ+XpJR+tKRlktam76PKfDYzM9tfaUFF0gjgC8C5wERgpqSJVdkuA56JiJOBBcC1Kf2XwDsj4o3ALODm3D1fAt4HdKbPtJQ+D1geEZ3A8nRuZmaDqMyWyiSgNyLWRcQLwGJgelWe6cCidHwbMEWSIuInEfGzlL4KeHlq1RwPHB4R90Y2F/prwAU1ylqUSzczs0FSZlA5AXgqd74hpdXMExG7gW3AmKo8FwEPRMSulH9DnTKPi4in0/HPgeNqVUrSbEk9kno2b97cvycyM7OGDuqBekmvJ+sSu7w/96VWTM23OiNiYUR0RUTX2LE1t1g2M7MBKjOobAROzJ2PS2k180gaCRwBbEnn44BvApdGxOO5/OPqlPmL1D1G+t7UtCcxM7NCygwq9wOdksZLGgXMALqr8nSTDcQDXAzcGREh6UjgDmBeRPy4kjl1b22XNDnN+roU+FaNsmbl0s3MbJCUFlTSGMkcYCnwGHBrRKySdI2k81O2G4ExknqBP2XvjK05wMnAxyQ9mD7HpmsfAL4C9AKPA99L6fOBqZLWAr+bzs3MbBB5QckBLihpZtauvKCkmZkNCgcVMzNrGgcVMzNrGgcVMzNrGgcVMzNrGgcVMzNrGgcVMzNrGgcVMzNrGgcVMzNrGgcVMzNrGgcVMzNrGgcVMzNrGgcVMzNrGgcVMzNrmpFFMqX9T347nf4wIr5dXpXMzKxV9dlSkfS3wFzg0fT5oKS/KbtiZmbWeop0f50HTI2ImyLiJmAa8I4ihUuaJmm1pF5J82pcHy1pSbq+QlJHSh8j6S5Jz0m6Ppf/sNxOkA9K+qWkz6Vr75G0OXftvUXqaGZmzVOo+ws4Etiajo8ocoOkEcAXgKnABuB+Sd0R8Wgu22XAMxFxsqQZwLXAJcBO4GrgDekDQETsAE7L/YyVwD/nylsSEXMKPpOZmTVZkZbK3wI/kfRVSYuAlcBfF7hvEtAbEesi4gVgMTC9Ks90YFE6vg2YIkkR8XxE3E0WXGqSNAE4FvhRgbqYmdkg6DOoRMQtwGSyFsHtwJkRsaRA2ScAT+XON6S0mnkiYjewDRhToGyAGWQtk8ilXSTpIUm3STqx1k2SZkvqkdSzefPmgj/KzMyKqBtUJL0ufb8JOJ4sKGwAXpXShtoM4Jbc+beBjog4BVjG3hbQPiJiYUR0RUTX2LFjB6GaZmbto9GYyp8Cs4HP1LgWwDl9lL0RyLcWxqW0Wnk2SBpJNl6zpY9ykXQqMDIiVu6pUET+vq8An+qrHDMza666QSUiZqfDcyNin7ENSYcWKPt+oFPSeLLgMQN4V1WebmAWcA9wMXBnVXdWPTPZt5WCpOMj4ul0ej7wWIFyzMysiYrM/voPoLq7q1baPiJit6Q5wFJgBHBTRKySdA3QExHdwI3AzZJ6yWaXzajcL2k9cDgwStIFwFtzM8d+H3h71Y/8YHpJc3cq6z0Fns3MzJpI9RoGkn6dbCD9H8laGEqXDgf+PiJeNyg1LFFXV1f09PQMdTXMzFqKpJUR0VXrWqOWytvI/mt/HPDZXPoO4M+aVjszMxs2Go2pLAIWSbooIm4fxDqZmVmL6nNMJSJul3Qe8Hrg0Fz6NWVWzMzMWk+RBSX/nmzplD8hG1f5PeDVJdfLzMxaUJFlWn4rIi4lW6PrE8CZwIRyq2VmZq2oSFCpvKPyX5JeBbxI9oa9mZnZPoq8p/JtSUcC/xd4gOxt+i+XWSkzM2tNDYOKpEOA5RHxLHC7pO8Ah0bEtsGonJmZtZaG3V8R8SuyPVEq57scUMzMrJ4iYyrLJV0kSX1nNTOzdlYkqFwOfAPYJWm7pB2StpdcLzMza0FFXn48bDAqYmZmra9IS8XMzKwQBxUzM2saBxUzM2uaQkFF0tmS/igdj027OZqZme2jyIKSfwl8FLgqJb2MbOOuPkmaJmm1pF5J82pcHy1pSbq+QlJHSh8j6S5Jz0m6vuqef0tlPpg+xzYqy8zMBk+RlsqFZHu+Pw8QET8D+pwRJmkE2YuT5wITgZmSJlZlu4xsocqTgQXAtSl9J3A18JE6xb87Ik5Ln019lGVmZoOkSFB5IbI9hwNA0isKlj0J6I2IdRHxArAYmF6VZzqwKB3fBkyRpIh4PiLuZu9ilkXULKsf95uZ2QEqElRulXQDcKSk9wE/oNiCkicAT+XON6S0mnkiYjewDRhToOx/SF1fV+cCR6GyJM2W1COpZ/PmzQV+lJmZFdVnUImIT5P9l//twGuBj0XE35VdsQbeHRFvBP5n+vxhf26OiIUR0RURXWPHji2lgmZm7arPN+rTTK8fRcSydP5ySR0Rsb6PWzcCJ+bOx6W0Wnk2SBoJHAFsaVRoRGxM3zsk/RNZN9vXBlKWmZk1V5Hur28Av8qdv5TS+nI/0ClpvKRRwAyguypPNzArHV8M3JnGb2qSNFLSMen4ZcA7gEcGUpaZmTVfkU26RqaBdgAi4oUUJBqKiN2S5gBLgRHATRGxStI1QE9EdAM3AjdL6gW2kgUeACStBw4HRkm6AHgr8CSwNAWUEew7vlO3LDMzGxxFgspmSeenIICk6cAvixQeEd8FvluV9rHc8U7g9+rc21Gn2DfXyV+3LDMzGxxFgsr7ga+nlxBFNsPq0lJrZWZmLanI0vePA5MlvTKdP1d6rczMrCUVmf01GrgI6ABGVl4LiYhrSq2ZmZm1nCKzv75F9rb6brKlWiqftrdg2ZqhroKZ2UGlyJjKuIiYVnpNWtB1y9dy5dQJQ10NM7ODRpGWyn9IemPpNTEzs5ZXpKVyNvAeSU8Au8hmgEVEnFJqzQ5SC5at4brla/ecd8y7A4C5UzrdajGztqe+XjqX9Opa6RHxZCk1GkRdXV3R09Mz4Ps75t3B+vnnNbFGZmYHP0krI6Kr1rUiU4qfTIUcCxza5LqZmdkwUmTnx/MlrQWeAH4IrAe+V3K9WsLcKZ1DXQUzs4NKkYH6TwKTgTURMR6YAtxbaq1ahMdQzMz2VSSovBgRW4BDJB0SEXcBNfvSzMysvRWZ/fVsWqLl38nWANuEX340M7MairRUpgP/DVwJfB94HHhnmZUyM7PWVGT2V75VsqjEupiZWYur21KRdHf63iFpe+6zQ9L2IoVLmiZptaReSfNqXB8taUm6vkJSR0ofI+kuSc+lJfcr+X9N0h2SfipplaT5uWvvkbRZ0oPp895+/DuYmVkT1A0qEXF2+j4sIg7PfQ6LiMP7KljSCOALwLnARGCmpIlV2S4DnomIk4EFwLUpfSdwNfCRGkV/OiJeB5wOnCXp3Ny1JRFxWvp8pa86NoMXlTQz26vhmIqkEZJ+OsCyJwG9EbEubUe8mGx8Jm86e7vUbgOmSFJEPB8Rd5MFlz0i4r/S7DNSmQ8A4wZYv6bIL9liZtbuGgaViHgJWC3ppAGUfQLZLpEVG1JazTwRsRvYBowpUrikI8kmDCzPJV8k6SFJt0k6cQB1NjOzA1BkSvFRwCpJ95GbShwR55dWqz5IGgncAnw+Ital5G8Dt0TELkmXk7WAzqlx72xgNsBJJw0kVnpRSTOzeooElasHWPZGIN9aGJfSauXZkALFEcCWAmUvBNZGxOcqCekFzYqvAJ+qdWNELEz309XV1Xg1zTqunDphT/DwopJmZnsVmVL8wwGWfT/QKWk8WfCYAbyrKk83MAu4B7gYuDP6WDZZ0l+RBZ/3VqUfHxFPp9PzgccGWG8zMxugInvUTwb+DvgfwChgBPB8XzPAImK3pDnA0nTPTRGxStI1QE9EdAM3AjdL6gW2kgWeys9dDxwOjJJ0AfBWYDvw58BPgQckAVyfZnp9UNL5ZNsebwXeU/Df4IB4UUkzs72K7KfSQ/bH/htka35dCkyIiKvKr165DnQ/FTOzdtRoP5Uiy7QQEb3AiIh4KSL+AfCe9WZmtp8iA/X/JWkU8KCkTwFPUzAYmZlZeykSHP4w5ZtDNqX4ROCiMitlZmatqUhL5c3AHRGxHfhEyfUxM7MWVqSl8k5gjaSbJb0jvU9iZma2nz6DSkT8EXAy2eyvmcDjkgZlsUYzM2sthVodEfGipO8BAbwcuICqlw/NzMz6bKlIOlfSV4G1ZAP0XwF+veR6mZlZCyrSUrkUWAJcHhG7Sq6PmZm1sCJrf80cjIqYmVnr80uMZmbWNA4qZmbWNA4qZmbWNHXHVCQ9TDaFuKaIOKWUGpmZWctqNFD/jvR9Rfq+OX2/u7zqmJlZK6sbVCLiSQBJUyPi9NyleZIeAOaVXTkzM2stRcZUJOms3MlvFbwPSdMkrZbUK2m/ICRptKQl6foKSR0pfYykuyQ9J+n6qnveLOnhdM/nlbZ/lHS0pGWS1qbvo4rU0czMmqdIcLgM+KKk9WmL3y8Cf9zXTZJGAF8AzgUmAjMlTaxR9jMRcTKwALg2pe8ErgY+UqPoLwHvAzrTp7Jh2DxgeUR0AstxS8rMbNAVWVByZUScCpwKnBoRp0XEAwXKngT0RsS6iHgBWAxMr8ozHViUjm8DpkhSRDwfEXeTBZc9JB0PHB4R90a2D/LXyNYhqy5rUS7dzMwGSZG1v46TdCOwOCK2SZoo6bICZZ8APJU735DSauaJiN3ANmBMH2VuqFPmcRHxdDr+OXBcneeZLalHUs/mzZsLPEZjC5atOeAyzMyGiyLdX18FlgKvSudrgA+VVJ+mSK2YmtOhI2JhRHRFRNfYsWMP+Gddt3ztAZdhZjZcFAkqx0TErcCvYE+L4qUC920k23q4YlxKq5knbf51BLCljzLH1SnzF6l7rNJNtqlAHc3MrImKBJXnJY0h/Ze/pMlk3VR9uR/olDRe0ihgBtBdlacbmJWOLwbuTK2MmlL31nZJk9Osr0uBb9Uoa1YuvekWLFtDx7w76Jh3B8CeY3eFmVm7U4O/4VkG6U3A3wFvAB4BxgIXR8RDfRYuvR34HDACuCki/lrSNUBPRHRLOpTspcrTga3AjIhYl+5dDxwOjAKeBd4aEY9K6iLrkns58D3gTyIiUuC7FTgJeBL4/YjY2qh+XV1d0dPT09djNNQx7w7Wzz/vgMowM2slklZGRFeta0WWvn9A0v8CXgsIWB0RLxb5wRHxXeC7VWkfyx3vBH6vzr0dddJ7yAJcdfoWYEqRepmZWTmKLig5iWxK8ZvI3je5tLwqtZa5Uzrd7WVmlhSZUnwz8GngbOA306dms6cdXTl1gmeAmZklRbYT7gImNhpANzMzg2JB5RHg14Gn+8rYThYsW7NPC6UyE2zulE6unDphqKplZjakigSVY4BHJd0H7KokRsT5pdWqBVw5dcKe4OEZYGZmmSJB5eNlV8LMzIaHIlOKfzgYFWllc6d0DnUVzMwOCo22E747Is6WtIN919ES2fJah5deOzMzayl1pxRHxNnp+7CIODz3OcwBZV+eUmxmlikypgKApGOBQyvnEfH/SqmRmZm1rD6DiqTzgc+QLX2/CXg18Bjw+nKrdnDzlGIzs/0Vaal8EpgM/CAiTpf0FuAPyq3Wwc9Tis3M9ldk7a8X02KNh0g6JCLuwsu0mJlZDUWCyrOSXgn8O/B1SdcBz5dbrdbiRSXNzDJFgsp04L+BK4HvA48D7yyzUq3Gi0qamWWKvPyYb5UsKrEuZmbW4uq2VCTtkLQ999mR/y5SuKRpklZL6pU0r8b10ZKWpOsrJHXkrl2V0ldLeltKe62kB3Of7ZI+lK59XNLG3LW39/cfo7+8rbCZ2b763E54wAVLI4A1wFRgA9me9TMj4tFcng8Ap0TE+yXNAC6MiEskTQRuIdsc7FXAD4AJEfFSVfkbgTMi4klJHweei4hPF61jM7YTrqgEFs8CM7PhrtF2woV2fpT0JkkflPQnkk4v+HMnAb0RsS4iXgAWk43P5E1nb5fabcAUSUrpiyNiV0Q8AfSm8vKmAI9HxJMF62NmZiUrsvPjx8j+8I8hWwb/q5L+okDZJwBP5c43pLSaeSJiN7At/Zwi984ga83kzZH0kKSbJB1V53lmS+qR1LN58+YCj9FYpQuswl1gZtbOirz8+G7g1IjYCSBpPvAg8Fcl1qshSaOA84GrcslfIntRM9L3Z4A/rr43IhYCCyHr/jrQuvglSDOzvYoElZ+Rrfm1M52PJhvL6MtG4MTc+bga91XybJA0EjgC2FLg3nOBByLiF5WE/LGkLwPfKVDHA+blWszM9ioSVLYBqyQtI2sFTAXuk/R5gIj4YJ377gc6JY0nCwgzgHdV5ekGZgH3ABcDd0ZESOoG/knSZ8kG6juB+3L3zaSq60vS8RFR2fL4QrJtkEtX3VJxMDGzdlYkqHwzfSr+rUjBEbFb0hxgKTACuCkiVkm6BuiJiG7gRuBmSb3AVrLAQ8p3K/AosBu4ojLzS9IryALb5VU/8lOSTiMLfOtrXB8U1y1f66BiZm2rzynFko6NiE1Vaa+NiNWl1mwQNHNKMeztCvO4ipkNZ42mFBdpqfxI0tURcWsq7MPAZcDEJtaxpXlcxcwsU6SlcjzZbKmdwHFke6l8OCKeK7965Wp2SwU8rmJmw98BvfyYBr+/D5wJdACLhkNAaabq5VquW77W76qYWVsqsvPjD8imFb+BbJrvjZL+PSI+UnblWkX1DDBw15eZtaciy7RcHxGXRsSzEfEwWYtlW8n1ajnVb9a7tWJm7ahI99e/SDpb0h+lpKOAfyy3Wq3nyqkTWD//POZO6dyT5taKmbWbImt//SXwUfYuiTIKB5WaqmeBubViZu2mSPfXhWTrbD0PEBE/Aw4rs1KtqlZrZf3889xaMbO2UeQ9lRfS0ikBe95otxqqWyrggXszay9Fgsqtkm4AjpT0PrKVf79cbrVak2eBmVm7K7Tzo6SpwFsBAUsjYlnZFRsMZbz8WKu1Ag4uZjZ8NHr5sbTthFtBGUGlIh9cvBaYmQ0nB7r2l/WT1wIzs3bllkpJLRXwqsVmNjwd0NpfNQo7UdL/PvBqDX+VVonfUzGzdlEoqEgaK+kDkn5EtknXcQXvmyZptaReSfNqXB8taUm6vkJSR+7aVSl9taS35dLXS3pY0oOSenLpR0taJmlt+j6qSB3LNndKZ82BezOz4ahuUJF0mKRZkpaSbeX7G8D4iPiNIotJShoBfIFsP/mJwExJ1XuwXAY8ExEnAwuAa9O9E8l2gXw9MA34Yiqv4i0RcVpV82sesDwiOoHl6XzIeQzFzNpJo4H6TWTB5C+Au9MLkBf2o+xJQG9ErAOQtBiYTrZFcMV04OPp+DbgeklK6YsjYhfwRNpueBLZXvb1TAd+Jx0vImtRfbQf9W0qD9abWTtq1P11FTAa+CJwlaTf6GfZJwBP5c43pLSaeSJiN9nqx2P6uDeAf5W0UtLsXJ7j0t4vAD+nYBddWSpLtuQH6R1QzGy4qxtUIuJzETGZrAUA8C/AqyR9VNJQ/mU8OyLeRNatdoWk367OENmUtprT2iTNltQjqWfz5s2lVbJ64y7IFpg8a/7y0n6mmdlQK7L0/bqI+JuIeCPQBRwOfLdA2RvJNvWqGJfSauaRNBI4AtjS6N6IqHxvAr5J1i0G8Iu09XFlC+RNdZ5nYUR0RUTX2LFjCzzGwFw5dQJnjD96v/SNz+7krPnLueSGRj15ZmatqdFA/cmSzsqnRcQjwPfIBs/7cj/QKWm8pFFkA+/dVXm6gVnp+GLgztTK6AZmpNlh44FO4D5Jr5B0WKrfK8iWjnmkRlmzgG8VqGOpllx+5j4rFldsfHYnK57Y6qnGZjbsNGqpfA7YXiN9G9lMrYbSGMkcYCnwGHBrRKySdI2k81O2G4ExaSD+T0kztiJiFXAr2aD+94ErIuIlsnGSuyX9J9kkgjsi4vuprPnAVElrgd9N50PuyqkTOOHIQ2teu275WgcWMxtW6r5RL+n+iPjNOtceTt1hLa3sN+rzzpq/nI3P7qx57bDRI3j4E0Uaf2ZmQ2+gb9Qf2eDayw+oRm3ox/Om1OwKA9ix6yXOmr/crRYza3mNgkpP2j9lH5LeC6wsr0rDV6OusI3P7uS65Wu55IZ7HFzMrGU1evnxQ8A3Jb2bvUGki2yP+v68BGk5P543hQXL1nDT3evYseul/a6veGIrK57YCvhtfDNrPX2uUizpLcAb0umqiLiz9FoNksEcU6kl/w5LLWeMP5rJrxnj4GJmB5UB7aci6VDg/cDJwMPAjWlGlzVJZYyl3oKTlVbLveu2OLiYWUtoNKayiKy762Gyt9c/PSg1aiOVPe1rvSSZt+KJrV7p2MxaQqOgMjEi/iAibiB7MXG/5VCsOZZcfiZnjD+67uywCg/im9nBrtFA/YuVg4jYnS0ebGVZcvmZANy7bsuegfpqle6w21Y+xY/nTRnM6pmZFdKopXKqpO3pswM4pXIsqdab9tYElaVdGrVaNj67kwXL1nj9MDM76HiP+iGc/dWXStCo13KBvTPEwFOQzWxwNJr95aByEAeVikZLvORVAsy967bs6U4zM2s2B5U6WiWowP47SfZl7pRO7l23BcABxsyaakDvqdjBpdK1VTSw5PMtWLbG77qY2aBwUGkhlYDQaIZYLZUAU7nHLRgzK4u7v1qk+6tapfXRn+BSrdJFVhmHAQcaM+ubx1TqaOWgUnHJDfcw+TVjmvbGfX4sxrPKzKyWIRtTkTQNuA4YAXwlIuZXXR8NfA14M9ne9JdExPp07SrgMuAl4IMRsVTSiSn/cUAACyPiupT/48D7gM2p+D+LiO+W+XwHg3zLIt/qGGgLJh+c8t1llXIdaMyskdJaKpJGAGuAqcAGsj3rZ0bEo7k8HwBOiYj3S5oBXBgRl0iaCNwCTAJeBfwAmAAcCxwfEQ+kvepXAhdExKMpqDwXEYXXKBsOLZV6Kt1j0Pg9l4Hy9GWz9jVULZVJQG9ErEuVWAxMJ9t3vmI68PF0fBtwvbL1YKYDiyNiF/BE2sN+UkTcAzwNEBE7JD0GnFBVprFvS6IZ4y/V8vu+VLrg8t1mDjZm7anMoHIC8FTufANwRr08aX2xbcCYlH5v1b0n5G+U1AGcDqzIJc+RdCnQA3w4Ip6prpSk2cBsgJNOOqnfD9WKKgGmshhls1sw+QCTLzcfbKqDjrvPzIanlpxSLOmVwO3AhyKisg7Zl4BPko21fBL4DPDH1fdGxEJgIWTdX4NS4YNE9R/y/PsrZXSV5YNNddCpbtXkjx10zFpXmUFlI3Bi7nxcSquVZ4OkkcARZAP2de+V9DKygPL1iPjnSoaI+EXlWNKXge807UmGqVp/uPNjMc2cVVatXrCpfDcKOu5WMzt4lRlU7gc6JY0nCwgzgHdV5ekGZgH3kO3ZcmdEhKRu4J8kfZZsoL4TuC+Nt9wIPBYRn80XJOn4iHg6nV4IPFLScw1rtQJN9eyvwdgwrFHQqTeG4/dtzIZeaUEljZHMAZaSTSm+KSJWSboG6ImIbrIAcXMaiN9KFnhI+W4lG4DfDVwRES9JOhv4Q+BhSQ+mH1WZOvwpSaeRdX+tBy4v69naSb1uqGZMXx6oemM4+bTqFle9AOSuNrPm8suPw3RK8WCq13IY7GAzUNUvfDZqAbkLzsxv1NfloFKuRrO/BqMLrUz593TAwcjai4NKHQ4qQ6eyAVm9P8atHnTqqReM+gpQlWN31dnBwEGlDgeVg1ejoNMq3WplOGP80UD/g5EnMlgzOajU4aDSmorM/mrnwFNEX+NI+eP+BC4Hq/bgoFKHg8rwVXT2l4NPc5UVrPLHDlxDz0GlDgcVq166pugfPwejoTMYgctjWY05qNThoGIDVa8LLn/sYDR8VI9l5Y+bFbj6kxeGdvsJB5U6HFRssBUZD2p0PFxnxVn/HehMQhh4YHJQqcNBxVpN9ay4gf7XrltMBrB+/nkDum/Idn40s+Zq1iB1XxMZ8sf9CVwOVuaWilsqZk1TVrDKHztwNd/cKZ396gpzS8XMBsVgDB4PRuBql7GsgXZ/NeKgYmYtZShmPQ1G4BouQc5BxcysDwfjC5f1AlN/Zn+VwUHFzKzFHMwvYx4y1BUwM7Pho9SgImmapNWSeiXNq3F9tKQl6foKSR25a1el9NWS3tZXmZLGpzJ6U5mjynw2MzPbX2lBRdII4AvAucBEYKakiVXZLgOeiYiTgQXAteneiWRbC78emAZ8UdKIPsq8FliQynomlW1mZoOozJbKJKA3ItZFxAvAYmB6VZ7pwKJ0fBswRZJS+uKI2BURTwC9qbyaZaZ7zkllkMq8oLxHMzOzWsoMKicAT+XON6S0mnkiYjewDRjT4N566WOAZ1MZ9X4WAJJmS+qR1LN58+YBPJaZmdXTdrO/ImIhsBBA0mZJTw6wqGOAXzatYq3Bz9we/Mzt4UCe+dX1LpQZVDYCJ+bOx6W0Wnk2SBoJHAFs6ePeWulbgCMljUytlVo/az8RMbbw01SR1FNvmYLhys/cHvzM7aGsZy6z++t+oDPNyhpFNvDeXZWnG5iVji8G7oxsMbJuYEaaHTYe6ATuq1dmuueuVAapzG+V+GxmZlZDaS2ViNgtaQ6wFBgB3BQRqyRdA/RERDdwI3CzpF5gK1mQIOW7FXgU2A1cEREvAdQqM/3IjwKLJf0V8JNUtpmZDaK2XqX4QEiancZn2oafuT34mdtDWc/soGJmZk3jZVrMzKxpHFTMzKxpHFT6qa/1zFqVpJskbZL0SC7taEnLJK1N30eldEn6fPo3eEjSm4au5gMn6URJd0l6VNIqSXNT+rB9bkmHSrpP0n+mZ/5ESq+5dl6j9flaTVrq6SeSvpPOh/UzS1ov6WFJD0rqSWml/247qPRDH2uPtbqvkq2zljcPWB4RncDydA7Z83emz2zgS4NUx2bbDXw4IiYCk4Er0v+ew/m5dwHnRMSpwGnANEmTqb92Xs31+VrUXOCx3Hk7PPNbIuK03Pso5f9uR4Q/BT/AmcDS3PlVwFVDXa8mPl8H8EjufDVwfDo+Hlidjm8AZtbK18ofsnebprbLcwO/BjwAnEH2ZvXIlL7n95xs+v6Z6XhkyqehrvsAnnVc+iN6DvAdQG3wzOuBY6rSSv/ddkulf4qsZzacHBcRT6fjnwPHpeNh9++QujhOB1YwzJ87dQM9CGwClgGPU3/tvHrr87WazwH/B/hVOm+0XuBweeYA/lXSSkmzU1rpv9ttt/aXDUxEhKRhOf9c0iuB24EPRcR2SXuuDcfnjuxF4tMkHQl8E3jd0NaoXJLeAWyKiJWSfmeIqzOYzo6IjZKOBZZJ+mn+Ylm/226p9E+R9cyGk19IOh4gfW9K6cPm30HSy8gCytcj4p9T8rB/boCIeJZseaMzSWvnpUv559rzzNp3fb5WchZwvqT1ZNtlnANcx/B+ZiJiY/reRPYfD5MYhN9tB5X+KbKe2XCSX5stv55aN3BpmjEyGdiWa1K3DGVNkhuBxyLis7lLw/a5JY1NLRQkvZxsDOkx6q+dV299vpYREVdFxLiI6CD7/+ydEfFuhvEzS3qFpMMqx8BbgUcYjN/toR5MarUP8HZgDVk/9J8PdX2a+Fy3AE8DL5L1p15G1o+8HFgL/AA4OuUV2Sy4x4GHga6hrv8An/lssn7nh4AH0+ftw/m5gVPI1sZ7KP2R+VhKfw3Zoq29wDeA0Sn90HTem66/Zqif4QCf/3eA7wz3Z07P9p/ps6ryt2owfre9TIuZmTWNu7/MzKxpHFTMzKxpHFTMzKxpHFTMzKxpHFTMzKxpHFTMzKxpHFTMzKxp/j+NUUge91tcowAAAABJRU5ErkJggg==","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["n_components = 1000\n","X_train: (12888, 1000), y_train: (12888,)\n","X_test: (4296, 1000), y_test: (4296,)\n","Accuracy: 0.7199720670391061\n","Macro-F1 score: 0.686003205410374\n","Model6: 0.686003205410374\n","\n","Accuracy: 0.6431564245810056\n","Macro-F1 score: 0.5741824982582959\n","KNN: 0.5741824982582959 \n","\n","Time taken for 1000 Principal Components: 1003.674681186676s \n","\n"]},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAY4AAAD4CAYAAAD7CAEUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAihklEQVR4nO3df5RdZX3v8feHRAIoP2OulYTbiZLojVIVpwjC6nU1DQaxxAqWoK1UabGrUBHrraFeS8H+gFvbiAv0kgpKqZpAtDYVhRsDtaXVyAS6hESTjEmURK0j4ZcIgYTP/WPvCSeTMzP7TM6ZOXPO57XWrDnn2c/e8905MN959vNLtomIiKjqoIkOICIiJpckjoiIaEgSR0RENCSJIyIiGpLEERERDZk60QGMhxe+8IXu6emZ6DAiIiaVdevW/dT2jKHlXZE4enp66Ovrm+gwIiImFUnfr1eeR1UREdGQliYOSQslbZTUL2lJnePTJK0oj6+V1FOWT5d0l6SfSbp2yDkHS1omaZOk70o6u5X3EBER+2rZoypJU4DrgAXAduAeSatsb6ipdgHwsO3jJS0GrgbOBZ4CPgy8svyq9SHgJ7bnSjoIOKZV9xAREftrZYvjJKDf9hbbTwPLgUVD6iwCbipfrwTmS5LtJ2zfTZFAhno38FcAtp+1/dPWhB8REfW0MnHMBB6seb+9LKtbx/Zu4FFg+nAXlHRU+fIjku6VdKukFw1T90JJfZL6BgYGxngLEREx1GTrHJ8KzAL+w/aJwDeAj9araHuZ7V7bvTNm7DearLKlqzeN+dyIiE7UysSxAziu5v2ssqxuHUlTgSOBh0a45kPAz4Evlu9vBU5sRrDDuWbN5lZePiJi0mll4rgHmCNptqSDgcXAqiF1VgHnl6/PAe70COu8l8f+GXhDWTQf2DBc/YiIaL6WjaqyvVvSxcAdwBTgRtvrJV0J9NleBdwA3CypH9hJkVwAkLQNOAI4WNJbgNPLEVkfLM/5GDAAvKvZsS9dvWmflkbPktsAuGT+HC5dMLfZPy4iYlJRN2zk1Nvb67HOHO9ZchvbrjqzyRFFRLQ/Sets9w4tn2yd4xERMcGSOEZxyfw5Ex1CRERbSeIYRfo0IiL2lcQRERENSeKIiIiGJHFERERDkjgiIqIhSRwREdGQJI6IiGhIEkdERDQkiSMiIhqSxBEREQ1J4oiIiIYkcUREREOSOCIioiFJHBER0ZCWJg5JCyVtlNQvaUmd49MkrSiPr5XUU5ZPl3SXpJ9JunaYa6+S9EAr44+IiP21LHFImgJcB5wBzAPOkzRvSLULgIdtHw8sBa4uy58CPgx8YJhrvxX4WSvijoiIkbWyxXES0G97i+2ngeXAoiF1FgE3la9XAvMlyfYTtu+mSCD7kPQC4P3An7cu9IiIGE4rE8dM4MGa99vLsrp1bO8GHgWmj3LdjwB/A/y8OWFGREQjJlXnuKRXAy+1/Y8V6l4oqU9S38DAQOuDi4joEq1MHDuA42rezyrL6taRNBU4EnhohGueAvRK2gbcDcyV9C/1KtpeZrvXdu+MGTPGdAMREbG/ViaOe4A5kmZLOhhYDKwaUmcVcH75+hzgTtse7oK2P2n7WNs9wGnAJttvaHrkERExrKmturDt3ZIuBu4ApgA32l4v6Uqgz/Yq4AbgZkn9wE6K5AJA2ao4AjhY0luA021vaFW8ERFRjUb4A79j9Pb2uq+vb6LDiIiYVCSts907tHxSdY5HRMTES+KIiIiGVOrjkHQW8Cvl26/b/ufWhRQREe1s1BaHpL8CLgE2lF/vlfSXrQ4sIiLaU5UWx5nAq20/CyDpJuA+4E9aGVhERLSnqn0cR9W8PrIFcURExCRRpcXxV8B9ku4CRNHXsd8S6RER0R1GTRy2P18u6/HLZdEHbf+4pVFFRETbGvZRlaSXl99PBF5MsbrtduDYsiwiIrrQSC2O9wMXUixhPpSBX21JRBER0daGTRy2LyxfnmF7nw2VJB3S0qgiIqJtVRlV9R8VyyIiogsM2+KQ9AsUO/QdKuk1FCOqoFix9rBxiC0iItrQSH0cbwR+h2IDpr+tKX+cTP6LiOhaI/Vx3ATcJOls218Yx5giIqKNVZnH8QVJZwKvAA6pKb+ylYFFRER7qrLI4f8FzgX+kKKf423AL7Y4roiIaFNVRlW93vY7gYdtXwGcAsytcnFJCyVtlNQvab9lSiRNk7SiPL5WUk9ZPl3SXZJ+JunamvqHSbpN0nclrZd0VaW7jIiIpqmSOAbncPxc0rHAMxQzyUckaQpwHXAGMA84T9K8IdUuoEhIxwNLgatrfuaHgQ/UufRHbb8ceA1wqqQzKtxDREQ0SZXE8c+SjgL+GrgX2AZ8rsJ5JwH9trfYfhpYDiwaUmcRcFP5eiUwX5JsP2H7bp5LWgDY/rntu8rXT5fxzKoQS0RENMmIneOSDgLW2H4E+IKkLwOH2H60wrVnAg/WvN8OvG64OrZ3S3oUmA78dLSLl8ns14FrKsQSERFNMmKLo9y86bqa97sqJo2WkjQV+DzwcdtbhqlzoaQ+SX0DAwPjG2BERAer8qhqjaSzJWn0qvvYARxX835WWVa3TpkMjgQeqnDtZcBm2x8broLtZbZ7bffOmDGjkbgjImIEVRLHe4BbgV2SHpP0uKTHKpx3DzBH0mxJBwOLgVVD6qwCzi9fnwPcadsjXVTSn1MkmPdViCEiIpqsygTAw8dy4bLP4mLgDmAKcKPt9ZKuBPpsrwJuAG6W1A/spEguAEjaRrEu1sGS3gKcDjwGfAj4LnBv2Qi61vanxhJjREQ0rsrWsWNm+yvAV4aU/WnN66coJhTWO7dnmMs2+sgsIiKaqMqjqoiIiL2SOCIioiGVEoek0yS9q3w9Q9Ls1oYVERHtqsoih5cDHwQuK4ueB/xDK4OKiIj2VaXF8RvAWcATALZ/CIxppFVEREx+VRLH0+XcCgNIen5rQ4qIiHZWJXHcIul64ChJvwd8Dfi71oYVERHtqsoEwI9KWkAx+e5lwJ/aXt3yyCIioi2NmjjKEVT/NpgsJB0qqcf2tlYHFxER7afKo6pbgWdr3u8pyyIiogtVSRxTy02TgL0bKB3cupAiIqKdVUkcA5LOGnwjaREVNlqKiIjOVGWRw98HPivpWooFBh8E3tnSqCIiom2N2uKw/T3bJwPzgP9h+/W2+1sfWntYunrTRIcQEdFWqoyqmgacDfQAUwc3ArR9ZUsjaxPXrNnMpQvmTnQYERFto8qjqn8CHgXWAbtaG05ERLS7Koljlu2FLY+kjSxdvYlr1mze+75nyW0AXDJ/TlofEdH1qoyq+g9JJ4zl4pIWStooqV/SkjrHp0laUR5fK6mnLJ8u6S5JPys75WvPea2k+8tzPq7BZ2dNdOmCuWy76ky2XXUmwN7XSRoREdUSx2nAujIBfLv8pf3t0U6SNAW4DjiDomP9PEnzhlS7AHjY9vHAUuDqsvwp4MPAB+pc+pPA7wFzyq+uag1FREy0Ko+qzhjjtU8C+m1vAZC0HFgEbKipswj4s/L1SuBaSbL9BHC3pONrLyjpxcARtr9Zvv974C3AV8cY46gumT+nVZeOiJiUqgzH/b7t7wNPUiytvneJ9VHMpJjzMWh7WVa3ju3dFJ3w00e55vZRrgmApAsl9UnqGxgYqBBufXk8FRGxryo7AJ4laTOwFfg6sI0W/oXfLLaX2e613TtjxoyJDiciomNU6eP4CHAysMn2bGA+8M0K5+0Ajqt5P6ssq1tH0lTgSOChUa45a5RrRkREC1VJHM/Yfgg4SNJBtu8Ceiucdw8wR9JsSQcDi4FVQ+qsAs4vX58D3FnuNliX7R8Bj0k6uRxN9U6KeSYRETFOqnSOPyLpBcC/UqxZ9RPK/cdHYnu3pIuBO4ApwI2210u6EuizvQq4AbhZUj+wkyK5ACBpG3AEcLCktwCn294A/AHwGeBQikdmbf/YLCKik2iEP/CLCsUe409RLHD4DorHSZ8tWyGTQm9vr/v6+iY6jIiISUXSOtv7PWGqMqrqCdt7bO+2fZPtj0+mpNEMWegwIuI5wyYOSXeX3x+X9FjN1+OSHhu/ECde7fIjERHdbtg+Dtunld8PH79wIiKi3Y3YOV4uG7Le9svHKZ62kYUOIyLqGzFx2N5TrlH1323/YLyCageXLpi7N0H0LLlt74KHERHdrspw3KOB9ZK+Rc0wXNtnDX9KRER0qiqJ48Mtj6LNZaHDiIjnjJo4bH99PAJpZ+nTiIh4TpVFDk+WdE+5qdLTkvZ023DciIh4TpW1qq4FzgM2Uyzz8bsUGzRFREQXqpI4sN0PTClnkH+a7LoXEdG1qnSO/7xc3fY/Jf0f4EdUTDgREdF5qiSA3y7rXUwxHPc44OxWBhUREe2rSovjtcBtth8DrmhxPBER0eaqtDh+Hdgk6WZJby536ouIiC5VZVn1dwHHA7dSjK76nqRPtTqwiIhoT5VaD7afkfRVwBRDct9CMSw3IiK6TJUJgGdI+gzFPI6zgU8Bv1Dl4pIWlosk9ktaUuf4NEkryuNrJfXUHLusLN8o6Y015ZdKWi/pAUmfl3RIlVgiIqI5qvRxvBP4EvAy279j+yu2d492Urkk+3XAGcA84DxJ84ZUuwB42PbxwFLg6vLceRT7j7+CYs7IJyRNkTQTeC/Qa/uVFHuZLyYiIsZNlT6O82x/yfauBq99EtBve4vtp4HlwKIhdRYBN5WvVwLzJaksX257l+2tQH95PSgerx1adtIfBvywwbgiIuIAtHIi30zgwZr328uyunXKVsyjwPThzrW9A/go8AOKiYiP2v5/9X64pAsl9UnqGxgYaMLtREQETLIZ4JKOpmiNzAaOBZ4v6bfq1bW9zHav7d4ZM2aMZ5gRER2tlYljB8Us80GzyrK6dcpHT0cCD41w7q8BW20P2H4G+CLw+pZEHxERdQ07HFfS/RTDb+uy/UujXPseYI6k2RS/9BcDbx9SZxVwPvAN4BzgTtuWtAr4nKS/pWhZzAG+BTwLnCzpMOBJYD7QN0ocERHRRCPN43hz+f2i8vvN5fd3VLmw7d2SLgbuoBj9dKPt9ZKuBPpsrwJuAG6W1A/spBwhVda7BdgA7AYusr0HWCtpJXBvWX4fsKzarUZERDPIHrZRUVSQ7rP9miFl99o+saWRNVFvb6/7+tIwiYhohKR1tnuHllfp45CkU2vevL7ieRER0YGqJIALKCbgbZO0DfgE8O6WRtVmlq7eNNEhRES0jSoTANfZfhXwKuBVtl9t+97Wh9Y+rlmzeaJDiIhoG1XWqnqRpBsoZnI/KmmepAvGIbaIiGhDVVbH/QzwaeBD5ftNwAqKEVEda+nqTfu0NHqW3AbAJfPncOmCuRMVVkTEhKuSOF5o+xZJl8HeYbZ7WhzXhLt0wdy9CaJnyW1su+rMCY4oIqI9VOkcf0LSdMrJgJJOplhTKiIiulCVFsf7KWZ4v1TSvwMzKGZ5R0REFxo1cdi+V9L/BF4GCNhYrhMVERFdqNLWsRR7YfSU9U+UhO2/b1lUbSCd4xER9Y2aOCTdDLwU+E9gsFPcQEcnjnSOR0TUV6XF0QvM82iLWkVERFeoMqrqAeAXWh1IO7tk/pyJDiEiom1UmscBbJD0LWDvvuO2z2pZVG0mfRoREc+pkjj+rNVBtLulqzcleURElKoMx/36eATSzq5ZszmJIyKiNNLWsXfbPk3S4+y7hawA2z6i5dFFRETbGbZz3PZp5ffDbR9R83V41aQhaaGkjZL6JS2pc3yapBXl8bWSemqOXVaWb5T0xpryoyStlPRdSd+RdEpDd1zR0tWb6Fly2975G4OvszdHRHS7qhMAkfTfgEMG39v+wSj1pwDXAQuA7cA9klbZ3lBT7QLgYdvHS1oMXA2cK2kexf7jrwCOBb4maW657/g1wO22z5F0MHBY1XtoROZxRETUV2U/jrMkbQa2Al8HtgFfrXDtk4B+21tsPw0sBxYNqbMIuKl8vRKYL0ll+XLbu2xvBfqBkyQdCfwK5ZLutp+2/UiFWCIiokmqzOP4CHAysMn2bGA+8M0K580EHqx5v70sq1vH9m6KVXenj3DubGAA+LSk+yR9StLz6/1wSRdK6pPUNzAwUCHc4V0yf04eUUVElKokjmdsPwQcJOkg23dRzCafCFOBE4FP2n4N8ASwX98JgO1ltntt986YMeOAfuilC+Zm+9iIiFKVxPGIpBcA/wp8VtI1FL+wR7MDOK7m/ayyrG4dSVOBI4GHRjh3O7Dd9tqyfCVFIomIiHFSJXEsAp4ELgVuB74H/HqF8+4B5kiaXXZiL6bY16PWKuD88vU5wJ3lmlirgMXlqKvZwBzgW7Z/DDwo6WXlOfOBDbRIRlZFROyvygTA2tbFTcNW3P+83ZIuBu4ApgA32l4v6Uqgz/Yqik7umyX1AzspkgtlvVsoksJu4KJyRBXAH1K0fA4GtgDvqhpTozKyKiJifxpu0dvhJv4xCScA9vb2uq+vr+Hzhu7JMSh7ckREN5C0zvZ+fdrDJo5OMtbEUSstjojoNsMljkoTACWdCJxG0eK42/Z9TY6vLWUXwIiI/Y3a4pD0p8DbgC+WRW8BbrX9560NrXma1eJIwoiIbjJci6PKqKp3AL9s+3Lbl1NMBvztZgc4GWQuR0REtcTxQ2rWqAKmsf98jI6XXQAjIgpV+jgeBdZLWk3Rx7EA+JakjwPYfm8L45tw6eeIiNhXlT6O80c6brvy3I6J0ow+DsjIqojoLgcyquqrtn8y5GIvs72xadFFRMSkUSVx/JukD9u+BUDSH1HsozGvpZG1iTyqiojYV5XE8QZgmaS3AS8CvkOx10ZERHShKmtV/UjS7cBlwLPAEts/a3lkbWLoelVA+jkioquNmjgkfY1iSO4rKZY6v0HSv9r+QKuDi4iI9lNlHse1tt9p+xHb9wOnUAzR7RqDy6sPyvLqEdHNRk0ctr8k6TRJg8uXHw38Q2vDioiIdjVq4pB0OfBBij4OgINJ4oiI6FpVHlX9BnAW5Xaxtn8IHN7KoCIion1VGY77tG1LMoCk57c4praTkVUREc+p0uK4RdL1wFGSfg/4GvB3VS4uaaGkjZL6JS2pc3yapBXl8bWSemqOXVaWb5T0xiHnTZF0n6QvV4mjGdJBHhFRqLQDoKQFwOkU28beYXt1hXOmAJsoFkXcDtwDnGd7Q02dPwB+yfbvS1oM/IbtcyXNAz5PMdHwWIpkNXdw33FJ7wd6gSNsv3m0WJq1VhU8N5M8LY6I6HQHsh8Htlfb/l+2P1AlaZROAvptb7H9NLAcWDSkziJgcJHElcB8SSrLl9veZXsr0F9eD0mzgDOBT1WMo6myzEhEdLtKiWOMZgIP1rzfXpbVrWN7N8X8kOmjnPsx4I8pZrEPS9KFkvok9Q0MDIzxFiIiYqhWJo6mk/Rm4Ce2141W1/Yy2722e2fMmDEO0UVEdIcqo6r2Iek4YLHtvx6l6g6KJUoGzWL/nQMH62yXNBU4EnhohHPPAs6S9CaKXQmPkPQPtn+r0ftoVFbJjYgoVO0cnwG8DTiPorP6H0dbq6pMBJuA+RS/9O8B3m57fU2di4ATajrH32r7NyW9Avgcz3WOrwHmDHaOl+e+AfjAeHeOQ4bkRkR3aHgjJ0mHA28F3g7MBb4IzLY9q8oPtL1b0sXAHcAU4Ebb6yVdCfTZXgXcANwsqR/YCSwuz10v6RZgA7AbuKg2aUyEoS0OKBJIWhwR0W2GbXFIehL4FvC/gbvLSYBbbL9kPANshma1OM69/hus3bpzv/LXzT6GFe855YCvHxHRTsYyHPcyYBrwCeAySS9tVXCTxckvmd5QeUREJxo2cdj+mO2TeW7uxZeAYyV9UFJXPpu5dMFcXjf7mP3Kr1mzmXOv/8YERBQRMf6qLKu+xfZf2j6BcrY28JWWR9amhmtdrN26M8kjIrrCsIlD0vGSTq0ts/0A8FVgYasDa1eXLpjLJfPnTHQYERETZqQWx8eAx+qUPwosbUk0k9zarTuz6GFEdLyREseLyq1i91GW9bQsoklguL4OgJXrHqxbHhHRKUZKHEeNcOzQJscx6ax4zynMPOqQ/cp3PPJUlluPiI42UuLoK/ff2Iek3wVGXSuqG/z7kvkcPm1K3WM33r1lnKOJiBgfIyWO9wHvkvQvkv6m/Po6cAFwybhENwncf8XCui2Px3ftSasjIjrSSPM4/sv264ErgG3l1xW2T7H94/EJb3KYdfRhdcszvyMiOtFIw3EPkfQ+4GzgaeCTtu8cr8AmkxXvOWXYIboZaRURnWakR1U3UUz4ux84A/jouEQ0SY00v+ObWx4a52giIlpnpMQxz/Zv2b4eOAf4lXGKqeOs3bqTEy6/faLDiIhoipESxzODL8ptXWMUI83veHzXHuZ+6Ct5bBURk95IieNVkh4rvx4HfmnwtaR6M8qDor9juOTx9B5zzZrNnHrVmnGOKiKieUYaVTXF9hHl1+G2p9a8PmI8g5xshpscOGjHI09xwuW3p/UREZNSpa1jJ7tmbx1b1alXrWHHI0+NWGcwwZzz2uOyk2BEtJWxbOTUjB+6UNJGSf2SltQ5Pk3SivL4Wkk9NccuK8s3SnpjWXacpLskbZC0XlJbT0T89yXzh31sNWjHI0+x45Gn9j7CyryPiGh3LUsckqYA11EM5Z0HnCdp3pBqFwAP2z6eYsXdq8tz51HsP/4KiiXcP1FebzfwR7bnAScDF9W5ZlsZqc9jqB2PPLV3BFYSSES0q1a2OE4C+suNoJ4GlvPcboKDFlHMFwFYCcyXpLJ8ue1dtrcC/cBJtn9k+14A248D3wFmtvAemmJwguBw61oN9fiuPazdujMtkIhoS61MHDOB2jXGt7P/L/m9dcohv48C06ucWz7Weg2wtt4Pl3ShpD5JfQMDA2O/iya5dMFc7r9iIa+bfcyIHee1alsgp161Jp3pEdEWpk50AGMh6QXAF4D32a47NNj2MmAZFJ3j4xjeiFa85xQATrj8dh7ftafSOY/v2sPju/ZwzZrN3Hj3Fo449HnMOvqwvdeKiBhPrUwcO4Djat7PKsvq1dkuaSpwJPDQSOdKeh5F0vis7S+2JvTWu/+KhZx7/TfY/vDPAUYdfTVoMInseOQpTr1qDY89+Qzzjj0ySSQixk3LhuOWiWATMJ/il/49wNttr6+pcxFwgu3fl7QYeKvt35T0CuBzFP0kxwJrgDnAsxR9Ijttv69qLBM1HLcRg5MCqyaQoWYedQiPPfkMRxz6PKAY0RURcSCGG47b0nkckt5EsXf5FOBG238h6Uqgz/YqSYcAN1P0VewEFtveUp77IeDdFCOp3mf7q5JOA/6NYuHFZ8sf8ye2vzJSHJMhcQxaunoTN969pfJjrOEMTSSZJxIRjZqQxNEuJlPiGLR09SZWrntwzC2Qemo75R978hnefdpLkkwiYlhJHJMscQwabIEccejzmppEBg0mk9rWSTreIwKSOCZt4qh17vXfYMMPH21ZEqk1tHWSpBLRfZI4OiBx1Bp8lPXYk88ccH9Io4ZLKhnhFdFZkjg6LHHUqk0ig7/EW90iGUm9x19ptURMPkkcHZw46hmc49EOiaSew6dNqZtUhr7PsOKIiZPE0WWJY6ihm0dNxCOusahNMDBykslIsYjmSuLo8sRRT+3M9XZunTRqtEdl9Y7df8XC8Q80os0lcSRxVFabUKCzkspwRurwHynh1B6DPFqLzpLEkcTRFCMllcny+KuVmpGA6h3LzP+YCEkcSRzjYnCEFwz/y7FTWy2tNnQZmbEmo0bqpgXV3ZI4kjjaRu1seBj5F1eSzMQaaXDCgbaiGj0GWXNtvCVxJHFMSidcfntDfzl3+6OyblBv8MPQ9xOR1DpxMEYSRxJHVxhupFgSTrSTVvWF1Tt2II8bh0sck3IHwIjhjHVGeu08l7H+z5rHalHV0P9Wav9wGfpHTDOONVsSRwTN6QSuHRgAzf1LMkkp2kkeVUVMAoOP4Mb7eXwSVueYedQhDf+BlEdVEZPYRC0KOdLghInogE4ia9y2q85s+jVbmjgkLQSuodg69lO2rxpyfBrw98BrgYeAc21vK49dBlwA7AHea/uOKteMiOZpt9E/w/VFDX3f7qOqJvtgjJYlDklTgOuABcB24B5Jq2xvqKl2AfCw7eMlLQauBs6VNA9YDLwCOBb4mqTBwdujXTMiOlSnTEhsxmCMRpJas7WyxXES0G97C4Ck5cAioPaX/CLgz8rXK4FrJaksX257F7BVUn95PSpcMyKirU32BHhQC689E3iw5v32sqxuHdu7gUeB6SOcW+WaAEi6UFKfpL6BgYEDuI2IiKjVysQxoWwvs91ru3fGjBkTHU5ERMdoZeLYARxX835WWVa3jqSpwJEUneTDnVvlmhER0UKtTBz3AHMkzZZ0MEVn96ohdVYB55evzwHudDGxZBWwWNI0SbOBOcC3Kl4zIiJaqGWd47Z3S7oYuINi6OyNttdLuhLos70KuAG4uez83kmRCCjr3ULR6b0buMj2HoB612zVPURExP66Yua4pAHg+2M8/YXAT5sYzmTQbffcbfcLueducaD3/Iu29+sk7orEcSAk9dWbct/Juu2eu+1+IffcLVp1zx07qioiIlojiSMiIhqSxDG6ZRMdwATotnvutvuF3HO3aMk9p48jIiIakhZHREQ0JIkjIiIaksQxDEkLJW2U1C9pyUTH0yySjpN0l6QNktZLuqQsP0bSakmby+9Hl+WS9PHy3+Hbkk6c2DsYO0lTJN0n6cvl+9mS1pb3tqJcjYByxYIVZflaST0TGvgYSTpK0kpJ35X0HUmndPrnLOnS8r/rByR9XtIhnfY5S7pR0k8kPVBT1vDnKun8sv5mSefX+1nDSeKoo2YvkTOAecB55R4hnWA38Ee25wEnAxeV97YEWGN7DrCmfA/Fv8Gc8utC4JPjH3LTXAJ8p+b91cBS28cDD1PsDwM1+8QAS8t6k9E1wO22Xw68iuLeO/ZzljQTeC/Qa/uVFKtLDO7z00mf82eAoTtsNfS5SjoGuBx4HcWWFZcPJptKbOdryBdwCnBHzfvLgMsmOq4W3es/UWyMtRF4cVn2YmBj+fp64Lya+nvrTaYvigUx1wC/CnwZEMWM2qlDP3OKJW1OKV9PLetpou+hwfs9Etg6NO5O/px5btuFY8rP7cvAGzvxcwZ6gAfG+rkC5wHX15TvU2+0r7Q46qu878dkVjbNXwOsBV5k+0floR8DLypfd8q/xceAPwaeLd9PBx5xsQ8M7Htfw+0TM5nMBgaAT5eP5z4l6fl08OdsewfwUeAHwI8oPrd1dPbnPKjRz/WAPu8kji4l6QXAF4D32X6s9piLP0E6Zpy2pDcDP7G9bqJjGUdTgROBT9p+DfAEzz2+ADrycz6aYkfQ2RRbTj+f/R/pdLzx+FyTOOrr6H0/JD2PIml81vYXy+L/kvTi8viLgZ+U5Z3wb3EqcJakbcByisdV1wBHqdgHBva9r+H2iZlMtgPbba8t36+kSCSd/Dn/GrDV9oDtZ4AvUnz2nfw5D2r0cz2gzzuJo76O3fdDkiiWs/+O7b+tOVS7N8r5FH0fg+XvLEdnnAw8WtMknhRsX2Z7lu0eis/yTtvvAO6i2AcG9r/nevvETBq2fww8KOllZdF8im0KOvZzpnhEdbKkw8r/zgfvuWM/5xqNfq53AKdLOrpsqZ1ellUz0Z087foFvAnYBHwP+NBEx9PE+zqNohn7beA/y683UTzbXQNsBr4GHFPWF8UIs+8B91OMWJnw+ziA+38D8OXy9UsoNgjrB24FppXlh5Tv+8vjL5nouMd4r68G+srP+kvA0Z3+OQNXAN8FHgBuBqZ12ucMfJ6iD+cZipblBWP5XIF3l/feD7yrkRiy5EhERDQkj6oiIqIhSRwREdGQJI6IiGhIEkdERDQkiSMiIhqSxBEREQ1J4oiIiIb8f70UjGVrq8WBAAAAAElFTkSuQmCC","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["n_components = 1500\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.7206703910614525\n","Macro-F1 score: 0.6874806009063257\n","Model6: 0.6874806009063257\n","\n","Accuracy: 0.5870577281191807\n","Macro-F1 score: 0.5626946501547562\n","KNN: 0.5626946501547562 \n","\n","Time taken for 1500 Principal Components: 1350.4969444274902s \n","\n"]},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAY4AAAD6CAYAAAC2wKAfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAiTklEQVR4nO3df5hdVX3v8feHSRNAgUCICknoBEn0xp/gNIDweH1MA0GUaMVLqK2otLEVaoztbZNylYL9Aa01YkElFRRRSTBqOxWExoBWLAQmYIVEk4wkSiKWkR8BsSQkfO8fe004c3LmzD6T2efsmfN5Pc95Zp+1197zPRvOfLP2WnstRQRmZmZ5HdDqAMzMbHRx4jAzs4Y4cZiZWUOcOMzMrCFOHGZm1hAnDjMza0ihiUPSPEkbJfVKWlJj/wRJK9P+tZI6U/kkSbdL+pWkKwc5d7ekB4qM38zM9jWuqBNL6gCuAuYC24B7JHVHxIaKaucDj0fEcZIWAJcD5wDPAB8BXple1ef+HeBXeWM58sgjo7Ozc7gfxcysLa1bt+6XETG5urywxAHMBnoj4kEASSuA+UBl4pgP/FXaXgVcKUkR8TRwh6Tjqk8q6YXAh4GFwI15Auns7KSnp2e4n8PMrC1J+mmt8iJvVU0BHqp4vy2V1awTEbuBHcCkIc77MeAfgV+PTJhmZtaIUdU5Lum1wEsj4hs56i6U1COpp6+vr/jgzMzaRJGJYzswreL91FRWs46kccBhwKN1znky0CVpK3AHMFPSd2pVjIjlEdEVEV2TJ+9zi87MzIapyMRxDzBD0nRJ44EFQHdVnW7gvLR9NnBb1Jl1MSI+ExFHR0QncCqwKSLeOOKRm5nZoArrHI+I3ZIuBG4FOoBrI2K9pEuBnojoBq4BrpfUCzxGllwASK2KQ4Hxkt4GnFY1IsvMzFpA7TCteldXVwx3VNWy1ZtYPHfmCEdkZlZ+ktZFRFd1+ajqHG+FK9ZsbnUIZmal4sRhZmYNKfIBwFFr2epNA1oanUtuAmDRnBm+bWVmbc99HEPoXHITWy87c4QjMjMrP/dxmJnZiHDiGMKiOTNaHYKZWak4cQzBfRpmZgM5cZiZWUOcOMzMrCFOHGZm1hAnDjMza4gTh5mZNcSJw8zMGuLEYWZmDXHiMDOzhjhxmJlZQ5w4zMysIU4cZmbWECcOMzNriBOHmZk1xInDzMwaUmjikDRP0kZJvZKW1Ng/QdLKtH+tpM5UPknS7ZJ+JenKivoHS7pJ0o8lrZd0WZHxm5nZvgpLHJI6gKuAM4BZwLmSZlVVOx94PCKOA5YBl6fyZ4CPAH9W49Qfj4iXA8cDp0g6o4j4zcystiJbHLOB3oh4MCJ2ASuA+VV15gPXpe1VwBxJioinI+IOsgSyV0T8OiJuT9u7gHuBqQV+BjMzq1Jk4pgCPFTxflsqq1knInYDO4BJeU4uaSLwVmDN/gZqZmb5jcrOcUnjgBuAT0XEg4PUWSipR1JPX19fcwM0MxvDikwc24FpFe+nprKadVIyOAx4NMe5lwObI+KTg1WIiOUR0RURXZMnT24kbjMzq6PIxHEPMEPSdEnjgQVAd1WdbuC8tH02cFtERL2TSvprsgTzoZEN18zM8hhX1IkjYrekC4FbgQ7g2ohYL+lSoCciuoFrgOsl9QKPkSUXACRtBQ4Fxkt6G3Aa8CRwEfBj4F5JAFdGxOeK+hxmZjZQYYkDICJuBm6uKvtoxfYzwDsHObZzkNNqpOIzM7PG5Uocks4C3pDefjci/q24kMzMrMyG7OOQ9HfAImBDen1Q0t8WHZiZmZVTnhbHmcBrI+I5AEnXAfcBf1lkYGZmVk55R1VNrNg+rIA4zMxslMjT4vg74D5Jt5N1TL8B2GfCQjMzaw9DJo6IuEHSd4DfSkV/ERG/KDQqMzMrrUFvVUl6efp5AnAU2VxT24CjU5mZmbWhei2ODwMLgX+ssS+ANxUSkZmZldqgiSMiFqbNM9KDentJOrDQqMzMrLTyjKr6z5xlZmbWBgZtcUh6Cdl6GQdJOp7np/o4FDi4CbGZmVkJ1evjOB14D9l06J+oKH8KP/xnZta26vVxXAdcJ+kdEfG1JsZkZmYlluc5jq9JOhN4BXBgRfmlRQZmZmbllGeSw88C5wB/QtbP8U7gNwuOy8zMSirPqKrXR8S7gccj4hLgZGBmsWGZmVlZ5Ukc/c9w/FrS0cCzZE+Sm5lZG8ozyeG/SZoI/ANwL9lT4/9cZFBmZlZedROHpAOANRHxBPA1Sd8EDoyIHc0IzszMyqfuraq0eNNVFe93OmmYmbW3PH0cayS9Q5KGrmpmZmNdnsTxfuCrwE5JT0p6StKTeU4uaZ6kjZJ6Je2z+JOkCZJWpv1rJXWm8kmSbpf0K0lXVh3zOkn3p2M+5YRmZtZcQyaOiDgkIg6IiPERcWh6f+hQx0nqILvNdQYwCzhX0qyqaueTDfM9DlgGXJ7KnwE+AvxZjVN/BvhDYEZ6zRsqFjMzGzl51xwfjtlAb0Q8GBG7gBXA/Ko684Hr0vYqYI4kRcTTEXEHzw8FBkDSUcChEXFXRATwReBtBX4GMzOrUmTimAI8VPF+WyqrWScidgM7gElDnHPbEOc0M7MCFZk4WkrSQkk9knr6+vpaHY6Z2ZiRK3FIOlXSe9P2ZEnTcxy2HZhW8X5qKqtZR9I44DDg0SHOOXWIcwIQEcsjoisiuiZPnpwjXDMzyyPPJIcXA38BLE1FvwF8Kce57wFmSJouaTywAOiuqtMNnJe2zwZuS30XNUXEw8CTkk5Ko6neDfxrjljMzGyE5Jly5O3A8WTTjRARP5d0yFAHRcRuSRcCtwIdwLURsV7SpUBPRHQD1wDXS+oFHiNLLgBI2kq22uB4SW8DTouIDcAHgC8ABwHfSi8zM2uSPIljV0SEpACQ9IK8J4+Im4Gbq8o+WrH9DNk07bWO7RykvAd4Zd4YzMxsZOXp47hR0tXAREl/CHwbT3JoZta28qwA+HFJc4EngZcBH42I1YVHZmZmpTRk4kgjqL7XnywkHSSpMyK2Fh2cmZmVT55bVV8Fnqt4vyeVmZlZG8qTOMalKUMASNvjiwvJzMzKLE/i6JN0Vv8bSfOBXxYXkpmZlVme4bh/BHw5TW8usrml3l1oVGZmVlp5RlX9BDhJ0gvT+18VHpWZmZVWnlFVE4B3AJ3AuP51kyLi0kIjK4llqzexeO7MVodhZlYaefo4/pVs3YzdwNMVr7ZwxZrNrQ7BzKxU8vRxTI0Ir7JnZmZAvsTxn5JeFRH3Fx5NSSxbvWlAS6NzyU0ALJozw7etzKztqc4s5lkFaQNwHLAF2Ek2sioi4tXFhzcyurq6oqenZ1jHdi65ia2XnTnCEZmZlZ+kdRHRVV2ep8VxRgHxmJnZKJVnOO5PASS9CDiw8IhKZtGcGa0OwcysVPKsAHiWpM1kt6q+C2yljRZPcp+GmdlAeYbjfgw4CdgUEdOBOcBdhUZlZmallSdxPBsRjwIHSDogIm4H9uksMTOz9pCnc/yJNN3If5DNWfUIbfQAoJmZDZSnxTEf+B9gMXAL8BPgrUUGZWZm5TVk4oiIpyNiT0TsjojrIuJT6dbVkCTNk7RRUq+kJTX2T5C0Mu1fK6mzYt/SVL5R0ukV5YslrZf0gKQbJBU60mvZ6k1Fnt7MbNQZNHFIuiP9fErSkxWvpyQ9OdSJJXUAV5E9BzILOFfSrKpq5wOPR8RxwDLg8nTsLGAB8ApgHvBpSR2SpgAfBLoi4pVAR6pXGM9VZWY20KCJIyJOTT8PiYhDK16HRMShOc49G+iNiAfTqoEryG57VZoPXJe2VwFzlE2/Ox9YERE7I2IL0JvOB1m/zEGSxgEHAz/P91HNzGwk1O0cT62G9RHx8mGcewrZok/9tgEnDlYnInZL2gFMSuV3VR07JSLulPRx4Gdk/S7/HhH/PozY6vJcVWZmg6ubOCJiT+pjOCYiftasoAYj6XCy1sh04Angq5J+LyK+VKPuQmAhwDHHHNPQ71k8d+beBOG5qszMBsozqupwYL2kNZK6+185jtsOTKt4PzWV1ayTbj0dBjxa59jfBrZERF9EPAt8HXh9rV8eEcsjoisiuiZPnpwjXDMzyyPPcxwfGea57wFmSJpO9kd/AfC7VXW6gfOAO4GzgdsiIlJi+oqkTwBHAzOAu4HnyJaxPZjsVtUcYHjT3ubkuarMzAbKM8nhd4dz4tRncSFwK9nop2sjYr2kS4GeiOgGrgGul9QLPEYaIZXq3QhsIFt58IKI2AOslbQKuDeV3wcsH058eblPw8xsoDzrcZwE/BPwv4DxZEng6Zwjq0phf9bjMDNrV4Otx5Gnj+NK4FxgM3AQ8Adkz2eYmVkbypM4iIheoCM9Qf55sofyzMysDeXpHP+1pPHADyT9PfAwOROOmZmNPXkSwO+neheSzYo7DXhHkUGZmVl55WlxvA64KSKeBC4pOB4zMyu5PC2OtwKbJF0v6S3pQT0zM2tTeaZVfy9wHPBVstFVP5H0uaIDMzOzcsrVeoiIZyV9CwiyIblvIxuWa2ZmbWbIFoekMyR9gew5jncAnwNeUnBcZmZWUnlaHO8GVgLvj4idBcdjZmYll2euqnObEYiZmY0OfpDPzMwa4sRhZmYNceIwM7OGDNrHIel+suG3NUXEqwuJyMzMSq1e5/hb0s8L0s/r0893FReOmZmV3aCJIyJ+CiBpbkQcX7FriaR7gSVFB2dmZuWTp49Dkk6pePP6nMeZmdkYlCcBnA98WtJWSVuBTwPvKzSqklm2elOrQzAzK408kxyui4jXAK8BXhMRr42Ie4sPrTyuWLO51SGYmZVGnrmqXizpGmBFROyQNEvS+U2IzczMSijPraovALcCR6f3m4AP5Tm5pHmSNkrqlbRPZ7qkCZJWpv1rJXVW7FuayjdKOr2ifKKkVZJ+LOlHkk7OE0ujlq3eROeSm+hcchPA3m3ftjKzdpdnksMjI+JGSUsBImK3pD1DHSSpA7gKmAtsA+6R1B0RGyqqnQ88HhHHSVoAXA6cI2kWsAB4BVnC+rakmRGxB7gCuCUizk5roR+c/+Pmt3juTBbPnQlkSWPrZWcW8WvMzEadPC2OpyVNIj0MKOkkYEeO42YDvRHxYETsAlYA86vqzAeuS9urgDmSlMpXRMTOiNgC9AKzJR0GvAG4BiAidkXEEzliMTOzEZKnxfFhoBt4qaTvA5OBs3McNwV4qOL9NuDEweqklswOYFIqv6vq2CnA/wB9wOclvQZYByyKiKdzxGNmZiMgz6iqe4H/DbweeD/wioj4YdGBDWIccALwmfRQ4tMM8iCipIWSeiT19PX1NTNGM7MxLdfSsWS3nTpT/RMkERFfHOKY7cC0ivdTU1mtOtskjQMOAx6tc+w2YFtErE3lqxgkcUTEcmA5QFdX16Bzbg1m2epNA4bh9neSL5ozY2/fh5lZOxoycUi6Hngp8AOgv1M8gKESxz3ADEnTyf7oLwB+t6pON3AecCfZ7a/bIiIkdQNfkfQJss7xGcDdEbFH0kOSXhYRG4E5wAYK4M5xM7Pa8rQ4uoBZEdHQv9pTn8WFZEN5O4BrI2K9pEuBnojoJuvkvl5SL/AYWXIh1buRLCnsBi5II6oA/gT4chpR9SDw3kbiMjOz/ZMncTwAvAR4uNGTR8TNwM1VZR+t2H4GeOcgx/4N8Dc1yn9AlsyaZtGcGc38dWZmpZbrOQ5gg6S7gZ39hRFxVmFRlYz7NMzMnpcncfxV0UGU3bLVm5w8zMySIRNHRHy3GYGU2RVrNjtxmJkl9ZaOvSMiTpX0FAOXkBUQEXFo4dGZmVnp1FsB8NT085DmhVMefo7DzKw25R1lK+lFwIH97yPiZ0UFNdK6urqip6dn2Mf7OQ4za0eS1kXEPqNY86zHcZakzcAW4LvAVuBbIx6hmZmNCnlmx/0YcBKwKSKmkz2tfVf9Q8aWE6cf0eoQzMxKI0/ieDYiHgUOkHRARNxOkx/Aa7W1Wx5rdQhmZqWR5zmOJyS9EPgPsqk+HiGbldbMzNpQnsQxH3gGWAy8i2wG20uLDKoMPKrKzKy23KOqRrORGFXlhGFm7WawUVWDJo7BHvxjFD4AOJzEUd3i6OcEYmbtouHEMZaMRIsD8LMcZtZWBkscuVYAlHQCcCpZi+OOiLhvhOMrJfdzmJnta8gWh6SPkq2Z8fVU9DbgqxHx18WGNnL2t8XRn0Dc4jCzdjLsJ8fJRlL9VkRcHBEXkz0M+PsjHWCZuXVhZva8PInj51TMUQVMIFtD3MzM2lCePo4dwHpJq8n6OOYCd0v6FEBEfLDA+FrKfRxmZvvK08dxXr39EXHdiEZUgP3t4wDPkGtm7Wd/RlV9KyIeqTrZyyJi44hFV1JucZiZ7StPH8f3JP2f/jeS/hT4Rp6TS5onaaOkXklLauyfIGll2r9WUmfFvqWpfKOk06uO65B0n6Rv5oljuBbPncnWy87c29Lo33bSMLN2lqfF8UZguaR3Ai8GfgTMHuogSR3AVWR9ItuAeyR1R8SGimrnA49HxHGSFgCXA+dImgUsAF4BHA18W9LMiNiTjluU4ij06XW3OMzM9jVkiyMiHgZuAU4GOoHrIuJXOc49G+iNiAcjYhewgmzCxErzgf4+klXAHElK5SsiYmdEbAF60/mQNBU4E/hcjhj2S3WL48TpR7jFYWZtL88KgN8GTgReSfYH+5OSPp7j3FOAhyreb0tlNetExG6yEVyThjj2k8CfA8/liGFEeV0OM7N8fRxXRsS7I+KJiLifrOWxo+C4apL0FuCRiFiXo+5CST2Sevr6+ob1+5at3kTnkpv23qKC7HbVstWbhnU+M7OxIM+tqn+RdKqk96aiw4Ev5Tj3dmBaxfup7Pvg4N46ksaRrfXxaJ1jTwHOkrSV7NbXmyTVjCUilkdEV0R0TZ48OUe4+1o8d2bNZWOvWLOZc66+c1jnNDMb7YbsHJd0MdlSsS8DPg+MJ0scpwxx6D3ADEnTyf7oLwB+t6pON3AecCdwNnBbRISkbuArkj5B1jk+A7g7Iu4Elqa43gj8WUT83tAfc/hWvv9kwDPkmpn1yzOq6u3A8cC9ABHxc0mHDHVQROyWdCFwK9ABXBsR6yVdCvRERDdwDXC9pF7gMbLkQqp3I7AB2A1cUDGiqqk8ssrMbKA8T47fHRGzJd0bESdIegFwZ0S8ujkh7r+ReHIc/PS4mbWX/Xly/EZJVwMTJf0h8D7gn0c6wLJyi8PMbKBcKwBKmgucRrZs7K0RsbrowEaSWxxmZo3brxUAU6IYVcnCzMyKkec5DqvgZzjMrN05cTSosr/DzKwdNZw4JE2T9H+LCKaM/PS4mdlAuRKHpMmSPiDpe8B3yGbJbQuL585k0ZwZ+5RfsWazk4eZtaVBO8fTQ36/Q/a090zg68D0iJjapNjMzKyE6o2qegS4G/h/wB1pKpC3NyescrnrwUcbKjczG8vqJY6lZFOAfBq4QdLK5oRUPv3zVYEfADQzyzPlyLFkCeRcsskGLwa+ERGj5gb//j4AWP30eKUTpx8xILGYmY0Vgz0AmGda9Qcj4m8j4lVks+QeCtxcQIylNdj06mZm7WjQxCHpOEkDpk6PiAeAbwHzig6sbE46dlLN8rVbHvPoKjNrK/VaHJ8EnqxRvgNYVkg0JVav1eFOcjNrJ/USx4vTUrEDpLLOwiIyM7NSq5c4JtbZd9AIxzEqrHz/yRwyoWOf8rVbHuNVF9/SgojMzJqvXuLoSetvDCDpD4B1xYVUbu879dia5U/t3OO+DjNrC/USx4eA90r6jqR/TK/vAucDi5oSXQktnjuTKRMPrLlv1bqHmhyNmVnzDZo4IuK/I+L1wCXA1vS6JCJOjohfNCe8cjr7ddNqlm9/4hm3OsxszKs3HPdASR8C3gHsAj4TEbc1K7Ay8wgrM2tn9W5VXUf2wN/9wBnAx5sSkZmZlVq9xDErIn4vIq4Gzgbe0OjJJc2TtFFSr6QlNfZPkLQy7V8rqbNi39JUvlHS6alsmqTbJW2QtF5Sy/paVr7/5JqtjrVbHuOUy9a0ICIzs+aolzie7d+IiN2NnlhSB3AVWWtlFnCupFlV1c4HHo+I48geKrw8HTuLbH6sV5A9pf7pdL7dwJ9GxCzgJOCCGudsmm2P/7pm+fYnnuGcq+9scjRmZs1RL3G8RtKT6fUU8Or+bUm1niivNhvoTXNd7QJWAPOr6swnuyUGsAqYI0mpfEVE7IyILUAvMDsiHo6IewEi4ingR8CUvB92pH1/yZxBR1h5KhIzG6vqjarqiIhD0+uQiBhXsX1ojnNPASrHp25j3z/ye+ukVs0OYFKeY9NtreOBtTliKcxgI6wgWyXQt63MbKxpeM3xMpD0QuBrwIciombrR9JCST2Sevr6+gqLZfHcmWy97MxB929/4hmOXeo1ys1s7CgycWwHKv85PjWV1awjaRxwGPBovWMl/QZZ0vhyRHx9sF8eEcsjoisiuiZPnryfH2Vo9ZLHc+HWh5mNHUUmjnuAGZKmSxpP1tndXVWnGzgvbZ8N3BbZylLdwII06mo62QJSd6f+j2uAH0XEJwqMfVgWzZlRd//2J55h5kU3u/VhZqNaYYkj9VlcCNxK1ol9Y0Ssl3SppLNStWuASZJ6gQ8DS9Kx64EbgQ3ALcAFEbEHOAX4feBNkn6QXm8u6jM0aqjbVgC79gRXrNnsUVdmNmoNuXTsWLC/S8cOx6suvoWndu6pW2d8hzj+mMO99KyZldKwl4614bn/knk1p2CvtGtP7J2SfeZFbbUar5mNYk4cBbr/knlMmXgg4ztUt95TO/ewa08w86KbmXnRzb6NZWal5ltVTZLn1lWl/mSz6W9K04VjZm1msFtVThxNdM7Vd3Lfzx5n157817yyteIkYmbN5MRRgsTRr9HWR7/KJDL5kAl8f8mckQzLzGwAJ44SJQ6AUy5bQ99TOxtqfVQ7ZEIHO3c/t/e9R2iZ2Uhy4ihZ4uh3ymVrePJ/nmXn7uf2K4n0q+6I/+M3HsfiuTP3+7xm1n6cOEqaOCr1D8kdiQRSqTKZTBh3ADt3P+fWiZkNyYljFCSOfv23sWDkk0ilWsOEJ4w7gPsvmVfY7zSz0cOJYxQljkqVDwYWmUQqVSeU/lYKuFPerJ04cYzSxFGpFUmklnoPNLrFYjZ2OHGMgcRR6Zyr72TDz3cMGFXVymRSqT+xVLZUanHrxazcnDjGWOKopbJvpF9Zkslg6t0WG4wfhDRrDieONkgctbzq4lsG/CHu/8Nc9oRSz3CSTTUPUzYbmhNHmyaOwdRqnUD5WygjqfoByjyGSlLu47GxxInDiSOXZas38Znv9A4oq/xj2U6JZbhGokXUyHFOVlYUJw4njhHRP1HjYJxYWiPvgIRaik5s/dw3Nfo4cThxNEVli2WoPyxOMu1lfIdKndhG+pix0BJ04nDiKJ1arZc8X1wnHBsthnrmqejEtr/Ja7DEMW7YZzTbT8OdK6t6md3hfAGdfKwZ6v1/tmtP40srNHrMcH5HHoUmDknzgCuADuBzEXFZ1f4JwBeB1wGPAudExNa0bylwPrAH+GBE3JrnnDb2jcS98lddfAvAiP+LzwnJ2kFht6okdQCbgLnANuAe4NyI2FBR5wPAqyPijyQtAN4eEedImgXcAMwGjga+DfQPuq97zlp8q8qaZahRaY3Ie5yTleVxyISOhm9bteJW1WygNyIeTAGsAOYDlX/k5wN/lbZXAVdKUipfERE7gS2SetP5yHFOs5ZZPHdm0x8sbGRAQi3N6nx2gmuNrZedOeLnLDJxTAEeqni/DThxsDoRsVvSDmBSKr+r6tgpaXuoc5q1lVYkq+GYedHNpU5sI33MWE6UY7ZzXNJCYCHAMccc0+JozKzdnuOodduyUrNGVRWhyMSxHZhW8X5qKqtVZ5ukccBhZJ3k9Y4d6pwARMRyYDlkfRzD+whmZsMzWlqCw1FMOsrcA8yQNF3SeGAB0F1Vpxs4L22fDdwWWW99N7BA0gRJ04EZwN05z2lmZgUqrMWR+iwuBG4lGzp7bUSsl3Qp0BMR3cA1wPWp8/sxskRAqncjWaf3buCCiNgDUOucRX0GMzPbl58cNzOzmgYbjlvkrSozMxuDnDjMzKwhbXGrSlIf8NNhHn4k8MsRDGeklT0+cIwjoezxQfljLHt8UL4YfzMiJlcXtkXi2B+Semrd4yuLsscHjnEklD0+KH+MZY8PRkeM4FtVZmbWICcOMzNriBPH0Ja3OoAhlD0+cIwjoezxQfljLHt8MDpidB+HmZk1xi0OMzNriBPHICTNk7RRUq+kJS2MY5qk2yVtkLRe0qJUfoSk1ZI2p5+Hp3JJ+lSK+4eSTmhSnB2S7pP0zfR+uqS1KY6VaW4x0vxjK1P5WkmdTYpvoqRVkn4s6UeSTi7TNZS0OP33fUDSDZIObPU1lHStpEckPVBR1vA1k3Reqr9Z0nm1ftcIx/gP6b/zDyV9Q9LEin1LU4wbJZ1eUV7Y971WjBX7/lRSSDoyvW/JdWxYRPhV9SKbB+snwLHAeOC/gFktiuUo4IS0fQjZCoizgL8HlqTyJcDlafvNwLcAAScBa5sU54eBrwDfTO9vBBak7c8Cf5y2PwB8Nm0vAFY2Kb7rgD9I2+OBiWW5hmRrzWwBDqq4du9p9TUE3gCcADxQUdbQNQOOAB5MPw9P24cXHONpwLi0fXlFjLPSd3kCMD19xzuK/r7XijGVTyObd++nwJGtvI4Nf6ZW/eIyv4CTgVsr3i8FlrY6rhTLv5ItnbsROCqVHQVsTNtXky2n219/b70CY5oKrAHeBHwz/U//y4ov797rmb4oJ6ftcameCo7vsPSHWVXlpbiGPL+g2RHpmnwTOL0M1xDorPqj3NA1A84Frq4oH1CviBir9r0d+HLaHvA97r+Ozfi+14qRbNXT1wBbeT5xtOw6NvLyraraaq1eOGWQuk2TbkkcD6wFXhwRD6ddvwBenLZbEfsngT8H+leYmQQ8ERG7a8QwYNVHoH/VxyJNB/qAz6fbaZ+T9AJKcg0jYjvwceBnwMNk12Qd5bqG/Rq9Zq3+Lr2P7F/w1Iml6TFKmg9sj4j/qtpVmhjrceIYJSS9EPga8KGIeLJyX2T/BGnJ8DhJbwEeiYh1rfj9OY0ju1XwmYg4Hnia7DbLXi2+hocD88kS3NHAC4B5rYilEa28ZnlIuohsWYYvtzqWSpIOBv4S+GirYxkuJ47a8qxe2DSSfoMsaXw5Ir6eiv9b0lFp/1HAI6m82bGfApwlaSuwgux21RXARGWrOlbHsDc+DVz1sUjbgG0RsTa9X0WWSMpyDX8b2BIRfRHxLPB1sutapmvYr9Fr1pLvkqT3AG8B3pUSXJlifCnZPxL+K31vpgL3SnpJiWKsy4mjttKsNChJZAte/SgiPlGxq3L1xPPI+j76y9+dRmecBOyouLUw4iJiaURMjYhOsut0W0S8C7idbFXHWvHVWvWxMBHxC+AhSS9LRXPIFgkrxTUku0V1kqSD03/v/vhKcw0rNHrNbgVOk3R4almdlsoKI2ke2a3TsyLi11Wxt3xl0Yi4PyJeFBGd6XuzjWwAzC8o0XWsq1WdK2V/kY1u2EQ22uKiFsZxKtntgB8CP0ivN5Pd014DbAa+DRyR6gu4KsV9P9DVxFjfyPOjqo4l+1L2Al8FJqTyA9P73rT/2CbF9lqgJ13HfyEbmVKaawhcAvwYeAC4nmzkT0uvIXADWZ/Ls2R/3M4fzjUj62foTa/3NiHGXrL+gP7vy2cr6l+UYtwInFFRXtj3vVaMVfu38nzneEuuY6MvPzluZmYN8a0qMzNriBOHmZk1xInDzMwa4sRhZmYNceIwM7OGOHGYmVlDnDjMzKwhThxmZtaQ/w/kQw2IflegQAAAAABJRU5ErkJggg==","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 4h 49min 20s\n","Wall time: 58min 39s\n"]}],"source":["%%time\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","comps = [100, 500, 1000, 1500] # removed 2000 as it takes too long to compute\n","kernel = \"linear\"\n","\n","for n_components in comps:\n","    start = time.time()\n","    pca = KernelPCA(n_components=n_components, kernel=kernel, n_jobs=n_jobs, random_state=random_state)\n","    pca.fit(df_features.to_numpy())\n","    X_pca = pca.transform(df_features.to_numpy())\n","    \n","    print(f\"n_components = {n_components}\")\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore} \\n\")\n","    elapsed = time.time() - start\n","    print(f\"Time taken for {n_components} Principal Components: {elapsed}s \\n\")\n","\n","    var_values = pca.eigenvalues_ / sum(pca.eigenvalues_)\n","    plt.plot(np.arange(1, pca.n_components + 1), var_values, \"+\", linewidth=2)\n","    plt.ylabel(\"PCA explained variance ratio\")\n","    plt.show()"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, we observe that the KernelPCA offers comparable performance to PCA. However, it is significantly slower than PCA (e.g. KernelPCA  (n=1000) took a total of `1003.68s` for dimensionality reduction and training 2 models. It also enabled Model 6 to achieve a Macro-F1 score of `0.68600` while PCA (n=1000) took a total of `166.69s` for dimensionality reduction and training 4 models. It also enabled Model 6 to achieve a Macro-F1 score of  `0.68342`). Hence, using PCA is sufficient in improving our model performance compared to KernelPCA  while reducing computational time. KernelPCA is computationally expensive to use while providing minimal improvement in performance. Perhaps, we could consider using the Nystroem method to approximate a kernel map of the training data for evaluation."]},{"cell_type":"markdown","metadata":{},"source":["We will now try different kernels and evaluate the model performance. We will use 500 components instead as it is computationally expensive to transform 5000 features using the KernelPCA using different kernels. Moreover, as the Nystroem method supports different kernels, we will try them out and evaluate our classifier model."]},{"cell_type":"code","execution_count":121,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["n_components=1500, random_state=100, kernels=['linear', 'poly', 'rbf', 'sigmoid', 'cosine', 'chi2', 'additive_chi2', 'laplacian'], n_jobs=-1\n","kernel = linear\n","Nystroem Completed: 0.0010538101196289062s\n","PCA Completed: 11.90524435043335s\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.7192737430167597\n","Macro-F1 score: 0.6869730900738653\n","Model6: 0.6869730900738653\n","\n","Accuracy: 0.6538640595903166\n","Macro-F1 score: 0.543404913226483\n","KNN: 0.543404913226483 \n","\n","Time taken for 1500 Principal Components: 52.637346029281616s \n","\n","kernel = poly\n","Nystroem Completed: 0.005002260208129883s\n","PCA Completed: 10.990509033203125s\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model6: 0.384791636832307\n","\n","Accuracy: 0.6538640595903166\n","Macro-F1 score: 0.543404913226483\n","KNN: 0.543404913226483 \n","\n","Time taken for 1500 Principal Components: 51.34895992279053s \n","\n","kernel = rbf\n","Nystroem Completed: 0.003998756408691406s\n","PCA Completed: 12.414059400558472s\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model6: 0.384791636832307\n","\n","Accuracy: 0.6536312849162011\n","Macro-F1 score: 0.5415707130686219\n","KNN: 0.5415707130686219 \n","\n","Time taken for 1500 Principal Components: 54.308987855911255s \n","\n","kernel = sigmoid\n","Nystroem Completed: 0.004998207092285156s\n","PCA Completed: 11.385772943496704s\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model6: 0.384791636832307\n","\n","Accuracy: 0.6543296089385475\n","Macro-F1 score: 0.5442941948148103\n","KNN: 0.5442941948148103 \n","\n","Time taken for 1500 Principal Components: 53.66383337974548s \n","\n","kernel = cosine\n","Nystroem Completed: 0.004998922348022461s\n","PCA Completed: 14.82302188873291s\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.7192737430167597\n","Macro-F1 score: 0.6869730900738653\n","Model6: 0.6869730900738653\n","\n","Accuracy: 0.6538640595903166\n","Macro-F1 score: 0.543404913226483\n","KNN: 0.543404913226483 \n","\n","Time taken for 1500 Principal Components: 57.12529730796814s \n","\n","kernel = chi2\n","Nystroem Completed: 0.004146575927734375s\n","PCA Completed: 542.6790273189545s\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.6431564245810056\n","Macro-F1 score: 0.5167758997541018\n","Model6: 0.5167758997541018\n","\n","Accuracy: 0.6566573556797021\n","Macro-F1 score: 0.5679043031388002\n","KNN: 0.5679043031388002 \n","\n","Time taken for 1500 Principal Components: 585.7695593833923s \n","\n","kernel = additive_chi2\n","Nystroem Completed: 0.005160331726074219s\n","PCA Completed: 587.8147435188293s\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.7164804469273743\n","Macro-F1 score: 0.690348365545216\n","Model6: 0.690348365545216\n","\n","Accuracy: 0.6708566108007449\n","Macro-F1 score: 0.5677715374086773\n","KNN: 0.5677715374086773 \n","\n","Time taken for 1500 Principal Components: 630.3445649147034s \n","\n","kernel = laplacian\n","Nystroem Completed: 0.003998279571533203s\n","PCA Completed: 376.9414110183716s\n","X_train: (12888, 1500), y_train: (12888,)\n","X_test: (4296, 1500), y_test: (4296,)\n","Accuracy: 0.625465549348231\n","Macro-F1 score: 0.384791636832307\n","Model6: 0.384791636832307\n","\n","Accuracy: 0.6659683426443203\n","Macro-F1 score: 0.5724716504381037\n","KNN: 0.5724716504381037 \n","\n","Time taken for 1500 Principal Components: 419.02041721343994s \n","\n","CPU times: total: 4h 29min 1s\n","Wall time: 31min 44s\n"]}],"source":["%%time\n","from sklearn.kernel_approximation import Nystroem\n","\n","bs = 2048\n","epochs = 256\n","lr = 0.1\n","C = 0\n","beta_m = 0.6 #No pattern could be clearly determined (classic rec: 0.9)\n","beta_v = 0.999 #(as close to 1) (classic rec: 0.999)\n","err = 0.01 #0.01 (classic rec: 1e-8)\n","\n","n_components=1500\n","Kernels=[\"linear\", \"poly\", \"rbf\", \"sigmoid\", \"cosine\", \"chi2\", \"additive_chi2\", \"laplacian\"]\n","n_jobs=-1\n","random_state=100\n","print(f\"n_components={n_components}, random_state={random_state}, kernels={Kernels}, n_jobs={n_jobs}\")\n","\n","for kernel in Kernels:\n","    start = time.time()\n","    print(f\"kernel = {kernel}\")\n","    nystroem = Nystroem(kernel=kernel, gamma=None, n_components=n_components, random_state=100, n_jobs=-1)\n","    print(f\"Nystroem Completed: {time.time()-start}s\")\n","    X_nystroem = nystroem.fit_transform(df_features.to_numpy())\n","    pca = PCA(n_components=n_components, random_state=random_state) #n_components\n","    X_pca = pca.fit_transform(X_nystroem)\n","    print(f\"PCA Completed: {time.time()-start}s\")\n","#     print(X_pca)\n","#     print(X_pca.shape)\n","#     print(df_label.shape)\n","\n","    X_pcatrain, X_pcatest, y_pcatrain, y_pcatest = train_test_split(X_pca, df_label, \\\n","                                                                    test_size=0.25, random_state=100)\n","    print(f\"X_train: {X_pcatrain.shape}, y_train: {y_pcatrain.shape}\")\n","    print(f\"X_test: {X_pcatest.shape}, y_test: {y_pcatest.shape}\")\n","    \n","    w, b, l = train6(X_pcatrain, y_pcatrain.values, bs, epochs, lr, C, beta_m, beta_v, err)\n","    Model6 = score(y_pcatest.values,  predict(X_pcatest, w, b))\n","    print(f\"Model6: {Model6}\\n\")\n","    \n","    knn = KNN(n_neighbors=2)\n","    knn.fit(X_pcatrain, y_pcatrain)\n","    knnScore = score(y_pcatest, knn.predict(X_pcatest))\n","    print(f\"KNN: {knnScore} \\n\")\n","    elapsed = time.time() - start\n","    print(f\"Time taken for {n_components} Principal Components: {elapsed}s \\n\")"]},{"cell_type":"markdown","metadata":{},"source":["Based on the above, we can see that using the Nystroem Method significantly boost the computation time of performing Principal Component Analysis. However, we can see that it takes a significant amount of time to perform chi2, additive chi2 and laplacian PCA on the training data. Hence, it would be useful to use Nystroem Kernel approximation for only linear and cosine kernels. Moreover, cosine kernels and linear kernels produce the same results as cosine kernel functions is just a normalisation of the linear kernel function. Hence, there should be a negligible difference in performance."]},{"cell_type":"markdown","metadata":{},"source":["# Task 3: Try other machine learning models and race to the top!\n","In this course, you are exposed to many other machine learning models. For this task, you can apply any other machine learning models (taught in the course or not) to improve the hate speech classification performance! Nevertheless, you are NOT TO use any deep learning approach"]},{"cell_type":"markdown","metadata":{},"source":["## Initialising Setup"]},{"cell_type":"code","execution_count":122,"metadata":{},"outputs":[{"name":"stderr","output_type":"stream","text":["Intel(R) Extension for Scikit-learn* enabled (https://github.com/intel/scikit-learn-intelex)\n"]}],"source":["from sklearnex import patch_sklearn, unpatch_sklearn\n","patch_sklearn()\n","import numpy as np\n","import pandas as pd\n","\n","from sklearn.model_selection import train_test_split, cross_val_score, RepeatedStratifiedKFold\n","from sklearn.metrics import f1_score\n","from sklearn.decomposition import TruncatedSVD\n","from imblearn.metrics import classification_report_imbalanced\n","from imblearn.pipeline import make_pipeline\n","from sklearn.feature_selection import chi2, SelectPercentile\n","from sklearn.preprocessing import RobustScaler\n","from sklearn.kernel_approximation import AdditiveChi2Sampler\n","\n","from lightgbm import LGBMClassifier as LGBM\n","from catboost import CatBoostClassifier as CatBoost\n","from xgboost import XGBClassifier as XGB\n","from sklearn.tree import DecisionTreeClassifier as DecisionTree\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.ensemble import (RandomForestClassifier as RandomForest, AdaBoostClassifier as AdaBoost, HistGradientBoostingClassifier as HistGradBoost, StackingClassifier, VotingClassifier)"]},{"cell_type":"code","execution_count":123,"metadata":{},"outputs":[],"source":["def score(y, y_hat):\n","    accuracy = np.sum(y == y_hat) / np.shape(y)[0]\n","    f1score = f1_score(y, y_hat, average='macro')\n","\n","    print(f\"Accuracy: {accuracy}\")\n","    print(f\"Macro-F1 score: {f1score}\")\n","\n","    # Return Macro-F1 score of the model\n","    return f1score"]},{"cell_type":"code","execution_count":124,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>17179</th>\n","      <td>17180</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17180</th>\n","      <td>17181</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17181</th>\n","      <td>17182</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17182</th>\n","      <td>17183</td>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>17183</th>\n","      <td>17184</td>\n","      <td>0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>17184 rows × 5002 columns</p>\n","</div>"],"text/plain":["          id  label    0    1    2    3    4    5    6    7  ...  4990  4991  \\\n","0          1      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1          2      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2          3      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3          4      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4          5      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...      ...    ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","17179  17180      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17180  17181      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17181  17182      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17182  17183      1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","17183  17184      0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","       4992  4993  4994  4995  4996  4997  4998  4999  \n","0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...     ...   ...   ...   ...   ...   ...   ...   ...  \n","17179   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17180   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17181   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17182   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","17183   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[17184 rows x 5002 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 15.5 s\n","Wall time: 15.8 s\n"]}],"source":["%%time\n","df_train = pd.read_csv(r\"./source/train_tfidf_features.csv\")\n","display(df_train)"]},{"cell_type":"code","execution_count":125,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["0    10633\n","1     6551\n","Name: label, dtype: int64\n"]}],"source":["count_label = df_train['label'].value_counts(dropna=False) # Unique labels\n","print(count_label)"]},{"cell_type":"code","execution_count":126,"metadata":{},"outputs":[{"data":{"text/plain":["array([[0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       ...,\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.]])"]},"metadata":{},"output_type":"display_data"}],"source":["X = df_train.iloc[:, 2:].to_numpy()\n","display(X)"]},{"cell_type":"code","execution_count":127,"metadata":{},"outputs":[{"data":{"text/plain":["array([1, 0, 1, ..., 1, 1, 0], dtype=int64)"]},"metadata":{},"output_type":"display_data"}],"source":["y = df_train.iloc[:, 1].to_numpy()\n","display(y)"]},{"cell_type":"code","execution_count":128,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["X_train: (12888, 6750), y_train: (12888,)\n","X_test: (4296, 6750), y_test: (4296,)\n","CPU times: total: 3.62 s\n","Wall time: 2.68 s\n"]}],"source":["%%time\n","select = SelectPercentile(chi2, percentile=45).fit(X, y) #45\n","X_new = select.transform(X)\n","X_train, X_valid, y_train, y_valid = train_test_split(X_new, y, test_size=0.25, random_state=100)\n","scaler = RobustScaler(with_centering=False, unit_variance=True).fit(X_train)\n","X_trans = scaler.transform(X_train)\n","X_valid_trans = scaler.transform(X_valid)\n","chi2sampler = AdditiveChi2Sampler(sample_steps=2)\n","chi2sampler.fit(X_train)\n","X_train = chi2sampler.transform(X_train)\n","X_valid = chi2sampler.transform(X_valid)\n","print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n","print(f\"X_test: {X_valid.shape}, y_test: {y_valid.shape}\")"]},{"cell_type":"code","execution_count":129,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>9</th>\n","      <th>...</th>\n","      <th>6740</th>\n","      <th>6741</th>\n","      <th>6742</th>\n","      <th>6743</th>\n","      <th>6744</th>\n","      <th>6745</th>\n","      <th>6746</th>\n","      <th>6747</th>\n","      <th>6748</th>\n","      <th>6749</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>...</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","      <td>12888.000000</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>0.001152</td>\n","      <td>0.000544</td>\n","      <td>0.000500</td>\n","      <td>0.000169</td>\n","      <td>0.000207</td>\n","      <td>0.000201</td>\n","      <td>0.000203</td>\n","      <td>0.000103</td>\n","      <td>0.000198</td>\n","      <td>0.000253</td>\n","      <td>...</td>\n","      <td>-0.000233</td>\n","      <td>-0.000059</td>\n","      <td>-0.000255</td>\n","      <td>-0.000295</td>\n","      <td>-0.000065</td>\n","      <td>-0.000070</td>\n","      <td>-0.000025</td>\n","      <td>-0.000093</td>\n","      <td>-0.000317</td>\n","      <td>-0.000124</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>0.021911</td>\n","      <td>0.015039</td>\n","      <td>0.015213</td>\n","      <td>0.008599</td>\n","      <td>0.009622</td>\n","      <td>0.010386</td>\n","      <td>0.010322</td>\n","      <td>0.006740</td>\n","      <td>0.010068</td>\n","      <td>0.010859</td>\n","      <td>...</td>\n","      <td>0.006409</td>\n","      <td>0.003044</td>\n","      <td>0.006851</td>\n","      <td>0.007335</td>\n","      <td>0.003354</td>\n","      <td>0.003302</td>\n","      <td>0.002021</td>\n","      <td>0.003994</td>\n","      <td>0.007482</td>\n","      <td>0.004715</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>-0.200480</td>\n","      <td>-0.180467</td>\n","      <td>-0.201058</td>\n","      <td>-0.200648</td>\n","      <td>-0.196119</td>\n","      <td>-0.196125</td>\n","      <td>-0.191861</td>\n","      <td>-0.183098</td>\n","      <td>-0.194675</td>\n","      <td>-0.193563</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>0.529542</td>\n","      <td>0.518047</td>\n","      <td>0.524518</td>\n","      <td>0.459635</td>\n","      <td>0.488460</td>\n","      <td>0.699323</td>\n","      <td>0.560395</td>\n","      <td>0.452007</td>\n","      <td>0.549215</td>\n","      <td>0.505453</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8 rows × 6750 columns</p>\n","</div>"],"text/plain":["               0             1             2             3             4     \\\n","count  12888.000000  12888.000000  12888.000000  12888.000000  12888.000000   \n","mean       0.001152      0.000544      0.000500      0.000169      0.000207   \n","std        0.021911      0.015039      0.015213      0.008599      0.009622   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.529542      0.518047      0.524518      0.459635      0.488460   \n","\n","               5             6             7             8             9     \\\n","count  12888.000000  12888.000000  12888.000000  12888.000000  12888.000000   \n","mean       0.000201      0.000203      0.000103      0.000198      0.000253   \n","std        0.010386      0.010322      0.006740      0.010068      0.010859   \n","min        0.000000      0.000000      0.000000      0.000000      0.000000   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.699323      0.560395      0.452007      0.549215      0.505453   \n","\n","       ...          6740          6741          6742          6743  \\\n","count  ...  12888.000000  12888.000000  12888.000000  12888.000000   \n","mean   ...     -0.000233     -0.000059     -0.000255     -0.000295   \n","std    ...      0.006409      0.003044      0.006851      0.007335   \n","min    ...     -0.200480     -0.180467     -0.201058     -0.200648   \n","25%    ...      0.000000      0.000000      0.000000      0.000000   \n","50%    ...      0.000000      0.000000      0.000000      0.000000   \n","75%    ...      0.000000      0.000000      0.000000      0.000000   \n","max    ...      0.000000      0.000000      0.000000      0.000000   \n","\n","               6744          6745          6746          6747          6748  \\\n","count  12888.000000  12888.000000  12888.000000  12888.000000  12888.000000   \n","mean      -0.000065     -0.000070     -0.000025     -0.000093     -0.000317   \n","std        0.003354      0.003302      0.002021      0.003994      0.007482   \n","min       -0.196119     -0.196125     -0.191861     -0.183098     -0.194675   \n","25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n","max        0.000000      0.000000      0.000000      0.000000      0.000000   \n","\n","               6749  \n","count  12888.000000  \n","mean      -0.000124  \n","std        0.004715  \n","min       -0.193563  \n","25%        0.000000  \n","50%        0.000000  \n","75%        0.000000  \n","max        0.000000  \n","\n","[8 rows x 6750 columns]"]},"metadata":{},"output_type":"display_data"},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>9</th>\n","      <th>...</th>\n","      <th>6740</th>\n","      <th>6741</th>\n","      <th>6742</th>\n","      <th>6743</th>\n","      <th>6744</th>\n","      <th>6745</th>\n","      <th>6746</th>\n","      <th>6747</th>\n","      <th>6748</th>\n","      <th>6749</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.0</td>\n","      <td>4296.0</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>...</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","      <td>4296.000000</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>0.001682</td>\n","      <td>0.000683</td>\n","      <td>0.000361</td>\n","      <td>0.000212</td>\n","      <td>0.000348</td>\n","      <td>0.000103</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.000106</td>\n","      <td>0.000111</td>\n","      <td>...</td>\n","      <td>-0.000241</td>\n","      <td>-0.000038</td>\n","      <td>-0.000247</td>\n","      <td>-0.000192</td>\n","      <td>-0.000046</td>\n","      <td>-0.000035</td>\n","      <td>-0.000043</td>\n","      <td>-0.000105</td>\n","      <td>-0.000255</td>\n","      <td>-0.000076</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>0.026159</td>\n","      <td>0.016925</td>\n","      <td>0.011906</td>\n","      <td>0.009805</td>\n","      <td>0.013387</td>\n","      <td>0.006778</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.006971</td>\n","      <td>0.007276</td>\n","      <td>...</td>\n","      <td>0.006491</td>\n","      <td>0.002487</td>\n","      <td>0.006648</td>\n","      <td>0.006012</td>\n","      <td>0.003009</td>\n","      <td>0.002306</td>\n","      <td>0.002800</td>\n","      <td>0.004009</td>\n","      <td>0.006836</td>\n","      <td>0.003534</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>-0.198031</td>\n","      <td>-0.162984</td>\n","      <td>-0.196588</td>\n","      <td>-0.203212</td>\n","      <td>-0.197210</td>\n","      <td>-0.151168</td>\n","      <td>-0.183527</td>\n","      <td>-0.168384</td>\n","      <td>-0.190539</td>\n","      <td>-0.171554</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>0.523270</td>\n","      <td>0.456939</td>\n","      <td>0.451797</td>\n","      <td>0.464191</td>\n","      <td>0.630285</td>\n","      <td>0.444263</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.456905</td>\n","      <td>0.476867</td>\n","      <td>...</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8 rows × 6750 columns</p>\n","</div>"],"text/plain":["              0            1            2            3            4     \\\n","count  4296.000000  4296.000000  4296.000000  4296.000000  4296.000000   \n","mean      0.001682     0.000683     0.000361     0.000212     0.000348   \n","std       0.026159     0.016925     0.011906     0.009805     0.013387   \n","min       0.000000     0.000000     0.000000     0.000000     0.000000   \n","25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","50%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","75%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","max       0.523270     0.456939     0.451797     0.464191     0.630285   \n","\n","              5       6       7            8            9     ...  \\\n","count  4296.000000  4296.0  4296.0  4296.000000  4296.000000  ...   \n","mean      0.000103     0.0     0.0     0.000106     0.000111  ...   \n","std       0.006778     0.0     0.0     0.006971     0.007276  ...   \n","min       0.000000     0.0     0.0     0.000000     0.000000  ...   \n","25%       0.000000     0.0     0.0     0.000000     0.000000  ...   \n","50%       0.000000     0.0     0.0     0.000000     0.000000  ...   \n","75%       0.000000     0.0     0.0     0.000000     0.000000  ...   \n","max       0.444263     0.0     0.0     0.456905     0.476867  ...   \n","\n","              6740         6741         6742         6743         6744  \\\n","count  4296.000000  4296.000000  4296.000000  4296.000000  4296.000000   \n","mean     -0.000241    -0.000038    -0.000247    -0.000192    -0.000046   \n","std       0.006491     0.002487     0.006648     0.006012     0.003009   \n","min      -0.198031    -0.162984    -0.196588    -0.203212    -0.197210   \n","25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","50%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","75%       0.000000     0.000000     0.000000     0.000000     0.000000   \n","max       0.000000     0.000000     0.000000     0.000000     0.000000   \n","\n","              6745         6746         6747         6748         6749  \n","count  4296.000000  4296.000000  4296.000000  4296.000000  4296.000000  \n","mean     -0.000035    -0.000043    -0.000105    -0.000255    -0.000076  \n","std       0.002306     0.002800     0.004009     0.006836     0.003534  \n","min      -0.151168    -0.183527    -0.168384    -0.190539    -0.171554  \n","25%       0.000000     0.000000     0.000000     0.000000     0.000000  \n","50%       0.000000     0.000000     0.000000     0.000000     0.000000  \n","75%       0.000000     0.000000     0.000000     0.000000     0.000000  \n","max       0.000000     0.000000     0.000000     0.000000     0.000000  \n","\n","[8 rows x 6750 columns]"]},"metadata":{},"output_type":"display_data"}],"source":["display(pd.DataFrame(X_train).describe())\n","display(pd.DataFrame(X_valid).describe())"]},{"cell_type":"code","execution_count":130,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>0</th>\n","      <th>1</th>\n","      <th>2</th>\n","      <th>3</th>\n","      <th>4</th>\n","      <th>5</th>\n","      <th>6</th>\n","      <th>7</th>\n","      <th>8</th>\n","      <th>...</th>\n","      <th>4990</th>\n","      <th>4991</th>\n","      <th>4992</th>\n","      <th>4993</th>\n","      <th>4994</th>\n","      <th>4995</th>\n","      <th>4996</th>\n","      <th>4997</th>\n","      <th>4998</th>\n","      <th>4999</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 5001 columns</p>\n","</div>"],"text/plain":["         id    0    1    2    3    4    5    6    7    8  ...  4990  4991  \\\n","0     17185  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","1     17186  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","2     17187  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","3     17188  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4     17189  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","...     ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...   ...   \n","4291  21476  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4292  21477  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4293  21478  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4294  21479  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","4295  21480  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...   0.0   0.0   \n","\n","      4992  4993  4994  4995  4996  4997  4998  4999  \n","0      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","1      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","2      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","3      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4      0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","...    ...   ...   ...   ...   ...   ...   ...   ...  \n","4291   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4292   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4293   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4294   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","4295   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n","\n","[4296 rows x 5001 columns]"]},"metadata":{},"output_type":"display_data"}],"source":["df_test = pd.read_csv(r\"./source/test_tfidf_features.csv\")\n","display(df_test)"]},{"cell_type":"code","execution_count":131,"metadata":{},"outputs":[{"data":{"text/plain":["array([[0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       ...,\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.],\n","       [0., 0., 0., ..., 0., 0., 0.]])"]},"metadata":{},"output_type":"display_data"}],"source":["predict_label = df_test.iloc[:, 1:].to_numpy()\n","display(predict_label)"]},{"cell_type":"markdown","metadata":{},"source":["## LightGBM Decision Tree Classifier (Boosting Algorithm)"]},{"cell_type":"code","execution_count":132,"metadata":{},"outputs":[],"source":["lgbm = LGBM(boosting_type='gbdt', learning_rate=0.13, num_leaves=162, max_depth=193, colsample_bytree=0.6, subsample=0.6, min_child_samples=3, min_child_weight=0.02, \\\n","    min_split_gain=0.02, max_delta_step=0.6, n_estimators=115, scale_pos_weight=1.48, reg_lambda=0, random_state=100, reg_alpha=1.3, n_jobs=-1)\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=5, random_state=100)\n","# boosting_type='gbdt', num_leaves=40, max_depth=-1, learning_rate=0.2, n_estimators=100, class_weight=None, reg_lambda=0, random_state=100, reg_alpha=0, n_jobs=-1"]},{"cell_type":"code","execution_count":133,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.702 (0.012)\n","CPU times: total: 1.09 s\n","Wall time: 3min 39s\n"]}],"source":["%%time\n","n_scores = cross_val_score(lgbm, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":134,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7292830540037244\n","Macro-F1 score: 0.7152095030031194\n","CPU times: total: 1min 37s\n","Wall time: 11.2 s\n"]},{"data":{"text/plain":["0.7152095030031194"]},"execution_count":134,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","lgbm.fit(X_train, y_train)\n","score(y_valid, lgbm.predict(X_valid))"]},{"cell_type":"code","execution_count":135,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.80      0.76      0.68      0.78      0.72      0.52      2687\n","          1       0.63      0.68      0.76      0.65      0.72      0.51      1609\n","\n","avg / total       0.73      0.73      0.71      0.73      0.72      0.52      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, lgbm.predict(X_valid)))"]},{"cell_type":"markdown","metadata":{},"source":["## CatBoost Decision Tree Classifier (Boosting Algorithm)"]},{"cell_type":"code","execution_count":136,"metadata":{},"outputs":[],"source":["cboost = CatBoost(boosting_type=\"Plain\", loss_function=\"Logloss\", max_depth=16, learning_rate=0.2, grow_policy='Depthwise', colsample_bylevel=0.39,\\\n","     n_estimators=100, min_child_samples=58, scale_pos_weight=1.52, reg_lambda=1.33, bootstrap_type='MVS', mvs_reg=9.75, random_state=100, verbose=0)\n","# loss_function=\"Logloss\", max_depth=None, use_best_model=True, task_type=\"GPU\", devices='1', set max_depth to 4 due to memory error\n","#from catboost.utils import get_gpu_device_count\n","# print('I see %i GPU devices' % get_gpu_device_count())\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=5, random_state=100)"]},{"cell_type":"code","execution_count":137,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.705 (0.012)\n","CPU times: total: 719 ms\n","Wall time: 15min 38s\n"]}],"source":["%%time\n","n_scores = cross_val_score(cboost, X_train, y_train, scoring=\"f1_macro\", cv=cv, n_jobs=-1, error_score=\"raise\")\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":138,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7306797020484171\n","Macro-F1 score: 0.7124023309515912\n","CPU times: total: 3min 40s\n","Wall time: 31.3 s\n"]},{"data":{"text/plain":["0.7124023309515912"]},"execution_count":138,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","cboost.fit(X_train, y_train)\n","score(y_valid, cboost.predict(X_valid))"]},{"cell_type":"code","execution_count":139,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.78      0.79      0.64      0.78      0.71      0.51      2687\n","          1       0.64      0.64      0.79      0.64      0.71      0.49      1609\n","\n","avg / total       0.73      0.73      0.69      0.73      0.71      0.50      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, cboost.predict(X_valid)))"]},{"cell_type":"markdown","metadata":{},"source":["## XGBoost Decision Tree Classifier (Boosting Algorithm)"]},{"cell_type":"code","execution_count":140,"metadata":{},"outputs":[],"source":["xgboost = XGB(learning_rate=0.25, booster=\"gbtree\", gamma=0.6, min_child_weight=0.02, max_delta_step=0.6, \\\n","    colsample_bylevel=0.35, max_leaves=120, eval_metric=\"error\", tree_method=\"hist\", max_depth=12, subsample=0.65, \\\n","    scale_pos_weight=1.5,  grow_policy=\"depthwise\", n_estimators=100, reg_alpha=0, reg_lambda=0, random_state=100, verbosity=0, n_jobs=-1)\n","# num_parallel_tree=0, check GPU supported algorithms (tree_method=\"gpu_hist\", gpu_id=1), max_leaves=40, max_depth=31, \n","# gamma, min_child_weight, max_delta_step, subsample, colsample_bylevel, process_type=update, max_leaves, eval_metric=error\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":141,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.692 (0.014)\n","CPU times: total: 625 ms\n","Wall time: 5min 10s\n"]}],"source":["%%time\n","n_scores = cross_val_score(xgboost, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":142,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.715316573556797\n","Macro-F1 score: 0.6984855031373471\n","CPU times: total: 4min 14s\n","Wall time: 27.7 s\n"]},{"data":{"text/plain":["0.6984855031373471"]},"execution_count":142,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","xgboost.fit(X_train, y_train)\n","score(y_valid, xgboost.predict(X_valid))"]},{"cell_type":"code","execution_count":143,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.78      0.76      0.64      0.77      0.70      0.49      2687\n","          1       0.62      0.64      0.76      0.63      0.70      0.48      1609\n","\n","avg / total       0.72      0.72      0.68      0.72      0.70      0.49      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, xgboost.predict(X_valid)))"]},{"cell_type":"markdown","metadata":{},"source":["## Decision Tree Classifier"]},{"cell_type":"code","execution_count":144,"metadata":{},"outputs":[],"source":["dtrees = DecisionTree(criterion=\"gini\", splitter=\"best\", max_features=\"sqrt\", max_depth=20, class_weight=\"balanced\", max_leaf_nodes=40, random_state=100)\n","# criterion = \"log_loss\" or \"entropy\", min_samples_split, min_samples_leaf, ccp_alpha\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":145,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.592 (0.025)\n","CPU times: total: 547 ms\n","Wall time: 22 s\n"]}],"source":["%%time\n","n_scores = cross_val_score(dtrees, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":146,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6610800744878957\n","Macro-F1 score: 0.5812215122413674\n","CPU times: total: 453 ms\n","Wall time: 472 ms\n"]},{"data":{"text/plain":["0.5812215122413674"]},"execution_count":146,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","dtrees.fit(X_train, y_train)\n","score(y_valid, dtrees.predict(X_valid))"]},{"cell_type":"markdown","metadata":{},"source":["## Random Forest Classifier (Bagging Algorithm)"]},{"cell_type":"code","execution_count":147,"metadata":{},"outputs":[],"source":["randforest = RandomForest(n_estimators=100, criterion=\"gini\", max_features=\"sqrt\", max_depth=20, n_jobs=-1, max_leaf_nodes=40, verbose=0, random_state=100, warm_start=False, oob_score=True, class_weight=\"balanced\")\n","# criterion = \"log_loss\" or \"entropy\", min_samples_split, max_depth, min_samples_leaf, ccp_alpha, max_samples\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":148,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.675 (0.012)\n","CPU times: total: 672 ms\n","Wall time: 1min 5s\n"]}],"source":["%%time\n","n_scores = cross_val_score(randforest, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":149,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6873836126629422\n","Macro-F1 score: 0.676182388774003\n","CPU times: total: 3min 35s\n","Wall time: 37.4 s\n"]},{"data":{"text/plain":["0.676182388774003"]},"execution_count":149,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","randforest.fit(X_train, y_train)\n","score(y_valid, randforest.predict(X_valid))"]},{"cell_type":"code","execution_count":150,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.78      0.70      0.67      0.74      0.68      0.47      2687\n","          1       0.57      0.67      0.70      0.62      0.68      0.47      1609\n","\n","avg / total       0.70      0.69      0.68      0.69      0.68      0.47      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, randforest.predict(X_valid)))"]},{"cell_type":"markdown","metadata":{},"source":["## Adaboost Decision Tree Classifier (Boosting Algorithm)"]},{"cell_type":"code","execution_count":151,"metadata":{},"outputs":[],"source":["base = DecisionTree(criterion=\"gini\", splitter=\"best\", max_features=\"sqrt\", max_depth=35, class_weight=\"balanced\", max_leaf_nodes=40, random_state=100)\n","adaboost = AdaBoost(base_estimator=base, n_estimators=100, learning_rate=0.5, random_state=100)\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":152,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.679 (0.010)\n","CPU times: total: 641 ms\n","Wall time: 17min 5s\n"]}],"source":["%%time\n","n_scores = cross_val_score(adaboost, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":153,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7094972067039106\n","Macro-F1 score: 0.6914122791184261\n","CPU times: total: 1min 15s\n","Wall time: 1min 16s\n"]},{"data":{"text/plain":["0.6914122791184261"]},"execution_count":153,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","adaboost.fit(X_train, y_train)\n","score(y_valid, adaboost.predict(X_valid))"]},{"cell_type":"code","execution_count":154,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.77      0.76      0.62      0.77      0.69      0.48      2687\n","          1       0.61      0.62      0.76      0.62      0.69      0.47      1609\n","\n","avg / total       0.71      0.71      0.68      0.71      0.69      0.48      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, adaboost.predict(X_valid)))"]},{"cell_type":"markdown","metadata":{},"source":["## Histogram-based Gradient Boosting Classifier (Boosting Algorithm)"]},{"cell_type":"code","execution_count":155,"metadata":{},"outputs":[],"source":["histboost = HistGradBoost(loss=\"log_loss\", learning_rate=0.3, max_depth=40, max_leaf_nodes=50, warm_start=True, scoring=\"f1_macro\", verbose=10, random_state=100)\n","# max_iter, max_depth, min_samples_leaf (default = 20), l2_regularization, max_bins, monotonic_cst, validation_fraction, n_iter_no_change, tol\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)"]},{"cell_type":"code","execution_count":156,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Macro-F1: 0.673 (0.013)\n","CPU times: total: 734 ms\n","Wall time: 13min 11s\n"]}],"source":["%%time\n","n_scores = cross_val_score(histboost, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":157,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Binning 0.626 GB of training data: 5.445 s\n","Binning 0.070 GB of validation data: 0.152 s\n","Fitting gradient boosted rounds:\n","[1/100] 1 tree, 50 leaves, max depth = 35, train score: 0.43430, val score: 0.44029, in 3.159s\n","[2/100] 1 tree, 50 leaves, max depth = 40, train score: 0.61570, val score: 0.62550, in 3.207s\n","[3/100] 1 tree, 50 leaves, max depth = 37, train score: 0.65723, val score: 0.66230, in 3.393s\n","[4/100] 1 tree, 50 leaves, max depth = 40, train score: 0.67338, val score: 0.67306, in 3.218s\n","[5/100] 1 tree, 50 leaves, max depth = 40, train score: 0.68840, val score: 0.68126, in 3.261s\n","[6/100] 1 tree, 50 leaves, max depth = 40, train score: 0.69980, val score: 0.69568, in 3.157s\n","[7/100] 1 tree, 50 leaves, max depth = 40, train score: 0.70927, val score: 0.68963, in 3.081s\n","[8/100] 1 tree, 50 leaves, max depth = 40, train score: 0.71460, val score: 0.69914, in 3.330s\n","[9/100] 1 tree, 50 leaves, max depth = 40, train score: 0.71943, val score: 0.69456, in 3.171s\n","[10/100] 1 tree, 50 leaves, max depth = 40, train score: 0.72734, val score: 0.69638, in 3.107s\n","[11/100] 1 tree, 50 leaves, max depth = 40, train score: 0.73283, val score: 0.69918, in 3.187s\n","[12/100] 1 tree, 50 leaves, max depth = 40, train score: 0.73874, val score: 0.69570, in 2.957s\n","[13/100] 1 tree, 50 leaves, max depth = 40, train score: 0.74368, val score: 0.69223, in 3.181s\n","[14/100] 1 tree, 50 leaves, max depth = 40, train score: 0.74659, val score: 0.68874, in 3.115s\n","[15/100] 1 tree, 50 leaves, max depth = 40, train score: 0.75067, val score: 0.69218, in 3.008s\n","[16/100] 1 tree, 47 leaves, max depth = 40, train score: 0.75468, val score: 0.69108, in 2.760s\n","[17/100] 1 tree, 50 leaves, max depth = 40, train score: 0.75753, val score: 0.68899, in 2.916s\n","[18/100] 1 tree, 50 leaves, max depth = 40, train score: 0.76070, val score: 0.68798, in 3.091s\n","[19/100] 1 tree, 50 leaves, max depth = 40, train score: 0.76293, val score: 0.69070, in 2.929s\n","[20/100] 1 tree, 50 leaves, max depth = 40, train score: 0.76662, val score: 0.69108, in 3.021s\n","[21/100] 1 tree, 50 leaves, max depth = 40, train score: 0.77025, val score: 0.69108, in 2.931s\n","Fit 21 trees in 73.778 s, (1047 total leaves)\n","Time spent computing histograms: 54.578s\n","Time spent finding best splits:  0.964s\n","Time spent applying splits:      0.230s\n","Time spent predicting:           0.003s\n","Accuracy: 0.7183426443202979\n","Macro-F1 score: 0.6833562859456914\n","CPU times: total: 7min 30s\n","Wall time: 1min 13s\n"]},{"data":{"text/plain":["0.6833562859456914"]},"execution_count":157,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","histboost.fit(X_train, y_train)\n","score(y_valid, histboost.predict(X_valid))"]},{"cell_type":"code","execution_count":158,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.74      0.84      0.52      0.79      0.66      0.45      2687\n","          1       0.66      0.52      0.84      0.58      0.66      0.42      1609\n","\n","avg / total       0.71      0.72      0.64      0.71      0.66      0.44      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, histboost.predict(X_valid)))"]},{"cell_type":"markdown","metadata":{},"source":["## Advanced Ensemble Methods (Voting Classifier and StackingClassifier)"]},{"cell_type":"markdown","metadata":{},"source":["### Voting Classifier"]},{"cell_type":"code","execution_count":159,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7309124767225326\n","Macro-F1 score: 0.7058492812338966\n","CPU times: total: 17.6 s\n","Wall time: 3min 41s\n"]},{"data":{"text/plain":["0.7058492812338966"]},"execution_count":159,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","models = [('lgbm', lgbm), ('catboost', cboost), ('xgboost', xgboost), ('randforest', randforest), ('adaboost', adaboost), ('histboost', histboost)]\n","votes = VotingClassifier(estimators=models, voting='soft', n_jobs=-1, verbose=True)\n","votes = votes.fit(X_train, y_train)\n","score(y_valid, votes.predict(X_valid))"]},{"cell_type":"code","execution_count":160,"metadata":{},"outputs":[{"ename":"ValueError","evalue":"Number of features of the model must match the input. Model n_features_ is 6750 and input n_features is 5000","output_type":"error","traceback":["\u001b[1;31m---------------------------------------------------------------------------\u001b[0m","\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)","File \u001b[1;32m<timed exec>:1\u001b[0m, in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n","File \u001b[1;32mc:\\Users\\issac\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\ensemble\\_voting.py:368\u001b[0m, in \u001b[0;36mVotingClassifier.predict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    366\u001b[0m check_is_fitted(\u001b[39mself\u001b[39m)\n\u001b[0;32m    367\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvoting \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39msoft\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m--> 368\u001b[0m     maj \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39margmax(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpredict_proba(X), axis\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m    370\u001b[0m \u001b[39melse\u001b[39;00m:  \u001b[39m# 'hard' voting\u001b[39;00m\n\u001b[0;32m    371\u001b[0m     predictions \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_predict(X)\n","File \u001b[1;32mc:\\Users\\issac\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\ensemble\\_voting.py:409\u001b[0m, in \u001b[0;36mVotingClassifier.predict_proba\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    395\u001b[0m \u001b[39m\"\"\"Compute probabilities of possible outcomes for samples in X.\u001b[39;00m\n\u001b[0;32m    396\u001b[0m \n\u001b[0;32m    397\u001b[0m \u001b[39mParameters\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    405\u001b[0m \u001b[39m    Weighted average probability for each class per sample.\u001b[39;00m\n\u001b[0;32m    406\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m    407\u001b[0m check_is_fitted(\u001b[39mself\u001b[39m)\n\u001b[0;32m    408\u001b[0m avg \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39maverage(\n\u001b[1;32m--> 409\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_collect_probas(X), axis\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m, weights\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_weights_not_none\n\u001b[0;32m    410\u001b[0m )\n\u001b[0;32m    411\u001b[0m \u001b[39mreturn\u001b[39;00m avg\n","File \u001b[1;32mc:\\Users\\issac\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\ensemble\\_voting.py:384\u001b[0m, in \u001b[0;36mVotingClassifier._collect_probas\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    382\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_collect_probas\u001b[39m(\u001b[39mself\u001b[39m, X):\n\u001b[0;32m    383\u001b[0m     \u001b[39m\"\"\"Collect results from clf.predict calls.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 384\u001b[0m     \u001b[39mreturn\u001b[39;00m np\u001b[39m.\u001b[39masarray([clf\u001b[39m.\u001b[39mpredict_proba(X) \u001b[39mfor\u001b[39;00m clf \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mestimators_])\n","File \u001b[1;32mc:\\Users\\issac\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\ensemble\\_voting.py:384\u001b[0m, in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    382\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_collect_probas\u001b[39m(\u001b[39mself\u001b[39m, X):\n\u001b[0;32m    383\u001b[0m     \u001b[39m\"\"\"Collect results from clf.predict calls.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 384\u001b[0m     \u001b[39mreturn\u001b[39;00m np\u001b[39m.\u001b[39masarray([clf\u001b[39m.\u001b[39;49mpredict_proba(X) \u001b[39mfor\u001b[39;00m clf \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mestimators_])\n","File \u001b[1;32mc:\\Users\\issac\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lightgbm\\sklearn.py:997\u001b[0m, in \u001b[0;36mLGBMClassifier.predict_proba\u001b[1;34m(self, X, raw_score, start_iteration, num_iteration, pred_leaf, pred_contrib, **kwargs)\u001b[0m\n\u001b[0;32m    994\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mpredict_proba\u001b[39m(\u001b[39mself\u001b[39m, X, raw_score\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m, start_iteration\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m, num_iteration\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m,\n\u001b[0;32m    995\u001b[0m                   pred_leaf\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m, pred_contrib\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[0;32m    996\u001b[0m     \u001b[39m\"\"\"Docstring is set after definition, using a template.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 997\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39mpredict(X, raw_score, start_iteration, num_iteration, pred_leaf, pred_contrib, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    998\u001b[0m     \u001b[39mif\u001b[39;00m callable(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_objective) \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m (raw_score \u001b[39mor\u001b[39;00m pred_leaf \u001b[39mor\u001b[39;00m pred_contrib):\n\u001b[0;32m    999\u001b[0m         _log_warning(\u001b[39m\"\u001b[39m\u001b[39mCannot compute class probabilities or labels \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m   1000\u001b[0m                      \u001b[39m\"\u001b[39m\u001b[39mdue to the usage of customized objective function.\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[0;32m   1001\u001b[0m                      \u001b[39m\"\u001b[39m\u001b[39mReturning raw scores instead.\u001b[39m\u001b[39m\"\u001b[39m)\n","File \u001b[1;32mc:\\Users\\issac\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\lightgbm\\sklearn.py:800\u001b[0m, in \u001b[0;36mLGBMModel.predict\u001b[1;34m(self, X, raw_score, start_iteration, num_iteration, pred_leaf, pred_contrib, **kwargs)\u001b[0m\n\u001b[0;32m    798\u001b[0m n_features \u001b[39m=\u001b[39m X\u001b[39m.\u001b[39mshape[\u001b[39m1\u001b[39m]\n\u001b[0;32m    799\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_n_features \u001b[39m!=\u001b[39m n_features:\n\u001b[1;32m--> 800\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mNumber of features of the model must \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    801\u001b[0m                      \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mmatch the input. Model n_features_ is \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_n_features\u001b[39m}\u001b[39;00m\u001b[39m and \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    802\u001b[0m                      \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39minput n_features is \u001b[39m\u001b[39m{\u001b[39;00mn_features\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[0;32m    803\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_Booster\u001b[39m.\u001b[39mpredict(X, raw_score\u001b[39m=\u001b[39mraw_score, start_iteration\u001b[39m=\u001b[39mstart_iteration, num_iteration\u001b[39m=\u001b[39mnum_iteration,\n\u001b[0;32m    804\u001b[0m                              pred_leaf\u001b[39m=\u001b[39mpred_leaf, pred_contrib\u001b[39m=\u001b[39mpred_contrib, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n","\u001b[1;31mValueError\u001b[0m: Number of features of the model must match the input. Model n_features_ is 6750 and input n_features is 5000"]}],"source":["%%time\n","results = votes.predict(predict_label)\n","display(results)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["(4296, 1) (4296, 1)\n","\n","\n"]},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 2 columns</p>\n","</div>"],"text/plain":["         id  label\n","0     17185      1\n","1     17186      0\n","2     17187      1\n","3     17188      0\n","4     17189      0\n","...     ...    ...\n","4291  21476      1\n","4292  21477      1\n","4293  21478      1\n","4294  21479      0\n","4295  21480      0\n","\n","[4296 rows x 2 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 15.6 ms\n","Wall time: 10 ms\n"]},{"data":{"text/plain":["0    2750\n","1    1546\n","Name: label, dtype: int64"]},"execution_count":79,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","df_ids = df_test.iloc[:, 0].to_frame()\n","df_results = pd.DataFrame(results)\n","print(df_results.shape, df_ids.shape)\n","print(\"\\n\")\n","    \n","df_submission = pd.concat([df_ids, df_results], axis =1)\n","df_submission = df_submission.rename(columns={0: 'label'})\n","\n","display(df_submission)\n","df_submission['label'].value_counts()"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["df_submission.to_csv(f\"Voting_Ensemble_Predictions.csv\", index=False)"]},{"cell_type":"markdown","metadata":{},"source":["### Stacking Classifier"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]},{"cell_type":"markdown","metadata":{},"source":["## Other Machine Learning Models (Naive Bayes Classifier, Support Vector Machines, Logistic Regression)"]},{"cell_type":"markdown","metadata":{},"source":["### Initialise Setup"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["from sklearn.naive_bayes import MultinomialNB, ComplementNB\n","from sklearn.pipeline import make_pipeline\n","from imblearn.over_sampling import SMOTE, ADASYN\n","from imblearn.pipeline import make_pipeline as make_pipeline_imb\n","\n","from sklearn.kernel_approximation import Nystroem\n","from sklearn.svm import SVC, NuSVC, LinearSVC\n","from sklearn.neighbors import KNeighborsClassifier as KNN\n","from sklearn.linear_model import LogisticRegression, LogisticRegressionCV"]},{"cell_type":"markdown","metadata":{},"source":["### Naive Bayes Classifier"]},{"cell_type":"code","execution_count":161,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.715316573556797\n","Macro-F1 score: 0.7117328238097133\n","CPU times: total: 6.44 s\n","Wall time: 2.29 s\n"]},{"data":{"text/plain":["0.7117328238097133"]},"execution_count":161,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","bayes = make_pipeline_imb(SMOTE(sampling_strategy='minority', k_neighbors=4, random_state=100, n_jobs=-1), MultinomialNB(alpha=13)) # RandomUnderSampler(random_state=100), \n","bayes.fit(X_trans, y_train)\n","score(y_valid, bayes.predict(X_valid_trans))"]},{"cell_type":"code","execution_count":162,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.85      0.66      0.81      0.74      0.73      0.53      2687\n","          1       0.59      0.81      0.66      0.68      0.73      0.54      1609\n","\n","avg / total       0.75      0.72      0.75      0.72      0.73      0.53      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, bayes.predict(X_valid_trans)))"]},{"cell_type":"code","execution_count":163,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7278864059590316\n","Macro-F1 score: 0.7234021026296668\n","CPU times: total: 19.8 s\n","Wall time: 4.65 s\n"]},{"data":{"text/plain":["0.7234021026296668"]},"execution_count":163,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","cbayes = make_pipeline_imb(ADASYN(sampling_strategy='minority', n_neighbors=6, random_state=100, n_jobs=-1), ComplementNB(alpha=10)) #\n","cbayes.fit(X_trans, y_train)\n","score(y_valid, cbayes.predict(X_valid_trans))"]},{"cell_type":"code","execution_count":164,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.85      0.68      0.80      0.76      0.74      0.54      2687\n","          1       0.60      0.80      0.68      0.69      0.74      0.55      1609\n","\n","avg / total       0.76      0.73      0.76      0.73      0.74      0.55      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, cbayes.predict(X_valid_trans)))"]},{"cell_type":"markdown","metadata":{},"source":["### Support Vector Machines"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7539571694599627\n","Macro-F1 score: 0.742741476449474\n","CPU times: total: 4.84 s\n","Wall time: 5.2 s\n"]},{"data":{"text/plain":["0.742741476449474"]},"execution_count":83,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","lsvm = make_pipeline_imb(LinearSVC(penalty=\"l2\", loss=\"squared_hinge\", dual=False, C=1.0, class_weight=\"balanced\", verbose=0, random_state=100, max_iter=1000))\n","lsvm.fit(X_train, y_train)\n","score(y_valid, lsvm.predict(X_valid))"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.83      0.77      0.73      0.80      0.75      0.56      2687\n","          1       0.65      0.73      0.77      0.69      0.75      0.56      1609\n","\n","avg / total       0.76      0.75      0.74      0.76      0.75      0.56      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, lsvm.predict(X_valid)))"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.763268156424581\n","Macro-F1 score: 0.7402443255037597\n","CPU times: total: 12min 18s\n","Wall time: 2min 13s\n"]},{"data":{"text/plain":["0.7402443255037597"]},"execution_count":85,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","nu_svm = make_pipeline(NuSVC(nu=0.5, kernel=\"rbf\", gamma=\"scale\", class_weight=\"balanced\", verbose=True, random_state=100))\n","nu_svm.fit(X_train, y_train)\n","score(y_valid, nu_svm.predict(X_valid))"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.79      0.85      0.62      0.82      0.73      0.54      2687\n","          1       0.71      0.62      0.85      0.66      0.73      0.52      1609\n","\n","avg / total       0.76      0.76      0.71      0.76      0.73      0.53      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, nu_svm.predict(X_valid)))"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7476722532588455\n","Macro-F1 score: 0.717882084958926\n","CPU times: total: 3min 7s\n","Wall time: 32.2 s\n"]},{"data":{"text/plain":["0.717882084958926"]},"execution_count":87,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","svm = make_pipeline(SVC(C=1, kernel=\"sigmoid\", coef0=0.0, gamma=0.625, class_weight=\"balanced\", verbose=True, random_state=100)) #Nystroem(kernel='cosine', gamma=0.625, coef0=0, n_components=2500, random_state=100, n_jobs=-1), \n","svm.fit(X_train, y_train)\n","score(y_valid, svm.predict(X_valid)) # Nystroem Linear, Sigmoid, Cosine, RBF"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.77      0.86      0.56      0.81      0.70      0.50      2687\n","          1       0.70      0.56      0.86      0.63      0.70      0.47      1609\n","\n","avg / total       0.74      0.75      0.67      0.74      0.70      0.49      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, svm.predict(X_valid)))"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7674581005586593\n","Macro-F1 score: 0.7443730647596168\n","CPU times: total: 4min 36s\n","Wall time: 49.6 s\n"]},{"data":{"text/plain":["0.7443730647596168"]},"execution_count":89,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","ssvm = make_pipeline(SVC(C=6.89, kernel=\"rbf\", coef0=0.0, gamma=0.19, class_weight=None, verbose=0, cache_size=500, tol=0.0001, random_state=100))\n","ssvm.fit(X_train, y_train)\n","score(y_valid, ssvm.predict(X_valid))"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.79      0.85      0.62      0.82      0.73      0.54      2687\n","          1       0.72      0.62      0.85      0.67      0.73      0.52      1609\n","\n","avg / total       0.76      0.77      0.71      0.76      0.73      0.54      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, ssvm.predict(X_valid)))"]},{"cell_type":"markdown","metadata":{},"source":["### K-Nearest Neighbors Classifier"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6289571694599627\n","Macro-F1 score: 0.40545006537786327\n","CPU times: total: 40.2 s\n","Wall time: 8.77 s\n"]},{"data":{"text/plain":["0.40545006537786327"]},"execution_count":91,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","knn = make_pipeline_imb(KNN(n_neighbors=2, weights=\"distance\", n_jobs=-1)) #metric=\"cosine\"\n","knn.fit(X_train, y_train)\n","score(y_valid, knn.predict(X_valid))"]},{"cell_type":"markdown","metadata":{},"source":["### Logistic Regression"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.7583798882681564\n","Macro-F1 score: 0.7339416238693894\n","CPU times: total: 39.8 s\n","Wall time: 40.1 s\n"]},{"data":{"text/plain":["0.7339416238693894"]},"execution_count":92,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","LogReg = LogisticRegression(penalty='l2', solver='saga', random_state=100, max_iter=500, class_weight=None, C=2.5)\n","LogReg.fit(X_train, y_train)\n","score(y_valid, LogReg.predict(X_valid))"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.78      0.85      0.61      0.81      0.72      0.53      2687\n","          1       0.71      0.61      0.85      0.65      0.72      0.50      1609\n","\n","avg / total       0.75      0.76      0.70      0.75      0.72      0.52      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, LogReg.predict(X_valid)))"]},{"cell_type":"markdown","metadata":{},"source":["### Stacking Classifier"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["estimators = [('NuSVC', nu_svm), ('lsvm', lsvm), ('LGBM', lgbm), ('LogReg', LogReg)] # ('cbayes', cbayes), ('LogRegCV', LogRegCV), ('bayes', bayes), ('LGBM', lgbm), ('lsvm', lsvm)\n","cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)\n","# final = LogisticRegression(penalty='l2', solver='saga', random_state=100, max_iter=500, class_weight=None, C=2.5)\n","# final = LGBM(boosting_type='gbdt', num_leaves=200, max_depth=150, learning_rate=0.35, colsample_bytree=0.55, subsample=0.55, min_child_samples=2, \\\n","#     min_child_weight=0.1, min_split_gain=0.2, max_delta_step=0.1, n_estimators=120, class_weight=None, reg_lambda=0, random_state=100, reg_alpha=0, n_jobs=-1)\n","final = RandomForest(n_estimators=100, criterion=\"gini\", max_features=\"sqrt\", max_depth=20, n_jobs=-1, max_leaf_nodes=40, verbose=0, random_state=100, warm_start=False, oob_score=True, class_weight=None)\n","stack = StackingClassifier(estimators=estimators, final_estimator=final, cv=\"prefit\", stack_method=\"auto\", n_jobs=-1, passthrough=False, verbose=0)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6757448789571695\n","Macro-F1 score: 0.661509475263802\n","CPU times: total: 4min 20s\n","Wall time: 46.7 s\n"]},{"data":{"text/plain":["0.661509475263802"]},"execution_count":95,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","stack.fit(X_train, y_train)\n","score(y_valid, stack.predict(X_valid))"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["                   pre       rec       spe        f1       geo       iba       sup\n","\n","          0       0.76      0.70      0.63      0.73      0.67      0.45      2687\n","          1       0.56      0.63      0.70      0.59      0.67      0.44      1609\n","\n","avg / total       0.68      0.68      0.66      0.68      0.67      0.44      4296\n","\n"]}],"source":["print(classification_report_imbalanced(y_valid, stack.predict(X_valid)))"]},{"cell_type":"markdown","metadata":{},"source":["### Submission of Results"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"data":{"text/plain":["array([0, 0, 1, ..., 1, 0, 0], dtype=int64)"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 46.6 s\n","Wall time: 8.52 s\n"]}],"source":["%%time\n","# X_pred = chi2sampler.transform(scaler.transform(select.transform(predict_label)))\n","X_pred = chi2sampler.transform(select.transform(predict_label))\n","\n","results = ssvm.predict(X_pred)\n","display(results)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["(4296, 1) (4296, 1)\n","\n","\n"]},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>17185</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>17186</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17187</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17188</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17189</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>4291</th>\n","      <td>21476</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4292</th>\n","      <td>21477</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4293</th>\n","      <td>21478</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4294</th>\n","      <td>21479</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4295</th>\n","      <td>21480</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4296 rows × 2 columns</p>\n","</div>"],"text/plain":["         id  label\n","0     17185      0\n","1     17186      0\n","2     17187      1\n","3     17188      0\n","4     17189      0\n","...     ...    ...\n","4291  21476      0\n","4292  21477      0\n","4293  21478      1\n","4294  21479      0\n","4295  21480      0\n","\n","[4296 rows x 2 columns]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: total: 15.6 ms\n","Wall time: 12.1 ms\n"]},{"data":{"text/plain":["0    2920\n","1    1376\n","Name: label, dtype: int64"]},"execution_count":44,"metadata":{},"output_type":"execute_result"}],"source":["%%time\n","df_ids = df_test.iloc[:, 0].to_frame()\n","df_results = pd.DataFrame(results)\n","print(df_results.shape, df_ids.shape)\n","print(\"\\n\")\n","    \n","df_submission = pd.concat([df_ids, df_results], axis =1)\n","df_submission = df_submission.rename(columns={0: 'label'})\n","\n","display(df_submission)\n","df_submission['label'].value_counts()"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# df_submission.to_csv(f\"chi2_ssvm_rbf_Predictions.csv\", index=False)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["elapsed = time.time() - start_time"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Process finished --- 38540.790263175964 seconds ---\n"]}],"source":["print(\"Process finished --- %s seconds ---\" % elapsed)"]},{"cell_type":"markdown","metadata":{},"source":["### Appendix"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"ename":"NameError","evalue":"name 'CalibratedClassifierCV' is not defined","output_type":"error","traceback":["\u001b[1;31m---------------------------------------------------------------------------\u001b[0m","\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[1;32mc:\\Users\\issac\\Documents\\GitHub\\ml_stuff_2022\\2022-50-007-ml-project.ipynb Cell 261\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/issac/Documents/GitHub/ml_stuff_2022/2022-50-007-ml-project.ipynb#ch0000260?line=0'>1</a>\u001b[0m calibrated \u001b[39m=\u001b[39m CalibratedClassifierCV(base_estimator\u001b[39m=\u001b[39mstack, method\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39misotonic\u001b[39m\u001b[39m\"\u001b[39m, cv\u001b[39m=\u001b[39m\u001b[39m2\u001b[39m, n_jobs\u001b[39m=\u001b[39m\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m, ensemble\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n","\u001b[1;31mNameError\u001b[0m: name 'CalibratedClassifierCV' is not defined"]}],"source":["calibrated = CalibratedClassifierCV(base_estimator=stack, method=\"isotonic\", cv=2, n_jobs=-1, ensemble=True)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  6.2min\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  6.3min remaining: 68.8min\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  6.3min remaining: 43.9min\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  6.4min remaining: 31.9min\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  6.5min remaining: 24.7min\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  6.5min remaining: 19.6min\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  6.5min remaining: 15.9min\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  6.6min remaining: 13.1min\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  6.6min remaining: 11.0min\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  6.6min remaining:  9.3min\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  6.6min remaining:  7.9min\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  6.9min remaining:  6.9min\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed: 12.8min remaining: 10.8min\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed: 12.8min remaining:  9.2min\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed: 12.9min remaining:  7.7min\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed: 12.9min remaining:  6.5min\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed: 13.0min remaining:  5.4min\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed: 13.2min remaining:  4.4min\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed: 13.3min remaining:  3.5min\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed: 13.4min remaining:  2.7min\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed: 13.5min remaining:  1.9min\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed: 13.6min remaining:  1.2min\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed: 13.8min remaining:    0.0s\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed: 13.8min finished\n","Macro-F1: 0.761 (0.008)\n"]}],"source":["n_scores = cross_val_score(calibrated, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, verbose=10, error_score='raise')\n","print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["# X_trans = svd.transform(X_valid)\n","calibrated.fit(X_train, y_train)\n","y_pred = calibrated.predict(X_valid)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Accuracy: 0.6759744037230948\n","Macro-F1 score: 0.661960747208876\n"]},{"data":{"text/plain":["0.661960747208876"]},"execution_count":176,"metadata":{},"output_type":"execute_result"}],"source":["score(y_valid, y_pred)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABIwElEQVR4nO3dd3gU1dfA8e9JgoSO9E6o0kMJIEgVEOxKE5BXwYIduyJiQywoKvUnggKKBURQsaCCYkWE0EKT3kINnRBakvP+MZO4QMqmbDblfJ4nT2Zmp5zZlLMz9865oqoYY4zJuwL8HYAxxhj/skRgjDF5nCUCY4zJ4ywRGGNMHmeJwBhj8rggfweQVqVKldKQkBB/h2GMMTnKsmXLDqpq6aRey3GJICQkhPDwcH+HYYwxOYqI7EjuNbs1ZIwxeZwlAmOMyeMsERhjTB5nicAYY/I4SwTGGJPH+SwRiMgUETkgImuSeV1EZKyIbBaRCBFp6qtYjDHGJM+XVwTTgG4pvH41UMv9GgS868NYjDHGJMNniUBVfwcOp7DKjcBH6lgMFBeR8r6KxxhjcqqTJ0+yfft2n+3fn20EFYFdHvOR7rKLiMggEQkXkfCoqKgsCc4YY7KLESNG0L17d+Lj432y/xzRWKyqk1Q1TFXDSpdO8glpY4zJVY4ePcru3bsBePrppxk9ejQBAb75l+3PRLAbqOwxX8ldZowxeVpcXBytW7dmwIABABQvXpx27dr57Hj+rDU0F3hQRGYALYFjqrrXj/EYY4xfHT9+nKJFixIYGMgrr7xC5cqVU98oE/iy++hnwN/AZSISKSJ3isi9InKvu8r3wFZgMzAZuN9XsRhjTHYXERFB9erVmTt3LgA333wzYWFhWXJsn10RqGrfVF5X4AFfHd8YY3ICVUVEqFOnDjfccAM1a9bM8hhSTQQiEga0BSoAp4A1wHxVPeLj2IwxJlf77LPPGDNmDL/++ivBwcFMmTLFL3Eke2tIRAaKyHLgGaAAsAE4ALQBFojIhyJSJWvCNMaY3OfSSy+lSJEiHD9+3K9xpHRFUBC4QlVPJfWiiDTGeSp4pw/iMsaYXCc+Pp5Ro0ZRvHhxBg0aRLdu3ejatSsi4te4kr0iUNUJySUB9/WVqvqzb8IyxpjcR0T45Zdf+PPPP89b5m/p6jUkIs9ndiDGGJMbnTlzhldeeYVDhw4hIsyZM4cPP/zQ32GdJ73dR+/K1CiMMSaX2rRpEy+99BJz5swBoGDBgtniKsBTsm0EIpJc64XgNB4bY4xJQnR0NAsXLuT666+nQYMG/Pvvv1SvXt3fYSUrpSuCo0AtVS16wVcRwJ4ANsaYZCQUidu1y6mrmZ2TAKScCD4Cqibz2qc+iMUYY3KsI0eOEBkZCcAzzzzDwoULs6xEREaJ84BvzhEWFqbh4eH+DsMYYxLFxcXRsGFDKlasyPz58/0dTpJEZJmqJlmzwp9F54wxJkc7duwYxYoVIzAwkFdffZUqVXLmM7Y5YjwCY4zJbiIiIqhWrRpfffUVADfddBNNm+bModctERhjTBokjBJWt25devToQZ06dfwcUcZZIjDGGC99/PHHtGzZktOnT5MvXz4mT56cdxKBiHyb0rwxxuQFpUuXpmTJkpw4ccLfoWQqr3oNiUh5z9HDLpzPStZryBiTVeLi4njzzTcpXrw4997rjKmVMH5ATpNSryGvrggS/umLyKUi0siGlDTG5AUBAQH89ttv/P3334nLcmISSE2qiUBEfhWRoiJSAlgOTBaRt30fmjHGZL3Tp08zfPjwbF0kLrN5c0VQTFWPA92Bj1S1JdDZt2EZY4x/bN68mREjRiR2Cy1QIPeXVvMmEQSJSHmgN2CNxMaYXCc6OjrxH3+DBg3YsGEDd955p3+DykLeJILhwI/AZlVdKiLVgU2+DcsYY7LOK6+8Qq9evRJrBVWrVs3PEWUtqzVkjMmTDh8+zMmTJ6lcuTLHjh1j3bp1tGrVyt9h+UyGeg2JyBtuY3E+EflZRKJEpH/mh2mMMVkjLi6O1q1bc8cddwBQrFixXJ0EUuNN0bmrVPUpEbkZ2I7TaPw78LEvAzPGmMx29OhRihcvTmBgICNHjqRq1eQq7ectXjUWu9+vBWap6jEfxmOMMT6xatWq84rE3XjjjTRu3NivMWUX3iSCb0XkX6AZ8LOIlAZO+zYsY4zJHAlF4urVq0fv3r2pV6+enyPKflJNBKo6BGgNhKnqOeAkcKOvAzPGmIyaPn06LVq0SCwS995771G7dm1/h5XteDswTQWgs4gEeyz7yAfxGGNMpilbtixlypThxIkTBAcHp75BHpVq91EReQHoANQDvgeuBv5U1Z4+jy4J1n3UGJOcuLg4Xn/9dUqUKMF9993n73CylYwOVdkTCAVWqOpAESmL9RgyxmRDAQEB/PXXX5QvX97foeQo3iSCU6oaLyKxIlIUOABU9nFcxhjjlVOnTvH666/z0EMPUapUKebMmePVbaCwEfM5GH32ouWlCl9C+LAuvgg12/Km11C4iBQHJgPLcCqQ/p3iFi4R6SYiG0Rks4gMSeL1KiKyUERWiEiEiFyTluCNMWbr1q289tprzJ07F8DrtoCkkkBKy3OzVK8IVPV+d3KiiPwAFFXViNS2E5FAYALQBYgElorIXFVd57HaMOBzVX1XRBLaIELSeA7GmDzm+PHj/Pzzz9x8883Ur1+fTZs22cNhGZDsFYGINL3wCyiBU420qRf7boFTqG6rqp4FZnBxt1MFirrTxYA9aT8FY0xe8+qrr9K7d+/EInGWBDImpSuCt1J4TYErU9l3RWCXx3wk0PKCdV4EfhKRh4BCJDPOgYgMAgYBVKlSJZXDGmNyo0OHDnHy5EmqVKnC0KFDuemmm6hUqVKa9xMfr0xbtD3zA8zBkk0EqtoxC47fF5imqm+JSCtguog0UNX4C2KZBEwCp/toFsRljMlGEorEValShfnz51O0aFEuv/zyNO9n77FTPDFrFX9tPuSDKHMub6qPPuA2FifMXyoi96ewSYLdnN+7qJK7zNOdwOcAqvo3EAyU8mLfxpg84PDhwwAEBgYyatQo3norpRsVKft65W66vvM7K3Ye5fXuDSlV+JIk10tueW7mzQNlK1W18QXLVqhqk1S2CwI2Ap1wEsBSoJ+qrvVYZx4wU1WniUhd4GegoqYQlD1QZkzesGrVKtq3b8+UKVPo3r17uvdzLOYcw75ewzer9tC0SnHe7t2YkFKFMjHSnCGjD5QFiogk/HN2ewOlmjJVNVZEHsQZ3SwQmKKqa0VkOBCuqnOBx4HJIvIoTrvDgJSSgDEm94uLiyMwMJB69erRr18/GjVqlO59/bnpIE/MWsXB6DM8cVVt7m1fg6BAb3rN5y3eXBG8CVQF3nMX3QPsUtXHfRxbkuyKwJjc68MPP2TMmDEsWrQoQ7WBTp+L4/V5/zJt0XZqlC7E6Fua0LBSsUyMNOfJ6BXB0zg9dhIKd8wH3s+k2IwxJlGFChWoWLEi0dHR6U4Ea3Yf45GZK9l8IJoBrUMYcnUdgvMFZnKkuYuNWWyM8Zu4uDheffVVSpQowQMPPJChfcXGxfPe71t5Z/5GSha+hFG9Qmlbq3QmRZrzZfSKwBhjfCIgIIDFixdnuEjcjkMneezzVSzbcYTrGpVnxE0NKF4w7/X+SS9LBMaYLBUTE8Nrr73Gww8/nFgkLn/+/Onal6oyc+kuhn+7jsAAYUyfxtzYuGImR5z7eZ0IRKSgqsb4MhhjTO63bds23njjDapXr87AgQPTnQSiTpzhmTkRLFh/gNY1SjKqVygVihfI5GjzhlQTgYi0xmkcLgxUEZFQ4B6PYnTGGJOiY8eOMX/+fHr27En9+vXZvHkzlSunv5r9/HX7GTI7ghNnYnnuunoMbB1CQIBkYsR5izcdat8BugKHAFR1FdDOl0EZY3KX1157jX79+iUWiUtvEog+E8vTX0Rw90fhlC0azLcPteHONtUsCWSQV7eGVHWXyHlvdJxvwjHG5BZRUVHExMRQtWpVhg4dSo8ePdJVJC5B+PbDPPb5KiKPxHB/hxo80rk2lwTZw2GZwZtEsMu9PaQikg94GFjv27CMMTlZXFwcbdq0oXLlyixYsICiRYvSvHnzdO3rbGw8Y37eyLu/bqHipQWYeU8rmoeUyOSI8zZvEsG9wBicstK7gZ+AjHX4NcbkSocOHaJkyZIEBgby1ltvERISkqH9bdx/gkdnrmTtnuPcElaZ566vR+H81tkxs3nzjoqq3urzSIwxOdrKlStp3749U6dOpXv37lx33XXp3ld8vDJ10XZG/vAvRfIHMen/mnFV/XKZGK3x5E0i+EtEtgMzgdmqetSnERljcpSEInH169enf//+hIaGZmh/e446YwYs2nKIznXL8Fr3RpQukr4upsY7qba0qGptnLGF6wPLReRbEenv88iMMdne1KlTCQsL49SpU+TLl48JEyZQo0aNdO/v65W76Tr6d1bucsYMmHxbmCWBLOBVk7uqLlHVx3DGIT4MfOjTqIwxOUKVKlWoWrUqMTEZe9b0aMxZHvpsBQ/PWEmtMoWZ93Bb+rSowgW9FY2PePNAWVHgZqAPUAP4EichGGPymLi4OIYPH07p0qV58MEH6dSpE506dcrQPv/YFMWTsyI4GH2GJ7texj3tqtuYAVnMmzaCVcBXwHB3OEljTB4VEBDAsmXLqFgx4/V8PMcMqFmmMO/fHkaDinl7zAB/8SYRVLdRw4zJu06ePMkrr7zCo48+SunSpZkzZw6XXJKxyp6rI4/xyMwVbIk6ycArQni6m40Z4E/JJgIRGa2qjwBzReSiRKCqN/gyMGNM9rBjxw7efvttateuzYABAzKUBGLj4pn42xZGL9hEqcL5+fjOlrSpVSoTozXpkdIVwXT3+6isCMQYk30cPXqUn376id69e1OvXj02b96cofIQ4IwZ8OjMlSzfeZTrQysw4sYGFCuYL5MiNhmRbIuMqi5zJxur6m+eX0DjLInOGOMXr7/+Ov37908sEpeRJKCqfLZkJ1eP+YPNB6IZ06cx4/o2sSSQjXgzeP1yVW16wbIVqtrEp5Elw4aqNMY3Dhw4QExMDCEhIZw4cYKNGzfSrFmzDO0z6sQZhsyO4Od/D3BFzZK82dPGDPCXdA1VKSJ9gX5ANRGZ6/FSEZxnCYwxuURcXBxXXHEFVatWZcGCBRQpUiTDSeCntft4Zs5qTpyJ5fnr6jHAxgzItlJqI1gE7AVKAW95LD8BRPgyKGNM1oiKiqJ06dIEBgYyZsyYDBeJA2fMgJe/WcfM8F3Ur1CUGbc0plbZIhkP1vhMsolAVXcAO4BWWReOMSarrFixgnbt2jF16lR69uzJNddck+F9Lt1+mMc+X8nuI6d4oGMNHu5kYwbkBCndGvpTVduIyAnAsyFBAFXVoj6PzhiT6WJjYwkKCqJhw4bccccdGb4FBM6YAe8s2MjE37ZQ+dKCfH5PK8JszIAcI6Urgjbud7umMyaX+OCDDxg7diyLFy+mQIECjBkzJsP73Lj/BI/MWMm6vcfp07wyw66zMQNyGm9qDdUAIlX1jIh0ABoBH1k5amNynpCQEGrUqEFMTAwFCmSs9058vDLlr2288eMGiuQPYvJtYXSpVzaTIjVZyZvuoyuBMCAE+B74Gqivqhm/oZgO1n3UGO/FxcXx4osvUrp0aQYPHpxp+z1/zICyvN6jIaUKW7no7Cxd3Uc9xKtqrIjcDIxT1XEisiJzQzTG+EJAQACrVq1Kd5G4sBHzORh99qLlAhS4JJCRPRrSO6yylYvO4bxJBOfcZwpuB653l9kjgcZkU9HR0YwYMYLHHnuMMmXK8MUXX6S7PlBSSQCc3iPzHm5L1ZKFMhCpyS686dc1EKcL6Suquk1EqvFfHaIUiUg3EdkgIptFZEgy6/QWkXUislZEPvU+dGNMUnbu3Mno0aOZN28eQIYrhSbHkkDukeoVgaquE5EngNoi0gDYoKojU9tORAKBCUAXIBJYKiJzVXWdxzq1gGeAK1T1iIiUSe+JGJOXHTlyhB9//JE+ffpQr149tm7dSoUKFfwdlskhUr0icHsKbcL5p/4/YKOItPNi3y2Azaq6VVXPAjOAGy9Y525ggqoeAVDVA96HboxJMHLkSG6//XZ2794NYEnApIk3t4beAq5S1faq2g7oCrzjxXYVgV0e85HuMk+1ca40/hKRxSLSLakdicggEQkXkfCoqCgvDm1M7rd//362bdsGwLPPPsvixYszZeQwcLqGjvh2XeormlzBm8bifKq6IWFGVTeKSGY1FgcBtYAOQCXgdxFpeOEzCqo6CZgETvfRTDq2MTlWXFwcbdq0Oa9IXJMmmVMQ+PS5OB7/fBXfrd5LcL4ATp+Lv2idUoV90+5g/MObRBAuIu8DH7vztwLedOTfDVT2mK/kLvMUCfyjqueAbSKyEScxLPVi/8bkOQcOHEgsEjd27FiqVauWqfs/GnOWuz8KZ+n2Iwy7ti53tqlmXUPzAG9uDd0HrAMGu1/r3GWpWQrUEpFqInIJ0AeYe8E6X+FcDSAipXBuFW31JnBj8prly5dTvXp1vvjiCwCuvvpq6tSpk2n733U4hh7vLmLVrmOM79eEu9pWtySQR6RUdK4MMBSoCawGBqjqcW937D6E9iDwIxAITFHVtSIyHAhX1bnua1eJyDogDnhSVQ+l/3SMyX0SisQ1atSIu+66ixYtWmT6MVZHHmPgtKWci4vn47ta0qKaFYzLS5ItMSEiPwDLgN+B64DCqjowC2NLkpWYMHnJ+++/z5gxY1iyZEmGawMlZ+G/B3jg0+VcWvASPryjOTXLWJ3J3Ci9JSbKq+qz7vSPIrI880MzxqSkRo0a1KlTh1OnTvkkEXy2ZCfDvlpD3fJFmDKgOWWKBGf6MUz2l2JjsYhcilNWBCDQc15VbbhKYzJZXFwcw4YNo2zZsjzyyCN07NiRjh07ZvpxVJW3529k3C+baV+7NP+7tSmFrHR0npXST74Yzq0hz9aihKsCBar7Kihj8qqAgADWr1/P8eNeN8el2dnYeIbMiWDO8t3cElaZETc3IF+gjSKWl6U0ME1IFsZhTJ514sQJXn75ZZ544gnKlCnDrFmzyJfPN3Udj58+x/0fL+fPzQd5rEttHrqypvUMMsl3HxWRkJQ2FEelTI/ImDwmMjKScePG8cMPPwD4LAnsO3aa3hP/ZvHWQ4zqFcrgTrUsCRgg5VtDb4pIAM5ANMuAKCAYpztpR6AT8ALOQ2HGmDQ4fPgwP/zwA/369aNu3bps3bqV8uXL++x4/+47zsCpSzlxOpapA5vTtlZpnx3L5DzJXhGoai/gOeAynIJzf+AkhbuADcCVqjo/K4I0Jrd58803GThwYGKROF8mgUWbD9Lr3b+JV+Xze1pZEjAXSXWoyuzGniMwOdXevXs5deoU1atXJzo6mi1bthAaGurTY365IpKnvoigWqlCTBvYggrFffMsgsn+MjpUpTEmg+Li4mjbti0hISEsWLCAwoUL+zQJqCr/+3ULb/64gVbVSzLx/5pRrIANLGiSZonAGB/at28fZcuWJTAwkAkTJmR6kbikxMbF8/zctXz6z05ualyBkT0bkT8o0OfHNTmXdR42xkeWL19OjRo1mDVrFgBdu3aldu3aPj1mzNlY7pm+jE//2cn9HWrwdu/GlgRMqrwZoewKESnkTvcXkbdFpKrvQzMmZzp37hwAjRo14t5776VVq1ZZctyoE2foO2kxCzccYMRNDXiqWx0CAqx7qEmdN1cE7wIxIhIKPA5sAT7yaVTG5FCTJk2iSZMmxMTEEBQUxFtvvUXlypVT3zCDtkZF0/3dv9i4P5pJ/xdG/8vts5rxnjeJIFadrkU3AuNVdQJg5QmNSUKtWrWoX78+p0+fzrJjLttxmB7vLiLmTBwzBl1O53pls+zYJnfwprH4hIg8A/wf0NZ9yMy6HxiD0xto6NChlCtXjkcffdRnReKS88OavTw8YyUVihdg2sDmVC1ZKMuObXIPbxLBLUA/4A5V3SciVYA3fRuWMTlDQEAAGzdu5NSpU1l+7Cl/buPl79bRpHJx3r+9OSUK2TjCJn1SvTWkqvuA2UB+d9FB4EtfBmVMdnb8+HEef/xx9u/fj4gwa9Ysxo4dm2XHj49XXv52HcO/XUfXeuX49O7LLQmYDPGm19DdwBfAe+6iijhjDRuTJ+3Zs4d3332X+fOdCitBQVn3OM7pc3E89NkKPvhzGwNahzDh1qYE57PuoSZjvPkNfgBoAfwDoKqb3PGMjckzDh48yA8//ED//v2pU6cO27Zto2zZrG2UPRpzlrs/Cmfp9iMMu7Yud7apZtVDTabwptfQGVU9mzAjIkE4A9MYk2eMGjWKu+66iz179gBkeRLYdTiG7u8uYtWuY4zv14S72la3JGAyjTeJ4DcRGQoUEJEuwCzgG9+GZYz/7dmzh61btwIwbNgwli5dSoUKFbI8jojIo9z8v0Ucij7Lx3e15LpGWR+Dyd28SQRDcMYiWA3cA3wPDPNlUMb4W1xcHO3atWPQoEEAFC5cmIYNG2Z5HAv/PcAt7y0mf1AAs+9rRYtqJbI8BpP7edNGcBPwkapO9nEsxvjd3r17KVeuHIGBgfzvf/+jenX/Dc392ZKdDPtqDXXLF2HKgOaUKRLst1hM7ubNFcH1wEYRmS4i17ltBMbkOuHh4ecVibvqqquoWbNmlsehqoz6cQPPzFlN21qlmDmolSUB41PePEcwEGd4yllAX2CLiLzv68CMySoJReIaN27MAw88QOvWrf0Wy9nYeB7/fBXjF26mT/PKvH9bGIXy22cv41telaFW1XPAPGAGzvjFN/kwJmOyzMSJEwkNDU0sEvfmm29SqVIlv8Ry/PQ5Bk5bwpwVu3m8S21e696QoECrFG98L9WPGiJyNU6ZiQ7Ar8D7QG+fRmVMFqlbty5NmjThzJkzFCxY0G9x7D12ioFTl7L5QDRv9QqlRzP/JCOTN3lzzXkbMBO4R1XP+DgeY3wqNjaWZ555hnLlyvH444/Tvn172rdv79eY/t13nAFTlhJ9JpapA5vb4PImy6WaCFS1b1YEYkxWCAwMZPPmzYntAv721+aD3Dt9GQXzB/L5Pa2oV6Gov0MyeVCyNyBF5E/3+wkROe7xdUJEjmddiMZkzLFjx3j00UfPKxI3evRof4fFnOWRDJi6hArFC/Dl/VdYEjB+k2wiUNU27vciqlrU46uIqtpvrMkx9u7dy6RJk1iwYAGQtUXikqKqTFi4mcc+X0VY1RJ8fm8rKhQv4NeYTN7mTfXR6d4sS2bbbiKyQUQ2i8iQFNbrISIqImHe7NeY1ERFRfHRR86IqnXq1GH79u3ceuutfo4KYuPiefarNbz54wZualyBD+9oQbECNs6T8S9v+qbV95xxHyhrltpGIhIITACuBuoBfUWkXhLrFQEexq1uakxmePvttxk0aFBikbjSpf3fABtzNpZ7pi/j0392cn+HGrxzS2MuCbLuocb/UmojeEZETgCNPNsHgP3A117suwWwWVW3utVLZ+CMe3yhl4GRQNYN8mpypcjISLZs2QLAs88+y7Jly/xSJC4pUSfO0GfSYhZuOMCImxrwVLc6Vj3UZBsptRG8pqpFgDcvaB8oqarPeLHvisAuj/lId1kiEWkKVFbV71LakYgMEpFwEQmPiory4tAmr4mNjaV9+/bcc889gFMkrn79+qlslTW2REXT/d2/2LQ/mkn/F0b/y6v6OyRjzpNsq5mI1FHVf4FZ7j/s86jq8owcWEQCgLeBAamtq6qTgEkAYWFhNhaCSbR7924qVKhAUFAQ7733nl+LxCUlfPth7voonEARZgy6nNDKxf0dkjEXSan7xGPAIOCtJF5T4MpU9r0bqOwxX8ldlqAI0AD41b1ELgfMFZEbVDU8lX0bQ3h4OG3btmXq1Kn06dOHzp07+zuk88xbvZeHZ66kYvECTBvYnKolC/k7JGOSlGwiUNVB7veO6dz3UqCWiFTDSQB9gH4e+z8GlEqYF5FfgScsCZjUnD17lksuuYQmTZowePBg2rVr5++QLvLBn9sY8d06mlQuzvu3N7fB5U225k330V5uzx5EZJiIzBGRJqltp6qxwIPAj8B64HNVXSsiw0XkhowGbvKm//3vf4SGhnLy5EkCAwMZOXJktmkQBoiPV4Z/s46Xv11H13rl+PTuyy0JmGzPmydrnlPVWSLSBugMvAlMBFqmtqGqfo8zopnnsueTWbeDF7GYPK5BgwaEhYVlixIRYSPmczD6bJKvDbwihGHX1iMwwHoGmezPm07Mce73a4FJbg8f+4hjskRsbCyPPfYYo0aNAqBdu3ZMnz6d4sWL+zcwSDYJALxwfX1LAibH8OaKYLeIvAd0AUaKSH68HMfAmIwKDAxkx44d1ufeGB/y5h96b5z7/F1V9ShQAnjSl0GZvO3o0aMMHjyYffv2ISJ8/vnnvPVWUp3XjDGZwZuhKmOALUBXEXkQKKOqP/k8MpNn7d+/nylTpvDLL78AzlVBdnIuLp4pf27zdxjGZBpvRih7GLgbmOMu+lhEJqnqOJ9GZvKU/fv3M2/ePAYMGMBll13G9u3bKVWqVOobZrG/Nh/kxblr2XQg2t+hGJNpvLk1dCfQUlWfd3v8XI6TGIzJNO+88w733XdfYpG47JYEdh2O4d7py7j1/X84ExvP5NvCKFU46T4TyS03JrsS1ZQrNojIaqC5qp5254OBparaMAviu0hYWJiGh9szZ7nBrl27OHPmDDVr1iQ6Oppdu3ZRt25df4d1nlNn45j42xYm/raFABEe6FiDu9pWJzhf9rpdZUxqRGSZqiZZ6t+bXkNTgX9E5EtAcCqIfpCJ8Zk8KKFIXPXq1VmwYAGFCxfOVklAVZm3Zh+vfLee3UdPcX1oBZ65uo4NIGNyJW/GLH7bLf/QBqfG0EBVXeHrwEzuFBkZScWKFQkKCmLy5MnZrkgcwIZ9J3hx7lr+3nqIOuWKMHPQ5bSsXtLfYRnjM2kZs09wEoF16DbpsnTpUtq2bcu0adPo06cPnTp18ndI5zkWc453Fmxk+uIdFAkO4uWbGtC3eWWCAu2xGZO7edNr6HmgFzAbJwlMFZFZqjrC18GZ3OHMmTPkz5+fpk2b8thjj9G+fXt/h3SeuHhl5tJdvPnjvxw7dY5bW1blsS61udRqBJk8wpvG4g1AqEdjcQFgpapelgXxXcQai3OW8ePHM27cOJYvX06hQtmvDHP49sO8MHcta/ccp0W1Erx4fX3qVSjq77CMyXQZbSzeAwTz31CS+Tl/XAFjktWoUSNatWpFbGysv0M5z/7jp3l93r98uWI35YsFM65vE65rVN5KWZg8yZtEcAxYKyLzcdoIugBLRGQsgKoO9mF8JoeJjY3l8ccfp2LFijz11FO0a9cuW40XcCY2jil/bmfcL5uIjVMe7FiT+zvWoOAlaWkuMyZ38ea3/0v3K8GvvgnF5AZBQUHs2bOHfPny+TuUi/zy736Gf7OO7Ydi6FKvLMOurWujhhmDd91HP8yKQEzOdeTIEYYNG8Zzzz1HuXLlmDFjRraqD7Q1KpqXv13Hwg1RVC9diA/vaEH72qX9HZYx2YZdD5sMO3DgAB999BFt2rShb9++2SYJRJ+JZdwvm5jy5zbyBwUy7Nq63NYqhEuCrDuoMZ4sEZh02bdvH99//z133HFHYpG4kiWzx0NX8fHKVyt389q8f4k6cYZezSrxZLfLKFMk2N+hGZMtJfvRSESmu98fzrpwTE4xZswYHnjggcQicdklCayOPEbPiYt47PNVVCgWzJf3t+bNXqGWBIxJQbLPEYjIOpwxiucBHbjgiWJVPezr4JJizxH4z/bt2zl79iy1a9fm5MmTREZGctllfnmc5CIHo88w6scNzAzfRclCl/BUtzr0bFqJABsu0hgg/c8RTAR+BqoDyzg/Eai73OQRsbGxdOzYkerVq/Pzzz9TqFChbJEEzsXF8/HiHbw9fyOnzsZx5xXVGNy5FkWDs1+vJWOyq2QTgaqOBcaKyLuqel8WxmSykZ07d1K5cmWCgoKYMmVKtioS99fmg7z0zVo27o+mba1SvHB9PWqWKeLvsIzJcbzpPnqfiIQCbd1Fv6tqhG/DMtlBQpG4qVOn0rdvXzp27OjvkABnkJhXv1/PvDX7qFyiAJP+rxld6pW1p4KNSSdvis4NBgbx31CVn9hQlbnb6dOnCQ4OpmnTpjzxxBNceeWV/g4JOH+QGBF4vEtt7m5ng8QYk1HeFJ2LAFqp6kl3vhDwt6o2yoL4LmKNxb41duxYxo8fz4oVK7JNkbgLB4m5rlF5hl5T1waJMSYNMlp0ToA4j/k4bEyCXKtJkya0adMm2xSJ27DvBC99s5ZFW5xBYmYMupzLbZAYYzJVWoeqBLgJG6oy14iNjeWRRx6hcuXKPP3007Rt25a2bdumvqGPeQ4SUzh/EC/fWJ++LarYIDHG+EBah6oEG6oyVwkKCiIqKorChQv7OxTAGSTm8/BdvPnjBo7GnKVfyyo83uUyGyTGGB/yqsSEqi4Hlvs4FpNFDh06xLPPPsvzzz9PhQoV+OyzzwgI8P8n7WU7nEFi1uw+TouQErxwQz3qVyjm77CMyfWs1lAedOjQIT799FM6duzILbfc4vck4DlITLmiwYzt24TrbZAYY7KMTxOBiHQDxgCBwPuq+voFrz8G3AXEAlHAHaq6w5cx5VV79+7lu+++46677qJ27drs2LGDSy+91K8xJTVIzH0dalAov30+MSYr+ewvTkQCgQk4I5pFAktFZK6qrvNYbQUQpqoxInIf8AZwi69iysvGjBnD2LFjufbaaylfvrzfk4ANEmNM9uHNA2XdgZFAGZxuowKoqqY2wncLYLOqbnX3MwO4EUhMBKq60GP9xUD/NEVvUrRt2zbOnTtH7dq1ee6557jzzjspX758lsYQNmI+B6PPJvla9dKFmDawOR0uK5OlMRljzufNFcEbwPWquj6N+64I7PKYjwRaprD+nTiVTi8iIoNwnm6mSpUqaQwjb4qNjeXKK6+kRo0aLFiwgEKFClGrVq0sjyO5JADww8PtbJAYY7IBbxLB/nQkgTQRkf5AGNA+qddVdRIwCZwni30ZS063fft2qlatSlBQEFOnTqVGjRr+DilZlgSMyR68SQThIjIT+Ao4k7BQVecku4VjN1DZY76Su+w8ItIZeBZor6pnLnzdeG/JkiWJReL69etHhw4d/BLH0ZizfL1yD7OW7Up9ZWOM33mTCIoCMcBVHsuU/4rQJWcpUEtEquEkgD5AP88VRKQJ8B7QTVUPeBu0Od+pU6coUKAAzZo1Y8iQIXTu3DnLY4iLV/7YFMWsZZHMX7ufs3Hx1CufWjOSMSY78ObJ4oHp2bGqxorIg8CPON1Hp6jqWhEZDoSr6lzgTaAwMMvtM75TVW9Iz/HyqjFjxjBu3DhWrlxJ4cKFeemll7L0+FujovliWSRzlu9m3/HTFC+Yj34tq9CzWSUaVCxGyJDvsjQeY0zaedNrqBIwDrjCXfQH8LCqRqa2rap+D3x/wbLnPaaz/qNrLqGqiAhhYWFceeWVxMfHZ9mxo8/E8n3EXj4P30X4jiMECLSvXZrnr69Hp7plyB/0X1noUoUvSbLBuFRhKxlhTHbhTRnq+cCnwHR3UX/gVlXt4uPYkpTXy1DHxsYyePBgqlSpwpAhQ7LsuKrKP9sOMys8ku9X7+XUuTiqly5Er2aV6d60ImWL2uDwxmRnGS1DXVpVp3rMTxORRzIlMpNmQUFBHDlyhOLFi2fJ8XYfPcXsZZF8sSySnYdjKJw/iBsbV6BXWCWaVrnUykAYkwt4kwgOud07P3Pn+wKHfBeSudDBgwcZOnQoL774IhUqVOCTTz7xaX2g0+fi+HHtPr5YFsmfmw+iCq2ql+SRzrXo1qAcBS+xEhDG5Cbe/EXfgdNG8A5Ob6FFQLoakE36HDlyhJkzZ9K5c2d69+7tkySgqqyKPMas8F3MXbWHE6djqVi8AA9dWYtezSpRuUTBTD+mMSZ78KbX0A7AevJksd27d/Pdd98xaNAgatWqxY4dO3xyOyjqxBm+XBHJrPBINh2IJn9QAFc3KEevsMq0ql6SgAC79WNMbpdsIhCRp1T1DREZh3MlcB5VHezTyPK48ePHM3bsWK6//nrKly+fqUngXFw8v/x7gFnhkSzccIC4eKVJleK8enNDrgstT9HgfJl2LGNM9pfSFUFCWYm820Uni23ZsoVz585Rp04dnnvuOe66665MLRL3777jzAqP5KsVuzl08iyli+TnrjbV6BVWiZplimTacUz6nDt3jsjISE6fPu3vUEwOFhwcTKVKlciXz/sPdMkmAlX9xp2MUdVZnq+JSK/0hWiSExsbS6dOnahZsyYLFiygYMGCmVIn6FjMOeau2s2sZZFERB4jX6DQqU5ZeoVVon3t0jYGcDYSGRlJkSJFCAkJsd5YJl1UlUOHDhEZGUm1atW83s6bxuJngFleLDPpsG3bNkJCQggKCuLDDz/MlH/+cfHKn5sPMit8Fz+t28/Z2HjqlCvC89fV46YmFSlh4/9mS6dPn7YkYDJERChZsiRRUVFp2i6lNoKrgWuAiiIy1uOlojgjipkMWrJkCW3atGHq1KnceuuttG+fZPFVr20/eJJZy3YxZ/lu9h5zyj30bV6ZXmGVqV+hqP2DyQHsZ2QyKj2/QyldEezBaR+4AVjmsfwE8Giaj2QSeRaJe/bZZ7nqqqtS3ygZJ8/E8t3qvXwRHsmS7YcJEGhXuzTDrq1H53rnl3swxpikJHuDWFVXqeqHQEPgY1X90J3/Go9y1CZtRo8eTYMGDYiOjiYwMJAXXniB0qVLp2kfqso/Ww/xxKxVNH9lAU99EUFU9Bme6nYZi4Z0YtrAFlzbqLwlAZMmgYGBNG7cmAYNGtCrVy9iYmLStP2TTz5J/fr1efLJJ9N87FdfffW8+cKFC6d5H9568cUXGTVqFADPP/88CxYsACAkJISDBw+me78rV67k+++/T33FC3To0AF/l83xpo3gJ6AzEO3OF3CXtfZVULlRQpG4Fi1a0KVLF1Kr8ZTcEI8FLwmkdJH87DgUQ6FLArm+kVPuoVlVK/dgMqZAgQKsXLkSgFtvvZWJEyfy2GOPpbpdbGwsQUFBTJo0icOHDxMYmPYPIK+++ipDhw5N83YZNXz48DStn3CuSVm5ciXh4eFcc801mRFalvKmy0iwqiYkAdxpe8zUS7GxsQwaNIjXX38dgNatWzNx4kSKFEm5u2ZyQzzGnI2jfLFg3uoVytJhnRnZsxFhISUsCeRCHTp0YNq0aYDTtbRDhw58/PHHAMTExNChQwdmzpwJwLFjx+jQoQNz5jjDhBw8eJAOHTrwzTdO5799+/al6dht27Zl8+bNnDx5kjvuuIMWLVrQpEkTvv76awCmTZvGDTfcwJVXXkmnTp244YYbiI6OplmzZsycOZOoqCh69OhB8+bNad68OX/99RcA0dHRDBw4kIYNG9KoUSNmz57NkCFDOHXqFI0bN+bWW289L47bbruNr776KnH+1ltvTYzB08iRI2nYsCGhoaGJxRgnT55M8+bNCQ0NpUePHkle4QwYMIAvvvgicf6NN96gYcOGtGjRgs2bNyeuc++999KyZUueeuoplixZQqtWrWjSpAmtW7dmw4YNnD17lueff56ZM2fSuHFjZs6cmex7d+rUKfr06UPdunW5+eabOXXqVJp+Nr7gzRXBSRFpqqrLAUSkGeD/yHOIoKAgoqOjOXnyZKbtc8agVpm2L2MuFBsby7x58+jWrRuvvPIKV155JVOmTOHo0aO0aNEiceCj5cuXExERQYkSJQDndk7CFUW/fv149NFHadOmDTt37qRr166sX7+el19+mWLFirF69WrAKZ/So0cPxo8fn7itpzvvvJN33nmHm266iWPHjrFo0SI+/PDD89aZN28eX3/9Nf/88w8FCxbk8OHDAHTv3p27774bgGHDhvHBBx/w0EMPpXjuCbF99NFHPPLII3z77beA07V30aJFBAYGcvz4cf744w+CgoJYsGABQ4cOZfbs2QwfPpzw8HDGjx8PwNChQ5N879577z0KFizI+vXriYiIoGnTpun4KWUubxLBIzgDx+wBBCgH3OLLoHK6qKgohgwZwvDhw6lYsSKffPKJfWI3afbrr78mTufLl++8+YIFC543X6xYsfPmS5Uqdd58uXLlUj1ewqdycK4I7rzzTlq3bs3cuXMT76mfPn2anTt3AtClS5fEJHChBQsWsG7dusT548ePEx0dzYIFC5gxY0bi8ksvvTTFmNq3b8/9999PVFQUs2fPpkePHhfdmlmwYAEDBw6kYEHnRkVCTGvWrGHYsGEcPXqU6Ohounbtmup70Ldv38Tvjz76X5+YXr16Jd7yOnbsGLfffjubNm1CRDh37lyS+/rpp5+SfO9+//13Bg92CjM0atSIRo0apRqXr3lTa2ipiNQBLnMXbVDVpM/cAM4vyuzZs+nWrRu9evWyJGByBM82ggSqyuzZs7nsssvOW/7PP/9QqFChZPcVHx/P4sWLCQ7O+DgVt912Gx9//DEzZsxg6tSpqW/gGjBgAF999RWhoaFMmzbtvMSYHM+/Vc9pz3N97rnn6NixI19++SXbt29Pdmzw5N677Mjbx0ovA+oBTYG+InKb70LKmSIjI5k4cSIANWvWZMeOHfTqZQ9gm5yta9eujBs3LrFzw4oVK7za7qqrrmLcuHGJ8wkJpkuXLkyYMCFx+ZEjRwDniie5T9YDBgxg9OjRANSrV++i17t06cLUqVMT2wASbg2dOHGC8uXLc+7cOT755BOv4k5oc5k5cyatWiV9C/bYsWNUrFgRILENB6BIkSKcOHEicT65965du3Z8+umngHPVEhER4VVsvpRqIhCRF3DKUI8DOgJvYNVILzJhwgQef/xx9u7dCziX6hmR3FCONsSjyUrPPfcc586do1GjRtSvX5/nnnvOq+3Gjh1LeHg4jRo1ol69eokfkoYNG8aRI0do0KABoaGhLFy4EIBBgwbRqFGjixqLAcqWLUvdunUZODDp6vfdunXjhhtuICwsjMaNGyfeinn55Zdp2bIlV1xxBXXq1PEq7iNHjtCoUSPGjBnDO++8k+Q6Tz31FM888wxNmjQhNva/Z2s7duzIunXrEhuLk3vv7rvvPqKjo6lbty7PP/88zZo18yo2X/JmqMrVQCiwQlVDRaQsznMFeX6oys2bNxMbG0udOnWIiYlh3759VK9e3d9hmRxq/fr11K1b199hZDsxMTE0bNiQ5cuXZ/gDVl6R1O9SSkNVenNr6JSqxgOxIlIUOABUznCkOVxsbCydO3dO7IVQsGBBSwLGZLIFCxZQt25dHnroIUsCPuRNr6FwESkOTMYpNREN/O3LoLKzzZs3U6NGDYKCgpg+fXqmFIkzxiStc+fO7Nixw99h5HopXhGI02z+mqoeVdWJQBfgdlXNk0NV/vPPP9StWzex4alt27ZUqFDBz1EZY0zGpJgI1GlA+N5jfruq+r+JO4slPAzWvHlzXnjhBa6++mo/R2SMMZnHmzaC5SLS3OeRZFNvvfUWDRs25MSJEwQEBDBs2DBKlizp77CMMSbTeNNG0BLoLyLbgZM4Txerqvr/cTgfSigS16pVK7Zu3WoPhRljcq1krwhEpIo72RWoDlwJXA9c537PlWJjY7nrrrsSy+K2bt2aCRMm+LQsrjFpFTZiPiFDvrvoK2zE/Aztd9++ffTp04caNWrQrFkzrrnmGjZu3Jjs+gl/F3v27KFnz56A85DVgw8+mKE4Ro8eneYy2L/++ivXXXdd4vy8efMICwujXr16NGnShMcffxw4vwx1Zmjd+r9CzJ6luCdOnMhHH32U5v0lFfdvv/120QNusbGxlC1blj179mT4HFK6IvgKaKqqO0Rktqr2yPDRcoCgoCBOnz7NmTM25ILJvpKrTpvccm+oKjfffDO33357Yj2gVatWsX//fmrXrp3ithUqVDiviqc3x1JVAgKS/iw6evRo+vfvn1g/KK3WrFnDgw8+yHfffUedOnWIi4tj0qRJ6dpXahYtWpQ4nZFS3LGxsfz7779Jxt22bVsiIyPZsWMHVatWBZyutfXr18+UDispJQLPeyG5uoP8gQMHePrppxkxYgQVK1Zk+vTpdivI+NVL36xl3Z7j6dr2lveS7t1dr0JRXri+frLbLVy4kHz58nHvvfcmLgsNDSU6OppOnTpx5MgRzp07x4gRI7jxxhvP23b79u1cd911rFmzBoBdu3bRoUMHdu/eTf/+/XnhhRfYvn07Xbt2pWXLlixbtozvv/+e119/naVLl3Lq1Cl69uzJSy+9xNixY9mzZw8dO3akVKlSLFy4kJ9++okXXniBM2fOUKNGDaZOnUrhwoX54YcfeOSRRyhYsCBt2rRJjOeNN97g2WefTXyiODAwkPvuu++ic548eTKTJk3i7Nmz1KxZk+nTp1OwYEFmzZrFSy+9RGBgIMWKFeP3339n7dq1DBw4kLNnzxIfH8/s2bOpVasWhQsXJjo6+rxS3M888wzr16+ncOHCPPHEE2zZsoUHHniAqKgoChYsyOTJk6lTpw4DBgwgODiYFStWcMUVV3Dw4MFk4+7duzczZszg6aefBmDGjBmJRfIyKqXGYk1mOtc5fvw4X331FYsXLwZs3FiTN61ZsybJcgfBwcF8+eWXLF++nIULF/L444+nOrDSkiVLmD17NhEREcyaNStxBK5NmzZx//33s3btWqpWrcorr7xCeHg4ERER/Pbbb0RERDB48GAqVKjAwoULWbhwIQcPHmTEiBEsWLCA5cuXExYWxttvv83p06e5++67+eabb1i2bNl5Yy4kdy4X6t69O0uXLmXVqlXUrVuXDz74AHAGrPnxxx9ZtWoVc+fOBWDixIk8/PDDiQPQVKpU6bx9zZ07N7Fw3y23nF+gedCgQYwbN45ly5YxatQo7r///sTXEkpcv/322ynG3bdv38QrtTNnzvD999/To0fm3KhJ6YogVESO41wZFHCn4b/G4qKZEoGf7Ny5k2+++YYHHniAmjVrsnPnzlQHizEmq6T0yR0gZMh3yb42857MHa9CVRk6dCi///47AQEB7N69m/3796dY2rpLly6Jveu6d+/On3/+yU033UTVqlW5/PLLE9f7/PPPmTRpErGxsezdu5d169ZdVJZ58eLFrFu3jiuuuAKAs2fP0qpVK/7991+qVatGrVq1AOjfv3+ab/8kV6r6iiuuYMCAAfTu3Zvu3bsD0KpVK1555RUiIyPp3r174nFTEx0dzaJFi84rQul569mzxHVKwsLCiI6OZsOGDaxfv56WLVsmWwY8rVIaszhQVYuqahFVDXKnE+a9SgIi0k1ENojIZhEZksTr+UVkpvv6PyISkoFzSZP33nuPIUOGJH6KsCRg8rr69euzbNmyi5Z/8sknREVFsWzZMlauXEnZsmU5ffp0ivu68Ko6Yd6znPO2bdsYNWoUP//8MxEREVx77bVJ7ldV6dKlCytXrmTlypWsW7cu8ZN7Ws/lQgMGDGD8+PGsXr2aF154IfH4EydOZMSIEezatYtmzZpx6NAh+vXrl/ip/5prruGXX35Jdf/glOQuXrx4YvwrV65k/fr1ia97viepxZ1wVZCZt4XA+zLUaSYigcAE4GqcEtZ9ReTCGrJ3AkdUtSbwDjDSV/EAbNiwIXGwjGHDhrF69WqvBuwwJrvxRXXaK6+8kjNnzpz3qToiIoIdO3ZQpkwZ8uXLx8KFC70q+TB//nwOHz7MqVOn+OqrrxI/zXs6fvw4hQoVolixYuzfv5958+YlvuZZ0vnyyy/nr7/+Shw68uTJk2zcuJE6deqwfft2tmzZAsBnn32WuP2TTz7Jq6++mtjjKT4+PrECqqfkSlVv2bKFli1bMnz4cEqXLs2uXbvYunUr1atXZ/Dgwdx4441el48uWrQo1apVY9asWYCT2FatWpXkuqnF3bdvXz7++GN++eWXi9ppMsKb5wjSqwWwWVW3AojIDOBGYJ3HOjcCL7rTXwDjRUQ0tRuQ6RAbG0vXrl2pWbMmCxYsoECBAoSEhGT2YYzJEuHDMr/4r4jw5Zdf8sgjjzBy5EiCg4MJCQnhxRdfZPDgwTRs2JCwsDCvSjq3aNGCHj16EBkZSf/+/QkLC2P79u3nrRMaGkqTJk2oU6cOlStXPi9ZDBo0iG7duiW2FUybNo2+ffsm3lIZMWIEtWvXZtKkSVx77bUULFiQtm3bJiaPRo0aMXr0aPr27UtMTAwicl7X0gQJpapLly5Ny5YtE7d/8skn2bRpE6pKp06dCA0NZeTIkUyfPp18+fJRrlw5hg4d6vV7+8knn3DfffcxYsQIzp07R58+fQgNDb1ovdTirlu3LoUKFaJZs2YpDgyUVqmWoU73jkV6At1U9S53/v+Alqr6oMc6a9x1It35Le46By/Y1yBgEECVKlWapbcI1Z9//kmNGjUoX758urY3xpesDLXJLL4oQ+13qjpJVcNUNax06dLp3k+bNm0sCRhjzAV8mQh2c/64BZXcZUmuIyJBQDHgkA9jMsYYcwFfJoKlQC0RqSYilwB9gLkXrDMXuN2d7gn84ov2AWNyCvv1NxmVnt8hnyUCVY0FHgR+BNYDn6vqWhEZLiIJYx5/AJQUkc3AY8BFXUyNySuCg4M5dOiQJQOTbqrKoUOHCA4OTtN2Pmss9pXsNGaxMZnp3LlzREZGptpH35iUBAcHU6lSJfLly3fe8pQai33ZfdQYkwb58uWjWrVq/g7D5EE5oteQMcYY37FEYIwxeZwlAmOMyeNyXGOxiEQB6Xu0GEoBB1NdK3exc84b7Jzzhoycc1VVTfKJ3ByXCDJCRMKTazXPreyc8wY757zBV+dst4aMMSaPs0RgjDF5XF5LBL4ZvTp7s3POG+yc8wafnHOeaiMwxhhzsbx2RWCMMeYClgiMMSaPy5WJQES6icgGEdksIhdVNBWR/CIy0339HxEJ8UOYmcqLc35MRNaJSISI/CwiVf0RZ2ZK7Zw91ushIioiOb6roTfnLCK93Z/1WhH5NKtjzGxe/G5XEZGFIrLC/f2+xh9xZhYRmSIiB9wRHJN6XURkrPt+RIhI0wwfVFVz1RcQCGwBqgOXAKuAehescz8w0Z3uA8z0d9xZcM4dgYLu9H154Zzd9YoAvwOLgTB/x50FP+dawArgUne+jL/jzoJzngTc507XA7b7O+4MnnM7oCmwJpnXrwHmAQJcDvyT0WPmxiuCFsBmVd2qqmeBGcCNF6xzI/ChO/0F0ElEJAtjzGypnrOqLlTVGHd2Mc6IcTmZNz9ngJeBkUBuqO3szTnfDUxQ1SMAqnogi2PMbN6cswJF3eliwJ4sjC/TqervwOEUVrkR+Egdi4HiIpKhMXhzYyKoCOzymI90lyW5jjoD6BwDSmZJdL7hzTl7uhPnE0VOluo5u5fMlVX1u6wMzIe8+TnXBmqLyF8islhEumVZdL7hzTm/CPQXkUjge+ChrAnNb9L6954qG48gjxGR/kAY0N7fsfiSiAQAbwMD/BxKVgvCuT3UAeeq73cRaaiqR/0ZlI/1Baap6lsi0gqYLiINVDXe34HlFLnximA3UNljvpK7LMl1RCQI53LyUJZE5xvenDMi0hl4FrhBVc9kUWy+kto5FwEaAL+KyHace6lzc3iDsTc/50hgrqqeU9VtwEacxJBTeXPOdwKfA6jq30AwTnG23Mqrv/e0yI2JYClQS0SqicglOI3Bcy9YZy5wuzvdE/hF3VaYHCrVcxaRJsB7OEkgp983hlTOWVWPqWopVQ1R1RCcdpEbVDUnj3Pqze/2VzhXA4hIKZxbRVuzMMbM5s057wQ6AYhIXZxEEJWlUWatucBtbu+hy4Fjqro3IzvMdbeGVDVWRB4EfsTpcTBFVdeKyHAgXFXnAh/gXD5uxmmU6eO/iDPOy3N+EygMzHLbxXeq6g1+CzqDvDznXMXLc/4RuEpE1gFxwJOqmmOvdr0858eBySLyKE7D8YCc/MFORD7DSeal3HaPF4B8AKo6Eacd5BpgMxADDMzwMXPw+2WMMSYT5MZbQ8YYY9LAEoExxuRxlgiMMSaPs0RgjDF5nCUCY4zJ4ywRZDNulcyPPeaDRCRKRL71Z1xpJSLb3X7siMiiVNYdICIV0rj/kOSqM2ZEevYrIr8m9aCaiNyQUC1TRF4UkSfc6eHuw32IyCMiUjCNxxMR+UVEirrzcSKyUkTWiMisdOyvgoh84U439qze6XkOvuD5e+JLyVX0FJFRInKlr4+f3VkiyH5OAg1EpIA734UMPjWYWdynsNNMVVunssoAIE2JIKNEJNDXx1DVuar6ehLLn1fVBe7sI0Ca/nHj9CFfparH3flTqtpYVRsAZ4F70xjnHlXt6c42dvef8FqS55ADTQOSqrs0DvBZosspLBFkT98D17rTfYHPEl4QkULup5slbv31G93lISLyh4gsd79au8s7uJ9YvxCRf0Xkk6QqrbrrjPH4ZNnCXf6iiEwXkb9wHsIrLSKzRWSp+3WFu15JEflJnBr47+OUyE3Yd7TH9NMislpEVonI6yLSE6f20SfusQuISDMR+U1ElonIj+JWVnSXrxKRVcADSb1x7vn+LiLfiVPDfqI4dYcQkWgRecvdvpU4YzSscb8e8dhNkPs+rXfft4Lu9s+757xGRCZd8D7+XxLv3QARGZ9EjNNEpKeIDMZJgAvFqad/h4iM9ljvbhF5J4nTvBX4OqnzB/4AaopICRH5Spx69YtFpJG7z/ZunCvd358i7u/OGnGe3B0O3OK+fkvCOYhIMRHZ4fFeFhKRXSKST0RqiMgP7s/rDxGpk8Q5FxaRqe7PPkJEeiSxzlfuPtaKyCB3WaD7fq1xt33UXT5Y/htfY0Yy70Wi5Cp6quoOoKSIlEttH7mav2tv29dFtcajgUY45bGDgZU4Txl+677+KtDfnS6OU0umEM6nymB3eS2cpy5xtz2GU48kAPgbaJPEcX8FJrvT7XBroeNUdlwGFHDnP03YHqgCrHenxwLPu9PX4jzhWSrhnNzvVwOL+G9chBIexw5zp/O565R252/BeZoUIAJo506/SRL12t3zPY1Tvz4QmA/0dF9ToLc73QxY7b53hYG1QBMgxF3vCne9KcATnvG609OB61N57wYA4z3ex4T9TPOIabvH+1QYp/Z+Pnd+EdAwiXPcARTx/J1xvwfhJIj7cD7pvuAuvxJY6U5/43Fuhd1tQpKKOYlz+Bro6PFzed+d/hmo5U63xCnZcmHMI4HRHvOXJnH+Cb8PBYA1OBWBmwHzPbYr7n7fA+S/YFlYQkzJ/G0lnucFyycDPfz9t+/PL7siyIZUNQLnl7YvztWBp6uAISKyEucfUDDOP+R8OI/ZrwZm4QzQkWCJqkaqU41xpbvvpHzmHv93oKiIFHeXz1XVU+50Z2C8e/y57nqFcf4Bfuxu/x1wJIn9dwamqjsugqomVXP9MpxicfPdYwwDKrmxFHdjA+cfcXKWqFO/Ps49pzbu8jhgtjvdBvhSVU+qajQwB2jrvrZLVf9ypz/22L6jOCParcb551rf45jJvXdec+P4BbjO/VSdT1VXJ7FqCVU94TFfwH2vwnHq7nzgxjzd3e8vOJ96iwJ/AW+7VyPF1SnD7q2ZOAkA3AGd3J99a5zSJStx6lklVRu/MzDB41yT+v0Y7F6tLcYpqlYLp05SdREZJ05J7YTbYRE4V5H9gVh3n+GqelcazifBAbL41mR2k+tqDeUic4FROJ9wPcdKEJxPLxs8VxaRF4H9QCjOJ3/PgVg8K43GkfzP/cJ6IwnzJz2WBQCXq+p5A71I5o3rI8BaVW11wf6Lp2EfyZ3HaTc5pHl7EQkG/odz5bLLfb+DvThmWr0PDAX+BaYms06siATof2WWT6lqY88Vkvt5qOrrIvIdTjvAXyLSFe8H7ZkLvCoiJXA+qf+Cc0V19MLjp5WIdMBJFq1UNUZEfsW5wj0iIqFAV5y2j97AHThXne2A64FnxSm1nZak5ikYOJXqWrmYXRFkX1OAl5L4RPgj8FDC/WlxqoqCU0p7r/vP4f9wbouk1S3uPtvgVDQ8lsQ6P+Ex8IeINHYnfwf6ucuuBi5NYtv5wECPe+4l3OUncMpGA2wASotTVx73HnR9derpH3VjA+c+eXJaiFOtMsA9pz+TWOcP4CYRKSgihYCb3WUAVRKO757Tn/z3T/+g+ym4J+fz5r1Liue5o6r/4Hwa7odH29AFNuDc+krJH7jvkftP9qCqHheRGqq6WlVH4lT2vPB+/nnxeHKvWJYCY3BuVcap02C9TUR6uccS9x/3hebj0a4jIhf+fhQDjrhJoA5O2fCECqoBqjob5+qwqftzrayqC4Gn3W0Lp/J+pKQ2zq2oPMsSQTbl3soZm8RLL+PcBooQkbXuPDifVm93L63rcP6neG+dFpEVwEScGu9JGQyEuY106/ivh8pLQDs3pu44tyguPKcfcD5Vhru3EZ5wX5oGTHSXBeL8kx3pnstKnFsP4FRZnOCul9IlyFJgPLAe2AZ8mUQsy93jLgH+wbm3vMJ9eQPwgIisx0lo77qJaDLOP4wf3WN48ua9S8ok4AcRWeix7HPgr2RunwB8h1tqOgUvAs1EJAJ4nf/Krj/iNrxGAOe4eKS6hUC9hMbiJPY7E+jvfk9wK3Cn+/NaS9JDho4ALnWPvQpnDG1PP+A00q93413sLq+IM6bESpzbdM/g/I587N6iWwGMVdWjIhImTkeFi4hT0fNv4DIRiRSRO93l+YCaOLfV8iyrPmoAp9cQTmNmjv6DcD/9PqGq1/k5lHQT55mRd1T152ReL48zZm2XrI0s9xGRm4Gmqvqcv2PxJ7siMCabEJHiIrIR555/kkkAQJ1BSCa7jb8mY4KAt/wdhL/ZFYExxuRxdkVgjDF5nCUCY4zJ4ywRGGNMHmeJwBhj8jhLBMYYk8f9P0DSTkZ+PfWaAAAAAElFTkSuQmCC","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"}],"source":["disp = CalibrationDisplay.from_estimator(calibrated, X_valid, y_valid)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["from sklearn.model_selection import cross_val_score\n","from hyperopt import fmin, tpe, hp, STATUS_OK, Trials\n","from hyperopt.pyll import scope\n","import time \n","\n","def hyperopt(param_space, model, X_train, y_train, X_test, y_test, num_eval):\n","    \n","    start = time.time()\n","    \n","    def objective_function(params):\n","        clf = model(**params)\n","        cv = RepeatedStratifiedKFold(n_splits=8, n_repeats=3, random_state=100)\n","        n_scores = cross_val_score(clf, X_train, y_train, scoring='f1_macro', cv=cv, n_jobs=-1, verbose=100, error_score='raise')\n","        print('Macro-F1: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))\n","        clf.fit(X_train, y_train)\n","        result = score(y_test, clf.predict(X_test))\n","        return {'loss': -result, 'status': STATUS_OK}\n","\n","    trials = Trials()\n","    best_param = fmin(objective_function, \n","                      param_space, \n","                      algo=tpe.suggest, \n","                      max_evals=num_eval, \n","                      trials=trials)\n","    loss = [x['result']['loss'] for x in trials.trials]\n","    \n","    print(\"\")\n","    print(\"##### Results\")\n","    print(\"Score best parameters: \", min(loss)*-1)\n","    print(\"Best parameters: \", best_param)\n","    print(\"Time elapsed: \", time.time() - start)\n","    print(\"Parameter combinations evaluated: \", num_eval)\n","    \n","    return trials, best_param"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["LGBM_params = {\n","    'boosting_type': hp.choice('boosting_type', ['gbdt', 'dart']),\n","    'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(1)),\n","    'max_depth': scope.int(hp.quniform('max_depth', 5, 50, 1)),\n","    'num_leaves': scope.int(hp.quniform('num_leaves', 5, 50, 1)),\n","\n","    'colsample_bytree': hp.uniform('colsample_bytree', 0.4, 1.0),\n","    'subsample': hp.uniform('subsample', 0.4, 1.0),\n","\n","    'min_child_samples': scope.int(hp.quniform('min_child_samples', 10, 200, 10)),\n","    'min_child_weight': hp.loguniform('min_child_weight', np.log(1e-3), np.log(1)),\n","    'min_split_gain': hp.loguniform('min_split_gain', np.log(1e-8), np.log(1)),\n","    \n","    'max_delta_step': hp.uniform('min_delta_step', 0, 2),\n","    'n_estimators': scope.int(hp.quniform('n_estimators', 5, 100, 1)),\n","    # 'scale_pos_weight':hp.uniform('scale_pos_weight', 1.0, 5.0),\n","    # 'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),\n","    # 'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),\n","    'random_state': hp.choice('random_state', ['100']),\n","    'n_jobs': hp.choice('n_jobs', ['-1'])\n","}\n","# LGBM(boosting_type='gbdt', num_leaves=40, max_depth=-1, learning_rate=0.2, n_estimators=100, class_weight='balanced', reg_lambda=0, random_state=100, reg_alpha=0, n_jobs=-1)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  3.5min\n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  3.5min remaining: 38.3min\n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  3.5min remaining: 24.4min\n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  3.5min remaining: 17.5min\n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  3.5min remaining: 13.4min\n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  3.5min remaining: 10.6min\n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  3.5min remaining:  8.6min\n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  3.5min remaining:  7.1min\n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  3.5min remaining:  5.9min\n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  3.6min remaining:  5.1min\n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  3.7min remaining:  4.3min\n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  3.7min remaining:  3.7min\n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  6.8min remaining:  5.7min\n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  6.8min remaining:  4.8min\n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  6.8min remaining:  4.1min\n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  6.8min remaining:  3.4min\n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  6.9min remaining:  2.8min\n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  6.9min remaining:  2.3min\n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  6.9min remaining:  1.8min\n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  6.9min remaining:  1.4min\n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  6.9min remaining:   59.3s\n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  6.9min remaining:   37.7s\n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  7.0min remaining:    0.0s\n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  7.0min finished\n","\n","Macro-F1: 0.500 (0.007)                               \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree\n","Accuracy: 0.45782431646305993                         \n","Macro-F1 score: 0.42792317976141503                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  2.2min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  2.2min remaining: 24.4min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  2.2min remaining: 15.7min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  2.2min remaining: 11.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  2.2min remaining:  8.6min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  2.3min remaining:  6.8min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  2.3min remaining:  5.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  2.3min remaining:  4.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  2.3min remaining:  3.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.3min remaining:  3.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.3min remaining:  2.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.3min remaining:  2.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  4.5min remaining:  3.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  4.5min remaining:  3.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  4.6min remaining:  2.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  4.6min remaining:  2.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  4.6min remaining:  1.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  4.6min remaining:  1.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  4.6min remaining:  1.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  4.6min remaining:   55.6s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  4.6min remaining:   39.7s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  4.7min remaining:   25.3s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.7min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.7min finished               \n","\n","Macro-F1: 0.754 (0.009)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.6614310645724258                                                         \n","Macro-F1 score: 0.6443235856247369                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.8min                       \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.8min remaining: 20.3min    \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.9min remaining: 13.0min    \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.9min remaining:  9.4min    \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.9min remaining:  7.1min    \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.9min remaining:  5.7min    \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.9min remaining:  4.6min    \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.9min remaining:  3.9min    \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.9min remaining:  3.2min    \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.0min remaining:  2.7min    \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.0min remaining:  2.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.0min remaining:  2.0min    \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.5min remaining:  2.9min    \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.6min remaining:  2.6min    \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.7min remaining:  2.2min    \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.7min remaining:  1.8min    \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.7min remaining:  1.5min    \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.7min remaining:  1.2min    \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.7min remaining:   58.6s    \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.7min remaining:   44.7s    \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.7min remaining:   31.9s    \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.8min remaining:   20.4s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.8min remaining:    0.0s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.8min finished              \n","\n","Macro-F1: 0.714 (0.007)                                                             \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                           \n","Accuracy: 0.6218731820826061                                                        \n","Macro-F1 score: 0.612381183653646                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.        \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.9min                       \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.9min remaining: 20.5min    \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.9min remaining: 13.1min    \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.9min remaining:  9.4min    \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.9min remaining:  7.2min    \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.9min remaining:  5.7min    \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.9min remaining:  4.6min    \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.9min remaining:  3.8min    \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.9min remaining:  3.2min    \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.0min remaining:  2.7min    \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.0min remaining:  2.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.0min remaining:  2.0min    \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.7min remaining:  3.1min    \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.7min remaining:  2.6min    \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.7min remaining:  2.2min    \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.7min remaining:  1.9min    \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.7min remaining:  1.5min    \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.8min remaining:  1.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.8min remaining:   59.6s    \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.8min remaining:   45.3s    \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.8min remaining:   32.5s    \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.8min remaining:   20.9s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.9min remaining:    0.0s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.9min finished              \n","\n","Macro-F1: 0.690 (0.007)                                                             \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                           \n","Accuracy: 0.6259453170447935                                                        \n","Macro-F1 score: 0.6200569579243529                                                  \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.        \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.9min                       \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.9min remaining: 21.4min    \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.9min remaining: 13.6min    \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  2.0min remaining:  9.8min    \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  2.0min remaining:  7.4min    \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  2.0min remaining:  5.9min    \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  2.0min remaining:  4.8min    \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  2.0min remaining:  4.0min    \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  2.0min remaining:  3.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.0min remaining:  2.8min    \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.0min remaining:  2.4min    \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.1min remaining:  2.1min    \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.9min remaining:  3.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.9min remaining:  2.8min    \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.9min remaining:  2.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.9min remaining:  2.0min    \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.9min remaining:  1.6min    \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.9min remaining:  1.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.9min remaining:  1.0min    \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  4.0min remaining:   47.4s    \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  4.0min remaining:   34.0s    \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  4.0min remaining:   21.6s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.0min remaining:    0.0s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.0min finished              \n","\n","Macro-F1: 0.475 (0.008)                                                             \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                           \n","Accuracy: 0.4316463059918557                                                        \n","Macro-F1 score: 0.38656588033114114                                                 \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.        \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.6min                       \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.6min remaining: 17.6min    \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.6min remaining: 11.3min    \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.6min remaining:  8.0min    \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.6min remaining:  6.1min    \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.6min remaining:  4.8min    \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.6min remaining:  3.9min    \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.6min remaining:  3.2min    \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.6min remaining:  2.7min    \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.6min remaining:  2.3min    \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.6min remaining:  1.9min    \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.6min remaining:  1.6min    \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.2min remaining:  2.7min    \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.3min remaining:  2.4min    \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.3min remaining:  2.0min    \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.3min remaining:  1.7min    \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.3min remaining:  1.4min    \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.3min remaining:  1.1min    \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.3min remaining:   52.5s    \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.3min remaining:   39.9s    \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.3min remaining:   28.5s    \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.3min remaining:   18.1s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.3min remaining:    0.0s    \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.3min finished              \n","\n","Macro-F1: 0.363 (0.006)                                                             \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                           \n","Accuracy: 0.3787085514834206                                                        \n","Macro-F1 score: 0.29542691351201994                                                 \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.        \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.4min                     \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.4min remaining: 15.9min  \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.4min remaining: 10.1min  \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.4min remaining:  7.3min  \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.5min remaining:  5.5min  \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.5min remaining:  4.4min  \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.5min remaining:  3.5min  \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.5min remaining:  2.9min  \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.5min remaining:  2.4min  \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.5min remaining:  2.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.5min remaining:  1.7min  \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.5min remaining:  1.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  2.9min remaining:  2.4min  \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  2.9min remaining:  2.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  2.9min remaining:  1.7min  \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  2.9min remaining:  1.4min  \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  2.9min remaining:  1.2min  \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  2.9min remaining:   57.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  2.9min remaining:   45.7s  \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  2.9min remaining:   34.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  2.9min remaining:   24.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  2.9min remaining:   15.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.9min remaining:    0.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.9min finished            \n","\n","Macro-F1: 0.617 (0.006)                                                           \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                         \n","Accuracy: 0.5293775450843514                                                      \n","Macro-F1 score: 0.5248390955524873                                                \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.      \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  2.1min                     \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  2.1min remaining: 22.8min  \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  2.1min remaining: 14.5min  \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  2.1min remaining: 10.4min  \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  2.1min remaining:  7.9min  \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  2.1min remaining:  6.3min  \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  2.1min remaining:  5.1min  \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  2.1min remaining:  4.2min  \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  2.1min remaining:  3.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.1min remaining:  3.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.1min remaining:  2.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.2min remaining:  2.2min  \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  4.1min remaining:  3.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  4.1min remaining:  2.9min  \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  4.2min remaining:  2.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  4.2min remaining:  2.1min  \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  4.2min remaining:  1.7min  \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  4.2min remaining:  1.4min  \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  4.2min remaining:  1.1min  \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  4.2min remaining:   50.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  4.2min remaining:   35.9s  \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  4.2min remaining:   22.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.2min remaining:    0.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  4.2min finished            \n","\n","Macro-F1: 0.419 (0.009)                                                           \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                         \n","Accuracy: 0.41535776614310643                                                     \n","Macro-F1 score: 0.3617944429852414                                                \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.      \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.5min                     \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.5min remaining: 16.9min  \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.5min remaining: 10.8min  \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.5min remaining:  7.7min  \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.5min remaining:  5.9min  \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.5min remaining:  4.6min  \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.6min remaining:  3.8min  \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.6min remaining:  3.1min  \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.6min remaining:  2.6min  \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.6min remaining:  2.2min  \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.6min remaining:  1.9min  \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.6min remaining:  1.6min  \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.0min remaining:  2.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.0min remaining:  2.2min  \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.1min remaining:  1.8min  \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.1min remaining:  1.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.1min remaining:  1.3min  \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.1min remaining:  1.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.1min remaining:   48.5s  \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.1min remaining:   36.9s  \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.1min remaining:   26.5s  \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.1min remaining:   16.8s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.1min remaining:    0.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.1min finished            \n","\n","Macro-F1: 0.335 (0.003)                                                           \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                         \n","Accuracy: 0.3624200116346713                                                      \n","Macro-F1 score: 0.2660119555935098                                                \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.      \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.2min                     \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.2min remaining: 13.5min  \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.2min remaining:  8.6min  \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.2min remaining:  6.2min  \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.2min remaining:  4.7min  \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.3min remaining:  3.8min  \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.3min remaining:  3.0min  \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.3min remaining:  2.5min  \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.3min remaining:  2.1min  \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.3min remaining:  1.8min  \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.3min remaining:  1.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.3min remaining:  1.3min  \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  2.4min remaining:  2.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  2.5min remaining:  1.8min  \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  2.5min remaining:  1.5min  \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  2.5min remaining:  1.2min  \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  2.5min remaining:  1.0min  \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  2.5min remaining:   50.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  2.5min remaining:   39.5s  \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  2.5min remaining:   30.1s  \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  2.5min remaining:   21.5s  \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  2.5min remaining:   13.6s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.5min remaining:    0.0s  \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.5min finished            \n","\n","Macro-F1: 0.500 (0.007)                                                           \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                         \n","Accuracy: 0.4584060500290867                                                      \n","Macro-F1 score: 0.4311922333961832                                                \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.       \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  2.3min                      \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  2.3min remaining: 25.8min   \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  2.4min remaining: 16.5min   \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  2.4min remaining: 11.8min   \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  2.4min remaining:  9.0min   \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  2.4min remaining:  7.1min   \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  2.4min remaining:  5.7min   \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  2.4min remaining:  4.7min   \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  2.4min remaining:  4.0min   \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  2.4min remaining:  3.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.4min remaining:  2.8min   \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.4min remaining:  2.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  4.6min remaining:  3.9min   \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  4.7min remaining:  3.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  4.7min remaining:  2.8min   \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  4.7min remaining:  2.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  4.7min remaining:  1.9min   \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  4.7min remaining:  1.6min   \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  4.7min remaining:  1.2min   \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  4.8min remaining:   56.9s   \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  4.8min remaining:   40.7s   \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  4.8min remaining:   25.8s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  5.1min remaining:    0.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  5.1min finished             \n","\n","Macro-F1: 0.502 (0.008)                                                            \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                          \n","Accuracy: 0.45782431646305993                                                      \n","Macro-F1 score: 0.4319313881781615                                                 \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.       \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.6min                      \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.6min remaining: 17.6min   \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.6min remaining: 11.2min   \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.6min remaining:  8.0min   \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.6min remaining:  6.1min   \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.6min remaining:  4.9min   \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.7min remaining:  4.0min   \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.7min remaining:  3.4min   \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.7min remaining:  2.8min   \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.7min remaining:  2.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.8min remaining:  2.1min   \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.1min remaining:  2.1min   \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.0min remaining:  2.5min   \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.0min remaining:  2.2min   \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.0min remaining:  1.8min   \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.0min remaining:  1.5min   \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.0min remaining:  1.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.1min remaining:  1.0min   \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.1min remaining:   48.9s   \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.1min remaining:   37.3s   \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.1min remaining:   26.7s   \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.2min remaining:   17.1s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.4min remaining:    0.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.4min finished             \n","\n","Macro-F1: 0.647 (0.008)                                                            \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                          \n","Accuracy: 0.5474112856311809                                                       \n","Macro-F1 score: 0.5437972787110719                                                 \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.       \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.0min                      \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.1min remaining: 12.2min   \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.1min remaining:  7.8min   \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.2min remaining:  5.8min   \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.2min remaining:  4.4min   \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.2min remaining:  3.5min   \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.2min remaining:  3.0min   \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.2min remaining:  2.5min   \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.3min remaining:  2.2min   \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.5min remaining:  2.1min   \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  2.0min remaining:  2.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.2min remaining:  2.2min   \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  2.3min remaining:  1.9min   \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  2.3min remaining:  1.6min   \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  2.3min remaining:  1.4min   \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  2.4min remaining:  1.2min   \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  2.4min remaining:   59.8s   \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  2.4min remaining:   48.4s   \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  2.4min remaining:   38.3s   \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  2.5min remaining:   29.4s   \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  2.6min remaining:   22.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  2.8min remaining:   15.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.1min remaining:    0.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.1min finished             \n","\n","Macro-F1: 0.424 (0.008)                                                            \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                          \n","Accuracy: 0.41128563118091915                                                      \n","Macro-F1 score: 0.35372866848552936                                                \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.       \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.6min                      \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.6min remaining: 17.6min   \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.6min remaining: 11.3min   \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.6min remaining:  8.1min   \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.6min remaining:  6.2min   \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.6min remaining:  4.9min   \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.6min remaining:  3.9min   \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.6min remaining:  3.3min   \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.7min remaining:  2.8min   \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.8min remaining:  2.5min   \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.9min remaining:  2.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  2.0min remaining:  2.0min   \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.2min remaining:  2.7min   \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.2min remaining:  2.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.2min remaining:  1.9min   \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.2min remaining:  1.6min   \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.3min remaining:  1.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.3min remaining:  1.1min   \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.3min remaining:   51.7s   \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.3min remaining:   39.3s   \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.3min remaining:   28.2s   \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.4min remaining:   18.3s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.5min remaining:    0.0s   \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.5min finished             \n","\n","Macro-F1: 0.604 (0.008)                                                            \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                          \n","Accuracy: 0.5287958115183246                                                       \n","Macro-F1 score: 0.5236304791099311                                                 \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.       \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.2min                      \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.2min remaining: 13.2min   \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.2min remaining:  8.4min   \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.2min remaining:  6.1min   \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.2min remaining:  4.6min   \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.2min remaining:  3.7min   \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.2min remaining:  3.0min   \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.2min remaining:  2.5min   \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.2min remaining:  2.1min   \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.2min remaining:  1.7min   \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.2min remaining:  1.5min   \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.3min remaining:  1.3min   \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  2.2min remaining:  1.9min   \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  2.3min remaining:  1.6min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  2.3min remaining:  1.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  2.3min remaining:  1.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  2.3min remaining:   57.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  2.3min remaining:   46.2s     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  2.3min remaining:   36.6s     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  2.3min remaining:   27.8s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  2.3min remaining:   19.9s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  2.3min remaining:   12.6s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.4min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.4min finished               \n","\n","Macro-F1: 0.420 (0.007)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.40430482838859805                                                        \n","Macro-F1 score: 0.3460653720644091                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.6min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.7min remaining: 18.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.7min remaining: 11.8min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.7min remaining:  8.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.7min remaining:  6.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.7min remaining:  5.1min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.7min remaining:  4.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.7min remaining:  3.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.8min remaining:  2.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.8min remaining:  2.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.8min remaining:  2.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.8min remaining:  1.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.3min remaining:  2.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.3min remaining:  2.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.4min remaining:  2.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.4min remaining:  1.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.4min remaining:  1.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.4min remaining:  1.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.5min remaining:   54.4s     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.5min remaining:   41.4s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.5min remaining:   30.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.5min remaining:   19.2s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.6min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.6min finished               \n","\n","Macro-F1: 0.405 (0.008)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.401977894124491                                                          \n","Macro-F1 score: 0.3399406551243971                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  4.0min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  4.0min remaining: 44.4min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  4.0min remaining: 28.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  4.1min remaining: 20.4min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  4.1min remaining: 15.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  4.1min remaining: 12.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  4.1min remaining: 10.0min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  4.1min remaining:  8.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  4.1min remaining:  6.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  4.1min remaining:  5.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  4.2min remaining:  4.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  4.2min remaining:  4.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  7.6min remaining:  6.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  7.7min remaining:  5.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  7.8min remaining:  4.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  7.8min remaining:  3.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  7.8min remaining:  3.2min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  7.8min remaining:  2.6min     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  7.8min remaining:  2.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  7.8min remaining:  1.6min     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  7.9min remaining:  1.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  7.9min remaining:   42.8s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  7.9min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  7.9min finished               \n","\n","Macro-F1: 0.644 (0.009)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.5491564863292612                                                         \n","Macro-F1 score: 0.5454126679462572                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.6min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.6min remaining: 17.9min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.6min remaining: 11.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.6min remaining:  8.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.6min remaining:  6.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.7min remaining:  5.0min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.7min remaining:  4.0min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.7min remaining:  3.3min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.7min remaining:  2.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.7min remaining:  2.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.7min remaining:  2.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.7min remaining:  1.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.2min remaining:  2.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.2min remaining:  2.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.3min remaining:  2.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.3min remaining:  1.6min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.3min remaining:  1.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.3min remaining:  1.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.3min remaining:   52.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.3min remaining:   39.6s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.3min remaining:   28.3s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.3min remaining:   18.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.3min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.3min finished               \n","\n","Macro-F1: 0.687 (0.008)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.5817335660267597                                                         \n","Macro-F1 score: 0.5796161120950234                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.7min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.7min remaining: 18.4min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.7min remaining: 11.9min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.7min remaining:  8.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.7min remaining:  6.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.7min remaining:  5.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.7min remaining:  4.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.7min remaining:  3.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.8min remaining:  2.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.8min remaining:  2.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.8min remaining:  2.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.8min remaining:  1.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  3.2min remaining:  2.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  3.3min remaining:  2.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  3.4min remaining:  2.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  3.4min remaining:  1.7min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  3.4min remaining:  1.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  3.4min remaining:  1.1min     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  3.4min remaining:   54.3s     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  3.5min remaining:   41.4s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  3.5min remaining:   29.6s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  3.5min remaining:   18.8s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.5min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  3.5min finished               \n","\n","Macro-F1: 0.426 (0.008)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.40372309482257124                                                        \n","Macro-F1 score: 0.3466658163066665                                                   \n","[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.         \n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:  1.2min                        \n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed:  1.2min remaining: 13.5min     \n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed:  1.3min remaining:  8.9min     \n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed:  1.3min remaining:  6.4min     \n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed:  1.3min remaining:  4.9min     \n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed:  1.3min remaining:  3.9min     \n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed:  1.3min remaining:  3.2min     \n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed:  1.3min remaining:  2.6min     \n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed:  1.3min remaining:  2.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed:  1.4min remaining:  1.9min     \n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed:  1.4min remaining:  1.6min     \n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed:  1.4min remaining:  1.4min     \n","\n","[Parallel(n_jobs=-1)]: Done  13 out of  24 | elapsed:  2.4min remaining:  2.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  14 out of  24 | elapsed:  2.5min remaining:  1.8min     \n","\n","[Parallel(n_jobs=-1)]: Done  15 out of  24 | elapsed:  2.5min remaining:  1.5min     \n","\n","[Parallel(n_jobs=-1)]: Done  16 out of  24 | elapsed:  2.5min remaining:  1.3min     \n","\n","[Parallel(n_jobs=-1)]: Done  17 out of  24 | elapsed:  2.5min remaining:  1.0min     \n","\n","[Parallel(n_jobs=-1)]: Done  18 out of  24 | elapsed:  2.6min remaining:   51.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  19 out of  24 | elapsed:  2.6min remaining:   40.4s     \n","\n","[Parallel(n_jobs=-1)]: Done  20 out of  24 | elapsed:  2.6min remaining:   30.8s     \n","\n","[Parallel(n_jobs=-1)]: Done  21 out of  24 | elapsed:  2.6min remaining:   22.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  22 out of  24 | elapsed:  2.6min remaining:   14.1s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.6min remaining:    0.0s     \n","\n","[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  2.6min finished               \n","\n","Macro-F1: 0.428 (0.012)                                                              \n","[LightGBM] [Warning] Unknown parameter: colsample_by_tree                            \n","Accuracy: 0.4083769633507853                                                         \n","Macro-F1 score: 0.35176500993549253                                                  \n","100%|██████████| 20/20 [1:24:03<00:00, 252.15s/trial, best loss: -0.6443235856247369]\n","\n","##### Results\n","Score best parameters:  0.6443235856247369\n","Best parameters:  {'boosting_type': 0, 'colsample_by_tree': 0.8360954191582883, 'learning_rate': 0.17292549690407305, 'max_depth': 34.0, 'min_child_samples': 160.0, 'n_estimators': 94.0, 'n_jobs': 0, 'num_leaves': 17.0, 'random_state': 0, 'scale_pos_weight': 1.2405835005274635, 'subsample': 0.7206617671584264}\n"]},{"ename":"NameError","evalue":"name 'clf_best' is not defined","output_type":"error","traceback":["\u001b[1;31m---------------------------------------------------------------------------\u001b[0m","\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[1;32mc:\\Users\\issac\\Desktop\\SUTD\\School Year\\Year 2\\Term 5\\50.007 Machine Learning\\Project\\work\\2022-50-007-ml-project.ipynb Cell 249\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=0'>1</a>\u001b[0m iterations, best_params \u001b[39m=\u001b[39m hyperopt(LGBM_params, LGBM,  X_res, y_res, X_trans, y_valid, \u001b[39m20\u001b[39;49m)\n","\u001b[1;32mc:\\Users\\issac\\Desktop\\SUTD\\School Year\\Year 2\\Term 5\\50.007 Machine Learning\\Project\\work\\2022-50-007-ml-project.ipynb Cell 249\u001b[0m in \u001b[0;36mhyperopt\u001b[1;34m(param_space, model, X_train, y_train, X_test, y_test, num_eval)\u001b[0m\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=28'>29</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mScore best parameters: \u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mmin\u001b[39m(loss)\u001b[39m*\u001b[39m\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=29'>30</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mBest parameters: \u001b[39m\u001b[39m\"\u001b[39m, best_param)\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=30'>31</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mTest Score: \u001b[39m\u001b[39m\"\u001b[39m, score(y_test, clf_best\u001b[39m.\u001b[39mpredict(X_test)))\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=31'>32</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mTime elapsed: \u001b[39m\u001b[39m\"\u001b[39m, time\u001b[39m.\u001b[39mtime() \u001b[39m-\u001b[39m start)\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/issac/Desktop/SUTD/School%20Year/Year%202/Term%205/50.007%20Machine%20Learning/Project/work/2022-50-007-ml-project.ipynb#ch0000248?line=32'>33</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mParameter combinations evaluated: \u001b[39m\u001b[39m\"\u001b[39m, num_eval)\n","\u001b[1;31mNameError\u001b[0m: name 'clf_best' is not defined"]}],"source":["iterations, best_params = hyperopt(LGBM_params, LGBM,  X_res, y_res, X_trans, y_valid, 20)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["NuSVC_params = {\n","    'kernel': hp.choice('kernel', ['rbf', 'linear', 'sigmoid']),\n","    'nu': hp.uniform('nu', 0.1, 1.0),\n","    'gamma': hp.uniform('gamma', 1e-2, 1),\n","    'class_weight': hp.choice('class_weight', [None, 'balanced']),\n","    'max_iter': hp.choice('max_iter', [100, 200]),\n","    'random_state': hp.choice('random_state', [-1, 1000]),\n","}"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n","\n","[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 10.8min\n","\n","[Parallel(n_jobs=-1)]: Done   2 out of  24 | elapsed: 10.9min remaining: 120.4min\n","\n","[Parallel(n_jobs=-1)]: Done   3 out of  24 | elapsed: 10.9min remaining: 76.6min\n","\n","[Parallel(n_jobs=-1)]: Done   4 out of  24 | elapsed: 11.0min remaining: 54.8min\n","\n","[Parallel(n_jobs=-1)]: Done   5 out of  24 | elapsed: 11.0min remaining: 41.6min\n","\n","[Parallel(n_jobs=-1)]: Done   6 out of  24 | elapsed: 11.0min remaining: 32.9min\n","\n","[Parallel(n_jobs=-1)]: Done   7 out of  24 | elapsed: 11.0min remaining: 26.7min\n","\n","[Parallel(n_jobs=-1)]: Done   8 out of  24 | elapsed: 11.1min remaining: 22.1min\n","\n","[Parallel(n_jobs=-1)]: Done   9 out of  24 | elapsed: 11.1min remaining: 18.5min\n","\n","[Parallel(n_jobs=-1)]: Done  10 out of  24 | elapsed: 11.1min remaining: 15.5min\n","\n","[Parallel(n_jobs=-1)]: Done  11 out of  24 | elapsed: 11.1min remaining: 13.1min\n","\n","[Parallel(n_jobs=-1)]: Done  12 out of  24 | elapsed: 11.5min remaining: 11.5min\n","\n","  0%|          | 0/2 [11:31<?, ?trial/s, best loss=?]"]}],"source":["iterations, best_params = hyperopt(NuSVC_params, NuSVC,  X_res, y_res, X_trans, y_valid, 2)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]},{"cell_type":"markdown","metadata":{},"source":["END"]}],"metadata":{"kernelspec":{"display_name":"Python 3.10.5 64-bit","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.5"},"vscode":{"interpreter":{"hash":"d81bf16a70e6e7c42043031d5c6c9e6d55e73e0e666fe713b30c2db86836c05c"}}},"nbformat":4,"nbformat_minor":4}