transformer-model.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Todos\n\n- Save model\n- check for other necessray callbacks\n- run version 8 with 16 shift\n- Embedding\n    - evaluate embedding\n    - run new embedding version on full asset data\n    - search an try alternatives\n        - add sin / cos signals\n- Check for other LR scheduler\n- Read for decoder necessety ","metadata":{}},{"cell_type":"markdown","source":"## Kernel","metadata":{}},{"cell_type":"code","source":"from IPython.core.display import display, HTML, Javascript\nimport os\nif not os.path.exists(\"../input/g-research-crypto-forecasting/\"): os.chdir('/t/Datasets/kaggle_crypto/internal')\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":0.072966,"end_time":"2021-11-29T18:05:23.845682","exception":false,"start_time":"2021-11-29T18:05:23.772716","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:47:22.574372Z","iopub.execute_input":"2022-02-16T23:47:22.574947Z","iopub.status.idle":"2022-02-16T23:47:22.599286Z","shell.execute_reply.started":"2022-02-16T23:47:22.574856Z","shell.execute_reply":"2022-02-16T23:47:22.598380Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport traceback\nimport pdb\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport gresearch_crypto\nimport tensorflow as tf\nfrom tensorflow import keras\nimport matplotlib.pyplot as plt\nimport pandas as pd, numpy as np\nfrom scipy.stats import pearsonr\nfrom tensorflow.keras import layers\nfrom sklearn.metrics import r2_score\nimport tensorflow_probability as tfp\nimport tensorflow.keras.backend as K\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import RobustScaler\nfrom tensorflow.keras.preprocessing.sequence import TimeseriesGenerator\nfrom sklearn.metrics import mean_absolute_error as mae\nfrom sklearn.preprocessing import MinMaxScaler\n\npd.set_option('display.max_columns', None)","metadata":{"papermill":{"duration":9.261664,"end_time":"2021-11-29T18:05:34.914352","exception":false,"start_time":"2021-11-29T18:05:25.652688","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:47:22.622087Z","iopub.execute_input":"2022-02-16T23:47:22.622821Z","iopub.status.idle":"2022-02-16T23:47:29.905893Z","shell.execute_reply.started":"2022-02-16T23:47:22.622765Z","shell.execute_reply":"2022-02-16T23:47:29.904988Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"GPU\" #or \"TPU\"\n\nSEED = 42\n\n# LOAD STRICT? YES=1 NO=0 | see: https://www.kaggle.com/julian3833/proposal-for-a-meaningful-lb-strict-lgbm\nLOAD_STRICT = True\n\n# WHICH YEARS TO INCLUDE? YES=1 NO=0\nINC2021 = 0\nINC2020 = 0\nINC2019 = 0\nINC2018 = 0\nINC2017 = 0\nINCCOMP = 1\nINCSUPP = 0\n\n# TRAINING PARAMETERS\nDEBUG = True\nSINGLE_ASSET = True\nasset_id=3\nN_ASSETS = 14\nnum_shift = 5\nTIME2VEC_DIM=3\nWINDOW_SIZE = 64\nprediction_length = 1\nBATCH_SIZE = 64\nPCT_VALIDATION = 10 # last 10% of the data are used as validation set\nPCT_TEST = 10 # last 10% of the data are used as validation set","metadata":{"papermill":{"duration":0.050546,"end_time":"2021-11-29T18:05:35.179716","exception":false,"start_time":"2021-11-29T18:05:35.12917","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:47:29.907865Z","iopub.execute_input":"2022-02-16T23:47:29.908177Z","iopub.status.idle":"2022-02-16T23:47:29.918688Z","shell.execute_reply.started":"2022-02-16T23:47:29.908136Z","shell.execute_reply":"2022-02-16T23:47:29.917871Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except: print(\"failed to initialize TPU\")\n    else: DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\": strategy = tf.distribute.get_strategy()\nif DEVICE == \"GPU\": print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.069363,"end_time":"2021-11-29T18:05:35.290484","exception":false,"start_time":"2021-11-29T18:05:35.221121","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:47:29.920518Z","iopub.execute_input":"2022-02-16T23:47:29.921101Z","iopub.status.idle":"2022-02-16T23:47:30.105551Z","shell.execute_reply.started":"2022-02-16T23:47:29.921055Z","shell.execute_reply":"2022-02-16T23:47:30.103474Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"Num GPUs Available:  1\n","output_type":"stream"},{"name":"stderr","text":"2022-02-16 23:47:29.987944: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-02-16 23:47:30.097080: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n2022-02-16 23:47:30.097929: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n","output_type":"stream"}]},{"cell_type":"code","source":"def compute_target(df):\n    \"\"\"\n    \n    \"\"\"\n    R=list()\n    c=list(df['Close'])\n    for i in range(df.shape[0]):\n        future=c[min([i+16,df.shape[0]-1])]\n        past=c[min([i+1,df.shape[0]-1])]\n        R.append(future/past)\n    df['R']=R\n    df['R']=np.log(df['R'])\n    df['pred']=np.exp(df['R'])-1\n    return df\n\n\n# we will use weighted correlation as function to evaluate our model performance\n# https://stackoverflow.com/questions/38641691/weighted-correlation-coefficient-with-pandas\ndef wmean(x, w):\n    return np.sum(x * w) / np.sum(w)\n\ndef wcov(x, y, w):\n    return np.sum(w * (x - wmean(x, w)) * (y - wmean(y, w))) / np.sum(w)\n\ndef wcorr(x, y, w):\n    return wcov(x, y, w) / np.sqrt(wcov(x, x, w) * wcov(y, y, w))\n\ndef wcorr(x, y, w=1):\n    return wcov(x, y, w) / np.sqrt(wcov(x, x, w) * wcov(y, y, w))\n\n# ----------------------------------------------------------------\n# ----------------------------------------------------------------\n# def tf_wmean(x, w):\n#     w_sum = tf.math.reduce_sum(w)\n#     xw_sum = tf.math.reduce_sum(tf.math.multiply(x, w))\n#     return  tf.math.divide(xw_sum, w_sum)\n# \n# def tf_wcov(x, y, w):\n#     w_sum = tf.math.reduce_sum(w)\n#     xyw = tf.math.multiply(x - tf_wmean(x, w), y - tf_wmean(y, w))\n#     w_xy_sum= tf.math.reduce_sum(tf.math.multiply(w, xyw))\n#     return  tf.math.divide(w_xy_sum, w_sum)\n# \n# def tf_wcorr(x, y, w=1):\n#     if not w:\n#         w=1.\n#     x = tf.cast(x, tf.float32)\n#     y = tf.cast(y, tf.float32)\n#     w = tf.cast(w, tf.float32)\n#     \n#     mul_xwcov_y_wcov = tf.math.multiply(tf_wcov(x, x, w), tf_wcov(y, y, w))\n#     return tf.math.divide(tf_wcov(x, y, w), tf.math.sqrt(mul_xwcov_y_wcov))\n\n\nfrom functools import partial\n\ndef corr_loss(y_true, y_pred):\n    x = tf.cast(y_true, tf.float32)\n    y = tf.cast(y_pred, tf.float32)\n    mx = K.mean(x)\n    my = K.mean(y)\n    xm, ym = x-mx, y-my\n    r_num = K.sum(tf.multiply(xm,ym))\n    r_den = K.sqrt(tf.multiply(K.sum(K.square(xm)), K.sum(K.square(ym))))\n    r = r_num / r_den\n    r = K.maximum(K.minimum(r, 1.0), -1.0)\n    return - r\n\ndef combined_loss(y_true, y_pred, weight1=0.5, weight2=None):\n    if not weight2:\n        weight2 = 1 - weight1\n    loss1 = corr_loss(y_true, y_pred)\n    mae_loss_ = tf.keras.losses.MeanAbsoluteError()\n    loss2 = mae_loss_(y_true, y_pred)\n    return loss1*weight1 + loss2*weight2\n\n\ndef competition_weighted_correlation(a, b, weights=1):\n\n    #w = np.ravel(weights)\n    #a = np.ravel(a)\n    #b = np.ravel(b)\n    a = tf.cast(a, dtype=tf.float32)\n    b = tf.cast(b, dtype=tf.float32)\n    \n    len_a = tf.cast(tf.size(a), dtype=tf.float32)\n    len_b = tf.cast(tf.size(b), dtype=tf.float32)\n    #sum_w = tf.reduce_sum(w)\n    mean_a = tf.cast(tf.math.reduce_mean(a), dtype=tf.float32)\n    mean_b = tf.cast(tf.math.reduce_mean(b), dtype=tf.float32)\n    var_a = tf.cast(tf.math.reduce_sum(tf.math.square(a - mean_a)) / len_a, dtype=tf.float32)\n    var_b = tf.cast(tf.math.reduce_sum(tf.math.square(b - mean_b)) / len_b, dtype=tf.float32)\n\n    cov = tf.math.reduce_sum((a * b)) / len_a - (mean_a * mean_b)\n    corr = cov / tf.math.sqrt(var_a * var_b)\n\n    return corr","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:47:30.108262Z","iopub.execute_input":"2022-02-16T23:47:30.108548Z","iopub.status.idle":"2022-02-16T23:47:30.130954Z","shell.execute_reply.started":"2022-02-16T23:47:30.108507Z","shell.execute_reply":"2022-02-16T23:47:30.130074Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"import datatable as dt\nextra_data_files = {0: '../input/cryptocurrency-extra-data-binance-coin', 2: '../input/cryptocurrency-extra-data-bitcoin-cash', 1: '../input/cryptocurrency-extra-data-bitcoin', 3: '../input/cryptocurrency-extra-data-cardano', 4: '../input/cryptocurrency-extra-data-dogecoin', 5: '../input/cryptocurrency-extra-data-eos-io', 6: '../input/cryptocurrency-extra-data-ethereum', 7: '../input/cryptocurrency-extra-data-ethereum-classic', 8: '../input/cryptocurrency-extra-data-iota', 9: '../input/cryptocurrency-extra-data-litecoin', 11: '../input/cryptocurrency-extra-data-monero', 10: '../input/cryptocurrency-extra-data-maker', 12: '../input/cryptocurrency-extra-data-stellar', 13: '../input/cryptocurrency-extra-data-tron'}\n\n# Uncomment to load the original csv [slower]\n# orig_df_train = pd.read_csv(data_path + 'train.csv') \n# supp_df_train = pd.read_csv(data_path + 'supplemental_train.csv')\n# df_asset_details = pd.read_csv(data_path  + 'asset_details.csv').sort_values(\"Asset_ID\")\n\norig_df_train = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_train.jay').to_pandas()\ndf_asset_details = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_asset_details.jay').to_pandas()\nsupp_df_train = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_supplemental_train.jay').to_pandas()\nassets_details = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_asset_details.jay').to_pandas()\nasset_weight_dict = {assets_details['Asset_ID'].tolist()[idx]: assets_details['Weight'].tolist()[idx] for idx in range(len(assets_details))}\nasset_name_dict = {assets_details['Asset_ID'].tolist()[idx]: assets_details['Asset_Name'].tolist()[idx] for idx in range(len(assets_details))}\n\ndef load_training_data_for_asset(asset_id, load_jay = True):\n    dfs = []\n    if INCCOMP: dfs.append(orig_df_train[orig_df_train[\"Asset_ID\"] == asset_id].copy())\n    if INCSUPP: dfs.append(supp_df_train[supp_df_train[\"Asset_ID\"] == asset_id].copy())\n    \n    if load_jay:\n        if INC2017 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.jay').to_pandas())\n        if INC2018 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.jay').to_pandas())\n        if INC2019 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.jay').to_pandas())\n        if INC2020 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.jay').to_pandas())\n        if INC2021 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.jay').to_pandas())\n    else: \n        if INC2017 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'))\n        if INC2018 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'))\n        if INC2019 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'))\n        if INC2020 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'))\n        if INC2021 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'))\n    df = pd.concat(dfs, axis = 0) if len(dfs) > 1 else dfs[0]\n    df['date'] = pd.to_datetime(df['timestamp'], unit = 's')\n    if LOAD_STRICT: df = df.loc[df['date'] < \"2021-06-13 00:00:00\"]    \n    df = df.sort_values('date')\n    return df\n\ndef load_data_for_all_assets():\n    dfs = []\n    for asset_id in list(extra_data_files.keys()): dfs.append(load_training_data_for_asset(asset_id))\n    return pd.concat(dfs)","metadata":{"_kg_hide-input":true,"papermill":{"duration":24.69094,"end_time":"2021-11-29T18:06:00.13746","exception":false,"start_time":"2021-11-29T18:05:35.44652","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:47:30.132648Z","iopub.execute_input":"2022-02-16T23:47:30.133254Z","iopub.status.idle":"2022-02-16T23:47:53.683980Z","shell.execute_reply.started":"2022-02-16T23:47:30.133207Z","shell.execute_reply":"2022-02-16T23:47:53.683208Z"},"trusted":true},"execution_count":6,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<style type='text/css'>\n.datatable table.frame { margin-bottom: 0; }\n.datatable table.frame thead { border-bottom: none; }\n.datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n.datatable .bool    { background: #DDDD99; }\n.datatable .object  { background: #565656; }\n.datatable .int     { background: #5D9E5D; }\n.datatable .float   { background: #4040CC; }\n.datatable .str     { background: #CC4040; }\n.datatable .time    { background: #40CC40; }\n.datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n.datatable .frame tbody td { text-align: left; }\n.datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n.datatable th:nth-child(2) { padding-left: 12px; }\n.datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n.datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n.datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n.datatable .sp {  opacity: 0.25;}\n.datatable .footer { font-size: 9px; }\n.datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n</style>\n"},"metadata":{}}]},{"cell_type":"code","source":"test = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_example_test.jay').to_pandas()\nsample_prediction_df = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_example_sample_submission.jay').to_pandas()\nassets = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_asset_details.jay').to_pandas()\nassets_order = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_supplemental_train.jay').to_pandas().Asset_ID[:N_ASSETS]\n\nassets_order = dict((t,i) for i,t in enumerate(assets_order))\nprint(\"Loaded all data!\")","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:47:53.686882Z","iopub.execute_input":"2022-02-16T23:47:53.687135Z","iopub.status.idle":"2022-02-16T23:47:53.796614Z","shell.execute_reply.started":"2022-02-16T23:47:53.687097Z","shell.execute_reply":"2022-02-16T23:47:53.795818Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"Loaded all data!\n","output_type":"stream"}]},{"cell_type":"code","source":"train = load_data_for_all_assets().sort_values('timestamp').set_index(\"timestamp\")\nif DEBUG: \n    train = train[-500000:]  # [-2221694:]  #\nelse:\n    train = train[1000000:]\nprint(train.shape)","metadata":{"papermill":{"duration":17.031431,"end_time":"2021-11-29T18:06:17.210444","exception":false,"start_time":"2021-11-29T18:06:00.179013","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:47:53.798225Z","iopub.execute_input":"2022-02-16T23:47:53.798512Z","iopub.status.idle":"2022-02-16T23:48:04.476393Z","shell.execute_reply.started":"2022-02-16T23:47:53.798473Z","shell.execute_reply":"2022-02-16T23:48:04.475519Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"(500000, 10)\n","output_type":"stream"}]},{"cell_type":"code","source":"# Memory saving function credit to https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype.name\n        \n        if col_type not in ['object', 'category', 'datetime64[ns, UTC]', 'datetime64[ns]']:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":0.054816,"end_time":"2021-11-29T18:06:17.597492","exception":false,"start_time":"2021-11-29T18:06:17.542676","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:48:04.477650Z","iopub.execute_input":"2022-02-16T23:48:04.478322Z","iopub.status.idle":"2022-02-16T23:48:04.494372Z","shell.execute_reply.started":"2022-02-16T23:48:04.478276Z","shell.execute_reply":"2022-02-16T23:48:04.492116Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"def upper_shadow(df): return df['High'] - np.maximum(df['Close'], df['Open'])\ndef lower_shadow(df): return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef get_features(df, row = False):\n    df_feat = df\n    df_feat['spread'] = df_feat['High'] - df_feat['Low']\n    df_feat['mean_trade'] = df_feat['Volume']/df_feat['Count']\n    df_feat['log_price_change'] = np.log(df_feat['Close']/df_feat['Open'])\n    df_feat['upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['lower_Shadow'] = lower_shadow(df_feat)\n    df_feat[\"high_div_low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    df_feat['UPS'] = (df_feat['High'] - np.maximum(df_feat['Close'], df_feat['Open']))\n    df_feat['LOS'] = (np.minimum(df_feat['Close'], df_feat['Open']) - df_feat['Low'])\n    df_feat['LOGVOL'] = np.log(1. + df_feat['Volume'])\n    df_feat['LOGCNT'] = np.log(1. + df_feat['Count'])\n    return df_feat\n\n\n# A utility function to build features around lags.\ndef get_features_hist(df_feat, row=False):\n    \n    ### features to consider. See potential features...\n    ### note that we predicting returns 15 minutes ahead. This minutes price data is therefore not sufficient. We must roll the variables, 15, 30, 90, 250, 1250\n       \n    # df_feat = df[['Open', 'High', 'Low', 'Close', 'Volume', 'Count', 'VWAP', 'lr15', 'm', 'lr16', 'Asset_ID']].copy()\n\n    if df_feat.shape[0]<3750:\n        df_feat['beta_num'] = np.nan\n        df_feat['m2'] = np.nan\n    else:\n        df_feat['beta_num'] = (df_feat['lr15']*df_feat['m']).rolling(3750).mean().values\n        df_feat['m2'] = (df_feat['m']*df_feat['m']).rolling(3750).mean().values\n        \n    if row:\n        # first .iloc as far back as we need, compute feature, then downsize until .iloc[-1]\n        df_feat = df_feat.iloc[-1]\n#         mean_price = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n#         med_price = df_feat[['Open', 'High', 'Low', 'Close']].median()\n        df_feat['upper_shadow'] = df_feat['High'] / df_feat[['Close', 'Open']].max()\n        df_feat['lower_shadow'] = df_feat[['Close', 'Open']].min() / df_feat['Low']\n    else:\n#         mean_price = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n#         med_price = df_feat[['Open', 'High', 'Low', 'Close']].median(axis=1)\n        df_feat['upper_shadow'] = df_feat['High'] / df_feat[['Close', 'Open']].max(axis=1)\n        df_feat['lower_shadow'] = df_feat[['Close', 'Open']].min(axis=1) / df_feat['Low']\n        # df_feat = df_feat.drop('Asset_ID', axis=1)\n        \n    df_feat['beta'] = np.nan_to_num(df_feat['beta_num'] / df_feat['m2'], nan=0., posinf=0., neginf=0.)\n    df_feat['target_lag'] = df_feat['lr15'] - df_feat['beta']*df_feat['m']  # first 15 entries of target_lagg are NaN\n\n        ### Sense checks\n#         print((df_feat['Target'] - df_feat.groupby('Asset_ID')['target_lag'].shift(-16)).abs().mean())\n#         print(df_feat.loc[(df_feat.Target.isnull())&(df_feat.target_lag.notnull())].shape)\n#         print(df_feat.loc[(df_feat.Target.notnull())&(df_feat.target_lag.isnull())].shape)        \n        \n    df_feat['open2close'] = df_feat['Close'] / df_feat['Open']\n    df_feat['high2low'] = df_feat['High'] / df_feat['Low']\n           \n#     df_feat['high2mean'] = df_feat['High'] / mean_price\n#     df_feat['low2mean'] = df_feat['Low'] / mean_price\n#     df_feat['high2median'] = df_feat['High'] / med_price\n#     df_feat['low2median'] = df_feat['Low'] / med_price\n    df_feat['volume2count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n    df_feat['close2vwap'] = df_feat['Close'] / df_feat['VWAP']\n    \n    return df_feat.replace([np.inf, -np.inf], np.nan)\n\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)\n\ndef timestamp_to_date(timestamp):\n    return(datetime.fromtimestamp(timestamp))\n\ndef get_lead_features():\n    # get_features from ethereum\n    pass\n\n","metadata":{"papermill":{"duration":37.064504,"end_time":"2021-11-29T18:06:54.699951","exception":false,"start_time":"2021-11-29T18:06:17.635447","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:48:04.496868Z","iopub.execute_input":"2022-02-16T23:48:04.497442Z","iopub.status.idle":"2022-02-16T23:48:04.540206Z","shell.execute_reply.started":"2022-02-16T23:48:04.497398Z","shell.execute_reply":"2022-02-16T23:48:04.539200Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"## FILTER DATAFRAME IF ONLY WORKING WITH ONE ASSET\nif SINGLE_ASSET:\n    train = train[train[\"Asset_ID\"]==asset_id]\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:48:04.545259Z","iopub.execute_input":"2022-02-16T23:48:04.545816Z","iopub.status.idle":"2022-02-16T23:48:04.744903Z","shell.execute_reply.started":"2022-02-16T23:48:04.545751Z","shell.execute_reply":"2022-02-16T23:48:04.744171Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"24"},"metadata":{}}]},{"cell_type":"code","source":"train[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Target']] = train[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Target']].astype(np.float32)\nprint(train.shape)\ntrain['Target'] = train['Target'].fillna(0)\nVWAP_max = np.max(train[np.isfinite(train.VWAP)].VWAP)\nVWAP_min = np.min(train[np.isfinite(train.VWAP)].VWAP)\ntrain['VWAP'] = np.nan_to_num(train.VWAP, posinf=VWAP_max, neginf=VWAP_min)\ndf = train[['Asset_ID', 'Target']].copy()\ntimes = dict((t,i) for i,t in enumerate(df.index.unique()))\ndf['id'] = df.index.map(times)\ndf['id'] = df['id'].astype(str) + '_' + df['Asset_ID'].astype(str)\nids = df.id.copy()\ndel df\n\ntrain = train.sort_index()\nind = train.index.unique()\ndef reindex(df):\n    df = df.reindex(range(ind[0],ind[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df\ntrain = train.groupby('Asset_ID').apply(reindex).reset_index(0, drop=True).sort_index()\ngc.collect()\nprint(train.shape)\n\n# Matching records and marking generated rows as 'non-real'\nprint(train.shape)\ntrain['group_num'] = train.index.map(times)\ntrain = train.dropna(subset=['group_num'])\ntrain['group_num'] = train['group_num'].astype('int')\ntrain['id'] = train['group_num'].astype(str) + '_' + train['Asset_ID'].astype(str)\ntrain['is_real'] = train.id.isin(ids) * 1\ntrain = train.drop('id', axis=1)\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:48:04.746176Z","iopub.execute_input":"2022-02-16T23:48:04.746938Z","iopub.status.idle":"2022-02-16T23:48:05.237219Z","shell.execute_reply.started":"2022-02-16T23:48:04.746896Z","shell.execute_reply":"2022-02-16T23:48:05.236285Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"(35716, 10)\n(35717, 10)\n(35717, 10)\n(35716, 12)\n","output_type":"stream"}]},{"cell_type":"code","source":"train['asset_order'] = train.Asset_ID.map(assets_order)\ntrain = train.sort_values(by=['group_num', 'asset_order'])\n# train = reduce_mem_usage(train)\ngc.collect()","metadata":{"papermill":{"duration":38.374972,"end_time":"2021-11-29T18:09:38.027192","exception":false,"start_time":"2021-11-29T18:08:59.65222","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:48:05.238612Z","iopub.execute_input":"2022-02-16T23:48:05.238963Z","iopub.status.idle":"2022-02-16T23:48:05.429156Z","shell.execute_reply.started":"2022-02-16T23:48:05.238922Z","shell.execute_reply":"2022-02-16T23:48:05.428278Z"},"trusted":true},"execution_count":13,"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"23"},"metadata":{}}]},{"cell_type":"code","source":"if train.index.name == \"timestamp\":\n    train = train.reset_index()\ntrain['lr15'] = train.groupby('Asset_ID')['Close'].apply(lambda x: log_return(x, 15))\ntrain['lr16'] = train.groupby('Asset_ID')['Close'].apply(lambda x: log_return(x, 16))\ntrain = train.merge(df_asset_details[['Asset_ID','Weight']], how='left', on = 'Asset_ID')\ntrain['m'] = train['lr15']*train['Weight']\ntrain['m'] = train.groupby('timestamp')['m'].transform('sum') / np.sum(df_asset_details['Weight'])\n\ntrain.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:48:05.430694Z","iopub.execute_input":"2022-02-16T23:48:05.430980Z","iopub.status.idle":"2022-02-16T23:48:05.493682Z","shell.execute_reply.started":"2022-02-16T23:48:05.430942Z","shell.execute_reply":"2022-02-16T23:48:05.492960Z"},"trusted":true},"execution_count":14,"outputs":[{"execution_count":14,"output_type":"execute_result","data":{"text/plain":"     timestamp  Asset_ID    Count      Open      High       Low     Close  \\\n0   1621399380         3  13863.0  1.642700  1.649500  1.611000  1.623511   \n1   1621399440         3   9173.0  1.623793  1.654195  1.615200  1.645859   \n2   1621399500         3   9524.0  1.645627  1.673242  1.617400  1.664102   \n3   1621399560         3   7761.0  1.663457  1.669406  1.631900  1.640231   \n4   1621399620         3   6559.0  1.640461  1.664600  1.635679  1.658525   \n5   1621399680         3   5590.0  1.658125  1.671800  1.643700  1.657311   \n6   1621399740         3   8552.0  1.657763  1.660800  1.620500  1.639733   \n7   1621399800         3   7144.0  1.640014  1.656900  1.628995  1.648322   \n8   1621399860         3   8980.0  1.647378  1.649000  1.615000  1.628970   \n9   1621399920         3  10643.0  1.630148  1.635000  1.600000  1.610441   \n10  1621399980         3   9123.0  1.610199  1.620600  1.590000  1.599522   \n11  1621400040         3   8817.0  1.598771  1.608200  1.585420  1.594127   \n12  1621400100         3   9643.0  1.593979  1.609100  1.580200  1.602591   \n13  1621400160         3   9587.0  1.602049  1.620000  1.583223  1.604161   \n14  1621400220         3   8442.0  1.604467  1.635300  1.590800  1.625749   \n15  1621400280         3   9203.0  1.626747  1.654680  1.614000  1.646023   \n16  1621400340         3   7288.0  1.644431  1.665100  1.635800  1.660292   \n17  1621400400         3   7925.0  1.660577  1.690000  1.658100  1.683223   \n18  1621400460         3  10511.0  1.684452  1.715800  1.682200  1.705574   \n19  1621400520         3  10550.0  1.704350  1.707674  1.676489  1.680776   \n\n         Volume      VWAP    Target                date  group_num  is_real  \\\n0   10208457.00  1.627059  0.006264 2021-05-19 04:43:00          0        1   \n1    6408643.00  1.640546  0.010261 2021-05-19 04:44:00          1        1   \n2    6283014.50  1.647340  0.016606 2021-05-19 04:45:00          2        1   \n3    4516063.00  1.650989  0.013718 2021-05-19 04:46:00          3        1   \n4    4456316.50  1.651940  0.021053 2021-05-19 04:47:00          4        1   \n5    3973633.75  1.655983  0.023950 2021-05-19 04:48:00          5        1   \n6    4245567.50  1.643210  0.021678 2021-05-19 04:49:00          6        1   \n7    3585431.25  1.644723  0.026200 2021-05-19 04:50:00          7        1   \n8    5622760.00  1.628477  0.027583 2021-05-19 04:51:00          8        1   \n9    7331379.50  1.614531  0.028171 2021-05-19 04:52:00          9        1   \n10   6849127.00  1.604222  0.022879 2021-05-19 04:53:00         10        1   \n11   5408449.00  1.597099  0.030317 2021-05-19 04:54:00         11        1   \n12   6437179.50  1.597654  0.033699 2021-05-19 04:55:00         12        1   \n13   6343381.00  1.606019  0.038886 2021-05-19 04:56:00         13        1   \n14   4911103.00  1.616691  0.028503 2021-05-19 04:57:00         14        1   \n15   5982300.00  1.636194  0.025401 2021-05-19 04:58:00         15        1   \n16   5016601.50  1.654719  0.014743 2021-05-19 04:59:00         16        1   \n17   5371792.50  1.670483  0.008765 2021-05-19 05:00:00         17        1   \n18   7358236.50  1.701093  0.010435 2021-05-19 05:01:00         18        1   \n19   6785659.50  1.687330  0.003379 2021-05-19 05:02:00         19        1   \n\n    asset_order      lr15      lr16    Weight         m  \n0             0       NaN       NaN  4.406719  0.000000  \n1             0       NaN       NaN  4.406719  0.000000  \n2             0       NaN       NaN  4.406719  0.000000  \n3             0       NaN       NaN  4.406719  0.000000  \n4             0       NaN       NaN  4.406719  0.000000  \n5             0       NaN       NaN  4.406719  0.000000  \n6             0       NaN       NaN  4.406719  0.000000  \n7             0       NaN       NaN  4.406719  0.000000  \n8             0       NaN       NaN  4.406719  0.000000  \n9             0       NaN       NaN  4.406719  0.000000  \n10            0       NaN       NaN  4.406719  0.000000  \n11            0       NaN       NaN  4.406719  0.000000  \n12            0       NaN       NaN  4.406719  0.000000  \n13            0       NaN       NaN  4.406719  0.000000  \n14            0       NaN       NaN  4.406719  0.000000  \n15            0  0.013771       NaN  4.406719  0.001484  \n16            0  0.008731  0.022402  4.406719  0.000941  \n17            0  0.011425  0.022448  4.406719  0.001232  \n18            0  0.039064  0.024616  4.406719  0.004211  \n19            0  0.013327  0.024418  4.406719  0.001437  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>timestamp</th>\n      <th>Asset_ID</th>\n      <th>Count</th>\n      <th>Open</th>\n      <th>High</th>\n      <th>Low</th>\n      <th>Close</th>\n      <th>Volume</th>\n      <th>VWAP</th>\n      <th>Target</th>\n      <th>date</th>\n      <th>group_num</th>\n      <th>is_real</th>\n      <th>asset_order</th>\n      <th>lr15</th>\n      <th>lr16</th>\n      <th>Weight</th>\n      <th>m</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1621399380</td>\n      <td>3</td>\n      <td>13863.0</td>\n      <td>1.642700</td>\n      <td>1.649500</td>\n      <td>1.611000</td>\n      <td>1.623511</td>\n      <td>10208457.00</td>\n      <td>1.627059</td>\n      <td>0.006264</td>\n      <td>2021-05-19 04:43:00</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1621399440</td>\n      <td>3</td>\n      <td>9173.0</td>\n      <td>1.623793</td>\n      <td>1.654195</td>\n      <td>1.615200</td>\n      <td>1.645859</td>\n      <td>6408643.00</td>\n      <td>1.640546</td>\n      <td>0.010261</td>\n      <td>2021-05-19 04:44:00</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1621399500</td>\n      <td>3</td>\n      <td>9524.0</td>\n      <td>1.645627</td>\n      <td>1.673242</td>\n      <td>1.617400</td>\n      <td>1.664102</td>\n      <td>6283014.50</td>\n      <td>1.647340</td>\n      <td>0.016606</td>\n      <td>2021-05-19 04:45:00</td>\n      <td>2</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1621399560</td>\n      <td>3</td>\n      <td>7761.0</td>\n      <td>1.663457</td>\n      <td>1.669406</td>\n      <td>1.631900</td>\n      <td>1.640231</td>\n      <td>4516063.00</td>\n      <td>1.650989</td>\n      <td>0.013718</td>\n      <td>2021-05-19 04:46:00</td>\n      <td>3</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1621399620</td>\n      <td>3</td>\n      <td>6559.0</td>\n      <td>1.640461</td>\n      <td>1.664600</td>\n      <td>1.635679</td>\n      <td>1.658525</td>\n      <td>4456316.50</td>\n      <td>1.651940</td>\n      <td>0.021053</td>\n      <td>2021-05-19 04:47:00</td>\n      <td>4</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>1621399680</td>\n      <td>3</td>\n      <td>5590.0</td>\n      <td>1.658125</td>\n      <td>1.671800</td>\n      <td>1.643700</td>\n      <td>1.657311</td>\n      <td>3973633.75</td>\n      <td>1.655983</td>\n      <td>0.023950</td>\n      <td>2021-05-19 04:48:00</td>\n      <td>5</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1621399740</td>\n      <td>3</td>\n      <td>8552.0</td>\n      <td>1.657763</td>\n      <td>1.660800</td>\n      <td>1.620500</td>\n      <td>1.639733</td>\n      <td>4245567.50</td>\n      <td>1.643210</td>\n      <td>0.021678</td>\n      <td>2021-05-19 04:49:00</td>\n      <td>6</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>1621399800</td>\n      <td>3</td>\n      <td>7144.0</td>\n      <td>1.640014</td>\n      <td>1.656900</td>\n      <td>1.628995</td>\n      <td>1.648322</td>\n      <td>3585431.25</td>\n      <td>1.644723</td>\n      <td>0.026200</td>\n      <td>2021-05-19 04:50:00</td>\n      <td>7</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>1621399860</td>\n      <td>3</td>\n      <td>8980.0</td>\n      <td>1.647378</td>\n      <td>1.649000</td>\n      <td>1.615000</td>\n      <td>1.628970</td>\n      <td>5622760.00</td>\n      <td>1.628477</td>\n      <td>0.027583</td>\n      <td>2021-05-19 04:51:00</td>\n      <td>8</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>1621399920</td>\n      <td>3</td>\n      <td>10643.0</td>\n      <td>1.630148</td>\n      <td>1.635000</td>\n      <td>1.600000</td>\n      <td>1.610441</td>\n      <td>7331379.50</td>\n      <td>1.614531</td>\n      <td>0.028171</td>\n      <td>2021-05-19 04:52:00</td>\n      <td>9</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>1621399980</td>\n      <td>3</td>\n      <td>9123.0</td>\n      <td>1.610199</td>\n      <td>1.620600</td>\n      <td>1.590000</td>\n      <td>1.599522</td>\n      <td>6849127.00</td>\n      <td>1.604222</td>\n      <td>0.022879</td>\n      <td>2021-05-19 04:53:00</td>\n      <td>10</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>1621400040</td>\n      <td>3</td>\n      <td>8817.0</td>\n      <td>1.598771</td>\n      <td>1.608200</td>\n      <td>1.585420</td>\n      <td>1.594127</td>\n      <td>5408449.00</td>\n      <td>1.597099</td>\n      <td>0.030317</td>\n      <td>2021-05-19 04:54:00</td>\n      <td>11</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>1621400100</td>\n      <td>3</td>\n      <td>9643.0</td>\n      <td>1.593979</td>\n      <td>1.609100</td>\n      <td>1.580200</td>\n      <td>1.602591</td>\n      <td>6437179.50</td>\n      <td>1.597654</td>\n      <td>0.033699</td>\n      <td>2021-05-19 04:55:00</td>\n      <td>12</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>1621400160</td>\n      <td>3</td>\n      <td>9587.0</td>\n      <td>1.602049</td>\n      <td>1.620000</td>\n      <td>1.583223</td>\n      <td>1.604161</td>\n      <td>6343381.00</td>\n      <td>1.606019</td>\n      <td>0.038886</td>\n      <td>2021-05-19 04:56:00</td>\n      <td>13</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>1621400220</td>\n      <td>3</td>\n      <td>8442.0</td>\n      <td>1.604467</td>\n      <td>1.635300</td>\n      <td>1.590800</td>\n      <td>1.625749</td>\n      <td>4911103.00</td>\n      <td>1.616691</td>\n      <td>0.028503</td>\n      <td>2021-05-19 04:57:00</td>\n      <td>14</td>\n      <td>1</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>1621400280</td>\n      <td>3</td>\n      <td>9203.0</td>\n      <td>1.626747</td>\n      <td>1.654680</td>\n      <td>1.614000</td>\n      <td>1.646023</td>\n      <td>5982300.00</td>\n      <td>1.636194</td>\n      <td>0.025401</td>\n      <td>2021-05-19 04:58:00</td>\n      <td>15</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0.013771</td>\n      <td>NaN</td>\n      <td>4.406719</td>\n      <td>0.001484</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>1621400340</td>\n      <td>3</td>\n      <td>7288.0</td>\n      <td>1.644431</td>\n      <td>1.665100</td>\n      <td>1.635800</td>\n      <td>1.660292</td>\n      <td>5016601.50</td>\n      <td>1.654719</td>\n      <td>0.014743</td>\n      <td>2021-05-19 04:59:00</td>\n      <td>16</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0.008731</td>\n      <td>0.022402</td>\n      <td>4.406719</td>\n      <td>0.000941</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>1621400400</td>\n      <td>3</td>\n      <td>7925.0</td>\n      <td>1.660577</td>\n      <td>1.690000</td>\n      <td>1.658100</td>\n      <td>1.683223</td>\n      <td>5371792.50</td>\n      <td>1.670483</td>\n      <td>0.008765</td>\n      <td>2021-05-19 05:00:00</td>\n      <td>17</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0.011425</td>\n      <td>0.022448</td>\n      <td>4.406719</td>\n      <td>0.001232</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>1621400460</td>\n      <td>3</td>\n      <td>10511.0</td>\n      <td>1.684452</td>\n      <td>1.715800</td>\n      <td>1.682200</td>\n      <td>1.705574</td>\n      <td>7358236.50</td>\n      <td>1.701093</td>\n      <td>0.010435</td>\n      <td>2021-05-19 05:01:00</td>\n      <td>18</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0.039064</td>\n      <td>0.024616</td>\n      <td>4.406719</td>\n      <td>0.004211</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>1621400520</td>\n      <td>3</td>\n      <td>10550.0</td>\n      <td>1.704350</td>\n      <td>1.707674</td>\n      <td>1.676489</td>\n      <td>1.680776</td>\n      <td>6785659.50</td>\n      <td>1.687330</td>\n      <td>0.003379</td>\n      <td>2021-05-19 05:02:00</td>\n      <td>19</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0.013327</td>\n      <td>0.024418</td>\n      <td>4.406719</td>\n      <td>0.001437</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"def get_split_indices(indices):\n    train_end_idx = int(len(indices) * (1 - (PCT_VALIDATION+PCT_TEST)/100))\n    val_end_idx = int(len(indices) * (1 - (PCT_TEST)/100))\n    return indices[train_end_idx], indices[val_end_idx]\n\ntrain_end_idx = {}\nval_end_idx = {}\nif not SINGLE_ASSET:\n    for i in range(N_ASSETS):\n        X_asset_indices = train[train[\"Asset_ID\"]==i].index\n        train_end_idx[i], val_end_idx[i] = get_split_indices(X_asset_indices)\nelse:\n    X_asset_indices = train[train[\"Asset_ID\"]==asset_id].index\n    train_end_idx[asset_id], val_end_idx[asset_id] = get_split_indices(X_asset_indices)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:48:05.494947Z","iopub.execute_input":"2022-02-16T23:48:05.495518Z","iopub.status.idle":"2022-02-16T23:48:05.509062Z","shell.execute_reply.started":"2022-02-16T23:48:05.495478Z","shell.execute_reply":"2022-02-16T23:48:05.508130Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"# create train data of shape: Batch Size X Window Size X Num Assets --> bBATCH_SIZEon mean over all features\n# create train lables of shape: batch size X Num Assets  -->  based on target of t(win_len)\n# given data and index, train batches + labels are created\n\n\nclass sample_generator(keras.utils.Sequence):\n    def __init__(self, x_set, y_set, batch_size, length, prediction_length):\n        self.x, self.y = x_set, y_set\n        self.batch_size = batch_size\n        self.length = length\n        self.prediction_length = prediction_length\n        self.size = len(x_set)\n    def __len__(self): return int((len(self.x)-self.length) / float(self.batch_size))\n    def __getitem__(self, idx):\n        batch_x=[]\n        batch_y=[]\n        for i in range(self.batch_size):\n            start_ind = self.batch_size*idx + i\n            end_ind = start_ind + self.length \n            if (end_ind+self.prediction_length-1) <= self.size:\n                batch_x.append(self.x[start_ind : end_ind])\n                batch_y.append(self.y[end_ind -1: end_ind+self.prediction_length-1]) \n        x,y = np.array(batch_x), np.array(batch_y)\n        assert x.shape == (BATCH_SIZE, WINDOW_SIZE,len(features)), f\"Shape Missmatch of train data generator X at idx {str(idx)} and i {str(i)}\"\n        assert x.shape == (BATCH_SIZE, WINDOW_SIZE,len(features)), f\"Shape Missmatch of train data generator Y at idx {str(idx)} and i {str(i)}\"\n        return x,y\n\nclass multivariate_sample_generator(keras.utils.Sequence):\n    # generator which supports multivariate ex-features input into nbeats\n    # -> Single asset per sequence\n    # --> x_sequence.shape = batch_size x Window_size x 1\n    # --> y_sequence.shape = batch_size x 1\n    # --> e_sequence.shape = batch_size x Window_size x num_ex_var    \n    def __init__(self, x_set, y_set, e_set, batch_size, length, prediction_length):\n        self.x, self.y, self.e = x_set, y_set, e_set\n        self.batch_size = batch_size\n        self.length = length\n        self.prediction_length = prediction_length\n        self.size = len(x_set)\n        self.num_assets = self.x.shape[1]\n    def __len__(self): return int(np.ceil(len(self.x) / float(self.batch_size)))\n    def __getitem__(self, idx):\n        # idx: how many times the generator is already called\n        batch_x=[]\n        batch_y=[]\n        batch_e=[]\n        num_sample=0\n        while num_sample < self.batch_size:\n            start_ind = self.batch_size//self.num_assets*idx + num_sample//self.num_assets\n            end_ind = start_ind + self.length \n            if (end_ind+self.prediction_length-1) <= self.size:\n                for a in range(self.num_assets):\n                    batch_x.append(self.x[start_ind : end_ind, a])\n                    batch_e.append(self.e[start_ind : end_ind, a, :])\n                    batch_y.append(self.y[end_ind -1: end_ind+self.prediction_length-1, a]) \n                    num_sample += 1\n                    if num_sample >= self.batch_size: break;  ## TODO: FIND BETTER SOLUTION (here we throw aray samples if we break out of loop)\n        return [np.array(batch_x).reshape(-1,self.length,1), np.array(batch_e)], np.array(batch_y)\n    \n\ndef assert_data_validity_generator(gen, batch_size, test_shift_size=1, num_tests=200):\n    for i in range(num_tests):\n        num_batch = np.random.randint(0, len(gen), dtype=int)\n        num_sample = np.random.randint(0, batch_size-1, dtype=int)\n        assert gen[num_batch][0][num_sample+test_shift_size,-1,0] == gen[num_batch][1][num_sample,0,0], \"Data Missmatch between Series and Target in generator data\"\n    print(\"All tests passed.\")\n","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:48:05.512531Z","iopub.execute_input":"2022-02-16T23:48:05.513042Z","iopub.status.idle":"2022-02-16T23:48:05.532970Z","shell.execute_reply.started":"2022-02-16T23:48:05.513007Z","shell.execute_reply":"2022-02-16T23:48:05.532157Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"X_asset = train.copy()\nX_asset = get_features_hist(X_asset)\nX_asset[\"Target_shifted\"] = X_asset[\"Target\"].shift(num_shift)\nX_asset[\"Target\"].plot()\n\ntrain_end_idx[asset_id], val_end_idx[asset_id]\n# features = [\"Target_shifted\"]\nfeatures = [\"Target_shifted\", \"m\", \"upper_shadow\", \"lower_shadow\", \"open2close\", \"high2low\", \"volume2count\", \"close2vwap\"] \ntarget = [\"Target\"]\nX_asset.loc[X_asset.is_real==0, features] = 0 \nX_asset[features] = X_asset[features].fillna(0)\nscaler = MinMaxScaler()\nX_asset[features] = scaler.fit_transform(X_asset[features])  # TODO: NEEDS TO BE SEPARATED FOR TRAIN TEST\n \ntrain_generator = sample_generator(X_asset.loc[:train_end_idx[asset_id], features], \n                                   X_asset.loc[:train_end_idx[asset_id], target], \n                                   length = WINDOW_SIZE, batch_size = BATCH_SIZE, prediction_length=prediction_length)\nval_generator = sample_generator(X_asset.loc[train_end_idx[asset_id]:val_end_idx[asset_id], features], \n                                 X_asset.loc[train_end_idx[asset_id]:val_end_idx[asset_id], target], \n                                 length = WINDOW_SIZE, batch_size = BATCH_SIZE, prediction_length=prediction_length)\ntest_generator = sample_generator(X_asset.loc[val_end_idx[asset_id]:, features], \n                                  X_asset.loc[val_end_idx[asset_id]:, target], \n                                  length = WINDOW_SIZE, batch_size = BATCH_SIZE, prediction_length=prediction_length)\n\nprint(\"BATCH_SIZE: \", BATCH_SIZE)\nprint(f'Train batch shape: {train_generator[0][0].shape}')\nprint(f'Target batch shape: {train_generator[0][1].shape}')\n\nprint(\"Num batches of train, validation and test generator: \", len(train_generator),\", \", len(val_generator),\", \", len(test_generator))\nprint(\"Num Training Samples: \", len(train_generator)*BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:48:05.534632Z","iopub.execute_input":"2022-02-16T23:48:05.535193Z","iopub.status.idle":"2022-02-16T23:48:05.899126Z","shell.execute_reply.started":"2022-02-16T23:48:05.535152Z","shell.execute_reply":"2022-02-16T23:48:05.897138Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"BATCH_SIZE:  64\nTrain batch shape: (64, 64, 8)\nTarget batch shape: (64, 1, 1)\nNum batches of train, validation and test generator:  445 ,  54 ,  54\nNum Training Samples:  28480\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAYkAAAD4CAYAAAAZ1BptAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAz0klEQVR4nO3deXwUVbbA8d9JAmGTPSCyGDZFQEGJgAuogMjiiPNGRcdxUFF0RmdzdMQNHURFnXkuM46KiiKOouL4REEZQHAXCAKyyBIBBWQJ+74kOe+Prk6qO1Wd9JJ0Qs738+kPtdyqPt106lTde+uWqCrGGGOMl5RkB2CMMabisiRhjDHGlyUJY4wxvixJGGOM8WVJwhhjjK+0ZAeQSI0bN9bMzMxkh2GMMZXKggULtqlqhte6YypJZGZmkp2dnewwjDGmUhGRH/zWWXWTMcYYX5YkjDHG+LIkYYwxxpclCWOMMb4sSRhjjPFlScIYY4wvSxLGGGN8WZLw8d6ijew7nJfsMIwxJqksSXhYunE3f5i0iJHvfJvsUIwxJqkSkiREZICIrBSRHBEZ6bG+t4h8IyJ5InKZa3lXEflKRJaJyLciMtS17hURWSsii5xX10TEWhoHjuQDsGXPofJ6S2OMqZDiHpZDRFKBZ4ALgQ3AfBGZoqrLXcV+BK4Fbg/b/ADwa1VdLSInAAtEZLqq7nLW36Gqk+ON0RhjTGwSMXZTdyBHVdcAiMgkYAhQmCRUdZ2zrsC9oaquck3/JCJbgQxgVwLiMsYYE6dEVDc1B9a75jc4y6IiIt2B6sD3rsUPOdVQT4hIenxhGmOMiVaFaLgWkWbAROA6VQ1ebdwFdADOBBoCd/psO0JEskUkOzc3NyHxqGpC9mOMMZVdIpLERqCla76Fs6xURKQuMBW4R1W/Di5X1U0acBh4mUC1VjGqOk5Vs1Q1KyPDczj0mAmS0P0ZY0xlk4gkMR9oLyKtRaQ6cCUwpTQbOuXfBV4Nb6B2ri4QEQEuBZYmIFZjjDFRiDtJqGoecCswHfgOeEtVl4nIaBG5BEBEzhSRDcDlwPMisszZ/AqgN3CtR1fXf4vIEmAJ0BgYE2+sxhhjopOQJ9Op6jRgWtiyUa7p+QSqocK3ew14zWeffRIRmzHGmNhViIZrY4wxFZMlCQ/Wt8kYYwIsSURinZuMMVWcJQljjDG+LEkYY4zxZUnCGGOML0sSHmxUDmOMCbAkEYG1WxtjqjpLEsYYY3xZkjDGGOPLkkQE1jRhjKnqLEl4EGuMMMYYwJKEJ+vdZIwxAZYkIrALCmNMVWdJwhhjjC9LEsYYY3xZkjDGGOMrIUlCRAaIyEoRyRGRkR7re4vINyKSJyKXha0bJiKrndcw1/JuIrLE2efTzrOuy4Va51djjAESkCREJBV4BhgIdASuEpGOYcV+BK4FXg/btiFwP9AD6A7cLyINnNXPAjcC7Z3XgHhjjZZ1hTXGVHWJuJLoDuSo6hpVPQJMAoa4C6jqOlX9FigI2/YiYIaq7lDVncAMYICINAPqqurXqqrAq8ClCYjVGGNMFBKRJJoD613zG5xl8Wzb3JkucZ8iMkJEskUkOzc3t9RBG2OMKVmlb7hW1XGqmqWqWRkZGckOxxhjjimJSBIbgZau+RbOsni23ehMx7JPY4wxCZKIJDEfaC8irUWkOnAlMKWU204H+otIA6fBuj8wXVU3AXtEpKfTq+nXwHsJiLV0rHOTMcYACUgSqpoH3ErggP8d8JaqLhOR0SJyCYCInCkiG4DLgedFZJmz7Q7gQQKJZj4w2lkG8FvgRSAH+B74MN5YoyU2MIcxpopLS8ROVHUaMC1s2SjX9HxCq4/c5cYD4z2WZwOdExGfMcaY2FT6hmtjjDFlx5KEMcYYX5YkPFi7tTHGBFiSiMCG5TDGVHWWJCKwJ9QZY6o6SxIe7ALCGGMCLEkYY4zxZUnCGGOML0sSHqwpwhhjAixJRGC9m4wxVZ0lCWOMMb4sSRhjjPFlScIYY4wvSxIe7CY6Y4wJsCRhjDHGlyUJD9aryRhjAhKSJERkgIisFJEcERnpsT5dRN501s8VkUxn+dUissj1KhCRrs66Oc4+g+uaJCJWY4wxpRd3khCRVOAZYCDQEbhKRDqGFRsO7FTVdsATwKMAqvpvVe2qql2Ba4C1qrrItd3VwfWqujXeWI0xxkQnEVcS3YEcVV2jqkeAScCQsDJDgAnO9GSgr0ixSp2rnG2NMcZUEIlIEs2B9a75Dc4yzzKqmgfsBhqFlRkKvBG27GWnquk+j6QCgIiMEJFsEcnOzc2N9TOEsN5NxhgTUCEarkWkB3BAVZe6Fl+tqqcCvZzXNV7bquo4Vc1S1ayMjIwEx5XQ3RljTKWTiCSxEWjpmm/hLPMsIyJpQD1gu2v9lYRdRajqRuffvcDrBKq1jDHGlKNEJIn5QHsRaS0i1Qkc8KeElZkCDHOmLwM+Vg1U6ohICnAFrvYIEUkTkcbOdDXgYmApxhhjylVavDtQ1TwRuRWYDqQC41V1mYiMBrJVdQrwEjBRRHKAHQQSSVBvYL2qrnEtSwemOwkiFZgJvBBvrNGytgljTFUXd5IAUNVpwLSwZaNc04eAy322nQP0DFu2H+iWiNiMMcbErkI0XFdU1nBtjKnqLEkYY4zxZUnCGGOML0sSxhhjfFmSMMYY48uShIecrXsBOJpvfWCNMVWbJQkPD7y/HIBvN+xKbiDGGJNkliSMMcb4siRhjDHGlyUJY4wxvixJGGOM8WVJIgLBxuUwxlRtliSMMcb4siRhjDHGlyWJCGwUWGNMVWdJwhhjjK+EJAkRGSAiK0UkR0RGeqxPF5E3nfVzRSTTWZ4pIgdFZJHzes61TTcRWeJs87RI+Z/X24WEMaaqiztJiEgq8AwwEOgIXCUiHcOKDQd2qmo74AngUde671W1q/O62bX8WeBGoL3zGhBvrNFKQl4yxpgKJRFXEt2BHFVdo6pHgEnAkLAyQ4AJzvRkoG+kKwMRaQbUVdWvVVWBV4FLExBrVNQecm2MqeISkSSaA+td8xucZZ5lVDUP2A00cta1FpGFIvKJiPRyld9Qwj4BEJERIpItItm5ubnxfRJjjDEhkt1wvQlopaqnA7cBr4tI3Wh2oKrjVDVLVbMyMjISGtz+I/l2NWGMqdISkSQ2Ai1d8y2cZZ5lRCQNqAdsV9XDqrodQFUXAN8DJznlW5Swz3Lx2twfk/G2xhhTISQiScwH2otIaxGpDlwJTAkrMwUY5kxfBnysqioiGU7DNyLShkAD9RpV3QTsEZGeTtvFr4H3EhBr1Gav2JqMtzXGmAohLd4dqGqeiNwKTAdSgfGqukxERgPZqjoFeAmYKCI5wA4CiQSgNzBaRI4CBcDNqrrDWfdb4BWgJvCh8yp3Vt1kjKnK4k4SAKo6DZgWtmyUa/oQcLnHdu8A7/jsMxvonIj44mEpwhhTlSW74brCswsJY0xVZkmiBJYjjDFVmSWJEuw9dDTZIRhjTNJYkijBwh93JTsEY4xJGksSwKGj+UxftjnZYRhjTIVjSQIY++EKbpq4gPnrdpRcuJI6nJdP5sipPD59RbJDMcZUIpYkgB93HABg94Fjt/1h/Y6DADwz+/skR2KMqUwsSVB0w9yxPDL40fyCZIdgjKmELEm4HMtJosBu+DDGxMCSBFXjXgjLEcaYWFiSqCIsSRhjYmFJoorQKnG9ZIxJNEsSFJ1lC+XXKHHoaD7dHpzBzOVbyuX98gssSRhjomdJwq0cG6437jrI9v1HeGjad+XyfpYjjDGxsCRB/A3X+w/ncehofkJiKSv2XAxjTCwsSeC6TyLG7TvdP50+f5sT5XsS13tGy64kjDGxSEiSEJEBIrJSRHJEZKTH+nQRedNZP1dEMp3lF4rIAhFZ4vzbx7XNHGefi5xXk0TEWsLniHnbn3YfivFNY37LqNh9EsaYWMSdJJxnVD8DDAQ6AleJSMewYsOBnaraDngCeNRZvg34maqeSuAZ2BPDtrtaVbs6L3vYdBzKK0nk5RfQ8+FZvL/4p3J5P2NM2UrElUR3IEdV16jqEWASMCSszBBggjM9GegrIqKqC1U1eDRZBtQUkfQExBSTY7nevrw+2r7DeWzec4h73l1SPm9ojClTiUgSzYH1rvkNzjLPMqqaB+wGGoWV+QXwjaoedi172alquk986oJEZISIZItIdm5ubjyfAxFhyYbdce2j9Mo3IZXXlURKSuC/6RjOt8ZUKRWi4VpEOhGogrrJtfhqpxqql/O6xmtbVR2nqlmqmpWRkRFfHMDBEnoprdqyNyFXHMdqw3WKk8vzj6EsMX3ZZrLGzORwXsXuwWZMWUhEktgItHTNt3CWeZYRkTSgHrDdmW8BvAv8WlULx7FW1Y3Ov3uB1wlUayXFvsN57Nh/hKdmrqb/E58y8esfkhVKzMrtSkLK9/3Kw+j3l7Nt32Fy9x4uubAxx5hEJIn5QHsRaS0i1YErgSlhZaYQaJgGuAz4WFVVROoDU4GRqvpFsLCIpIlIY2e6GnAxsDQBscbkponZnPHgDJ6YuQqAiV9VjiRx6Gh+0dlvOR+zrctt5TRv7Q7Wbduf7DBMBRJ3knDaGG4FpgPfAW+p6jIRGS0ilzjFXgIaiUgOcBsQ7CZ7K9AOGBXW1TUdmC4i3wKLCFyJvBBvrLGav25nyHwiqlKCe4in221JOtz3EVljZgKw6+CRMnsfLwUFyiercln4486SC1cSybw42nvoKJkjp/LR0k1l+j5XPP8V50d5z485tqUlYieqOg2YFrZslGv6EHC5x3ZjgDE+u+2WiNii4XcMOJIX+sCeggSeJpdVigjGuPdQHgB/enNxGb1TqOCBtECVYePnAbBu7OByee9j2bptgacn/uPjHAZ0bpbkaExVkpAkUdUEc0TmyKn0O6VpcoPx8dynyX1MqTuP5uUXkJaawv7DgYRVO91+dtE6lh+IZSq2CtG7qSIpTc8ld6PszO+KRnF9ZnZOFO8TXVzRWrqxvLrylqzdPR+ycddBOt0/nU73T092OFHbuOtgskMwJmksScRgw07vg8bj01dGva+qcoa4NtcaQxPhGOo0ZqI0d812Nsc6/E8cLEm4CGXbkOwW7UOAtjh3MR/NLyi5cJjMkVOj3iZWfp/qrez1Pmsqj6qS0E3FNHTc1/R/4pNyf19LEjHKXrcjIfvJyw8cVncfOBpxn/e8u5R/z/2RT1fFd1d5tPYfziNz5FT+NSe0Kq3jqI949at1pd7PlEo4llN+gVa4hzXFG03O1r0cPGI3BVZWe5yOKOXJkoRLztZ9fLKqdOMIXvbcV57LX/libVTvucbpk37tK/O47LmvCntSzV65lcyRU9m6N3B5mV8QWF7as9kVm/b6rnv0oxVkjpxKQYHyj1mrmbtmu2/ZXQePAvCa694QVeXAkXxGvbesdMFEIS+/gFe+WFusR1kynP+32XQc9VHhvIgwZfFPfLdpT7Gyz875njsnf1tseadRHzHmg+Vxx5KIq5hDR/Pp97+fcorrMxlTEksSFDVEj/5gOc/Mjq9X0APvL/c8iIQLr1sONjQHq6EmfLkuZHnwhLa01WF7Dh31XffsnMBnbHP3NP4+YxVDx31drEwwSe119uM+oQ5Op3iEEuuQJXn5BVz/ynwuf/4rHnh/OS99Hl2yLQvrdxzkcFiy+v0bCxn41GfFyj760Qre9KhS238knxc/X1vYsyvoln9/U+JnXPbTbrbuSVwd9OGjZZ941+84wC3//qbCP4SrMtsb4W+7LFiSKAOflKJKyH0s3bz7ULHnbAfbHoJNEMFEllJCkpixfAtvZa+Pu23lX05PrRsmZAfidVV0RIol1uqQlVv28vGKrSz8cRdAsYNqRRDPmF13vhN6lTF1ySYeLOEKY/DTn3Pe43MSFkN5DJXy1/eXMXXJJj5fva3M36uquu//ynfwCUsSJL7HiFc99ry1O3wHiHts+grynG2Cx90vcgJVQJPm/RgSY6RD/9H8Am58NZu/TP426nGGMkdOZfH6XcWWB3tyub+j4OfLK1AOHMljwpfr+P0bC4uVK61fvTi3WPfhFK/LlFJasmE305ZsSng3YPdni/aGyljrkoMDTkoUt136XcmWT+tKIM7t+w9XyKH3567ZTtaYGeV+Np5IW/aU7xhiliTKwOPTV3Lr698UHkxXbt7LFc9/xZgPvvMs/59visZD3LTrUMgf1wGnkXGt03bxwmdrQtZv23e4sEpi5DvxPcPh0Y9W8M+PVzPwqc+KDUUSfMej+QV0uK+oTrvjqOncP2VZUcN0DMeFz3O2MW3J5pBlqTFeCV3+3Jf87J+f89t/f8PF//jct9z6HQeYvmyz73ovH7qGxAiO4xXuJ597KuK9Sz94Jbdi817yIvRwm7ZkEwOf+owPvk1cR4FlP+3m8ue+LFUVUp7TdnbnO0sqRJVhuL/9dyXb9h1h+U8lVwkD7Nx/hJ4Pz6pQ9x2Vdy87SxKUTd/zD77dxHWvzAcCVSkAa7bt44kZq1ixeY/vpX/vx2fzqquROCUlUH0VvKHrs9XbGPjUZ2xxEkPWmJl0f3gWh47mM3VJfAeG1BThb/9d5XkmGrwyufqFuRH3UZquvcNfmU+bu6ZyJK/At6dNqs8vc03uPtbvOBCybNu+w8xbG+gZFp7c/PR6bDY3TVxQqrJBD09bUTj9/KdrPMssXr+L2Su3sm1f6NleIqt6Hgu7H0dVmbxgA4eO5rPK+a2t2ly840K0Z/bvLNjAe4s28sCUZcxft5Ov12z3TYJBc1YWVbV+lqAqpw++/Ym35iemC3Xw97F172GemLGqxO/k09W5bN5ziOc+Sd4IBu8u3MCCH4p6PlqSSIKyqqsNdlcNVsV8kbOdp2at5n/+9SXpaf5f/f1TinoNpYgUjoEUtGLzXno8PIsXPys6UK3J3R93siupveOr77czL0I33bz8Am59fWGJ7zNrxVYKFP4+Y6Xv2alfdVOfv39Cr8dmhywb8s8vuOL5r3z/kLftO0zmyKmeZ9fuhLP7QOmrIPx6Xx3JL+C6l+dzuaunGvj/xkpTv/zj9gO8/MW6wvn5Yf8Hn+ds4/a3F/PQVO8r1SD3wJQlXdmoKn9+ezF/mLSo8MB67cvzOXvsx3zt9IZb+OPOiG0P1VITczS79fWF/OWd4j3H3A4eyafr6P8ye8VWjuQVkDlyKn+LcHPr795YyFOzVrPCI5m6BX+fac7v8cCRvHLvQvynNxfzi2eLelMGT4jKiyUJyrau1usOyQNH8rnn3dI1PkU6GxvjOijkFRTE/TlKanC/6oXivaDcftxxgM9zSn/2uGHnQd+bA99fHDraaUGB8tDUoobeNndN5ZEPv+PQ0fzCq6yxH67AywvOWb/XEO//+WYjmSOnkjlyKl1G/5fnP/m+cL4kHy3dzOyVW0MS3R8mLQIC1YM//1fh6PfsOVjUJuFOHhO//qHwN7Jz/xHPpNn78dlMXrChcL6gQDmcl1/YdTk4iGNJ7VDuPJX9Q+QrrnMfne277spxX/PD9v38/F9f8quX/K8s01IiH16WbtwdUzXO7BWBnncbdhYl+B927GfXgaNc98p8jji/qWfm5HA4L5/3Fm30vWIY+NRnXPbsl4XthX96c1HIyddX3we+4/9bFDjB6DhqOt0fmhl1zNFSVd/v5qhzb9W+w3nl0lXcRloDalZLLbN993xklufySGfksXh8+spyf2ZEuD5/j/JuUIVFHo3lUNT4+umqXH49fh5PDu3KC58V1XEXKDz/yRqe/8S72ifI62DvfkRteNvCIz6JxsvNrwWqq85qE/4k3oBlrnrv5Zv2kDlyKk9d2bUwkQT1fGRW4fI2GbXp0boR9w4+JeJ7PzT1O1796gem/7F3YYeDj5Zt5pRmdX23cXeoyMsv4NDRfC584hOu6XkiI3q3DSlb0nhV7jPwtdv2U69mtWJdotfvPEC4I3kFpAikpaYUthlFO0rw2wsCVU+L1++mRYNagHfDvio8PPU7Jnz1A8fXrUF1n6v37B92krN1Hy99vpZ3F27k3YUbuaFXGyBw4hMU7FyxN0LPuwNH8sgvUI6rUS2qzxS0dc8halRPZeq3m7jrP0t49Ben+pbtfP90erZpyKQRZ8X0XqVlSQK49uzMUnVbrcgSVf9bnqYu2cTUJf7PR3hy5qrCBvs/vrko7vebu3YH+QXKzgOJfbbGVxFuRgwXniDCl6/J3c+a3P284fRq87LzwNHCJLp5z6GQ9pFgm5ACn63O5Zy2jcn+YSfN6tUI2ceUxT/RqE4663cc5OFpK7iqe6uoDmzu9pwLfJ4/seynPTwwZRkPXNIJKErYpzavx/u/O7fU7xX01vz1XHFmy8Lu3cHqs72HjvLYR0XJvbNrEMkJztXjvsN57NrpX504+OnQTg4fLd1Mv1OasP9w0ZVdacZm6zgq8N7rxg7m/MdnIyK8/7tzeXZODiN6taVerdDveP2OA6SkCM3r12T3gaN0fzj0pPJOn84o//vfQCxfr9nBmA+W84d+7WNOTCWRithNLVZZWVmanZ0d9XazV2wtbGQ2x77urRuWe71ueTu+bg027zlEj9YNmet81ieGdgl5rsjoIZ0K75q/pMsJZTZ0yt8v78Kyn/Yw3mc0giUP9KeOM3z8tCWbueX1b/jnL0+ne+uGNDmuRsjVYLsmdcjZug+Ai09rxvXntublL9bxfgmxP/erboVXftFoWLs6O/YXP6lo3bg215+TyaBTm/G7Nxay/3AedWtWKzxZe/qq0wvbIt1+fnpznhjalfwCZfOeQ5wz9mMAVjw4IKTXYCxm3tabdk2Oi2lbEVmgqlme6xKRJERkAPAUkAq8qKpjw9anA68SeJDQdmCoqq5z1t0FDAfygd+r6vTS7NOLJQljji3N6tVgUxJGPi1L340eUCZDo9w9qEOxasPSipQk4m64FpFU4BlgINARuEpEOoYVGw7sVNV2wBPAo862HQk8E7sTMAD4l4iklnKfCfNFFI2txpjyc6wlCKDMxs561aNjRiIkondTdyBHVdeo6hFgEjAkrMwQYIIzPRnoK4GKxSHAJFU9rKprgRxnf6XZZ8LsSHAdtTHGlLe9ZTRCbCKSRHPAfafLBmeZZxlVzQN2A40ibFuafQIgIiNEJFtEsnNzY2t8TuQzq40xJhkiDeoZj0p/n4SqjlPVLFXNysjIiGkf+ZYjjDGVXFn1QUpEktgItHTNt3CWeZYRkTSgHoEGbL9tS7PPhCmP0TGNMaYySkSSmA+0F5HWIlKdQEP0lLAyU4BhzvRlwMca6FY1BbhSRNJFpDXQHphXyn0mjuUIYyqd6qkptG9SJ9lhHPPiThJOG8OtwHTgO+AtVV0mIqNF5BKn2EtAIxHJAW4DRjrbLgPeApYDHwG3qGq+3z7jjdX3M1iWMFVAvZqhN1v9vm/7Mn/Pbx/oz2d/uaDY8jsuOjlkvm1G7cLpn58eaH48M7NBse2+f3gQ68YOZt3Ywax6aCAzbjsvwRFH5/yTM5h3T9/C+eWjLyqM79x2jYuVP75u6E2N8+7uW+z/JVZ/7Fc2/592Mx3wm9cW8OHS6IaNNsl3UtM6rNqyL6Ztrz07k1ecp/8l0jU9T2Ti17F1Rfxlj1Z8/f32wkfa1q9VjV1RDDro5fuHB5FXUMCP2w/QNqMObe6eBgQSxJ/6tSe/QGl3z4fFtru8Wwvedo0XFY21jwyi9V3TCqdFhLz8AibNX8+9zoCGr9/Yg+6ZDXnp87V0aVmfyQs2MHnBBj694wJaNapFQYEiUnSX/GPTV7J4/S7PITz+MWs1f5+xinVjB3PluK/4ek3ojZKnt6rPwh930aVFPRY7Q7Icl55WbHiNn3U5ocSb8tz+94ouXNLlBPYdzqPr6BlA6BAjR/IK+Dwnl+tfCRyTnvtVNwZ0Ph4IjBaccVw6J9SvyaGj+fz1/eUhd9mf1qIe324o/bhW0Q5tEq5M75M4FlTmPHlS06pxub3wvgtD5teNHcxLw86MeX/BoSLCndq8Xsz7hMBdzNFKSxH+5/TmjL6kEx/ffn5csXz4h14sH31R4XyKQHpaKu2bHkdKivDs1WdwQr0a/KFve0SEtNQUVj80kLsGdqBVw1qF2905sEOxfU+++ayQM363T++4gPduOYdXrjsTESk8mw4OoZGWmsLVPVoxcmAHbji3NT1bNyItNYWbzmtLzzaNGHNpZ96++SxaNQrEkJIiiAg92zTinHaN+b/fns2ahwd5vvfv+rYvPEhOGnEWz/2qW8j6//zmbCaN6Mnfr+hSuKz3yaGdXC44OYN/XHV64fwtF5R8U1qd9DTSUlNIc8a1b16/Zsj66mkp1HUNldH3lCaF011a1ucEp3yNaqnFRsytFjZWfngSeHJoVzKOSwdgaFZLypIlCQJnEJXR/5zenDdHnMWUW89JdigANK2bHnF93w5NQubfuLGnb9neJ2WwasxAIFDtUL9W8Uvylq6DWmn96+ozOLut94B8QOFBqiRtGtdm3t19i333kR4b+8XIPnzwu3OLDdq24L4L+d+hXQsPNkG39w+tkgEY0buN7/4fu+w0TmlWl1rV03j52jMZ0On4YvEMPLUZX97Vl1TXaHzVnIP1wFOPL1xWu3rosG4L77uQrMyGhWfCN/ZqXbhu7SODaNWoFl1a1uf8k0P/j91EhJvPa8u9F3csNhR8jWqpnJnZMOK2pX1aYd0aRbG3a1KnMNkEBwOEwFAhQSkCN50XmhS6tKgfMl+/VjVm/fk8vrqrDwOd7yA4YGCd9DTuu7gj/76hR7FY2jcNDJPx0rCsYgd+t0GnNguZz/KoanMnrktPb878e/qxbuxgHr3sNN/9JoIlCQLjs1RGN/ZuQ4Pa1el8Qr2IB49EG3NpZ8/l//zlGRG369+paeH0M788g7M8DtY9Wjfkr5d04smhXamelsK6sYN5++azfQ++X4zsA8ApzerSuXnoCKjv33ouLRuGnt0NOrUZrzvJ6ZqeJxbbX/Bs8N7Bp/DpHRcUu1J7zPmDPK5GGk3q1uC0FvU52TkQPH9N4Ax28aj+xfa77K8X0bx+TTo3r8fQM1sV7geKtxUEtQtrlD2xUS3uHnQK68YOpo+TcO+46GQu7Ro4yXF/Qxd0aMJz14SeUZfkd32K6rRrVk+lad10Rl3ckXVjB9Mg7G+kXs1qNK9fkz4dmsT9PPVEO6ttI5755RmsHDOAma42ixrVUguvcGq4Rn5e88hgejoj+V7erQXN69cs9pnm3d2Pthl1aFavJg9e2pnfnN+WXu2LrkaGn9uazMbFr7Lq1azGurGD6XtK02Lr3Hq2aRRytXBH/5P56I+9QsrccVHxq7vyYEmC8mu4vnNAYv+T6zoHl5QU4e5BkYeWjlfw4Hl220b8ynVwdZ89nZnZkMZ1/K8mrnBdFg8+rZlnmfZN6zDs7MyIifvNEUVXIM3r1+TZq8/g1eu7c9kZLULKndqiXsTHoD7oSnbz7u7LN/ddyG0XnsRDP+/M8HNb06pRrZCYAXo7Bwb3GXO1tMB7BEdaDc671U4PPTMP7tfrwTwXOck0/MFU7pIv/DqLF3+dxW/Pb0tqCc9tKK3wWObe3Y/rz20dsqxZvcDvoGndGnwxsg/jr429yq+siAiDT2tGelrkRwB097hyefzyLnwxsg81qgW+0591OYF1YweHDDPeuE46dw7oEHI1lii39z+Jvww4mbTUFDoc7z3s+y97tEr4+0ZiQ4UT2ibRq33juIbd/uSO8znv8TnFlj/881P5ZY9WPOoMaXzByRnMXlm6O8TPbdeY127owQNTloU0tobXgZali7s04/lP1hT7wzgnrAdH9r39fB/YU5ozzgiPb2bC9d05uelxHB827PVA51K9rscZ+cM/P5XRHyxn6JktQ84ewzVx9Tq5ukfxK4yg4+vVIPvefjSsVZTEsk5syNKNewqHag4+2yA9LYXDER4Ks/C+C6lZvXhM/7q6GweO5BWrfnJ/f6kpQr+OoWen5XGq88vurTi+bo2Q+vXK6tXh3dl90LtjwDltG3PHRSdzdTkfkG/tU7yH0lXdi2JY8/Cgcn98qSUJKGwASoRWDWvRuE51tu0rGg+qTnpasez/8nXdmbl8C3VqpHHdy/M56HoiWf+OTbmo0/H8+e3AsM5POw1qdw3qQN9TmnDNS6GPMw2affv5vmP7h6tdPZX9R/K5qXcbqqel8I+PAw9UGX9tFgt+2Mkzs4seBbpu7GDyCxRVuP6c1sX2Ne6abuyJYtyYSyK0AXlVQQWdd1LkO+ov7dqco/kFIWPwn92uMR/9sXepY4vkYufqJ/xq6Z7Bp3BZtxa0dqobgn/EJR20w6twglJTxPPZAH7HhsKDRpxZwuvBPeFSPJJTZVWjWqrviUNKinDLBe3KOaLiwhusS9suk0iWJICTmhaNwV7Sc55LIiJk33thyNm03x6Df2xZmQ0Kr156tW/MuF9nFT49rXPzuoVVL+lpqSH1oOFae9SJRooT4PpzW/Oa02XzD33b06dDU/p0aMrkBRvYsqfocZipYVVaSx7oX/iks/6djsftH1edTq3qqQyfULw78nejB4Rcuruv3BaNupD6tWJvH0pJEYae2YrlP+3xvKrw8vRVpzPV49nX4Qaf1sy3zaVaagqdXT2Rgg2UI3q1Yc6qrSVWe0Ty3K+6kV+g3PL6N75linJEJe6mZyosa5NwLLi3H91bN+TaszMTvu9hJezzbyE9LQJ/8sGhQkpzdlcavzk/tPdG8EAtBPqRA5xxYlGPiim3Rn5y2HE1qvke0H/W5YRiDXVnOO9Rs3pqSJXVxOFFbRrxJAi3vw7pzJ89egZ5uaTLCTx/jWf38BBNj6tRYpmg1JRAF9DbLzqZD37Xi3d+c3aptw03oPPxdGgWOInxe/xmmtOW4LfemHjYlYSjUZ103rrprJgezF6SP/c/KeL6pq768OABNPiH79X1M5LS3gQ1cXh3pi/dTMZx6fTp0JT59/QLqXZrWrcG//1T75B+3vGIdI57ebcWLN6wKyHvU1aSeZbepnFtbr2gHUPP9O4Pf+eADtRJT+Pi0ypnV25TsVmSKGOf/eWCkAbHCdd3Z0mEA2LwhqaOzepy7+BTCocoKK0HL+3smSTCr0faZtThNtfZtle7jLsaLl5PX3m677rHXVdSpjgR4faL/K+M6teqzj2D438mVwXryWoqCEsSYaK9+7rTCXVZ9tOeYstfv7EH+w7lFbvh67yTMjwbYMdd040RExdwWbdAN04R4YZe0d/74NcQl8za6onDu8d041tFEm9blTGVlVVihmlar/Q9nRbc26/YXcRBZ7dtXKxBN5L+nY5n5ZgBIQ2gfnq0bkjHZt59qKHohq/HfnEaD/wscIZZO6yrZXke8yI1tld0lXnIllgloQONqcDsSiJMkygaKGunpyX0DL20vWDevOmsiOuvyGpZeLPW0fwCDuUVcN05mfztv6sKyySqQTySd35zFht2Hizz9ylLAzofzyMfflfsprpjWUW7g9oklyWJONSolsr157Rm1Za9TF+2JdnheKqWmsLN5xUfrKw8jgPdTmxIN//70iqFlg1rseaR+EbYNKYys+qmKHjd4dygdvVSdaGsaOxc0YQLVq3Zb8O4WZKIwmd/uaBwELcLK/ldp1alYMIFu/naT8O4WZKIQkqKFN6wVJlvXHpyaNcyGZzMVG7VnIECb7uwdDcimqohriOdiDQUkRkistr5t/gg6IFyw5wyq0VkmLOslohMFZEVIrJMRMa6yl8rIrkissh53RBPnIl0XvsMft+3PQ8O8R4uuzK4NMp7L0zVkOLcKR5+d76p2uI9HR4JzFLV9sAsZz6EiDQE7gd6AN2B+13J5G+q2gE4HThHRAa6Nn1TVbs6rxfjjDNhUlKE2y48qdI+g8IYY6IRb5IYAkxwpicAl3qUuQiYoao7VHUnMAMYoKoHVHU2gKoeAb4BWnhsb4wxJkni7QLbVFU3OdObAa/W3ObAetf8BmdZIRGpD/wMeMq1+Bci0htYBfxJVd37cG87AhgB0KpV2Yz9PuvP53HwSH7JBSuB01rE9wxnY0zVUmKSEJGZgNetw/e4Z1RVRSTqe8tEJA14A3haVdc4i98H3lDVwyJyE4GrlD5e26vqOGAcQFZWVpncH9s2o07JhSqBxaP6k16t8ja4G2PKX4lJQlX7+a0TkS0i0kxVN4lIM2CrR7GNwPmu+RbAHNf8OGC1qj7pes/trvUvAo+VFKcpWb0oR5Q1xph4TyunAMOc6WHAex5lpgP9RaSB02Dd31mGiIwB6gF/dG/gJJygS4Dv4ozTGGNMDOJNEmOBC0VkNdDPmUdEskTkRQBV3QE8CMx3XqNVdYeItCBQZdUR+Casq+vvnW6xi4HfA9fGGacxxpgYxNVw7VQL9fVYng3c4JofD4wPK7MBnxEAVPUu4K54YjPGGBM/a8U0xhjjy5KEhzrpNjiuMcaAJQlPfU/xfpCQMcZUNZYkPFTFp5EZY4wXSxIeaqeX7glxxhhzrLMk4aF+raLB+2pVt4RhjKm6LEl4cPfLnTi8e9LiMMaYZLMkUYJGtdOTHYIxxiSNJQkP7nZre5SjMaYqsyRhjDHGl9015sF98dC0bo1SbXNjr9ZccLLdX2GMObZYkihBjWql6910z+COZRyJMcaUP6tuMsYY48uShDHGGF+WJIwxxviyJGGMMcZXXElCRBqKyAwRWe3828Cn3DCnzGoRGeZaPkdEVjpPpVskIk2c5eki8qaI5IjIXBHJjCdOY4wxsYn3SmIkMEtV2wOznPkQItIQuB/oAXQH7g9LJleralfntdVZNhzYqartgCeAR+OMMyp2A50xxgTEmySGABOc6QnApR5lLgJmqOoOVd0JzAAGRLHfyUBfETt0G2NMeYs3STRV1U3O9GagqUeZ5sB61/wGZ1nQy05V032uRFC4jarmAbuBRl4BiMgIEckWkezc3Nw4PkqR4PMkWjSomZD9GWNMZVXizXQiMhM43mPVPe4ZVVURifZxPVer6kYROQ54B7gGeDWaHajqOGAcQFZWVkIfF3TlmS0TuTtjjKl0SkwSqtrPb52IbBGRZqq6SUSaAVs9im0EznfNtwDmOPve6Py7V0ReJ9Bm8aqzTUtgg4ikAfWA7aX5QIlgFVvGGBMQb3XTFCDYW2kY8J5HmelAfxFp4DRY9wemi0iaiDQGEJFqwMXAUo/9XgZ8rGoPFTXGmPIW79hNY4G3RGQ48ANwBYCIZAE3q+oNqrpDRB4E5jvbjHaW1SaQLKoBqcBM4AWnzEvARBHJAXYAV8YZpzHGmBjElSRUdTvQ12N5NnCDa348MD6szH6gm89+DwGXxxNbIti1izGmqrM7rj0I1ihhjDFgScIYY0wEliQ8KFbPZIwxYEnCGGNMBJYkPFibhDHGBFiSMMYY48uShDHGGF+WJCKw5mtjTFVnScKDjd1kjDEBliSMMcb4siThoVpq4GtJS7VLCmNM1RbvAH/HpBt7tWH/4TyuP6d1skMxxpiksiThoWb1VO4adEqywzDGmKSz6iZjjDG+LEkYY4zxZUnCGGOMr7iShIg0FJEZIrLa+beBT7lhTpnVIjLMWXaciCxyvbaJyJPOumtFJNe17gav/RpjjClb8V5JjARmqWp7YJYzH0JEGgL3Az2A7sD9ItJAVfeqatfgi8DjT//j2vRN1/oX44zTGGNMDOJNEkOACc70BOBSjzIXATNUdYeq7gRmAAPcBUTkJKAJ8Fmc8RhjjEmgeJNEU1Xd5ExvBpp6lGkOrHfNb3CWuV1J4MrBPVzSL0TkWxGZLCIt/QIQkREiki0i2bm5uTF8BGOMMX5KTBIiMlNElnq8hrjLOQf4WMfEuxJ4wzX/PpCpqqcRuPKY4LlV4H3HqWqWqmZlZGTE+PbGGGO8lHgznar281snIltEpJmqbhKRZsBWj2IbgfNd8y2AOa59dAHSVHWB6z23u8q/CDxWUpwACxYs2CYiP5SmrIfGwLYYty1vFmviVZY4ofLEWlniBIv1RL8V8d5xPQUYBox1/n3Po8x04GFXz6f+wF2u9VcRehVBMPE4s5cA35UmGFWN+VJCRLJVNSvW7cuTxZp4lSVOqDyxVpY4wWKNJN4kMRZ4S0SGE+iddAWAiGQBN6vqDaq6Q0QeBOY724xW1R2ufVwBDArb7+9F5BIgD9gBXBtnnMYYY2IQV5JwqoX6eizPBm5wzY8Hxvvso43HsrsIvdowxhiTBHbHdZFxyQ4gChZr4lWWOKHyxFpZ4gSL1ZeE9jo1xhhjitiVhDHGGF+WJIwxxviyJAGIyAARWSkiOSJSbPypcophnYgscQY0zHaWeQ6gKAFPO/F+KyJnuPZTbDDFBMQ2XkS2ishS17KExSYi3ZzPnuNsG9NzY33ifEBENroGixzkWneX854rReQi13LP34OItBaRuc7yN0WkeixxOvtqKSKzRWS5iCwTkT84yyvU9xohzgr3vYpIDRGZJyKLnVj/Gmn/IpLuzOc46zNj/QwJjPUVEVnr+l67OsuT9neFqlbpF5AKfA+0AaoDi4GOSYhjHdA4bNljwEhneiTwqDM9CPgQEKAnMNdZ3hBY4/zbwJlukIDYegNnAEvLIjZgnlNWnG0HJjDOB4DbPcp2dP6v04HWzm8gNdLvAXgLuNKZfg74TRzfaTPgDGf6OGCVE1OF+l4jxFnhvlfnc9ZxpqsBc53P77l/4LfAc850cGigmD5DAmN9BbjMo3zS/q7sSiIwMm2Oqq5R1SPAJAIDF1YEfgMoDgFe1YCvgfoSuOO9xMEUY6GqnxK4XyXhsTnr6qrq1xr4Zb+K90CRscbpZwgwSVUPq+paIIfAb8Hz9+CchfUBJnt85lhi3aSq3zjTewncMNqcCva9RojTT9K+V+e72efMVnNeGmH/7u96MtDXiSeqz5DgWP0k7e/KkkTpBiAsDwr8V0QWiMgIZ5nfAIp+MZfnZ0lUbM2d6fDliXSrc4k+Xoru/I82zkbALlXNS3ScTjXH6QTOJivs9xoWJ1TA71VEUkVkEYEhgmYQOPP3239hTM763U485fL3FR6rqga/14ec7/UJEUkPj7WUMSXs/9+SRMVxrqqeAQwEbhGR3u6VztlAheyvXJFjA54F2gJdgU3A35MaTRgRqQO8A/xRVfe411Wk79Ujzgr5vapqvgaeT9OCwJl/h+RG5C88VhHpTOAm4g7AmQSqkO5MXoQBliQCAxC6hyJv4SwrV6q60fl3K/AugR/4FueyEQkdQNEv5vL8LImKbaMzXSYxq+oW54+xAHiBwPcaS5zbCVzip4Utj5mIVCNw4P23qgYfuFXhvlevOCvy9+rEtwuYDZwVYf+FMTnr6znxlOvflyvWAU71nqrqYeBlYv9eE/d3FUtDxrH0IjA0yRoCDVTBxqhO5RxDbeA41/SXBNoSHie0EfMxZ3owoY1Y87SoEWstgQasBs50wwTFmElog3DCYqN4A9ugBMbZzDX9JwJ1zQCdCG2cXEOgYdL39wC8TWgD6G/jiFMI1BM/Gba8Qn2vEeKscN8rkAHUd6ZrEniI2cV++wduIbTh+q1YP0MCY23m+t6fBMYm+++q3A6EFflFoOfAKgL1l/ck4f3bOD+4xcCyYAwE6kdnAauBma7/fAGeceJdAmS59nU9gYa2HOC6BMX3BoEqhaME6jaHJzI2IAtY6mzzT5yRABIU50Qnjm8JjFrsPrjd47znSlw9P/x+D87/0zwn/reB9Di+03MJVCV9CyxyXoMq2vcaIc4K970CpwELnZiWAqMi7R+o4cznOOvbxPoZEhjrx873uhR4jaIeUEn7u7JhOYwxxviyNgljjDG+LEkYY4zxZUnCGGOML0sSxhhjfFmSMMYY48uShDHGGF+WJIwxxvj6fz97TxfICm7PAAAAAElFTkSuQmCC\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":"# <span class=\"title-section w3-xxlarge\" id=\"training\">Training 🏋️</span>\n<hr>\n\nOur model will be trained for the number of FOLDS and EPOCHS you chose in the configuration above. Each fold the model with lowest validation loss will be saved and used to predict OOF and test. Adjust the variable `VERBOSE`. The variable `VERBOSE=1 or 2` will display the training and validation loss for each epoch as text. ","metadata":{"papermill":{"duration":0.038491,"end_time":"2021-11-29T18:09:46.955605","exception":false,"start_time":"2021-11-29T18:09:46.917114","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Time2Vec(keras.layers.Layer):\n    def __init__(self, kernel_size=1):\n        super(Time2Vec, self).__init__(trainable=True, name='Time2VecLayer')\n        self.k = kernel_size\n    \n    def build(self, input_shape):  # build automatically executed before layer is called for the first time --  mostly used to instantiate weights\n        # trend\n        self.wb = self.add_weight(name='wb',shape=(input_shape[1],),initializer='uniform',trainable=True)\n        self.bb = self.add_weight(name='bb',shape=(input_shape[1],),initializer='uniform',trainable=True)\n        # periodic\n        self.wa = self.add_weight(name='wa',shape=(1, input_shape[1], self.k),initializer='uniform',trainable=True)\n        self.ba = self.add_weight(name='ba',shape=(1, input_shape[1], self.k),initializer='uniform',trainable=True)\n        super(Time2Vec, self).build(input_shape)\n    \n    def call(self, inputs, **kwargs):  # where the layer logic lives\n        bias = self.wb * inputs + self.bb\n        dp = K.dot(inputs, self.wa) + self.ba\n        wgts = K.sin(dp) # or K.cos(.)\n\n        ret = K.concatenate([K.expand_dims(bias, -1), wgts], -1)\n        ret = K.reshape(ret, (-1, inputs.shape[1]*(self.k+1)))\n        return ret\n    \n    def compute_output_shape(self, input_shape):\n        return (input_shape[0], input_shape[1]*(self.k + 1))\n    \n\n# https://towardsdatascience.com/time2vec-for-time-series-features-encoding-a03a4f3f937e\nclass T2V(keras.layers.Layer):\n    \n    def __init__(self, output_dim=None, **kwargs):\n        self.output_dim = output_dim\n        super(T2V, self).__init__(**kwargs)\n        \n    def build(self, input_shape):        \n        self.W = self.add_weight(name='W',\n                      shape=(input_shape[-1], self.output_dim),\n                      initializer='uniform',\n                      trainable=True)        \n        self.P = self.add_weight(name='P',\n                      shape=(input_shape[1], self.output_dim),\n                      initializer='uniform',\n                      trainable=True)        \n        self.w = self.add_weight(name='w',\n                      shape=(input_shape[1], 1),\n                      initializer='uniform',\n                      trainable=True)        \n        self.p = self.add_weight(name='p',\n                      shape=(input_shape[1], 1),\n                      initializer='uniform',\n                      trainable=True)        \n        super(T2V, self).build(input_shape)\n        \n    def call(self, x):\n        \n        original = self.w * x + self.p\n        sin_trans = K.sin(K.dot(x, self.W) + self.P)\n        \n        return K.concatenate([sin_trans, original], -1)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:48:05.900707Z","iopub.execute_input":"2022-02-16T23:48:05.901038Z","iopub.status.idle":"2022-02-16T23:48:05.919205Z","shell.execute_reply.started":"2022-02-16T23:48:05.900994Z","shell.execute_reply":"2022-02-16T23:48:05.918366Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"# https://towardsdatascience.com/the-time-series-transformer-2a521a0efad3\nfrom tensorflow_addons.layers import MultiHeadAttention\n\nclass AttentionBlock(keras.Model):\n    def __init__(self, name='AttentionBlock', num_heads=2, head_size=128, ff_dim=None, dropout=0, **kwargs):\n        super().__init__(name=name, **kwargs)\n\n        if ff_dim is None:\n            ff_dim = head_size\n\n        self.attention = MultiHeadAttention(num_heads=num_heads, head_size=head_size, dropout=dropout)\n        self.attention_dropout = keras.layers.Dropout(dropout)\n        self.attention_norm = keras.layers.BatchNormalization(epsilon=1e-6)\n\n        self.ff_conv1 = keras.layers.Conv1D(filters=ff_dim, kernel_size=1, activation='relu')\n        # self.ff_conv2 at build()\n        self.ff_dropout = keras.layers.Dropout(dropout)\n        self.ff_norm = keras.layers.BatchNormalization(epsilon=1e-6)\n\n    def build(self, input_shape):\n        self.ff_conv2 = keras.layers.Conv1D(filters=input_shape[-1], kernel_size=1) \n\n    def call(self, inputs):\n        x = self.attention([inputs, inputs])\n        x = self.attention_dropout(x)\n        x = self.attention_norm(inputs + x)\n\n        x = self.ff_conv1(x)\n        x = self.ff_conv2(x)\n        x = self.ff_dropout(x)\n\n        x = self.ff_norm(inputs + x)\n        return x\n    \n\nclass ModelTrunk(keras.Model):\n    def __init__(self, name='ModelTrunk', time2vec_dim=1, num_heads=2, head_size=128, ff_dim=None, num_layers=1, dropout=0, **kwargs):\n        super().__init__(name=name, **kwargs)\n        self.time2vec = Time2Vec(kernel_size=time2vec_dim)\n        if ff_dim is None:\n            ff_dim = head_size\n        self.dropout = dropout\n        self.attention_layers = [AttentionBlock(num_heads=num_heads, head_size=head_size, ff_dim=ff_dim, dropout=dropout) for _ in range(num_layers)]\n\n        \n    def call(self, inputs):\n        time_embedding = keras.layers.TimeDistributed(self.time2vec)(inputs)\n        x = K.concatenate([inputs, time_embedding], -1)\n        for attention_layer in self.attention_layers:\n            x = attention_layer(x)\n\n        return K.reshape(x, (-1, x.shape[1] * x.shape[2])) # flat vector of features out\n    \n\ndef build_model_new(\n    input_shape,  # shape of time series sample\n    head_size, # size of multi head attention\n    num_heads,  # number of multi-head attention \n    ff_dim,  # \n    num_transformer_blocks, # \n    mlp_units, # list of N dense layers with i neurons\n    dropout=0,  # transformer block dropout rate\n    mlp_dropout=0,  # \n    time2vec_dim=3\n):\n    inputs = keras.Input(shape=input_shape)\n    \n    x = ModelTrunk(num_heads=num_heads, head_size=head_size, ff_dim=ff_dim, num_layers=num_transformer_blocks, time2vec_dim=time2vec_dim, dropout=dropout)(inputs)\n\n    # x = layers.GlobalAveragePooling1D(data_format=\"channels_first\")(x)\n    for dim in mlp_units:\n        x = layers.Dense(dim, activation=\"relu\")(x)\n        x = layers.Dropout(mlp_dropout)(x)\n    outputs = layers.Dense(1, kernel_initializer=\"normal\")(x)\n    return keras.Model(inputs, outputs)\n\n\n# https://github.com/tatp22/multidim-positional-encoding/blob/master/positional_encodings/tf_positional_encodings.py\nclass TFPositionalEncoding1D(tf.keras.layers.Layer):\n    def __init__(self, channels: int, dtype=tf.float32):\n        \"\"\"\n        Args:\n            channels int: The last dimension of the tensor you want to apply pos emb to.\n        Keyword Args:\n            dtype: output type of the encodings. Default is \"tf.float32\".\n        \"\"\"\n        super(TFPositionalEncoding1D, self).__init__()\n\n        self.channels = int(np.ceil(channels / 2) * 2)\n        self.inv_freq = np.float32(\n            1\n            / np.power(\n                10000, np.arange(0, self.channels, 2) / np.float32(self.channels)\n            )\n        )\n\n    @tf.function\n    def call(self, inputs):\n        \"\"\"\n        :param tensor: A 3d tensor of size (batch_size, x, ch)\n        :return: Positional Encoding Matrix of size (batch_size, x, ch)\n        \"\"\"\n        if len(inputs.shape) != 3:\n            raise RuntimeError(\"The input tensor has to be 3d!\")\n        _, x, org_channels = inputs.shape\n\n        dtype = self.inv_freq.dtype\n        pos_x = tf.range(x, dtype=dtype)\n        sin_inp_x = tf.einsum(\"i,j->ij\", pos_x, self.inv_freq)\n        emb = tf.expand_dims(tf.concat((tf.sin(sin_inp_x), tf.cos(sin_inp_x)), -1), 0)\n        emb = emb[0]  # A bit of a hack\n        return tf.repeat(emb[None, :, :org_channels], tf.shape(inputs)[0], axis=0)        \n        ","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:48:05.920891Z","iopub.execute_input":"2022-02-16T23:48:05.921422Z","iopub.status.idle":"2022-02-16T23:48:06.097689Z","shell.execute_reply.started":"2022-02-16T23:48:05.921379Z","shell.execute_reply":"2022-02-16T23:48:06.096733Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"# https://keras.io/examples/timeseries/timeseries_transformer_classification/\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\n\ndef transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):\n    # Normalization and Attention\n    x = layers.BatchNormalization(epsilon=1e-6)(inputs)\n    x = layers.MultiHeadAttention(\n        key_dim=head_size, num_heads=num_heads, dropout=dropout\n    )(x, x)\n    x = layers.Dropout(dropout)(x)\n    res = x + inputs\n\n    # Feed Forward Part\n    x = layers.BatchNormalization(epsilon=1e-6)(res)\n    x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation=\"relu\")(x)\n    x = layers.Dropout(dropout)(x)\n    x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)\n    return x + res\n\n# self.time2vec = Time2Vec(kernel_size=time2vec_dim)\n\ndef build_model(\n    input_shape,  # shape of time series sample\n    head_size, # size of multi head attention\n    num_heads,  # number of multi-head attention \n    ff_dim,  # \n    num_transformer_blocks, # \n    mlp_units, # list of N dense layers with i neurons\n    dropout=0,  # transformer block dropout rate\n    mlp_dropout=0,  # \n    time2vec_dim=1\n):\n    inputs = keras.Input(shape=input_shape)\n    \n    time_embedding = TFPositionalEncoding1D(channels=input_shape[-1])(inputs)\n    x = tf.keras.layers.Add()([inputs, time_embedding])\n    \n    # x = inputs\n    for _ in range(num_transformer_blocks):\n        x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout)\n\n    x = layers.GlobalAveragePooling1D(data_format=\"channels_first\")(x)\n    for dim in mlp_units:\n        x = layers.Dense(dim, activation=\"relu\")(x)\n        x = layers.Dropout(mlp_dropout)(x)\n    outputs = layers.Dense(1, kernel_initializer=\"normal\")(x)\n    return keras.Model(inputs, outputs)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:57:05.289538Z","iopub.execute_input":"2022-02-16T23:57:05.289880Z","iopub.status.idle":"2022-02-16T23:57:05.316472Z","shell.execute_reply.started":"2022-02-16T23:57:05.289841Z","shell.execute_reply":"2022-02-16T23:57:05.315274Z"},"trusted":true},"execution_count":34,"outputs":[]},{"cell_type":"code","source":"input_shape = train_generator[0][0].shape[1:]\n\nmodel = build_model(\n    input_shape,\n    head_size=256,\n    num_heads=4,\n    ff_dim=4,\n    num_transformer_blocks=8,\n    mlp_units=[128, 128],\n    mlp_dropout=0.1,\n    dropout=0.1,\n    time2vec_dim=TIME2VEC_DIM\n)\n\n# tfp.stats.correlation\n#tf_corr_new = tfp.stats.correlation(x, y=None, sample_axis=0, event_axis=-1, keepdims=False, name=None)\n# tf.contrib.metrics.streaming_pearson_correlation(logits,labels)\n\n# model.compile(\n#     optimizer=\"Adam\",\n#     loss={\"head1\": \"mse\", \"head2\": \"mse\"},\n#     loss_weights={\"head1\": HEAD1_WEIGHT, \"head2\": HEAD2_WEIGHT},\n#     metrics={\"head1\": [\"mae\"], \"head2\": [\"mae\"]}\n# )\n# \nloss_func = combined_loss\nmodel.compile(\n    loss=loss_func,\n    optimizer=keras.optimizers.Adam(learning_rate=1e-2),\n    metrics=[\"mae\", tfp.stats.correlation],\n)\nmodel.summary()\n","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:57:07.134095Z","iopub.execute_input":"2022-02-16T23:57:07.134622Z","iopub.status.idle":"2022-02-16T23:57:08.533446Z","shell.execute_reply.started":"2022-02-16T23:57:07.134585Z","shell.execute_reply":"2022-02-16T23:57:08.532463Z"},"trusted":true},"execution_count":35,"outputs":[{"name":"stdout","text":"Model: \"model_1\"\n__________________________________________________________________________________________________\nLayer (type)                    Output Shape         Param #     Connected to                     \n==================================================================================================\ninput_2 (InputLayer)            [(None, 64, 8)]      0                                            \n__________________________________________________________________________________________________\ntf_positional_encoding1d (TFPos (None, 64, 8)        0           input_2[0][0]                    \n__________________________________________________________________________________________________\nadd (Add)                       (None, 64, 8)        0           input_2[0][0]                    \n                                                                 tf_positional_encoding1d[0][0]   \n__________________________________________________________________________________________________\nbatch_normalization_16 (BatchNo (None, 64, 8)        32          add[0][0]                        \n__________________________________________________________________________________________________\nmulti_head_attention_8 (MultiHe (None, 64, 8)        35848       batch_normalization_16[0][0]     \n                                                                 batch_normalization_16[0][0]     \n__________________________________________________________________________________________________\ndropout_18 (Dropout)            (None, 64, 8)        0           multi_head_attention_8[0][0]     \n__________________________________________________________________________________________________\ntf.__operators__.add_16 (TFOpLa (None, 64, 8)        0           dropout_18[0][0]                 \n                                                                 add[0][0]                        \n__________________________________________________________________________________________________\nbatch_normalization_17 (BatchNo (None, 64, 8)        32          tf.__operators__.add_16[0][0]    \n__________________________________________________________________________________________________\nconv1d_16 (Conv1D)              (None, 64, 4)        36          batch_normalization_17[0][0]     \n__________________________________________________________________________________________________\ndropout_19 (Dropout)            (None, 64, 4)        0           conv1d_16[0][0]                  \n__________________________________________________________________________________________________\nconv1d_17 (Conv1D)              (None, 64, 8)        40          dropout_19[0][0]                 \n__________________________________________________________________________________________________\ntf.__operators__.add_17 (TFOpLa (None, 64, 8)        0           conv1d_17[0][0]                  \n                                                                 tf.__operators__.add_16[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_18 (BatchNo (None, 64, 8)        32          tf.__operators__.add_17[0][0]    \n__________________________________________________________________________________________________\nmulti_head_attention_9 (MultiHe (None, 64, 8)        35848       batch_normalization_18[0][0]     \n                                                                 batch_normalization_18[0][0]     \n__________________________________________________________________________________________________\ndropout_20 (Dropout)            (None, 64, 8)        0           multi_head_attention_9[0][0]     \n__________________________________________________________________________________________________\ntf.__operators__.add_18 (TFOpLa (None, 64, 8)        0           dropout_20[0][0]                 \n                                                                 tf.__operators__.add_17[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_19 (BatchNo (None, 64, 8)        32          tf.__operators__.add_18[0][0]    \n__________________________________________________________________________________________________\nconv1d_18 (Conv1D)              (None, 64, 4)        36          batch_normalization_19[0][0]     \n__________________________________________________________________________________________________\ndropout_21 (Dropout)            (None, 64, 4)        0           conv1d_18[0][0]                  \n__________________________________________________________________________________________________\nconv1d_19 (Conv1D)              (None, 64, 8)        40          dropout_21[0][0]                 \n__________________________________________________________________________________________________\ntf.__operators__.add_19 (TFOpLa (None, 64, 8)        0           conv1d_19[0][0]                  \n                                                                 tf.__operators__.add_18[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_20 (BatchNo (None, 64, 8)        32          tf.__operators__.add_19[0][0]    \n__________________________________________________________________________________________________\nmulti_head_attention_10 (MultiH (None, 64, 8)        35848       batch_normalization_20[0][0]     \n                                                                 batch_normalization_20[0][0]     \n__________________________________________________________________________________________________\ndropout_22 (Dropout)            (None, 64, 8)        0           multi_head_attention_10[0][0]    \n__________________________________________________________________________________________________\ntf.__operators__.add_20 (TFOpLa (None, 64, 8)        0           dropout_22[0][0]                 \n                                                                 tf.__operators__.add_19[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_21 (BatchNo (None, 64, 8)        32          tf.__operators__.add_20[0][0]    \n__________________________________________________________________________________________________\nconv1d_20 (Conv1D)              (None, 64, 4)        36          batch_normalization_21[0][0]     \n__________________________________________________________________________________________________\ndropout_23 (Dropout)            (None, 64, 4)        0           conv1d_20[0][0]                  \n__________________________________________________________________________________________________\nconv1d_21 (Conv1D)              (None, 64, 8)        40          dropout_23[0][0]                 \n__________________________________________________________________________________________________\ntf.__operators__.add_21 (TFOpLa (None, 64, 8)        0           conv1d_21[0][0]                  \n                                                                 tf.__operators__.add_20[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_22 (BatchNo (None, 64, 8)        32          tf.__operators__.add_21[0][0]    \n__________________________________________________________________________________________________\nmulti_head_attention_11 (MultiH (None, 64, 8)        35848       batch_normalization_22[0][0]     \n                                                                 batch_normalization_22[0][0]     \n__________________________________________________________________________________________________\ndropout_24 (Dropout)            (None, 64, 8)        0           multi_head_attention_11[0][0]    \n__________________________________________________________________________________________________\ntf.__operators__.add_22 (TFOpLa (None, 64, 8)        0           dropout_24[0][0]                 \n                                                                 tf.__operators__.add_21[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_23 (BatchNo (None, 64, 8)        32          tf.__operators__.add_22[0][0]    \n__________________________________________________________________________________________________\nconv1d_22 (Conv1D)              (None, 64, 4)        36          batch_normalization_23[0][0]     \n__________________________________________________________________________________________________\ndropout_25 (Dropout)            (None, 64, 4)        0           conv1d_22[0][0]                  \n__________________________________________________________________________________________________\nconv1d_23 (Conv1D)              (None, 64, 8)        40          dropout_25[0][0]                 \n__________________________________________________________________________________________________\ntf.__operators__.add_23 (TFOpLa (None, 64, 8)        0           conv1d_23[0][0]                  \n                                                                 tf.__operators__.add_22[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_24 (BatchNo (None, 64, 8)        32          tf.__operators__.add_23[0][0]    \n__________________________________________________________________________________________________\nmulti_head_attention_12 (MultiH (None, 64, 8)        35848       batch_normalization_24[0][0]     \n                                                                 batch_normalization_24[0][0]     \n__________________________________________________________________________________________________\ndropout_26 (Dropout)            (None, 64, 8)        0           multi_head_attention_12[0][0]    \n__________________________________________________________________________________________________\ntf.__operators__.add_24 (TFOpLa (None, 64, 8)        0           dropout_26[0][0]                 \n                                                                 tf.__operators__.add_23[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_25 (BatchNo (None, 64, 8)        32          tf.__operators__.add_24[0][0]    \n__________________________________________________________________________________________________\nconv1d_24 (Conv1D)              (None, 64, 4)        36          batch_normalization_25[0][0]     \n__________________________________________________________________________________________________\ndropout_27 (Dropout)            (None, 64, 4)        0           conv1d_24[0][0]                  \n__________________________________________________________________________________________________\nconv1d_25 (Conv1D)              (None, 64, 8)        40          dropout_27[0][0]                 \n__________________________________________________________________________________________________\ntf.__operators__.add_25 (TFOpLa (None, 64, 8)        0           conv1d_25[0][0]                  \n                                                                 tf.__operators__.add_24[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_26 (BatchNo (None, 64, 8)        32          tf.__operators__.add_25[0][0]    \n__________________________________________________________________________________________________\nmulti_head_attention_13 (MultiH (None, 64, 8)        35848       batch_normalization_26[0][0]     \n                                                                 batch_normalization_26[0][0]     \n__________________________________________________________________________________________________\ndropout_28 (Dropout)            (None, 64, 8)        0           multi_head_attention_13[0][0]    \n__________________________________________________________________________________________________\ntf.__operators__.add_26 (TFOpLa (None, 64, 8)        0           dropout_28[0][0]                 \n                                                                 tf.__operators__.add_25[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_27 (BatchNo (None, 64, 8)        32          tf.__operators__.add_26[0][0]    \n__________________________________________________________________________________________________\nconv1d_26 (Conv1D)              (None, 64, 4)        36          batch_normalization_27[0][0]     \n__________________________________________________________________________________________________\ndropout_29 (Dropout)            (None, 64, 4)        0           conv1d_26[0][0]                  \n__________________________________________________________________________________________________\nconv1d_27 (Conv1D)              (None, 64, 8)        40          dropout_29[0][0]                 \n__________________________________________________________________________________________________\ntf.__operators__.add_27 (TFOpLa (None, 64, 8)        0           conv1d_27[0][0]                  \n                                                                 tf.__operators__.add_26[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_28 (BatchNo (None, 64, 8)        32          tf.__operators__.add_27[0][0]    \n__________________________________________________________________________________________________\nmulti_head_attention_14 (MultiH (None, 64, 8)        35848       batch_normalization_28[0][0]     \n                                                                 batch_normalization_28[0][0]     \n__________________________________________________________________________________________________\ndropout_30 (Dropout)            (None, 64, 8)        0           multi_head_attention_14[0][0]    \n__________________________________________________________________________________________________\ntf.__operators__.add_28 (TFOpLa (None, 64, 8)        0           dropout_30[0][0]                 \n                                                                 tf.__operators__.add_27[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_29 (BatchNo (None, 64, 8)        32          tf.__operators__.add_28[0][0]    \n__________________________________________________________________________________________________\nconv1d_28 (Conv1D)              (None, 64, 4)        36          batch_normalization_29[0][0]     \n__________________________________________________________________________________________________\ndropout_31 (Dropout)            (None, 64, 4)        0           conv1d_28[0][0]                  \n__________________________________________________________________________________________________\nconv1d_29 (Conv1D)              (None, 64, 8)        40          dropout_31[0][0]                 \n__________________________________________________________________________________________________\ntf.__operators__.add_29 (TFOpLa (None, 64, 8)        0           conv1d_29[0][0]                  \n                                                                 tf.__operators__.add_28[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_30 (BatchNo (None, 64, 8)        32          tf.__operators__.add_29[0][0]    \n__________________________________________________________________________________________________\nmulti_head_attention_15 (MultiH (None, 64, 8)        35848       batch_normalization_30[0][0]     \n                                                                 batch_normalization_30[0][0]     \n__________________________________________________________________________________________________\ndropout_32 (Dropout)            (None, 64, 8)        0           multi_head_attention_15[0][0]    \n__________________________________________________________________________________________________\ntf.__operators__.add_30 (TFOpLa (None, 64, 8)        0           dropout_32[0][0]                 \n                                                                 tf.__operators__.add_29[0][0]    \n__________________________________________________________________________________________________\nbatch_normalization_31 (BatchNo (None, 64, 8)        32          tf.__operators__.add_30[0][0]    \n__________________________________________________________________________________________________\nconv1d_30 (Conv1D)              (None, 64, 4)        36          batch_normalization_31[0][0]     \n__________________________________________________________________________________________________\ndropout_33 (Dropout)            (None, 64, 4)        0           conv1d_30[0][0]                  \n__________________________________________________________________________________________________\nconv1d_31 (Conv1D)              (None, 64, 8)        40          dropout_33[0][0]                 \n__________________________________________________________________________________________________\ntf.__operators__.add_31 (TFOpLa (None, 64, 8)        0           conv1d_31[0][0]                  \n                                                                 tf.__operators__.add_30[0][0]    \n__________________________________________________________________________________________________\nglobal_average_pooling1d_1 (Glo (None, 64)           0           tf.__operators__.add_31[0][0]    \n__________________________________________________________________________________________________\ndense_3 (Dense)                 (None, 128)          8320        global_average_pooling1d_1[0][0] \n__________________________________________________________________________________________________\ndropout_34 (Dropout)            (None, 128)          0           dense_3[0][0]                    \n__________________________________________________________________________________________________\ndense_4 (Dense)                 (None, 128)          16512       dropout_34[0][0]                 \n__________________________________________________________________________________________________\ndropout_35 (Dropout)            (None, 128)          0           dense_4[0][0]                    \n__________________________________________________________________________________________________\ndense_5 (Dense)                 (None, 1)            129         dropout_35[0][0]                 \n==================================================================================================\nTotal params: 312,865\nTrainable params: 312,609\nNon-trainable params: 256\n__________________________________________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"# input_shape = train_generator[0][0].shape[1:]\n# \n# model = build_model_new(\n#     input_shape,\n#     head_size=256,\n#     num_heads=4,\n#     ff_dim=4,\n#     num_transformer_blocks=8,\n#     mlp_units=[128, 128],\n#     mlp_dropout=0.1,\n#     dropout=0.1,\n#     time2vec_dim=TIME2VEC_DIM\n# )\n# \n# model.compile(\n#     loss=\"mae\",\n#     optimizer=keras.optimizers.Adam(learning_rate=1e-2),\n#     metrics=[\"mae\", tf_wcorr],\n# )\n# model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:57:08.535592Z","iopub.execute_input":"2022-02-16T23:57:08.535942Z","iopub.status.idle":"2022-02-16T23:57:08.540893Z","shell.execute_reply.started":"2022-02-16T23:57:08.535898Z","shell.execute_reply":"2022-02-16T23:57:08.539817Z"},"trusted":true},"execution_count":36,"outputs":[]},{"cell_type":"code","source":"def lr_scheduler(epoch, lr, warmup_epochs=15, decay_epochs=100, initial_lr=1e-6, base_lr=1e-3, min_lr=5e-5):\n    if epoch <= warmup_epochs:\n        pct = epoch / warmup_epochs\n        return ((base_lr - initial_lr) * pct) + initial_lr\n\n    if epoch > warmup_epochs and epoch < warmup_epochs+decay_epochs:\n        pct = 1 - ((epoch - warmup_epochs) / decay_epochs)\n        return ((base_lr - min_lr) * pct) + min_lr\n\n    return min_lr\n\n","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:57:08.542526Z","iopub.execute_input":"2022-02-16T23:57:08.542898Z","iopub.status.idle":"2022-02-16T23:57:08.553774Z","shell.execute_reply.started":"2022-02-16T23:57:08.542833Z","shell.execute_reply":"2022-02-16T23:57:08.552739Z"},"trusted":true},"execution_count":37,"outputs":[]},{"cell_type":"code","source":"callbacks = [keras.callbacks.EarlyStopping(monitor='val_correlation', mode='max',patience=5, restore_best_weights=True)]\ncallbacks += [keras.callbacks.LearningRateScheduler(lr_scheduler, verbose=1)]\nfilepath = \"./transformer_model\"\ncallbacks += [keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False)]\n\nEPOCHS = 10\nhistory = model.fit(\n            train_generator, \n            validation_data = (val_generator),\n            epochs=EPOCHS,\n            batch_size=BATCH_SIZE,\n            callbacks=callbacks)\n\nmodel.evaluate(test_generator, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:57:08.711875Z","iopub.execute_input":"2022-02-16T23:57:08.714694Z","iopub.status.idle":"2022-02-17T00:04:22.650268Z","shell.execute_reply.started":"2022-02-16T23:57:08.714645Z","shell.execute_reply":"2022-02-17T00:04:22.648841Z"},"trusted":true},"execution_count":38,"outputs":[{"name":"stdout","text":"Epoch 1/10\n\nEpoch 00001: LearningRateScheduler setting learning rate to 1e-06.\n445/445 [==============================] - 38s 67ms/step - loss: 0.1175 - mae: 0.2359 - correlation: 8.3939e-04 - val_loss: 0.1016 - val_mae: 0.2231 - val_correlation: 0.0198\n\nEpoch 00001: val_loss improved from inf to 0.10163, saving model to ./transformer_model\n","output_type":"stream"},{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/keras/utils/generic_utils.py:497: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n  category=CustomMaskWarning)\n","output_type":"stream"},{"name":"stdout","text":"Epoch 2/10\n\nEpoch 00002: LearningRateScheduler setting learning rate to 6.76e-05.\n445/445 [==============================] - 29s 64ms/step - loss: 0.0323 - mae: 0.0772 - correlation: 0.0126 - val_loss: 0.0566 - val_mae: 0.1813 - val_correlation: 0.0680\n\nEpoch 00002: val_loss improved from 0.10163 to 0.05662, saving model to ./transformer_model\n","output_type":"stream"},{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/keras/utils/generic_utils.py:497: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n  category=CustomMaskWarning)\n","output_type":"stream"},{"name":"stdout","text":"Epoch 3/10\n\nEpoch 00003: LearningRateScheduler setting learning rate to 0.0001342.\n445/445 [==============================] - 28s 63ms/step - loss: 0.0040 - mae: 0.0603 - correlation: 0.0523 - val_loss: -0.0088 - val_mae: 0.0622 - val_correlation: 0.0798\n\nEpoch 00003: val_loss improved from 0.05662 to -0.00882, saving model to ./transformer_model\n","output_type":"stream"},{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/keras/utils/generic_utils.py:497: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n  category=CustomMaskWarning)\n","output_type":"stream"},{"name":"stdout","text":"Epoch 4/10\n\nEpoch 00004: LearningRateScheduler setting learning rate to 0.00020080000000000003.\n445/445 [==============================] - 28s 64ms/step - loss: -0.0131 - mae: 0.0517 - correlation: 0.0779 - val_loss: 0.0369 - val_mae: 0.0578 - val_correlation: -0.0161\n\nEpoch 00004: val_loss did not improve from -0.00882\nEpoch 5/10\n\nEpoch 00005: LearningRateScheduler setting learning rate to 0.00026740000000000005.\n445/445 [==============================] - 29s 65ms/step - loss: -0.0228 - mae: 0.0402 - correlation: 0.0859 - val_loss: -0.0216 - val_mae: 0.0120 - val_correlation: 0.0551\n\nEpoch 00005: val_loss improved from -0.00882 to -0.02156, saving model to ./transformer_model\n","output_type":"stream"},{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/keras/utils/generic_utils.py:497: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n  category=CustomMaskWarning)\n","output_type":"stream"},{"name":"stdout","text":"Epoch 6/10\n\nEpoch 00006: LearningRateScheduler setting learning rate to 0.00033400000000000004.\n445/445 [==============================] - 29s 65ms/step - loss: -0.0245 - mae: 0.0409 - correlation: 0.0899 - val_loss: -0.0022 - val_mae: 0.0582 - val_correlation: 0.0626\n\nEpoch 00006: val_loss did not improve from -0.02156\nEpoch 7/10\n\nEpoch 00007: LearningRateScheduler setting learning rate to 0.0004006000000000001.\n445/445 [==============================] - 28s 62ms/step - loss: -0.0593 - mae: 0.0252 - correlation: 0.1438 - val_loss: -0.0386 - val_mae: 0.0599 - val_correlation: 0.1371\n\nEpoch 00007: val_loss improved from -0.02156 to -0.03858, saving model to ./transformer_model\n","output_type":"stream"},{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/keras/utils/generic_utils.py:497: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n  category=CustomMaskWarning)\n","output_type":"stream"},{"name":"stdout","text":"Epoch 8/10\n\nEpoch 00008: LearningRateScheduler setting learning rate to 0.0004672000000000001.\n445/445 [==============================] - 28s 62ms/step - loss: -0.0664 - mae: 0.0202 - correlation: 0.1531 - val_loss: -0.1068 - val_mae: 0.0389 - val_correlation: 0.2525\n\nEpoch 00008: val_loss improved from -0.03858 to -0.10681, saving model to ./transformer_model\n","output_type":"stream"},{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/keras/utils/generic_utils.py:497: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.\n  category=CustomMaskWarning)\n","output_type":"stream"},{"name":"stdout","text":"Epoch 9/10\n\nEpoch 00009: LearningRateScheduler setting learning rate to 0.0005338000000000001.\n445/445 [==============================] - 27s 61ms/step - loss: -0.0733 - mae: 0.0164 - correlation: 0.1629 - val_loss: -0.0973 - val_mae: 0.0245 - val_correlation: 0.2192\n\nEpoch 00009: val_loss did not improve from -0.10681\nEpoch 10/10\n\nEpoch 00010: LearningRateScheduler setting learning rate to 0.0006004000000000001.\n445/445 [==============================] - 27s 61ms/step - loss: -0.0962 - mae: 0.0170 - correlation: 0.2093 - val_loss: -0.0520 - val_mae: 0.0268 - val_correlation: 0.1308\n\nEpoch 00010: val_loss did not improve from -0.10681\n54/54 [==============================] - 1s 24ms/step - loss: -0.0334 - mae: 0.0294 - correlation: 0.0961\n","output_type":"stream"},{"execution_count":38,"output_type":"execute_result","data":{"text/plain":"[-0.033364370465278625, 0.02939314767718315, 0.09612186998128891]"},"metadata":{}}]},{"cell_type":"code","source":"# tf.keras.utils.plot_model(get_model(), show_shapes=True)","metadata":{"papermill":{"duration":6.240302,"end_time":"2021-11-29T18:09:55.791092","exception":false,"start_time":"2021-11-29T18:09:49.55079","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:55:13.609947Z","iopub.execute_input":"2022-02-16T23:55:13.610955Z","iopub.status.idle":"2022-02-16T23:55:13.616031Z","shell.execute_reply.started":"2022-02-16T23:55:13.610905Z","shell.execute_reply":"2022-02-16T23:55:13.615245Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"# EPOCHS = 100\n# seed = 1 \n# \n# tf.random.set_seed(seed)\n# estop = keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 5, verbose = 0, mode = 'min',restore_best_weights = True)\n# # scheduler = keras.optimizers.schedules.ExponentialDecay(1e-3, (0.5 * len(X_train) / BATCH_SIZE), 1e-3)\n# # lr = keras.callbacks.LearningRateScheduler(scheduler, verbose = 1)\n# mcp_save = keras.callbacks.ModelCheckpoint('model_multivariate_min_val_loss', save_best_only=True, monitor='val_loss', mode='min')\n# plateau = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, verbose=1, mode='min')\n# \n# history = model.fit(train_generator, validation_data = (val_generator), epochs = EPOCHS, callbacks = [estop, mcp_save, plateau], verbose=1)\n# model.save(\"model_multivariate\")\n\n","metadata":{"papermill":{"duration":791.992821,"end_time":"2021-11-29T18:23:07.875583","exception":false,"start_time":"2021-11-29T18:09:55.882762","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-02-16T23:55:13.617549Z","iopub.execute_input":"2022-02-16T23:55:13.618597Z","iopub.status.idle":"2022-02-16T23:55:13.625113Z","shell.execute_reply.started":"2022-02-16T23:55:13.618550Z","shell.execute_reply":"2022-02-16T23:55:13.624308Z"},"trusted":true},"execution_count":26,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['val_loss'], label= \"validation_loss\")\nplt.plot(history.history['loss'], label= \"train_loss\")\nplt.legend(loc=\"upper left\", bbox_to_anchor=(1, 0.5))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:55:13.626919Z","iopub.execute_input":"2022-02-16T23:55:13.627403Z","iopub.status.idle":"2022-02-16T23:55:13.849532Z","shell.execute_reply.started":"2022-02-16T23:55:13.627357Z","shell.execute_reply":"2022-02-16T23:55:13.848718Z"},"trusted":true},"execution_count":27,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAe8AAAD6CAYAAACI5c9RAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0CElEQVR4nO3deVzVVf7H8ddhF0EEQVHABdeMxIXcy6VNy6y0xjZbbLemqaZmqplpWmfs175OOrZp1rRom5m2u2+45gaSoqAioIKoLALn98cFMwMVufAF7vv5ePDwcvne7/fDLX3fs3zPMdZaREREpP7wcroAERERqRqFt4iISD2j8BYREalnFN4iIiL1jMJbRESknlF4i4iI1DNuCW9jzDBjTJIxJsUY82AFPz/bGLPSGFNsjLncHdcUERHxVD7VPYExxht4DTgPSAeWG2O+sNZuOOqw7cANwP0ne97w8HDbtm3b6pYnIuJRVqxYkW2tjXC6DqlZ1Q5voDeQYq3dAmCM+R9wCXAkvK21qWU/Kz3Zk7Zt25bExEQ3lCci4jmMMducrkFqnju6zaOAtKO+Ty97TkRERGpAnZqwZoy51RiTaIxJzMrKcrocERGROskd4b0DiDnq++iy56rMWjvJWptgrU2IiNCQjYiISEXcEd7LgY7GmHbGGD/gSuALN5xXREREKlDt8LbWFgN3AXOAjcBH1tr1xpjHjTEjAYwxZxpj0oErgInGmPXVva6IiIincsdsc6y1s4BZxzz3yFGPl+PqThcREZFqqlMT1kREROTEGl54Fx2Cb/8J+3Sro4iINEwNL7wP7YHlk+HLu8Fap6sRERFxu4YX3k1j4LzHYMtPsGqq09WIiIi4XcMLb4Be46DNAJjzd9i/0+lqRERE3KphhreXF4x8BUqKYOZ96j4XEZEGpWGGN0Cz9jD0b5D8Nayb7nQ1IiIibtNwwxug73iI6gWzHoADWitdREQahoYd3l7ecMlrUJgHX//F6WpERETcomGHN0Dz02DQX2H9DNg40+lqREREqq3hhzfAwHugxRnw1X2Qv8/pakRERKrFM8Lb2xcueRUOZsOcvzldjYiISLV4RngDtOoOA/4Eq6dByndOVyMiInLKPCe8wTX2Hd4JvrzHNYlNRESkHvKs8PYNcM0+z02H7x51uhoREZFT4lnhDRDTG/re4dq8JHWB09WIiIhUmeeFN8DQv0NoW/jij64tREVEROoRzwxvv8Zw8cuwdwv89C+nqxEREakSzwxvgNhB0OsGWPwapK9wuhoREZGT5rnhDXDe4xDcEj6/E4oLna5GRETkpHh2eAeEwIgXIWsjzHvW6WpEREROimeHN0Cn86HblbDgecj42elqRERETkjhDTDs39Ao1NV9XlLsdDUiIiLHpfAGCAyDi56DXWtg0ctOVyMiInJcCu9yXS+B00bCTxMgK9npakRERCql8D7ahc+CX6Cr+7y0xOlqREREKqTwPlpwCxg2AdKXwbJJTlcjIiJSIYX3sbqNgY7nw/ePw96tTlcjIiLyOwrvYxkDI14A4w1f3g3WOl2RiIjIbyi8KxISDec/AVvnwcp3na5GRETkNxTelel1A7Q9C+b8HXJ3OF2NiIjIEQrvyhgDI18GWwIz71H3uYiI1BkK7+MJi4Wh/4DN38Daj5yuRkREBFB4n1if2yC6N8z+KxzIdLoaERERhfcJeXnDJa9C0UGYdb/T1YiIiCi8T0pEZxj8IGz43PUlIiLiIIX3yep/N0R2g6/uh0N7na5GREQ8mML7ZHn7wiWvQf5emPOw09WIiIgHc0t4G2OGGWOSjDEpxpgHK/i5vzHmw7KfLzXGtHXHdWtdy24w8F5Y8wFs/tbpakRExENVO7yNMd7Aa8BwoCtwlTGm6zGH3QTss9Z2AF4Anq7udR1z9gMQ0QW+/BMU7He6GhER8UDuaHn3BlKstVustUXA/4BLjjnmEqB8ndFPgHOMMcYN167QwcLimjo1+Pi7us/zdsF3/6y564iIiFTCHeEdBaQd9X162XMVHmOtLQZygWZuuPbvbNtzkKHP/cT0Fek1cXqX6AToOx4S34Kt82vuOiIiIhWoUxPWjDG3GmMSjTGJWVlZp3SOliGNiA0P4sEZa1m2tQZnhQ/5m2sFti/uct0DLiIiUkvcEd47gJijvo8ue67CY4wxPkAIsOfYE1lrJ1lrE6y1CREREadUjJ+PF29c24uY0EBum5pIanYNBatfIIx8Bfalwg9P1cw1REREKuCO8F4OdDTGtDPG+AFXAl8cc8wXwPVljy8HfrC25nb6CAn05a0bzsQC495ZTu6hwzVzobYDIeEmWPI6pC2vmWuIiIgco9rhXTaGfRcwB9gIfGStXW+MedwYM7LssDeBZsaYFOA+4He3k7lb2/DGTBqbQNq+Q9z+3gqKiktr5kLnPeba//vzO6G4sGauISIichS3jHlba2dZaztZa9tba58qe+4Ra+0XZY8LrLVXWGs7WGt7W2u3uOO6J9K7XRgTRnVj8ZY9/OOzddRIY98/GC5+EbKTYO7/uf/8IiIix6hTE9Zqwuhe0fxxaAc+TExj4rwa+szQ4Vzofg0seAF2ramZa4iIiJRp8OENcO+5nbioW0uenr2J2et21cxFLngKGoe7us9LamiMXUREBA8Jby8vw3NXxBMf3ZR7PlzN2vQc91+kUShc9Dxk/AwLX3T/+UVERMp4RHgDBPh689/rEmjW2J+b3k1kZ06++y9y2gg4/TLX2HfmJvefX0REBA8Kb4CIYH/euuFM8otKuOndRA7UxDKqw58BvyBX93lpifvPLyIiHs+jwhugc2Qwr13Tk+Tdedz9wSpKSt08Az0oAob/H+xIhKVvuPfcIiIieGB4AwzqFMGjF3flh02ZPPXVRvdf4IzLodNw+P4J1/rnuv9bRETcyCPDG2Bsv7bcOKAtby3cytQl29x7cmNc93637AYz74WXe8Cy/8LhAvdeR0REPJLHhjfA3y/qytAuzXn0i/XMTT61jVAqFRwJ4+bA2M+gaWuYdT+8FA9L/gNFh9x7LRER8SgeHd7eXoaXr+pBx+ZB3DVtJUkZee69gDHQfgjc+DVcPxPCO8LsB10hvugV7UYmIiKnxKPDGyDI34e3bjiTAD9vxr2znKy8GhifNgbanQU3zHQFeYvT4Zu/w4tnwPznodDNHxpERKRB8/jwBmjVtBFvXp/AnoOF3Do1kYLDNXiLV5v+cN1ncNO30KonfP+YK8TnPgMFuTV3XRERaTAU3mW6RTflxTHdWbU9h/s/XkOpu28hO1ZMb7j2E7jlB4jpCz8+6QrxH/8N+ftq9toiIlKvKbyPMiyuJQ8O78LMtbt48bvk2rloVC+4+n9w61xoexbMnQAvdoMfnoRDe2unBhERqVcU3se47exY/pAQzcs/pDB9RXrtXbhVd7hyGty+ENoPhXnPulri3/4TDmbXXh0iIlLnKbyPYYzhyUvPoF9sMx6csZZlW2u59RsZB394F8Yvhk7DYOFLrhCf8zfI2127tYiISJ2k8K6An48Xb1zbi5jQQG6bmkhqtgO3dDU/DS5/E+5cBqeNhCWvw0vd4OsHYX8NbWsqIiL1gsK7EiGBvrx1w5lYYNw7y8k95NAe3RGdYNREuCsR4kbDskmu+8S/uh9ya7FbX0RE6gyF93G0DW/MxGt7kbbvELe/t4Ki4lLnimnWHi59Hf64AuLHwIq34aXu8OU9sM/Ny7uKiEidpvA+gT6xzZgwqhuLt+zhH5+tw9oavoXsRMLawchX4O5V0HMsrHoPXukJn98Fe7c4W5uIiNQKhfdJGN0rmj8O7cCHiWlMnFdHArJpaxjxAvxpNSSMg7UfwSsJ8OkdkJ3idHUiIlKDFN4n6d5zO3FRt5Y8PXsTs9fVoQljIdFw4TPwpzXQ5zZYPwNeOxOm30Lhrg18tXYXr/6wmZ05+U5XKiIibmIc7wauREJCgk1MTHS6jN8oOFzClZOWsCljPx/d1o9u0U2dLul3bF4Gu+c8S9j6qfjYQr4q6cMrxZexxbTm0h5R3D6oPR2aBzldpojUEGPMCmttgtN1SM1SeFdRVl4hl762kMMlpXx25wBaNW3kdEkApO09xIyVO5ixKp1tew7RyvcgTzT/iUG5n+FdfIiVzUZwd+ZF7CxuwrDTIxk/uANnRIc4XbaIuJnC2zMovE9BUkYeo/+ziJiwQD65vR+N/X0cqWN/wWG+/nkX01fuYNnWvRgD/WKbMapnNMPiIgny93EtsTr/OVj6BqU+ASyIvJ57t/djT4EXZ3UM547B7ekX2wxjjCO/g4i4l8LbMyi8T9Hc5CzGvbOcwZ0imHRdAt5etRN+xSWlLEjJZsbKHcxZn0FhcSmx4Y0Z3SuaS3tEEVVZT0B2Cnz7D0iaRWlIG76LvpOHN8WSfbCI7jFNGT+4Peee1gKvWvo9RKRmKLw9g8K7GqYuTuUfn69n3IB2PHJx1xq91qaM/cxYuYPPVu0gM6+QkEa+jIxvxaieUXSPaXryLedffoQ5D0PmBkpj+jE7+k/8e40/aXvz6dQiiNsHtefi+Fb4emsuo0h9pPD2DArvanrsy/W8vTCVJy6NY2zfNm49d1ZeIV+s2cn0Fels2LUfHy/D0C7NGdUzmiFdIvD38T61E5cUw6opR3YuK42/mm9a3soLi/eTtDuP6NBG3HZ2LFckxBDge4rXEBFHKLw9g8K7mkpKLbdMSWRuchZv3XAmgzpFVOt8BYdL+H5jJtNXpjM3OYuSUkt8dAijekZzcXwrwhr7ualyoCAX5j0DS94Abz9KB97HT6FX8OqCdFZuzyE8yI9xA9txbd82NAnwdd91RaTGKLw9g8LbDQ4UFnP5fxaxY18+08f3p1OL4Cq93lrLyu37+GTFDmau3UleQTGRTQK4rGcUo3pE0bGK56uyPb/At4/AppkQ0hp73mMsbXQ2r8/dwrzkLIL9fRjbrw3jBrYjPMi/ZmsRkWpReHsGhbeb7MzJ55LXFuLn7cVndw4gIvjEIXfs7V2NfL0ZHhfJ6F7R9I1tVmuT4I7YOg9mPwy7f4aYvjDs36yjPf/56RdmrduFn7cXY86M4dazY4kODazd2kTkpCi8PYPC243Wpufwh4mLOa1lEz64pW+F48WV3d41uuz2LqduOzuitMS1XvoPT8DBLIi/Gs55hF8Kg5k49xc+XbUDa2Fk91bcMah9zfcKiEiVKLw9g8LbzWav28Xt761kRLeWvHxlD7y8TMW3d0U0ZnTPE9ze5aSC/a77w5e8Dl4+MPBe6HcXOw8ZJs/fygfLtpN/uITzu7Zg/JAOdI9p6nTFIoLC21MovGvAf376hadnb+KG/m3x8/Hi01U7yMorpGlg+e1d0cRHh9SPhVH2bnWNh2/8AppEw3mPQdxo9h46zDsLt/LOolT2FxTTv30zxg/uwIAOWvBFxEkKb8+g8K4B1lr+On0tHyWm4+ttGNLZdXvX0C7N8fOpp/dPpy6A2Q9BxlqI7g3D/g3RCRwoLOb9pduYPH8rmXmFdIsOYfzg9pzfNVILvog4QOHtGRTeNeRwSSk/JWXRq02oe2/vclJpCax+3zUefmA3dBsD5/wTQqIoOFzCjJU7mDjvF7btOUT7iMbcPqg9l/aI0oIvIrVI4e0ZFN5SdYV5sOAFWPQqGC8YeA/0vxv8AikuKWXWugxe/zGFTRl5RDVtxC1ntWPMma1p5KcFX0RqmsLbMyi85dTt2wbf/RPWfwpNouDcRyHucvDywlrLj0mZvP7jLyRu20ezxn7cOaQDNw5oqzFxkRqk8PYM1erPNMaEGWO+NcZsLvsztJLjZhtjcowxM6tzPaljQtvAFe/AjbOhcQTMuAXePA/SlmGMYWiXFnxyR38+uq0fXVs14fGZG3hw+s8cLil1unIRkXqtuoORDwLfW2s7At+XfV+RZ4Cx1byW1FVt+sEtP8Kl/4HcdFeAf3IT5KQB0LtdGFPG9ebuoR34MDGNm99N5EBhscNFi4jUX9UN70uAd8sevwtcWtFB1trvgbxqXkvqMi8v6H41/HEFnP0X11KrrybAD09B4QGMMdx3fmcmjDqDBSnZjJm4mMz9BU5XLSJSL1U3vFtYa3eVPc4AWlTzfFLf+QfB0L/BXYnQZQTM+z9XiK/+AEpLubJ3ayZfn8DW7INc9voiUjL1mU5EpKpOGN7GmO+MMesq+Lrk6OOsa+ZbtWa/GWNuNcYkGmMSs7KyqnMqcVrTGLj8TRj3DTRpBZ/dDlMvhZLDDOncnA9v7UdhcSmjXl/E0i17nK5WRKReOWF4W2vPtdbGVfD1ObDbGNMSoOzPzOoUY62dZK1NsNYmRERUb2tNqSNa94GbvoMLn4Wtc+H7xwA4IzqET8f3JzzYn7FvLmPm2p0OFyoiUn9Ut9v8C+D6ssfXA59X83zSEHl5Qe9b4MxbYNErkPQ1ADFhgcy4oz/xMSHc9f4q/jtvC3X11kURkbqkuuE9ATjPGLMZOLfse4wxCcaYyeUHGWPmAx8D5xhj0o0xF1TzulIfnf8ktIyHT28/MhO9aaAfU2/qw0VntOSpWRt57MsNlJQqwEVEjkeLtEjt2rsFJg6CiM5w49fg7QtAaanlX7M2MnnBVi44vQUvXdmjwi1VReT4tEiLZ9Ci01K7wmJh5MuQvvzI+DeAl5fh7yO68s+Lu/LNht1c9d8l7D1Y5GChIiJ1l8Jbat/pl/1u/LvcjQPa8Z9rerJh535G/2cR2/YcdKhIEZG6S+Etzjj/SYjs9pvx73LD4lry/i19yDlUxKjXF7E6LceZGkVE6iiFtzjDN8C1LnppCXxyI5Qc/s2Pe7UJY/od/Qn09+bKSYv5dsNuZ+oUEamDFN7inGbtKxz/LhcbEcSMOwbQqUUwt01NZOri1NqvUUSkDlJ4i7PiRsGZN1c4/g0QEezP/27ty5DOzfnH5+uZ8PUmSnUrmYh4OIW3OO/8pyod/wYI9PNh4theXNOnNW/M/YV7PlxNYXGJA4WKiNQNCm9x3gnGvwF8vL148tI4/jKsM1+s2cn1by0jN//3x4mIeAKFt9QNJxj/BjDGMH5wB14c050V2/ZxxRuL2JGTX8uFiog4T+EtdccJxr/LXdojindv7M2unAJGvb6Q9Ttza7FIERHnKbylbjnB+He5/h3C+eSO/ngZw5iJS5i/WVvIiojnUHhL3XIS49/lOkcG8+n4AUSHNuLGt5fzcWLlYS8i0pD4OF2AyO+Uj39/cqNr/Pv8Jys9NDIkgI9u78cd763ggU/WsjOngLvP6YAxphYLFqnbVqxY0dzHx2cyEIcabfVBKbCuuLj45l69emVWdIDCW+qmuFGwbaFr/LvNAOg8vNJDmwT48vYNvXlwxlpe+C6ZnTn5PHlZHL7e+jdKBMDHx2dyZGTkaREREfu8vLy0UEIdV1paarKysrpmZGRMBkZWdIz+dZO66yTHvwH8fLx47op4/ji0Ax8mpnHzu4kcKCyupUJF6ry4iIiI/Qru+sHLy8tGRETk4uopqfiYWqxHpGqqMP4NrlvJ/nx+Z/496gwWpGQzZuJiMvcX1E6tInWbl4K7fin771VpRiu8pW47ifu/j3VV79ZMvi6BrdkHuez1RaRk5tVwkSIitUvhLXXfSd7/fbQhXZrz4a39KCwuZdTri1i6ZU8NFykiUnsU3lI/VGH8u9wZ0SF8Or4/4cH+jH1zGTPX7qzhIkXEXQIDA3sApKam+g4bNiy2omN69+7ded68eYHHO8/jjz/ePC8v70jWDRo0qEN2dra3u+ocPXp027fffjvUXec7WQpvqR+qOP5dLiYskOm396dbdAh3vb+K/87bgrUa+hOpL9q2bXt49uzZW0719RMnTmxx4MCBI1k3d+7clPDw8Hq/s5FuFZP6owr3fx8ttLEf793ch/s+Ws1TszaSV3CY+87vXMPFitRND3yyJiY5I++4rdWq6hQZfOiZy+OP2yU2fvz4qJiYmKKHHnooC+C+++5r5ePjY+fPnx+cm5vrXVxcbB555JGd1157bc7Rr0tKSvIbMWJEx82bN68/cOCAufLKK9tt2LChUfv27QsKCgqOLOhwzTXXtF6zZk3jgoICr4svvnjfCy+8sPPJJ59snpmZ6Tto0KBOoaGhxUuXLk2Oioo6IzExcWPLli2LH3300RbTpk0LBxg7dmzWI488kpmUlOQ3fPjwjr179z6QmJgY1KJFi6I5c+akBAUFnfBT/+effx784IMPxpSUlBAfH39oypQp2xo1amTHjx8fNWfOnKbe3t528ODB+ydNmpT+1ltvhf773/9u5eXlZYODg0sSExOTqvKeq+Ut9cspjH8DBPh68+pVPRndM5pXfkxhicbARWrVNddcs3fGjBlh5d9//vnnobfeemv2V199lbJhw4aNc+fOTX744YejS0tLKz3Hs88+27xRo0alW7ZsWf/kk0/u3LBhQ+Pynz3//PM71q1bt3HTpk3rFy5cGLx06dJGf//73zObN29+eO7cuclLly5NPvpc8+fPD3z//febrVixYmNiYuLGKVOmRCxcuLARwPbt2wPuvvvuzJSUlPUhISElU6ZMOWG3+KFDh8xtt93W7sMPP/wlOTl5Q3FxMc8880xERkaG96xZs0I3b968Pjk5ecO//vWvXQATJkxo+c033yQnJSVtmD17dkpV30+1vKX+Of8pSFvmGv++fQE0jTmpl3l5GZ649HQSt+3lzx+tYfY9ZxEc4FvDxYrULSdqIdeUAQMG5O/Zs8cnNTXVd9euXT4hISElMTExxbfcckvMkiVLgry8vMjMzPRLT0/3ad26dYWLNCxYsCDo7rvvzgTo06dPfqdOnQ6V/+zdd98Ne+edd8KLi4tNVlaW75o1awL69OlT6baDP/30U9CFF16Y06RJk1KAiy66aN+PP/4YfMUVV+RERUUV9u/fPx+gR48eh1JTU/1P9PutWbMmIDo6urBbt26FADfccMOe1157rflDDz2U6e/vXzpmzJi2I0aMyBkzZkwuQEJCwoFrrrmm7ejRo/ddc801+6ryXoJa3lIf/Wb8e9xJj38DBPr58Pwf4tmVm88TMzfUXI0i8jsjR47c995774VOmzYtbNSoUXsnTpwYtmfPHp+ff/5546ZNmzY0a9bscH5+fpVzadOmTX6vvvpqi7lz5yYnJydvGDp0aG5BQcEp55ufn9+RLnJvb29bXFx8yust+/r6snr16o2XX375vpkzZzYdPHhwR4D3339/+5NPPrkzLS3Nr1evXl0zMjKqNIlO4S3105H7v5fB949X6aW92oRx+6D2fJSYzrcbdtdQgSJyrGuvvXbv9OnTw2bOnBk6duzYfbm5ud7h4eGH/f397Zdffhm8c+dOv+O9fuDAgQemTZsWBrB8+fKA5OTkQIB9+/Z5N2rUqDQsLKwkLS3N56effgopf03jxo1LcnNzf5d1Q4YMOTBr1qymeXl5Xvv37/eaNWtW6JAhQ055UYj4+PiCHTt2+K1bt84fYMqUKc3OOuusvNzcXK+9e/d6jxkzJveNN95I27RpUyDA+vXr/YcOHXrwxRdf3BkaGlq8ZcuW4/7ux1K3udRfR9Y/f7ls/fNhJ/3Se87txI9JWTw0Yy09W59Ns6AT9oqJSDUlJCQUHDx40KtFixZFbdq0OXzzzTfvHT58eIdOnTp17dat26F27dodd0nE+++/P/PKK69sFxsbe3qHDh0KunbtehCgX79++XFxcYfat28f17Jly6JevXodKH/N9ddfnz1s2LBOLVq0KDp63HvgwIGHrr766j09e/Y8DVwT1gYMGJCflJRUpRAtFxgYaN94443UK664on35hLX7778/KzMz02fEiBEdCgsLDcATTzyRBnDvvfdGp6am+ltrzcCBA/f37du30i7+ipi6ettMQkKCTUxMdLoMqesOF8Cb50FuGtw2/6THvwE2Zexn5CsLGdIlgjeu7aWdyKRBMMassNYmHP3cmjVrUuPj47OdqklOzZo1a8Lj4+PbVvQzdZtL/VY+/l1SXOXx7y6RTfjz+Z2Ys343n67aUXM1ioi4mcJb6r9qjH/ffFYsZ7YN5Z+fr2dnTpV6rUTEg4wdO7Z1ly5duh799dJLLzVzqh6NeUvDEDcKUhdUefzb28vw3BXdGfbSPB74ZA1Tx/XBy0vd5yLyW1OnTt3udA1HU8tbGo4L/gWRZ8BnJ7/+OUDrZoH8Y0RXFqbs4d3FqTVXn4iImyi8peHwDYAr3j2l8e8rz4xhSOcIJny9iZTMAyd+gYiIgxTe0rA0aw8jX6ry+LcxhqdHd6ORnzf3fbSawyWVL9EoIuI0hbc0PHGjIeEm1/h30uyTflnzJgE8dekZrE3P5fUff6nBAkU8S3Z2tveECRMiqvq6U92+06ltOmuTwlsaplMc/76oW0su6d6KV37YzNr0nJqrT8SD7Nmzx/vNN99sfuzzhw8ff2iroWzfWRM021wapvLx74mDXOPfN84C75PbhOTxkXEs3bKX+z5aw8w/DiTAt8of/EXqrs/ujCFzg1u3BKV510Nc+lqln5L//Oc/R6elpfl36dKlq4+Pj/X39y8NCQkp2bJlS0Bqauq6c889t/2uXbv8CgsLvW6//fbd999/fzZA+fad+/fv96oP23TWJrW8peE6xfHvkEBfnrmiGymZB3hmTp39uytSbzz33HPpMTExhZs2bdowYcKE9A0bNgS+/vrr21NTU9cBTJs2LXX9+vUbV69evWHixIktKtqkoz5s01mb1PKWhi1uNKRWff3zszpGcF2/Nry5YCvnnNac/u3Da7hQkVpynBZybenWrdvBLl26FJV///TTT7f46quvmgJkZGT4rl+/PiAyMvLg0a+pD9t01qZqtbyNMWHGmG+NMZvL/vzdJyFjTHdjzGJjzHpjzFpjzJjqXFOkyk5x/PvB4V1oF96YBz5eS17Byd92JiLHFxgYeOR2jpkzZwbPnTs3ODExcVNSUtKG0047Lb+ibUHrwzadtam63eYPAt9bazsC35d9f6xDwHXW2tOBYcCLxpim1byuyMk7xfu/A/18eK5s7+/Hv9Te3yKnKiQkpOTgwYMV5k1OTo53SEhISXBwcOmqVasC1qxZ09hd163tbTprU3W7zS8BBpc9fhf4Cfjr0QdYa5OPerzTGJMJRAA51by2yMkrH//+ZBx8eQ+MeB58TrwNaM/WoYwf3IFXf0zhvK4tOP/0yJqvVaSBiYyMLOnVq9eBjh07nu7v718aERFx5BP06NGjcydNmhQRGxt7emxsbEF8fPzB452rKmp7m87aVK0tQY0xOdbapmWPDbCv/PtKju+NK+RPt9YedxUMbQkqNeKHJ2HeM9Cqp2s3stA2J3xJUXEpl762kN37C5hz79mEa+9vqcO0JWjDUa0tQY0x3xlj1lXwdcnRx1nXp4BKPwkYY1oCU4EbKwtuY8ytxphEY0xiVlbWiUoTqbqhf4cx78GeX2DiWbBp1glf4ufjxQtjupNXUMxDM36mOh94RUTc4YThba0911obV8HX58DuslAuD+fMis5hjGkCfAX8zVq75DjXmmStTbDWJkREVHkxHpGTc9rFcNtPENoW/ncVfPOPE46Dd44M5oELOvPtht1MX6m9v0Xqgrq2TWdtqu6Y9xfA9cCEsj8/P/YAY4wf8CkwxVr7STWvJ+IeYbEw7huY87DrNrK0pXD52xASVelLxg1sx7cbd/PYF+vpGxtGdKh717kQkaqpa9t01qbqzjafAJxnjNkMnFv2PcaYBGPM5LJj/gCcDdxgjFld9tW9mtcVqT7fANfEtdFvwu71rm70lO8qPdy193c8pdbywMdrKS1V97nUG6WlpaXaqL4eKfvvVencsGqFt7V2j7X2HGttx7Lu9b1lzydaa28ue/yetdbXWtv9qK/V1bmuiFudcTnc+hMERcJ7l7smtZVWvJxyTFggj1zclcVb9vDOotRaLVOkGtZlZWWFKMDrh9LSUpOVlRUCrKvsGK2wJgIQ3hFu/g6+fsA1G337EleLPLjF7w79Q0IM36zfzdOzN3F2p3A6NA92oGCRk1dcXHxzRkbG5IyMjDi0LHZ9UAqsKy4uvrmyA6p1q1hN0q1i4phV0+CrP4N/MFz+JrQ7+3eHZOYVcMEL84gJC2T6Hf3x9da/h1I3VHSrmDQ8+hdH5Fg9roFbfoCAEJhyiaslXvrboafmwQH86zLX3t+v/lCn9y8QkQZI4S1SkRZdXePgcaNdY+DTLoeDe35zyPAzWnJZjyhe/TGFNWk5jpQpIp5J4S1SGf8gGPVfGPECpC6ANwa6xsKP8ujI02ke7M+9H62m4HDFk9xERNxN4S1yPMZAwji4+VvXWuhvXwgLX4ayuSIhjXx55vJ4tmQd5OnZmxwuVkQ8hcJb5GS0jIfb5kKXi+Dbf8D/roZ813a/AzuGc0P/try9MJVFKVo+WkRqnsJb5GQFhMAfpsCwp2Hzt/DG2ZC+AoC/DutCbHhj7v94Dfu197eI1DCFt0hVGAN9b4dxswELb10ASyfSyNeL58d0Z3deIY99ob2/RaRmKbxFTkV0Atw2DzqcA1//BT6+nu4RhjsHt2f6ynRmr8twukIRacAU3iKnKjAMrvwAzn0MNs6ESYP5Y9cC4qKa8LdPfyb7QKHTFYpIA6XwFqkOLy8YeA/c8BUczsf37fN4M249eYWHtfe3iNQYhbeIO7TpB7fNhzb9aTH3r3wV/R4LN2zj4xXpTlcmIg2QwlvEXYIi4NrpMORvdMiYxTdBjzLtyzmk7T3kdGUi0sAovEXcycsbBv0Fc91ntPQ9xAc8zBfvPqe9v0XErRTeIjUhdjDe4xdyoNkZ3Jn7LJsn3wCH852uSkQaCIW3SE0JjiTiztnMDLmGzjs/o/CNIZCtHchEpPoU3iI1yHj70ufmF7jL62EK9u7AThoE66Y7XZaI1HMKb5EaFhHsz4hR1zMs/yl2+beDT8bBV3+GooNOlyYi9ZTCW6QWDIuLpF/PbgzZ8xcy426B5ZPh/2Jh2hWw/E3I3eF0iSJSj5i6uohEQkKCTUxMdLoMEbfZX3CYYS/MI8DPm68v88M/6QtI/hr2pboOiOwGnYZB52HQsodrARiRKjLGrLDWJjhdh9QshbdILVqUks3Vk5dyQ/+2PDrydNe+4FlJrhBPngNpS8GWQlAL6HSBK8xjB4NfY6dLl3pC4e0ZfJwuQMST9O8Qzo0DXHt/R4c24srerQlq3gWad4GB98LBPZDyLSR9Des/g5VTwCcA2p3tCvJOwyAkyulfQ0Qcppa3SC0rOFzCdW8tY9nWvQT5+zCqZxRj+7ahY4vg3x5YXATbF0HS7GO618+ATsPVvS4VUsvbMyi8RRxgrWVVWg5TF2/jq7W7KCoppV9sM67r14Zzu7bA19vr2BdAdrKrRZ48+7fd6x3Ph87D1b0ugMLbUyi8RRyWfaCQjxLTmLZkOzty8mnRxJ+re7fhqt4xNG8SUPGLDu2Fzd+6WuQp30PhfvD2h9hBv46Vh0TX7i8idYLC2zMovEXqiJJSyw+bMpmyOJX5m7Px8TIMi4tkbN829G4XhjGm4hcWF8H2xa4WedLXsG+r6/ny7vVOw6CVutc9hcLbMyi8ReqgrdkHeW/JNj5OTGN/QTFdIoO5tm8bLusRRWP/48wzLe9eT57tGitPW6LudQ+j8PYMCm+ROiy/qITPV+9gyuJtbNi1nyB/H0b3jGJsvzZ0aB584hMc6V6fXda9nuvqXm93tmvCm7rXGxyFt2dQeIvUA9ZaVm7PYeriVGb9nEFRSSn92zdjbN82nNe1BT7HTnCrSMlh2Laogu71btD5QlervGU8VNY9L/WCwtszKLxF6pnsA4V8uDyN95e6JrhFNgng6j6tubJ3DM2DK5ngdixrIXuza8Jb0te/zl5vEuUK8c7Doe1Z4ONfs7+MuJ3C2zMovEXqqZJSy/cbdzN1ybbfTHC7rl9bzmwbWvkEt4oczHat8JY0C375AQ4fAr9g6HCOq1Xe8TwIDKu5X0bcRuHtGRTeIg3AlqwDvLdkOx+vSCOvbILb2H5tuLT7CSa4VeRwAWyd6wrypNlwIAOMN7TuB13KutfDYmvmF5FqU3h7BoW3SANyqKiYL1bvPDLBLdjfh9G9orm2bxs6NA+q+glLS2HXKtg0y9W9nrne9XxEl7Jx8gshqpduQ6tDFN6eQeEt0gC5JrjtY+ribUcmuA3o4Jrgdu5pJznBrSL7Ul0hnjQLUheCLYHGzV0Lw3S5CNoNAr9At/4uUjUKb8+g8BZp4MonuE1bso2duQW0DAng6t6tGVOVCW4Vyd8Hm79zBXnKd65V3nwaQfshrq71TsMgqLn7fhE5KQpvz6DwFvEQxSWl/LAp88gEN19vw7C4llzTpzV9jreC20mdvAi2LShrlX8NuWmAgegzy2avXwgRnXUbWi1QeHsGhbeIBzp2gltsRGOu7t2aUT2jCWvsV72TWwsZP//avb5rtev5sNhf7yeP6Qve2pG4Jii8PYPCW8SD5ReV8NXPu3h/6TZWbs/Bz9uLYXGRXO2O1ni53B2/3k++dR6UFEGjUOh4gSvIO5wD/iexWpycFIW3Z6hWeBtjwoAPgbZAKvAHa+2+Y45pA3wKeAG+wCvW2jdOdG6Ft0jt2pSxn/8tS2P6ynRXazy8MVf1bs3oXm5ojZcrzHMt05r0NWye4xo39/aDNv2hRRyEd4TwTtCsIzQOVzf7KVB4e4bqhvf/AXuttROMMQ8Codbavx5zjF/ZdQqNMUHAOqC/tXbn8c6t8BZxRnlr/INl21mxbV/NtMYBSopdG6ckfQ1b5sKezVBc8OvPA5q6gjy806+hHt4JQtuAt697amiAFN6eobrhnQQMttbuMsa0BH6y1nY+zvHNgFVAX4W3SN1XK63xcqWlrolu2ZtdO6NlJ8OeFNefB3b/epyXj2v8/NhQb9YBGjV1b031kMLbM1Q3vHOstU3LHhtgX/n3xxwXA3wFdAAesNa+Vsn5bgVuBWjdunWvbdu2nXJtIuI+lbXGr+rdmr6xbmyNV1pAzq9Bnp1cFvCbYe8vUFr863FBLVxd7keHenhHCIk54UIy+UUlFBaX0DTQzR9KapnC2zOcMLyNMd8BkRX86G/Au0eHtTFmn7U29DjnagV8Blxsrd1d2XGglrdIXZWUkccHy7bXTmv8REoOw75tR7XUy0I9KwkKcn49ziegLNQ7QHgnSsI6sM0rmhV5YazMKGJ1Wi7Ju/MAGNGtJbecFUtcVEjt/i5uovD2DLXabV72mreAWdbaT453nMJbpG7LLyph1s+7eP+o1vgFcZFcXVut8eOxFg7tgexkbFYyeTs2kr9zI745KTQt3IUXpUcO3Uk42QFtKG7ankzfKL7dVsqO4mCio9twycDuDIzrgKlHy78qvD1DdcP7GWDPURPWwqy1fznmmOiyY/KNMaHAUmC0tfbn451b4S1Sf5S3xmesTGe/w63xvQeLWJOew5q0sq/0XPYeLALAz8eL7i0DGByeR0JQNh28Mwg9tBVT3g1fdOB35yvGmyL/ZgSERuIV1BwaR7i+gpq7loZtHP7r48Bmjt+/rvD2DNUN72bAR0BrYBuuW8X2GmMSgNuttTcbY84DngMsYIBXrbWTTnRuhbdI/VPeGv9g2XYSj2qNX9U7hn6xzdzeGs8vKmHdztwjIb0mLYftew8BrrvMOjYPIj66KfExTeke05ROLYLx86mkFV3eWj+YBQcyKd6/m/WbU9iYsgVzMItWvnl0Csongly8DmW57lf/HePaOrVxcwiKKAv3iF8fHxv+NbBfusLbM2iRFhGpEce2xtuFN+aq3jFc3ivmlFrjxSWlbM48UBbUOUfGqUtKXf+GRTVtRHxMCN2imxIf3ZQzokMIqup2qBWw1rIwZQ+T5m9hXnIWgX7e/KFXNDf3Difa9wAczIQDma7QLwv+3z2uoEUPgH9IWbAf05oP7wBxo0+pXoW3Z1B4i0iNKjhcNja+9ORb49Za0vfls/pI13cO63bsJ/9wCQBNAnyOtKbjo5vSLSakepusnKSNu/Yzef5Wvlizg5JSy/C4ltxydizdY5oe/4VFh1whfzC7LNAz4UBZwB95XPYhoCDHtXzsTXNOqUaFt2dQeItIrUnencf7S3/fGj+/ayRb9xysdJw6rlWTI2HdLbopbZsFOjohLiO3gHcWpTJt6TbyCorp3TaMW86O5ZwuzfHyqmZdxUVw+KBrCdlToPD2DApvEal15a3xD5ZtZ3nqrysqVzRO3TkyGN9T3X+8hh0oLObD5Wm8tWArO3LyiY1ozM0DYxnVM4oAX29HalJ4ewaFt4g4Knl3Hku37KFji2DiotwzTl3biktKmbUug0nzfmHdjv00a+zHdf3aMrZfm1qfba/w9gwKbxERN7HWsmTLXv47fws/bMokwNeLy3tFc9PAWNqFN66VGhTenqH+fcQVEamjjDH0a9+Mfu2bsXl3HpPnb+Wj5elMW7qd87u24NazY+nVJszpMqUBUMtbRKQGZeYVMGXRNqYu2UZu/mF6tm7KrWfHcl7XSLyrO7mtAmp5ewaFt4hILThUVMzHielMXrCFtL35tGkWyM0D23F5rxga+blvcpvC2zMovEVEalFJqWXO+gwmztvCmrQcQgN9Gdu3DWP7tSUiuPorrim8PYPCW0TEAdZaErftY9K8LXy3cTe+3l6M7hnFTQNj6dA86JTPq/D2DJqwJiLiAGMMZ7YN48y2YfySdYA3F2xl+op0PliWxkXdWvLqVT2c3ZlN6jSFt4iIw9pHBPGvy87gvvM6MXXxNopLSxXcclwKbxGROiI8yJ97z+vkdBlSD9TNNQdFRESkUgpvERGRekbhLSIiUs8ovEVEROoZhbeIiEg9o/AWERGpZxTeIiIi9YzCW0REpJ6ps2ubG2OygG3VOEU4kO2mcuo7vRe/pffjt/R+/KohvBdtrLURThchNavOhnd1GWMStTi/i96L39L78Vt6P36l90LqC3Wbi4iI1DMKbxERkXqmIYf3JKcLqEP0XvyW3o/f0vvxK70XUi802DFvERGRhqoht7xFREQapAYX3saYYcaYJGNMijHmQafrcZIxJsYY86MxZoMxZr0x5k9O1+Q0Y4y3MWaVMWam07U4zRjT1BjziTFmkzFmozGmn9M1OckYc2/Z35N1xpgPjDEBTtckUpkGFd7GGG/gNWA40BW4yhjT1dmqHFUM/Nla2xXoC9zp4e8HwJ+AjU4XUUe8BMy21nYB4vHg98UYEwXcDSRYa+MAb+BKZ6sSqVyDCm+gN5Bird1irS0C/gdc4nBNjrHW7rLWrix7nIfrH+coZ6tyjjEmGrgImOx0LU4zxoQAZwNvAlhri6y1OY4W5TwfoJExxgcIBHY6XI9IpRpaeEcBaUd9n44Hh9XRjDFtgR7AUodLcdKLwF+AUofrqAvaAVnA22XDCJONMY2dLsop1todwLPAdmAXkGut/cbZqkQq19DCWypgjAkCpgP3WGv3O12PE4wxI4BMa+0Kp2upI3yAnsB/rLU9gIOAx84RMcaE4uqlawe0AhobY651tiqRyjW08N4BxBz1fXTZcx7LGOOLK7inWWtnOF2PgwYAI40xqbiGU4YaY95ztiRHpQPp1trynphPcIW5pzoX2GqtzbLWHgZmAP0drkmkUg0tvJcDHY0x7YwxfrgmnHzhcE2OMcYYXGOaG621zztdj5OstQ9Za6OttW1x/X/xg7XWY1tW1toMIM0Y07nsqXOADQ6W5LTtQF9jTGDZ35tz8OAJfFL3+ThdgDtZa4uNMXcBc3DNFn3LWrve4bKcNAAYC/xsjFld9tzD1tpZzpUkdcgfgWllH3S3ADc6XI9jrLVLjTGfACtx3aWxCq22JnWYVlgTERGpZxpat7mIiEiDp/AWERGpZxTeIiIi9YzCW0REpJ5ReIuIiNQzCm8REZF6RuEtIiJSzyi8RURE6pn/BxztHzw5H00gAAAAAElFTkSuQmCC\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":"plt.plot(history.history['val_mae'], label= \"validation metric (mae)\")\nplt.plot(history.history['mae'], label= \"train metric (mae)\")\nplt.legend(loc=\"upper left\", bbox_to_anchor=(1, 0.5))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:55:13.850899Z","iopub.execute_input":"2022-02-16T23:55:13.851422Z","iopub.status.idle":"2022-02-16T23:55:14.096647Z","shell.execute_reply.started":"2022-02-16T23:55:13.851383Z","shell.execute_reply":"2022-02-16T23:55:14.095727Z"},"trusted":true},"execution_count":28,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAhgAAAD4CAYAAACuX5+7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9DElEQVR4nO3dd3zV9fXH8de5NzuEkEUCCRAwAwKIkBBkKLIUR8Ut4mzds85Wqz+raFtt1VorVay2YtWqVWsVtzIcCBJE9jBAwswkhCRk3dzP74/vDYQQIMBNvjc35/l48Mi933Hv4QreN58pxhiUUkoppbzJYXcBSimllPI/GjCUUkop5XUaMJRSSinldRowlFJKKeV1GjCUUkop5XUBdhfQXGxsrElOTra7DKWU6lCWLFlSYoyJs7sOpRr5XMBITk4mJyfH7jKUUqpDEZF8u2tQqqlWdZGIyGQRWSciuSJybwvnTxaRH0TEJSIXtHC+q4hsFZFnvVG0UkoppXzbYQOGiDiBGcDpQAZwiYhkNLtsM3AV8PpBXuYR4KujL1MppZRSHUlrWjCygVxjzEZjTB3wBjCl6QXGmDxjzHLA3fxmEckE4oHPvFCvUkoppTqA1gSMRGBLk+dbPccOS0QcwJPA3Ye57joRyRGRnOLi4ta8tFJKKaV8WFtPU70J+MgYs/VQFxljXjDGZBljsuLidBC0Ukop1dG1ZhbJNqBXk+dJnmOtMRI4SURuAroAQSJSaYw5YKCoUkoppfxHawLGYiBVRPpiBYupwLTWvLgx5tLGxyJyFZCl4UIppZTyf4ftIjHGuIBbgE+BNcBbxphVIjJdRM4GEJHhIrIVuBCYKSKr2rLoFtXshi8ehtIN7f7WSimllNqfGGPsrmE/WVlZ5qgW2qoogGeGQepEuOgV7xemlFI+TESWGGOy7K5DqUb+sxdJRAKMvg1W/w+2fG93NUoppVSn5j8BA2DUrdAlAT69H3ysZUYppZTqTPwrYASFw7jfwNbvrZYMpZRSStnCvwIGwNDLoHsGfPEQuOrsrkYppZTqlPwvYDicMOkRKNsEOS/ZXY1SSinVKflfwABImQD9ToH5j0P1LrurUUoppTod/wwYIlYrRvUu+PpJu6tRSimlOh3/DBgAPY6HE6bBouehLN/uapRSSqlOxX8DBsC4+0GcMOcRuytRSimlOhX/DhiRiTDyZljxH9i2xO5qlFJKqU7DvwMGwJjbITwOPvs/XXxLKaWUaif+HzCCI+CUeyH/W1j3sd3VKKWUUp2C/wcMgGFXQmwafP4gNNTbXY1SSinl9zpHwHAGwqTpUPoTLHnZ7mqUUkopv9c5AgZA2mToMwbmPQY1u+2uRimllPJrnSdgiMCpj8CeEvj2aburUUoppfxa5wkYAInDYPBF8N0MKN9qdzVKKaWU3+pcAQNggme66pzf2V2JUkop5bc6X8Do1htOvAGW/Rt2LLO7GqWUUsovdb6AATDmTgiN0sW3lFJKqTbSqoAhIpNFZJ2I5IrIvS2cP1lEfhARl4hc0OT4CSLynYisEpHlInKxN4s/aqHdYOyvYdN8yP3C7mqUUkopv3PYgCEiTmAGcDqQAVwiIhnNLtsMXAW83uz4HuAKY8xAYDLwtIh0O8aavSPrFxDdz2rFaHDZXY1SSinlV1rTgpEN5BpjNhpj6oA3gClNLzDG5BljlgPuZsfXG2N+8jzeDhQBcV6p/FgFBMHEh6F4Dfz4mt3VKKWUUn6lNQEjEdjS5PlWz7EjIiLZQBCw4UjvbTMDfga9ToS5v4PaSrurUUoppfxGuwzyFJEewL+Anxtj3C2cv05EckQkp7i4uD1KanxjOPVRqCyEBX9tv/dVSiml/FxrAsY2oFeT50meY60iIl2BD4H7jTELW7rGGPOCMSbLGJMVF9fOPSi9hsPAc2HBM1BR0L7vrZRSSvmp1gSMxUCqiPQVkSBgKvB+a17cc/1/gVeMMW8ffZltbMJvrV1W5+riW0oppZQ3HDZgGGNcwC3Ap8Aa4C1jzCoRmS4iZwOIyHAR2QpcCMwUkVWe2y8CTgauEpEfPb9OaIvfyDGJ7gvZ18HSV6Fwtd3VKKWUUh2eGB9baCorK8vk5OS0/xvv2QnPnABJ2XCZ7za2KKVUS0RkiTEmy+46lGrUOVfybElYNJx8D+R+Dhvm2F2NUkop1aFpwGgq+zprr5LPHgR3g93VKKWUUh2WBoymAoJh4kNQuAKWv2l3NUoppVSHpQGjuYHnQWImfPkI1O2xuxqllFKqQ9KA0Vzj4lsV22HhDLurUUoppTokDRgt6TMK+p8F3zwNlUV2V6OUUkp1OBowDmbiw+CqgXmP2V2JUkop1eFowDiY2BRrS/clL0PxerurUUoppToUDRiHMvbXEBQOX/zW7kqUUkqpDkUDxqGEx8KYO2DdR5D3jd3VKKWUUh2GBozDOfFG6JoEnz0A7gN2mldKKaVUCzRgHE5gKEx4ELYvhZXv2F2NUkop1SFowGiNwRdCjyHw5cNQX2N3NUoppZTP04DRGg4HTHoEyrfA9zPtrkYppZTyeRowWqvfWEibDF89CVWldlejlFJK+TQNGEdi4sNQVwFf/dHuSpRSSimfpgHjSHTvD8OuhMUvQukGu6tRSimlfJYGjCN1yn0QEAJfPGR3JUoppZTP0oBxpCLiYfQvYc37sHmh3dUopZRSPkkDxtEYeTNE9LAW3zLG7mqUUkopn6MB42gEhcO4+2HrYlj9nt3VKKWUUj6nVQFDRCaLyDoRyRWRe1s4f7KI/CAiLhG5oNm5K0XkJ8+vK71VuO1OmAbdB1pjMVy1dlejlFJK+ZTDBgwRcQIzgNOBDOASEclodtlm4Crg9Wb3RgO/BUYA2cBvRSTq2Mv2AQ4nnPoIlOXB4pfsrkYppZTyKa1pwcgGco0xG40xdcAbwJSmFxhj8owxy4Hmu4GdBnxujNlpjCkDPgcme6Fu35AyAY4bD/Mfh+oyu6tRSimlfEZrAkYisKXJ862eY63RqntF5DoRyRGRnOLi4la+tI+Y9AjUlMNXT9hdiVJKKeUzfGKQpzHmBWNMljEmKy4uzu5yjkzCIBh6KXz/gtVdopRSSqlWBYxtQK8mz5M8x1rjWO7tOMbdD+KEL6fbXYlSSinlE1oTMBYDqSLSV0SCgKnA+618/U+BU0UkyjO481TPMf/StSeMuhVWvgNbl9hdjVJKKWW7wwYMY4wLuAUrGKwB3jLGrBKR6SJyNoCIDBeRrcCFwEwRWeW5dyfwCFZIWQxM9xzzP6Nvg/DuuviWUkopBYjxsS/DrKwsk5OTY3cZRyfnHzD7Drj4NRhwlt3VKKU6ERFZYozJsrsOpRr5xCBPvzH0CohNhy9+Cw31dlejlFJK2UYDhjc5A+DUR6E0F+b/0e5qlFJKKdtowPC2tFNhyDT4+gnIX2B3NUoppZQtNGC0hTP+CN36wLvXQfUuu6tRSiml2p0GjLYQHAHnvwi7t8OHd+qsEqWUUp2OBoy2kpQF435jrY2x7A27q1FKKaXalQaMtjTmDugzBj66G0o32F2NUkop1W40YLQlhxPOm2n9fPdanbqqlFKq09CA0dYik+Bnf4FtS2DeY3ZXo5RSSrULDRjtYeC5cMJl8PWTkPeN3dUopZRSbU4DRns5/XGI7ueZulpmdzVKKaVUm9KA0V6Cu8D5f4fKQvjgdp26qpRSyq9pwGhPiZkw7n5Y/R78+Jrd1SillFJtRgNGexv9S0g+CT76lU5dVUop5bc0YLQ3hxPOnQnOQHjnanDV2V2RUkop5XUaMOwQmQhnPwPbl8K839tdjVJKKeV1GjDskjEFhl0B3zwNm76yuxqllFLKqzRg2GnyYxBzHLx7PezZaXc1SimllNdowLBTULi162pVMXzwS526qpRSym9owLBbz6Ew/gFY8z4s/Zfd1SillFJeoQHDF4y6DfqeDB//Gkp+srsapZRS6pi1KmCIyGQRWSciuSJybwvng0XkTc/5RSKS7DkeKCKzRGSFiKwRkfu8XL9/cDisqasBwfDONTp1VSmlVId32IAhIk5gBnA6kAFcIiIZzS67GigzxqQAfwYe9xy/EAg2xgwGMoHrG8OHaqZrTzj7WdjxI8x91O5qlFJKqWPSmhaMbCDXGLPRGFMHvAFMaXbNFGCW5/HbwAQREcAA4SISAIQCdcBur1TujwacBZlXwbfPwMb5dlejlFJKHbXWBIxEYEuT51s9x1q8xhjjAsqBGKywUQXsADYDTxhjDpiPKSLXiUiOiOQUFxcf8W/Cr5z2e4hJgf/q1FWllFIdV1sP8swGGoCeQF/gLhHp1/wiY8wLxpgsY0xWXFxcG5fk44LC4YKXoKoE3r9Vp64qpZTqkFoTMLYBvZo8T/Ica/EaT3dIJFAKTAM+McbUG2OKgG+BrGMt2u/1GAITfwtrZ8OSl+2uRimllDpirQkYi4FUEekrIkHAVOD9Zte8D1zpeXwBMMcYY7C6RcYDiEg4cCKw1huF+70Tb4Z+4+CT+6B4vd3VKKWUUkfksAHDM6biFuBTYA3wljFmlYhMF5GzPZe9BMSISC5wJ9A4lXUG0EVEVmEFlX8aY5Z7+zfhlxwOOOc5CAz17Lpaa3dFSimlVKuJ8bE+/qysLJOTk2N3Gb5j7UfwxiUw6lY4VaevKqVaJiJLjDHaBa18hq7k6ev6nwFZV8OCv8KGOXZXo5RSSrWKBoyO4NRHITYd/nsjVJXaXY1SSil1WAF2F+BvSitr+d1Ha3C7DSGBToIDHAQ3/gxwEBzgJDjQQYjn595jAQ7Pc+vx3nsDnAQHBhN03os4XpoA798CU18HEbt/q0oppdRBacDwspcX5PHfpdvoFRVGrauBWpeb2no3Na6GY17S4rrAi/nNun/xh0d+xeyg0/cLJE2DTGM46RUVxq3jU3A4NIwopZRqXxowvKjW1cC/v9/M+PTuvHTV8P3OGWNwuQ019Z7Q4XJT28Lj/c67Gqit3/e4pq4vuWvXc1fFLJyJY9js7LXf/eXV9dTWN1DnclNZ66KoopaMnl2ZlBFv0yeilFKqs9KA4UUfLt9BSWUdV45KPuCciBDoFAKdDiKO5U1O+hc8N4pfVf4Jrv3S2oG1Ba4GN+OenMezc35i4oDuiHapKKWUakc6yNOLZi3Io19cOGNSYtvuTSLiYcoMKFwBXzx80MsCnA5uOiWFZVvL+eqnkrarRymllGqBBgwvWbq5jGVby7lqVHLbj3lInwzDr4WFMyD3y4Nedv6wJHpEhvDXL3/C19Y7UUop5d80YHjJrAV5dAkO4LxhSe3zhqc+AnED4L0brY3RWhAU4OCGsceRk1/Gok26M6tSSqn2owHDC4oqavhwxQ4uyEyiS3A7DWsJDIXzX4TqXfC/mw+66+rFw3sR2yWYv875qX3qUkoppdCA4RWvL9pMfYPhipF92veNEwbBpOmw/hNY/GKLl4QEOrn+5H58m1vKkvyy9q1PKaVUp6UB4xjVudy8tmgzY9Pi6BfXpf0LGHE9pEyCzx6AojUtXjJtRG+iwgKZMTe3nYtTSinVWWnAOEYfr9xBcUUtV7UwNbVdiMA5f4PgCHj7aqivOeCS8OAArh7Tlzlri1i5rdyGIpVSSnU2GjCO0awFeSTHhDE2Lc6+Irp0hyl/g6JV8MVDLV5yxahkIkICeHaOtmIopZRqe7rQ1jFYsbWcHzbv4v/OyrB/Oe60U2HEDbDoOUiZAKmT9jvdNSSQn49K5pk5uawvrCAt/piW+1JKdQBLlizpHhAQ8CIwCP0HpfI+N7DS5XJdk5mZWdT8pAaMY/DygjzCgpxcmNVOU1MPZ+LDsOlra+rqjQuslo0mfj66Ly9+s4kZc3P5y9ShNhWplGovAQEBLyYkJAyIi4srczgcuhiO8iq32y3FxcUZBQUFLwJnNz+vifYolVbW8sHy7Zw3LJGuIYF2l2MJDLGmrtbsbnHqalR4EJef2IcPlm1nU0mVTUUqpdrRoLi4uN0aLlRbcDgcJi4urhyrhezA8+1cj994Y/EW6lxurhyZbHcp+4vPgFMfhZ8+g+9fOOD0NSf1I9Dp4G86o0SpzsCh4UK1Jc+frxazhAaMo+BqcPPqwnzGpMSS6otjGbKvhdTT4JP74IPboaJg76m4iGAuye7Nf5duY8vOPfbVqJRSyq9pwDgKn60uZEd5TYu7pvoEETj/7zD8alj6L3hmKMx51Oo6Aa4f2w8RmPnVBpsLVUqp/YWFhQ0FyMvLC5w8eXK/lq7Jzs5O/+qrr8IO9TrTp0/vXlFRsfc7buzYsSklJSVO71Z75JrX1dzFF1/cZ8mSJSGtfb1vv/029KKLLmrTVR5///vfxz399NMxR3qfBoyj8PK3eSRFhTK+f/fDX2yXkEg4409w8/eQNhm++hM8cwIsfJ4e4Q4uyOzFW4u3UlB+4LoZSillt+Tk5PpPPvlk49HeP3PmzPjKysq933Hz58/PjY2NbfBOdUeveV1NuVwu3nzzzfzMzMxW/4/50Ucf7XHHHXccMIPDm2699dbSmTNnxh/pfa0KGCIyWUTWiUiuiNzbwvlgEXnTc36RiCQ3OXe8iHwnIqtEZIWItDqZ+aLV23fzfd5OrhjZB6fdU1NbI+Y4uPCfcO1ciB8In/wanh3O3T2W4TYNvPDVUf/9VUqpQ7rpppsS//CHP+xdJOjOO+/s+eCDD8aXl5c7Ro4cmZaRkTEgLS0t49VXX+3W/N5169YFpaamDgSorKyUs846q1+/fv0GTpo06biampq9//O99NJLew8aNGhASkrKwDvuuKMnwKOPPtq9qKgocOzYsWkjRoxIA0hMTBy8Y8eOAICHHnooPjU1dWBqaurA6dOnd298v379+g2cOnVqn5SUlIGjR49OraysPOB/8ueff37ypZde2nvIkCH9k5KSBs+ePTviwgsvTO7Xr9/A888/P7nxunfffbfrCSec0D8jI2PA6aef3q+8vNzRUl1hYWFDr7322qT09PSML7/8skvT1pm33367a0ZGxoD09PSMkSNHpjWvpayszLFmzZqwkSNHVjd+vuedd15yZmZmes+ePQfPmjWr2w033JCUlpaWcdJJJ6XW1tYKwN13391j0KBBA1JTUwdecsklfdxuNwCrVq0KPumkk1IHDhw4IDMzM33p0qUhABEREe6kpKTauXPnHrLVqLnDTlMVEScwA5gEbAUWi8j7xpjVTS67GigzxqSIyFTgceBiEQkAXgUuN8YsE5EYoP5ICvQ1sxbkERLo4KKsXnaXcmQSh8EV78OGOfDFb4n59Gbmd03lwe8voGTcccR2Cba7QqVUG7rn7WW91hdUHNEXxOGkJUTs+dMFQ7Yc7Pyll1668/bbb+993333FQP873//i/r000/Xh4WFuT/88MPc6Oho944dOwJGjBjRf9q0abscjpb/zfvEE090Dw0NdW/cuHHVokWLQkePHp3ReO6pp57aFh8f3+ByuRg1alT6okWLQh944IGi5557Ln7+/Pnre/To4Wr6Wl9//XXY66+/HrNkyZI1xhgyMzMHTJgwoSI2NrZh8+bNIa+++urGUaNG5Z9xxhn9XnnllaibbrrpgK2oy8vLA5YuXbr29ddf7zZ16tSUOXPmrM3MzKw+/vjjByxYsCC0b9++9b///e97fPXVV+u7du3qvv/++xMeeeSR+CeeeGJH87qqq6sdI0aMqPr73/++FeD//u//ANi+fXvALbfckjxv3ry1/fv3ryssLDyge+ebb74JT09Pr256LD8/P3jBggXrf/jhh5Dx48f3nzVr1obnn39+66RJk4576623Ii+//PJd99xzT9ETTzyxA+Ccc87p+8Ybb0ROmzat/Jprrunzwgsv5A8ePLh2zpw54TfeeGPvhQsXrgcYNmxY1bx58yLGjRvX6sF7rVkHIxvINcZsBBCRN4ApQNOAMQV4yPP4beBZERHgVGC5MWYZgDGmtLWF+aKyqjre+3Eb5w1LpFtYkN3lHDkRaxGufuNg5dt0/3w6Lzn/wKYXviR26h+hp66NoZTyntGjR1eXlpYG5OXlBe7YsSMgMjKyISUlpb62tlZuv/32pIULF3ZxOBwUFRUFbd26NaB3796ull7nm2++6XLbbbcVAYwYMaI6LS1t75fcrFmzol9++eVYl8slxcXFgcuWLQsZMWJEdUuvAzBv3rwuZ5xxxq6uXbu6Ac4888yyuXPnRlx44YW7EhMTa0eNGlUNMHTo0D15eXkt/svrzDPP3OVwOBg2bNiemJiY+uzs7GqAtLS06g0bNgTn5+cHbdiwISQ7O7s/QH19vWRmZla29FpOp5OrrrrqgJ0o582bF56dnV3Rv3//OoD4+PgDune2bdsWGBMTs98/2idOnFgeHBxssrOzqxsaGuSCCy7YDTBw4MDqTZs2BQF8/PHHEU899VRCTU2NY9euXQEZGRnV5eXlFUuXLu1y4YUXHtf4WnV1dXtbcLp37+5au3btEfVAtCZgJAJNE+pWYMTBrjHGuESkHIgB0gAjIp8CccAbxpg/Nn8DEbkOuA6gd+/eR1J/u3ozZwu1LrfvDu5sLYcDjr+IwIwpvP3CdCYUzoIXToFB58P4ByC6xXFVSqkO7FAtDW3p7LPPLnv11VejCgoKAs8777ydADNnzowuLS0NWLFixZrg4GCTmJg4uLq6+ojHBK5duzbo2WefjV+yZMmauLi4hvPPPz+5pqbmqMcWBgUF7Z3S63Q6zcFqCgkJMZ5r9rvH4XDgcrnE6XSaMWPG7P7ggw82teI93QEBR7fmZVhYmLu2tna/GoODg/fWFhAQYBpbhRpr27Nnj9x11119Fi1atDolJaX+zjvv7FlTU+NoaGggIiLCtXbt2tUtvBU1NTWO0NBQ95HU19aDPAOAMcClnp/nisiE5hcZY14wxmQZY7Li4mzc0+MQGtyGf32Xz4i+0fRP6Gp3Od4REMyg837NybV/ZlHSL2DtR/BsNnz0K6gstrs6pZQfuOyyy3a+88470bNnz466/PLLywDKy8udsbGx9cHBweaDDz6I2L59+yGbhMeMGVP52muvRQMsXrw4ZP369WEAZWVlztDQUHd0dHTDli1bAubNmxfZeE94eHhDeXn5Ad9x48aNq/zoo4+6VVRUOHbv3u346KOPosaNG1fhzd/zKaecUpWTk9Nl5cqVwQC7d+92LF++PPhQdbX0Gt9//33E2rVrgwBa6iIZPHhwzcFaWQ5mz549DoCEhARXeXm544MPPogCiI6OdiclJdX94x//iAJwu9189913oY33rV+/PnjQoEEHbRlqSWsCxjag6YCDJM+xFq/xjLuIBEqxWju+MsaUGGP2AB8Bw46kQF/xxZpCtu2qtm/X1DbSP6ErIzP6cu3WyVRe/z0MvQwWv2jNOJn3ONS22KqnlFKtkpWVVVNVVeWIj4+v69OnTz3ANddcs3PZsmXhaWlpGbNmzYrp27fvIWdN3H333UVVVVXOfv36Dbz//vsTMzIyqgBGjhxZPWjQoD3HHXfcoIsuuqhf026IK6+8smTy5Ml7B1M2GjNmzJ5p06aVDhs2bEBmZuaAyy+/vHj06NFH9MV5OD179nTNnDkzb+rUqf3S0tIysrKy+q9YsSLkUHW19BrPPPNM3rnnnpuSnp6ece655x7QtDx06NCaiooKZ1lZWasbC2JjYxsuvfTS4gEDBgwcN25c2pAhQ/Yu6/zvf/974z//+c/Y9PT0jNTU1IHvvPNOt8Zzixcv7nL22Wfvbu37AIgxh17kzRMY1gMTsILEYmCaMWZVk2tuBgYbY27wDPI8zxhzkYhEAV9itV7UAZ8AfzbGfHiw98vKyjI5OTlH8ntoF9P+vpC8kiq++tU4Apz+Nbt3xdZyfvbsN9xzWjo3j0uB4vUwZzqs+QDCu8PYX0HmVeD0kSXRlVIHEJElxpispseWLVuWN2TIkBK7alJt7+GHH+4eERHhvvPOO9vsv/O3334b+qc//Snhvffea7HLZ9myZbFDhgxJbn78sN+UxhgXcAvwKbAGeMsYs0pEpotI4+YmLwExIpIL3Anc67m3DHgKK5T8CPxwqHDhq9YXVrBgQymXjezjd+ECYHBSJKekx/HSN5vYU+eCuDS4+FW4+guISYGP7oYZ2bDqvwfsb6KUUso+99xzT3FwcPARjY04UkVFRYGPP/54856LwzpsC0Z788UWjPv/u4L/LNnKwvsmEB3eAWePtMKS/J2c/9x3PHDmAK45qUlLnDHWviZfPARFq6HnMJj0MPQ92bZaj1hDPZRugIh4CI2yuxql2oS2YCi7HKwFQ7drP4zy6nre/WEbU4b09NtwAZDZJ5qR/WJ44auNXHZiH0ICPeOJRCDtNEiZCMvegLm/g1k/s55PfAgSBtta9wFqK6BwFexYDgXLoWAFFK2BhlprddNTH4Whl1u/L6WUUm1GA8Zh/CdnC9X1DR1/amor3DohhWl/X8R/crZwefNdYh1OGHopDDoPvv87fP0kPH8SHH8xjPsNRLXpUvgHMsbaxK1gBRQs8/xcATubrEwaGgUJx1ubv3UfAD++Du/fCsvehJ/9BWJT2rdmpZTqRDRgHEKD2/DKd/lk9YliUGLk4W/o4Eb2iyGzTxTPz9/IxcN7ExTQwniTwFAYfRsMuxy++TMsfB5WvQvDr4WT74awaO8X5m6A0lwrQOxoEib2NGn5jUq2WlOGTLN+JgyGrj33b6kYMs3a/O3z/4PnRsHJ98DoX0KA/7ZMKaWUXTRgHMK8dUVs3rmHe05Lt7uUdiEi3DI+hZ//czHvLd3GRcMPsRx6aBRMmg7Z18G8P8Ci56wv7zG3w4gbIegoVySuq4LC1fu6NwqWW89dnllkziCI6w/pk63WiYTB1h4rIa0IgA4HZF5pbf72ya9h7qOw8h04+xnolX109SqllGqR/02J8KKXF+QR3zWYyYMS7C6l3ZySFsfgxEhmzMvF1dCKgcmRSTBlBtzwLfQZDV9Ot7aHX/IyNLS46u8+lUWQ+4XVEvKfn8Nfs+D3ifDSRPjwTqtlJDAMsn4B5zxvvcd92+CGr633HHE99BnVunDRVEQ8XPgyXPIG1O6Gl06FD+/eu529Uso7SkpKnI899thRrZ7Y3turd6Zt1NuLziI5iA3FlUx4cj53TkrjtgmpdpfTrj5dVcD1/1rC0xefwDlDE4/s5vwF8PlvYev3EJMKE38L6WfAzk37t0oUrIDKwn33deu9r0Wi8Vdkr7YfjFlbAXMehUUzIaIHnPkE9D+zbd9TqTbgi7NI1q1bF3TWWWel/vTTT6uan6uvrycw0HfW1klMTByck5OzpvnmaGBto36ky3mffvrp/R588MEdjTudtoWKigpHdnZ2/zVr1rS4vHd7Oep1MDqrVxbkEeR0cEm27+6N0lYmDYgnPT6CZ+fm4nYfYQDtMwqu/sxaRwPgzcvgdwnwbCa8/XNY8AxUFMJxE+C0P8CVs+HXeXD7Cpj6Gpxyr/UF3613+8z0CI6A0x+Ha76wun3emAZvXg67d7T9eyvl5+66666kLVu2BPfv3z/j+uuvT5o9e3ZEZmZm+vjx41NSU1MHAUycOPG4gQMHDkhJSRn4xBNPxDbe27i9um6j7v1t1NuLjsFoQUVNPW8v2cpZx/cgLqLzbWPucAg3j0/htn8v5ZNVBZwxuMeRvYAIDPgZpJ0Oy163ponGD7RaJeL6Q4APfqZJWXD9fCsAzXscNs6HSQ/BsKussRtKdXTv3dyLotXe/SLqnrGHc2YcdBO1J598cutZZ50V2riB1uzZsyNWr14dtnTp0lWNu4S+9tprefHx8Q2VlZUydOjQjMsuu6wsISFhv51DdRt1726j3l40YLTgnSVbqarrHFNTD+bMwT14+vP1/HVOLqcPSkCOpjXBGQDDrvB+cW3FGQgn3QUZ58Ds22H2HbD8LWtKa1znGOirVFs7/vjjqxrDBcDjjz8e/+GHH3YDKCgoCFy1alVIQkJCVdN7dBt1726j3l40YDTj9kxNPaFXN4b06mZ3ObZxOoSbxqVw93+WMWdtERMGxNtdUvuJOQ6ueN9aN+PT38Bzo63gcdKdvtn6olRrHKKloT2FhYXtHT0+e/bsiPnz50fk5OSsjYiIcGdnZ6e3tEW6bqPu3W3U24u2/TbzdW4JG0uq/G7X1KMx5YSeJEWF8sycXHxtMHCbE7EWFrslBzKmwPzH4PkxkP+d3ZUp1WFERkY2VFVVHfR7ZteuXc7IyMiGiIgI99KlS0OWLVsW3pb16Dbq7UsDRjOzFuQR2yX4yMcd+KFAp4MbTzmOZVt28U1uJ93OoEscXPASXPo21NfAPyfDB7dD9S67K1PK5yUkJDRkZmZWpqamDrz++uuTmp8///zzy10ul/Tr12/gPffck9h06/C2oNuoty+dptpEXkkV456cx63jU7lz0iH/jHUata4Gxv5xHr1jwnjr+pF2l2Ov2kprUbGFf7O2sT/jjzDgbN3XRPkEX5ym2pn5wjbq7UWnqbbCK9/l4xTh0hGdb2rqwQQHOLl+bD++37STRRtL7S7HXsFd4LTfwTVfWi0bb10Bb1wK5Ue8i7FSys/58jbq7UUDhkdVrYv/5Gzh9ME9iO/qkwNybTN1eG9iuwTx7Nxcu0vxDYnD4Nq51lLpG+bAjBHWBnBunxxnpZSyQVhYmLn55psPmErrTeeee+7u9PT0usNfaQ8NGB7vLt1GRa2Lq0a1866gHUBokJNrT+rH1z+VsHTzAbO2OidnoLVR2k3fWWtofHQ3/OM0a98UpXyH2+12ax+eajOeP18t/utKAwZgjOGVBXkMSuzKsN5Rdpfjky49sQ/dwgKZoa0Y+4vuC5f/F86dae34OvMk+PIRa0CoUvZbWVxcHKkhQ7UFt9stxcXFkcDKls7rOhjAgg2l/FRUyRMXDjm6BaU6gS7BAfxidF+e+nw9q7aXM7Cn/29f32oiMGQqpEyy1s34+glY/Z61QFfyGLurU52Yy+W6pqCg4MWCgoJB6D8olfe5gZUul+ualk7qLBLg2ldyWJJfxoJ7xxMS2G6b93U45dX1jHlsDielxfK3SzPtLsd3bZhjTWXdlQ9DL4dTH7H2OVGqDbU0i0QpO3X6RLtl5x6+XFPIJdm9NFwcRmRoIFeOSubjlQX8VFhhdzm+67jxcNNCGHWbtRros9mw8h3wsTCvlFJtqdMHjFcX5iMiXHaiDu5sjV+M6UtIgJO/zdtgdym+LSjMarm4bi507Qlv/wJevxh2+cRqzUop1eZaFTBEZLKIrBORXBG5t4XzwSLypuf8IhFJbna+t4hUisjdXqrbK6rrGnhj8RZOGxhPj8jQw9+giA4P4rITe/O/H7eRV9Kmi+75hx5DrHUzTvs95H1tTWld8Cw01B/+XqWU6sAOGzBExAnMAE4HMoBLRCSj2WVXA2XGmBTgz8Djzc4/BXx87OV613s/bqO8up4rRybbXUqHcu3J/QhwOnhOWzFaxxkAI2+2uk2SR8Nn98PMkyHvW7srU0qpNtOaFoxsINcYs9EYUwe8AUxpds0UYJbn8dvABPFMxxCRc4BNwCqvVOwlxhhmLcijf0IE2X2j7S6nQ+keEcIlw3vxzg9b2bbLJ/fY8U1RfWDaW3Dxa1BbAS+fAe9eD5VFdlemlFJe15qAkQg07Tje6jnW4jXGGBdQDsSISBfg18DDh3oDEblORHJEJKe4uLi1tR+TRZt2sraggqtGJevU1KNw/djjEIHntRXjyIjAgLPg5kXWFvAr34G/ZsKimdDgsrs6pZTymrYe5PkQ8GdjTOWhLjLGvGCMyTLGZMXFxbVxSZZZC/KIDA1kygnNs5JqjZ7dQrkgM4k3c7ZQuFsXlTpiQeEw4UFrJdDEYfDxr+Dvp8CW7+2uTCmlvKI1AWMb0KvJ8yTPsRavEZEAIBIoBUYAfxSRPOB24DcicsuxlXzstu+q5rPVhUwd3ovQIJ2aerRuHJtCg9vw96822l1KxxWbCpe/Bxe+DFWl8NIk+N/NUKUbYCqlOrbWBIzFQKqI9BWRIGAq8H6za94HrvQ8vgCYYywnGWOSjTHJwNPA740xz3qn9KP36sJ8jDE6NfUY9Y4JY8qQnry2aDOllbV2l9NxicDAc+GWxdbaGcvesLpNFr8E7ga7q1NKqaNy2IDhGVNxC/ApsAZ4yxizSkSmi8jZnstewhpzkQvcCRwwldVX1NRbU1MnDoinV3SY3eV0eDeNS6HG1cBL32yyu5SOL7iLtXbGDd9AwmD48E54cQJsW2J3ZUopdcQ63VLh/8nZwj1vL+f1a0YwKiW2zd6nM7n59R+Yv66Yb389nsiwQLvL8Q/GwIq3rSmtlUWQeZU1ZiNMZzyplulS4crXdKqVPI0xzPouj7T4Low8LsbucvzGLeNSqKx18fKCPLtL8R8icPyFcEsOnHgj/PCK1W3ywyvgbnFnZKWU8imdKmD8sLmMldt2c8VInZrqTQN6dGXigHj+8e0mKmp0hUqvCukKk/8A138Fcenw/q3wj9NgxzK7K1NKqUPqVAHj5QX5RIQEcO5QnZrqbbeOT6G8up5XF262uxT/lDAIfv4xnPMc7NwIL5wCH90D1bvsrkwppVrUaQJG4e4aPl6xg4uyehEeHGB3OX5nSK9unJwWx4tfb6S6Tmc+tAkROGEa3LoEsq6GxS/Cs1nw4791p1allM/pNAHjtYX5NBjDFSN1ampbuXV8CqVVdfz7e23FaFOh3eDMJ+DaudCtD7x3A/zzdCj0qdX4lVKdXKcIGLWuBl7/fjPj0rvTJybc7nL81vDkaEb0jWbmVxuoqddWjDbX8wS4+nM4+69QvA6ePwk++Q3U7La7MqWU6hwB46MVOyiprOPKUcl2l+L3bpuQSuHuWt5estXuUjoHhwOGXWF1mwy7HBb+DZ4dbk1x1W4TpZSNOkXAeHlBPv3iwjlJ171oc6OOi2Fo7248N28D9Q06nbLdhEXDz/4C13wJEQnwztUw62dQtNbuypRSnZTfB4wft+xi2ZZdXDkyGYdDp6a2NRHh1vEpbNtVzX+XNt+yRrW5pEy4dg6c+RQUrIDnR8PnD0LtIfcbVEopr/P7gDFrQR5dggM4PzPJ7lI6jXHp3RnYsyt/m5tLg1ub6dudwwnDr7a6TYZMhW//AjOyYdV72m2ilGo3fh0wiitqmb18OxdkJtFFp6a2m8ZWjLzSPcxevt3ucjqv8FiYMgN+8RmERsN/roRXz4OSXLsrU0p1An4dMP79/WbqG3Rqqh1OzUggtXsXZszNxa2tGPbqPQKumwen/xG25sBzI+HL6bBnp92VKaX8mN8GjPoGN68tyufktDj6xXWxu5xOx+EQbhmfwvrCSj5bXWB3OcoZACOut/Y2GXgefP0kPJEGb14G6z6GBl3iXSnlXX4bMD5ZWUDh7lquGqWtF3Y56/ie9I0N54H3VvLo7NUs3FiKS2eW2CsiHs6bCTd8C9nXQf538O+p8NQA+PR+XaxLKeU1frtd+/nPLaCkspa5d52is0ds9OOWXfz58/V8t6GUugY33cICGZ/enUkZ8ZycFqfLttutoR5++hx+fA3WfwJuF/QYAidcCoMugHDddbij0O3ala/xy4Cxcls5Z/31Gx44cwDXnNTPS5WpY1FZ6+Kr9cV8vrqQOWuLKK+uJ8jpYFRKDJMy4pk4IJ74riF2l9m5VZVYC3T9+BoULAdHIKRPtsJGykRwBtpdoToEDRjK1/hlwLj7P8v4cPkOFv5mApGh+j9FX+NqcLM4r4zPVxfy+ZoCtuysBmBIUqQVNjLiSY+PQERbnmxTsMLaRG35m7CnBMLj4PiLrc3W4gfaXZ1qgQYM5Wv8LmCUVtYy8rE5XJSVxKPnDPZiZaotGGNYX1jJF2sK+Wx1Icu27AKgV3QokwYkMDGjO9nJ0QQ4/Xa4kG9rqIfcL6xWjXWfgLve6kIZMg0GX6hdKD5EA4byNX4XMGbMzeVPn67j8ztOJjU+wouVqfZQtLuGL9YU8fnqAr7dUEqdy01kaCDj+3dn4oB4xqbH6ZomdqkqhZWeLpQdy6wulLTTrC6U1EnahWIzDRjK1/hVwHA1uDn5j3PpGxfOa9ec6OXKVHurqnXx9U/FfL66iDlrCynbY43bOPG4xnEb3ekRGWp3mZ1TwUpY5ulCqSqGsNh9XSgJg+yurlPSgKF8jV8FjI9X7ODG137ghcszOXVggpcrU3ZyNbhZkl/GF2sK+Xx1IXmlewAYnBi5d5DogB46bqPdtdSFknC81aqhXSjtSgOG8jWtChgiMhn4C+AEXjTGPNbsfDDwCpAJlAIXG2PyRGQS8BgQBNQB9xhj5hzqvY4lYFw88zu27apm/j3jcOrUVL9ljCG3qJLPPWHjxy27MAaSokKZOCCeSRnxZPeNJlDHbbSvPTv3zULZ8aN2obQzDRjK1xw2YIiIE1gPTAK2AouBS4wxq5tccxNwvDHmBhGZCpxrjLlYRIYChcaY7SIyCPjUGJN4qPc72oCRV1LFKU/M477T+3P92OOO+H7VcRVV1DBnTRFfrCnk659KqHW5iQgJYJxnvY1T0uOICNEvt3ZVuAp+fB2WvwVVRdqF0g40YChf05qAMRJ4yBhzmuf5fQDGmD80ueZTzzXfiUgAUADEmSYvLlbbdSnQwxhTe7D3O5YWjFXby0nqFkZkmH6ZdFZ76lx8/VMJX6wu5Mu1ReysqiPQKZzYL4axaXFkJUczsGdXbd1oLw31kPulpwvl4yZdKI2zUGLtrtBvaMBQvqY1AeMCYLIx5hrP88uBEcaYW5pcs9JzzVbP8w2ea0qavc4NxpiJLbzHdcB1AL17987Mz88/5t+YUg1uw9LNjettFLKxuAqA0EAnJ/TqxvDkKLKSoxnau5u2cLSHPTth5TtW2Ni+FBwB1gJeScOt0NHjeOgSDzqO5qhowFC+pl0ChogMBN4HTjXGbDjU+3lrqXClmivcXUNOXhmL83aSk7+T1dt34zbgEBjQoyvDk6PJSo5ieHK0rira1gpXw7LXYc1sKNu073h4HCQMtgJHwmBrzY3ofuBw2ldrB6EBQ/maNu8iEZEkYA7wc2PMt4crSAOGai+VtS6Wbi5jcV4ZOXk7Wbp5F9X1DYC10NfwPtFkJUczPDmK4+K66J42baWm3BqzsWO5tYJowXIoWmN1pwAEhlmrhzaGjoTjIT4DAnWKclMaMJSvaU3ACMAa5DkB2IY1yHOaMWZVk2tuBgY3GeR5njHmIhHpBswHHjbGvNuagjRgKLvUN7hZs2P33sCxOG8nJZV1AHQLCySrT9TewDEoMZLgAP1XdZtx1UHJOitw7A0eK6C23DovDohNaxI6PMGjE0+L1YChfE1rp6meATyNNU31H8aY34nIdCDHGPO+iIQA/wKGAjuBqcaYjSLyAHAf8FOTlzvVGFN0sPfSgKF8hTGG/NI9VpdKXhmL83fuHccRFOBgSFLk3sCR2TtaBxe3NWNgV/6BoWP31n3XdE3cv4slYTBEJXeKcR0aMJSv8auFtpRqayWVtSzJb2zhKGPltnJcbuvvUHp8xN4xHFnJUSR2C9WFv9pDVSkUrtg/eJSsB2N1dxHc9cDQEdcfAoLsrdvLNGAoX6MBQ6ljUF3XwI9bdlmBI7+MH/LLqKx1AdAjMmRvC0dWn2jSEyJ0Abj2Ul8NRav3Dx2FK6HeWgEWRyB07+8JHcdbjyN6WLNYQiI7ZIuHBgzlazRgKOVFDW7D2oLdLMm3Bo8u3rSTgt01AEQEBzCsTxRZfaLITI7ihF7dCAvSjdvajbsBdm60BpHuDR7Lrb1UmgoIhYh46JJg/WwMHhEJ1q8unp+hUT4VRDRgKF+jAUOpNmSMYduu6r3TYxfn7WR9YSUATocwsGdXMvtYLRxZyVE6PdYOFYVWl0plIVQUQMWOJo8LrMe1uw+8zxm0L4S0FEAaH4fFgKPtF3bTgKF8jQYMpdpZ+Z56fthcRk6+NXh02dZd1NS7AWs/FauFI5qsPlGkxWu3ik+oq9oXNvb+3GGFk8qCfWGkZteB9zoCILx7k9DhaRVp3koSHndM631owFC+RgOGUjarb3CzavtucvJ2WgNI88sorrBW048IDmCop1slq08UJ/TWbhWfVl/TJIQUWAGkaYtIYzDZU3rgveKA5DFw5QdH9dYaMJSv0YChlI8xxrBlZ7XVwpFfxpK8MtYXVWCM1a2S0cPTreIZPJoQqd0qHY6rztoEbm83jCeMhEbByJuO6iU1YChfowFDqQ6gvNrqVlmSZ3Wt/LhlX7dKYrdQT9iIIlNnq3RaGjCUr9G2VqU6gMjQQMald2dcenfA6lZZvX231cKRv5PvNpTyvx+3A1a3ygm9u+0dOHpCr26EB+tfdaVU+9IWDKX8gDGGrWXVnjEc1uDRdYX7ulUG9Iggq0/03q6VHpG6j4e/0RYM5Ws0YCjlp3bX1LN08y6W5FljOZpu5tYjMoReUWHERgQR1yWYuIh9v2I9z2PCgwkKaPvplco7NGAoX6Ptpkr5qa4hgYxNi2NsWhwArgY3a3ZUkJO/k2VbdlGwu4Z1BRV8U1HC7hpXi68RFRa4X+hoGkZimzyOCgvScR9Kqf1owFCqkwhwOhicFMngpMgDztXUN1BSWUtJZR3FFbX7flXWUFJRR3FlLUs376Koombv4NKmnA4hJjxov9BxsGDSNSRA92hRqhPQgKGUIiTQSVJUGElRYYe8zhhDVV0DxRW1lFTW7h9GGo9V1rK+sILiitq9G8E1FeR0WOEjIpjuEcH0iQ4jOTacvrHhJMeG06NrCA5tDVGqw9OAoZRqNRGhS3AAXYID6Bsbfshr3W5DeXX9viDSPJBU1pJXUsX89cXUufa1igQHOOgTE0afGE/oiAknOTaMvrHhxEdo+FCqo9CAoZRqEw6HEBUeRFR4EKnxEQe9zu027NhdQ35JFZtKq8grqWJTyR42lVQxf10xdQ37wkdIoIM+0VbgSI4Np2+M1eqRHBNOfNdg7XpRyodowFBK2crhEBK7hZLYLZRRKbH7nWtwG3aUV5NXsmdv+MgrqSK3qJI5a4uob9jXBRMa6KRPTNjerpbkmDCSPa0gcRG+Fz6MMdS63FTXNVBdb/1yipB8mJYhpToKDRhKKZ/ldMjesSFjUg8MH9t3VbOppIq80irySvaQV1rFuoIKPl9duN/4j/AgJ308XS3JnlaPxu6X2C5BB4SPBrehxvOlX13XsN/j6nrr+Z66ls67m5x3UV3vpqZJgNjv2voGmq8SMLR3N/570+g2+zyVak8aMJRSHZLTIfSKDqNXdBgnE7ffOVeDm227qskr3ePpcrFCyOrtu/l0VSENTcJHl+AAYrsEWa0JnhBQ6zpwpkxr6gkLdBIS5CQsyElooJOQQOtnXETwvudBDsKCAvaeCw10EBpknYuLCD7mz0UpX6EBQynldwKcDvrEhNMnJnzvOiCN6hvcbCur3q/LZeeeeuuLvjEgBAYQGuRoEgoaw8D+AaLxeWigk0CnLkqmVFMaMJRSnUqg02GN0YgNh3S7q1HKf7UqcovIZBFZJyK5InJvC+eDReRNz/lFIpLc5Nx9nuPrROQ0L9aulFJKKR912IAhIk5gBnA6kAFcIiIZzS67GigzxqQAfwYe99ybAUwFBgKTgb95Xk8ppZRSfqw1LRjZQK4xZqMxpg54A5jS7JopwCzP47eBCWINy54CvGGMqTXGbAJyPa+nlFJKKT/WmoCRCGxp8nyr51iL1xhjXEA5ENPKe5VSSinlZ3xi2LOIXCciOSKSU1xcbHc5SimllDpGrQkY24BeTZ4neY61eI2IBACRQGkr78UY84IxJssYkxUXF9f8tFJKKaU6mNYEjMVAqoj0FZEgrEGb7ze75n3gSs/jC4A5xhjjOT7VM8ukL5AKfO+d0pVSSinlqw67DoYxxiUitwCfAk7gH8aYVSIyHcgxxrwPvAT8S0RygZ1YIQTPdW8BqwEXcLMxpqGNfi9KKaWU8hFimi+GbzMRKQbyj+ElYoESL5XT0elnsT/9PPann8c+/vBZ9DHGaB+z8hk+FzCOlYjkGGOy7K7DF+hnsT/9PPann8c++lko5X0+MYtEKaWUUv5FA4ZSSimlvM4fA8YLdhfgQ/Sz2J9+HvvTz2Mf/SyU8jK/G4OhlFJKKfv5YwuGUkoppWymAUMppZRSXuc3AUNEJovIOhHJFZF77a7HTiLSS0TmishqEVklIr+0uya7iYhTRJaKyGy7a7GbiHQTkbdFZK2IrBGRkXbXZCcRucPz92SliPxbRELsrkkpf+AXAUNEnMAM4HQgA7hERDLsrcpWLuAuY0wGcCJwcyf/PAB+Cayxuwgf8RfgE2NMf2AInfhzEZFE4DYgyxgzCGu14qn2VqWUf/CLgAFkA7nGmI3GmDrgDWCKzTXZxhizwxjzg+dxBdYXSKK9VdlHRJKAM4EX7a7FbiISCZyMtbw/xpg6Y8wuW4uyXwAQ6tmoMQzYbnM9SvkFfwkYicCWJs+30om/UJsSkWRgKLDI5lLs9DTwK8Btcx2+oC9QDPzT02X0ooiE212UXYwx24AngM3ADqDcGPOZvVUp5R/8JWCoFohIF+Ad4HZjzG6767GDiJwFFBljlthdi48IAIYBzxljhgJVQKcdsyQiUVitnX2BnkC4iFxmb1VK+Qd/CRjbgF5Nnid5jnVaIhKIFS5eM8a8a3c9NhoNnC0ieVhdZ+NF5FV7S7LVVmCrMaaxRettrMDRWU0ENhljio0x9cC7wCiba1LKL/hLwFgMpIpIXxEJwhqk9b7NNdlGRASrj32NMeYpu+uxkzHmPmNMkjEmGevPxRxjTKf9F6oxpgDYIiLpnkMTgNU2lmS3zcCJIhLm+XszgU486FUpbwqwuwBvMMa4ROQW4FOsUeD/MMassrksO40GLgdWiMiPnmO/McZ8ZF9JyofcCrzmCeMbgZ/bXI9tjDGLRORt4Aes2VdL0WXDlfIKXSpcKaWUUl7nL10kSimllPIhGjCUUkop5XUaMJRSSinldRowlFJKKeV1GjCUUkop5XUaMJRSSinldRowlFJKKeV1/w/ypIULVDYLPwAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":"history.history.keys()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:55:14.101395Z","iopub.execute_input":"2022-02-16T23:55:14.101776Z","iopub.status.idle":"2022-02-16T23:55:14.107960Z","shell.execute_reply.started":"2022-02-16T23:55:14.101737Z","shell.execute_reply":"2022-02-16T23:55:14.106860Z"},"trusted":true},"execution_count":29,"outputs":[{"execution_count":29,"output_type":"execute_result","data":{"text/plain":"dict_keys(['loss', 'mae', 'correlation', 'val_loss', 'val_mae', 'val_correlation', 'lr'])"},"metadata":{}}]},{"cell_type":"code","source":"plt.plot(history.history['val_correlation'], label= \"validation metric W-Corr\")\nplt.plot(history.history['correlation'], label= \"train metric W-Corr\")\nplt.legend(loc=\"upper left\", bbox_to_anchor=(1, 0.5))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:55:14.109687Z","iopub.execute_input":"2022-02-16T23:55:14.110102Z","iopub.status.idle":"2022-02-16T23:55:14.345633Z","shell.execute_reply.started":"2022-02-16T23:55:14.110059Z","shell.execute_reply":"2022-02-16T23:55:14.344747Z"},"trusted":true},"execution_count":30,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAhYAAAD5CAYAAAB7ynytAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9MElEQVR4nO3dd3hUVf7H8fdJQoAUakKAACkkgYQmJNIRRFSKgl3E3nDXxe5aftZl3VVXV11XLIi9LHapggVp0hFRAgECBBIgJAQIpJA25/fHBIwhwAQmmZTP63nymLlz75lvRmA+uffc7zHWWkRERETcwcvTBYiIiEjdoWAhIiIibqNgISIiIm6jYCEiIiJuo2AhIiIibqNgISIiIm7j48pOxpjhwH8Ab2CKtfaZcs+/CJxd+tAPaGWtbXaiMYOCgmx4eHhl6xURqddWr16911ob7Ok6RI7npMHCGOMNTALOBdKAlcaY6dba9Uf2sdbeU2b/O4CeJxs3PDycVatWnVLRIiL1lTFmu6drEDkRVy6F9AaSrbVbrbWFwFRgzAn2vwr4nzuKExERkdrFlWARCqSWeZxWuu0YxpgwIAKYd/qliYiISG3j7smbY4HPrbUlFT1pjBlvjFlljFmVmZnp5pcWERERT3MlWOwE2pd53K50W0XGcoLLINbaydbaBGttQnCw5h6JiIjUNa4Ei5VAtDEmwhjjizM8TC+/kzGmM9AcWOreEkVERKS2OGmwsNYWAxOAucAG4FNrbaIxZqIxZnSZXccCU62WSxUREam3XOpjYa2dDcwut+3xco+fdF9ZIiIiUhup86aISDXJzi/iX3OS2JGV5+lSRKqMS2csRETk1BUUl/Dhsh38d95mDuQV0aZpI67tF+7pskSqhIKFiEgVcTgsM37dxXNzN5K2P59B0UE8OLwzXUObero0kSqjYCEiUgV+St7L099sYN3Og8S2acL7N3XjrBjdZi91n4KFiIgbrd91kGfmJLFwUyahzRrz4pU9GNMjFC8v4+nSRKqFgoWIiBvsPJDPv7/dyFdrdtKkUQMeGRnLtf3CaNTA29OliVQrBQsRkdOQnVfEq/OTeWdJCgDjB0Vy+5Aomvo18GxhIh6iYCEicgoOF5XwwdLtvPJjMgcPF3Fxz1DuO68Toc0ae7o0EY9SsBARqQSHwzJt7U6en7uJnQfyGRwTzIPDOxPXtomnSxOpERQsRKTKWGv5KTmL7Pwi+ndsSXN/X0+XdFoWbc7k6dlJrN99kK6hTfjXZd0ZEBXk6bJEahQFCxFxu8JiB9PX7uLNhVvZuOcQAMZA93bNOCs6iLNigunZvhk+3rWj+e+6ndk8OyeJRZv30q55Y/4z9gwu7N628nd6FOVDbiY0bAKNm1VJrSKepmAhIm5z8HAR/1u+g3d+SiH94GE6hQTy/OU9iAjyZ9HmTBZuymTSj8n8d14ygQ196B/VkrNigjkrOpj2Lfw8Xf4xUvfl8cJ3m/hqzU6a+TXg0VHOOz0a+pTe6eFwQP5+Z1jIzYTcDMjd+/vjnMwyz+2FQmfI4oKXIOFGj/1cIlXJeGox0oSEBLtq1SqPvLaIuNfu7Hze+SmFj5fvIKegmH6RLRk/OJIhMcEY88ff6rPzivhpy97SoLGXnQfyAYgI8j96NqNvZEv8G3ro956ifLL37uLzhWtY/ttGgkw254V70y/EQcOCrN9DwpH/2pJjxzBe4BcEAa3APwj8g//41aEfBEWdUnnGmNXW2oTT/ClFqoyChYicsg27D/Lmwq1MX7sLh7WM6t6W8YMi6dbOtZbV1lq2ZOaycFMmCzdnsmxrFoeLHDTwNsSHNT96NiOuTZPTazBVUgT7t8Oh3Sc8q2BzMzGFORWP4RtQGhJalQaE0sBwTHhoBY2bg1fVXOZRsJCaTsFCRCrFWsuSLVm8sXArCzdl0riBN1ee2Z6bB0ac9uWMguISVqXsZ+GmTBZsyiQp3XnpICjAl4FRzrMZg6KDCQ5sWFFhzuCQlVz6tQX2bnZ+vz/l2DMLZc4qWL8gUgv9WZJu2H44gODWoZzbuyvtQ8MgINi5n2/NuFSjYCE1nYKFiLikuMTBrN928+airazbeZCgAF9u6B/ONX3DaOZXNXd7ZBw8zKLNe1m4OZNFm/eyL7eQJuQyNPggQ1sdpKdfFm1LduK9b4szSBTl/n6wT2NoGQUtO/7+36bt/nBWwRrDgk2ZPPNNEknph+jerikPjehM/441904PBQup6TR5U0ROKLegmE9WpvLW4m3sPJBPZLA/T1/SjYt7hlZdu+riAti3lVZZyVyan8yljZOxbbdQsnczPvl74RBwCEqsIY1WZPuF4d3mIlpFdCGoQxdMUBQEtj3h5Yjf0rJ5+psNLNmSRfsWjfnvVT0Z1a2N1vQQOU0KFiJSoYxDh3lvSQofLttBdn4RCWHNeeLCOIbFhrjnw9fhgINpzksVe5PLXMJIhuxUsI7f9w0IwbSMwid2ZOnZh2hym4SzNCuQhVuzWbgpk5RNebAJQps5OCtmP4NjvOkfFUSTRn9srZ26L4/n5m5k+tpdNPdrwBMXxnF1nzB8fWrHra8iNZ0uhYjIHyRn5DBl0Va+/HknRQ4H58e15tazIokPa35qAxYcgj3rS0PD5t/nP+zbCsWHf9/PN6A0NJT9Kr2M0ejkXS13ZOWxYHMmizZlsmRLFjkFxXh7Gc5o34yzooPpG9mCuYl7+GBZCt5ehpsHRnDb4I7HBI+aTpdCpKZzKVgYY4YD/wG8gSnW2mcq2OcK4EnAAmutteNONKaChUjNYa1lZcp+Ji/cwvcbMmjo48Vl8e24ZVAkEUH+lR9wbzJsngub5sL2JeAocm738oHmEc6wEFQuRASEOLtouUFRiYM1Ow4cvdvkt53ZWAteBi6Pb88958bQumkjt7xWdVOwkJrupMHCGOMNbALOBdKAlcBV1tr1ZfaJBj4Fhlpr9xtjWllrM040roKFiOeVOCzfJqbzxsKt/JJ6gOZ+Dbi2XzjX9QsjKKCCOy+Op7gAUhbD5m+dX/u2OrcHd4bo8yBsAARFQ7Mw8K7+K7D7cgtZsS2LqFYBRLUKrPbXdycFC6npXPkb3htIttZuBTDGTAXGAOvL7HMrMMlaux/gZKFCRDzrcFEJn61O461FW0nJyqNDCz8mjunC5fHtaezr4oTMg7ucIWLTt7B1vvOODJ9GED4I+t7uDBTNw6r053BVC39fhndt4+kyROoFV4JFKJBa5nEa0KfcPjEAxpifcF4uedJaO8ctFYqI2+zLLeT9pSm8v3Q7+3IL6dGuKZPG9WJ419Z4n2xCpqME0laVnpWYC+m/Obc3bQ89xkLM+c5QUUP6PYiIZ7jrnKQPEA0MAdoBC40x3ay1B8ruZIwZD4wH6NChg5teWkROZntWLlMWbeOz1akcLnIwtHMrxp8VSZ+IFse03P6DvH2wZZ5zrkTy95C/D4w3tO8Dw56E6POhVazb5kaISO3nSrDYCbQv87hd6bay0oDl1toiYJsxZhPOoLGy7E7W2snAZHDOsTjVokXENb+kHmDywi18sy6dBl5eXNSzLbcOiiQ65DjzDKyFjPXOILH5W0hd7rzts3ELiD7XeXkj6hxny2oRkQq4EixWAtHGmAicgWIsUP6Oj6+Bq4B3jDFBOC+NbHVjnSLiIofD8kNSBm8u3MqKlH0ENvLhT4M7cmP/cFo1qeBOiMI82LagNEx85+wtAdC6Owy6z3lWIrQXeFVRMywRqVNOGiystcXGmAnAXJzzJ9621iYaYyYCq6y100ufO88Ysx4oAf5qrc2qysJF5I8OF5Xwxc9pvLVoG1v35hLarDGPjoplbO8OBJRfKXR/inPS5ea5sG0RlBRAA3/oeDYMfsB5ZqKJJjuKSOWpQZZILbc3p4APlm7ng2XOCZndQpty61mRjOzaGh/v0m6SJUWwY+nvd3Hs3ejc3iLSeUYipvSWUJ9K3GIqHqHbTaWmU0tvkVpqS2YOUxZt44uf0ygsdnBO51bcWnZC5sFdsOVH51mJLT9CwUHwagDhAyD+emegCIry9I8hInWMgoVILWKtZfm2fUxZtJXvN2Tg6+PFpb1CuXlABFENMmH7NzBtCWz/yXm5AyCgNcSNcd4OGjkEGtbuBlEiUrMpWIjUAsUlDr5Zl86bi7bya1o2LfwaMLGfF5e03E5A+mfw4RI4tNu5c+PmzssavcdD+EAI6XbCVT5FRNxJwUKkBsspXbL83UXJNDm4kRGBW3klLIX2B3/BrCmdHx3Q2nl5I6x/aevsTgoSIuIxChYiNdDufdl89/0c9q2fT4+SROb6bMavYR4UAgVhpV0uS8NE8wg1qBKRGkPBQqQmKMyDtJVkrvuRA0nzaZebyHWmEIDDLaNo1PFK59mIsH7QtJ2HixUROT4FCxFPOJwNqSsgZTF2+xLsrjV4OYpoaQ2ZhLE25CKiEs4jKG4IjQKCPV2tiIjLFCxEqkPuXmcfie2ld2yk/wbWgcP4kOQVxYLC4Wxq1I2ufc7nsgFdiPNr4OmKRUROiYKFSFU4uOv3ELF9CWQmObf7NKKoTTy/dLiZt9NCmZ/bgbDWwdw6MpKbe7TF10eTLkWkdlOwEHGn4kL47jFY/rrzsW8gdOgD3a8gvVk8b25pyser95BfVMKg6CAmnxXJwKigE68wKiJSiyhYiLjLwd3w2fXOFUHPvAV6XgMh3fh55yGmLNrKnNnpeHvlMbpHKLcMiiC2TRNPVywi4nYKFiLukLIYPrsRCnPhsrcpibuE79bvYcq0Fazavp8mjXy4bXBHbugfTkhFK4yKiNQRChYip8NaWPoKfPeEc0Gv66ezybbjthcWsG1vLu2aN+aJC+O4IqE9/uVXGBURqYP0L53IqSo4BNP+AuunQeyFMOZVHL6BPPj6Eg7mFzFpXC/O7xLy+wqjIiL1gIKFyKnI3AifXANZyXDuROh/JxjDF6tSWbPjAM9f3oNR3dt4ukoRkWqnYCFSWeu+hGkTwNcPrpsGEWcBkJ1fxDPfJBEf1pxLeoZ6uEgREc9QsBBxVUmRcy7FsknQrjdc8R40aXv06Re/28S+vELeG90bLy/dPioi9ZOChYgrDu2Bz26AHUug921w3lPg43v06Q27D/L+0hSu7tOBrqFNPVeniIiHKViInMz2pc7+FAWH4JI3ofsVf3jaWssT0xJp2rgB95/XyUNFiojUDC5NVzfGDDfGbDTGJBtjHqrg+RuMMZnGmF9Kv25xf6ki1cxaWPYavHcB+PrDLd8fEyoApq/dxYqUfTwwvDPN/HwrGEhEpP446RkLY4w3MAk4F0gDVhpjpltr15fb9RNr7YQqqFGk+hXkwPQ7IPFL6DQKLn4NGh17iSOnoJh/zNpA93ZNuSKhvQcKFRGpWVy5FNIbSLbWbgUwxkwFxgDlg4VI3bB3s/NW0r2b4JwnYMDd4FXxyb2Xf9hMxqECJl+XgLcmbIqIuHQpJBRILfM4rXRbeZcaY341xnxujNGvblI7rZ8Gk8+G3Ey49isYdO9xQ0VyxiHeXryNKxPac0b7ZtVbp4hIDeWuloAzgHBrbXfgO+C9inYyxow3xqwyxqzKzMx000uLuEFJMXz7GHx6HQTHwG0LIXLIcXe31vLE9ET8fL15YLgmbIqIHOFKsNgJlD0D0a5021HW2ixrbUHpwylAfEUDWWsnW2sTrLUJwcHBp1KviPvlZMAHF8GSlyHhZrjxG2ja7oSHfLMunZ+Ss7j//E60DGhYPXWKiNQCrsyxWAlEG2MicAaKscC4sjsYY9pYa3eXPhwNbHBrlSJVJXWF8yxF/gG4+A3oMfakh+QVFvPUzPXEtmnCuN4dqr5GEZFa5KTBwlpbbIyZAMwFvIG3rbWJxpiJwCpr7XTgTmPMaKAY2AfcUIU1i5w+a2HFmzD3/6BpKNzyHbTu5tKhk35MZlf2YV6+qqcWGBMRKcelBlnW2tnA7HLbHi/z/cPAw+4tTaSKFObCjLvht08hZjhc/Do0bu7Sodv25vLmwm1c0jOUhPAWVVuniEgtpM6bUr9kbXHeSpqxAYY+CgPvO+5dH+VZa/nbjER8fbx4aETnKi5URKR2UrCQ+iNpFnz1J/DygWu+gKhzKnX49xsymL8xk0dHxdKqSaMqKlJEpHZTsJC6r6QYfnwKFr8IbXvCFe9Ds8pNujxcVMLEmYnEhARwff/wqqlTRKQOULCQui0nE764CbYthPgbYPiz0KDyZxteX7CF1H35fHxrHxpowqaIyHEpWEjdlbbKeStp7l4YMwl6XnNKw6Tuy+O1+Vu4oHsb+ncMcnORIiJ1i4KF1D3Wwqq34JuHoEkbuPlbaHvGKQ/395nr8fYyPDIq1n01iojUUQoWUrcU5sGse2Ht/yDqXLhkMvid+m2h8zdm8O36PTw4vDNtmjZ2Y6EiInWTgoXUHfn74d0LYc86GPIwnPWAy7eSVqSguIS/zVhPZJA/Nw+McGOhIiJ1l4KF1B3f/w0y1sO4TyDm/NMebsqibWzbm8v7N/XG10cTNkVEXKF/LaVuSF0Bq9+Bvn92S6jYdSCfV+Ylc36XEM6K0YJ5IiKuUrCQ2q+kGGbeA01CYchDbhnyH7M34LCWR0fFuWU8EZH6QsFCar/lrznnVYx4FhoGnvZwPyXvZdavu/nL2VG0b+HnhgJFROoPBQup3Q6kwo9PQ8wI6HzBaQ9XVOLgiemJdGjhx/izIt1QoIhI/aJgIbXbnIcACyP/Bcac9nDvLUkhOSOHxy+Io1ED79OvT0SknlGwkNoraTYkzXTOq6jk2h8VyTh4mJe+38zQzq0YFhfihgJFROofBQupnQpyYPZfoVUc9L3dLUM+/U0ShcUOHr9AEzZFRE6V+lhI7bTgGTiYBpd9C94NTnu4Fdv28dWandwxNIrwIH83FCgiUj/pjIXUPunrYOmr0Ot66NDntIcrLnHw+LR1hDZrzO1DotxQoIhI/aVgIbWLw+HsWdG4GQx70i1DfrR8B0nph3h0VCyNfTVhU0TkdLgULIwxw40xG40xycaY43YgMsZcaoyxxpgE95UoUsbP70HaCjjvH6e1uNgRe3MKeP7bjQyKDmJ419ZuKFBEpH47abAwxngDk4ARQBxwlTHmmNltxphA4C5gubuLFAEgJxO+fwLCB0GPsW4Z8l9zksgvLOGJC7tg3HC7qohIfefKGYveQLK1dqu1thCYCoypYL+/A88Ch91Yn8jvvn3UuSz6qBfc0rPi5x37+XRVGjcPjCCqVYAbChQREVeCRSiQWuZxWum2o4wxvYD21tpZJxrIGDPeGLPKGLMqMzOz0sVKPbZ1Afw6FQbeDcExpz1cicPyxLREQpo05I5zok+/PhERAdwwedMY4wW8ANx3sn2ttZOttQnW2oTgYK0YKS4qLoBZ90LzCBh00j9mLvlkZSq/7czm/0bGEtBQd12LiLiLK8FiJ9C+zON2pduOCAS6AvONMSlAX2C6JnCK2/z0H8hKhlHPQ4PGpz3c/txC/jU3iT4RLRjdo60bChQRkSNcCRYrgWhjTIQxxhcYC0w/8qS1NttaG2StDbfWhgPLgNHW2lVVUrHUL1lbYOHz0OUSiBrmliGf/3Yjhw4X87cxmrApIuJuJw0W1tpiYAIwF9gAfGqtTTTGTDTGjK7qAqUesxZm3Qc+DWH4024Zct3ObD5esYPr+oXRuXUTt4wpIiK/c+nisrV2NjC73LbHj7PvkNMvSwRY9wVs/RFGPg+Bp99jwuGwPDZtHS39fbnn3NOfACoiIsdS502pmfIPwJyHoW0vSLjJLUN+8XMaa3Yc4KERsTRpdPrri4iIyLE0HV5qpnl/h7y9cPVn4HX6bbaz84t45pskenVoxiU9Q09+gIiInBKdsZCaJ201rHwLet8Gbc9wy5AvfreJfXmFTBzTFS8vTdgUEakqChZSs5QUw8y7ILANDH3ELUNu2H2Q95emcHWfDnQNbeqWMUVEpGK6FCI1y4rJkP4bXPE+NAw87eGsdXbYbNq4Afef18kNBYqIyInojIXUHNk74cd/QPR5EOueO5mnr93FipR9PDC8M838fN0ypoiIHJ+ChdQccx4ERwmMfM4ti4zlFBTzj1kb6N6uKVcktD/5ASIictoULKRm2DgHNsyAwQ9A83C3DPnyD5vJOFTAxDFd8daETRGRaqFgIZ5XmAuz/wrBnaHfBLcMmZxxiLcXb+PKhPac0b6ZW8YUEZGT0+RN8bwF/4LsHXDjN+Bz+vMgrLU8MT0RP19vHhiuCZsiItVJZyzEs/ash6WvQM9rIKy/W4b8Zl06PyVncf/5nWgZ0NAtY4qIiGsULMRzHA6YeQ80bALDJrplyLzCYp6auZ7YNk0Y17uDW8YUERHX6VKIeM4vH0LqMhjzKvi3dMuQk35MZlf2YV6+qic+3srNIiLVTf/yimfk7oXvHoewAXDGOLcMuSplH28u3MYlPUNJCG/hljFFRKRyFCzEM759DApyYNQLbulZsWLbPq57ewXtmjfm/0bFuqFAERE5FQoWUv22LYK1H8OAO6FV59MebumWLK5/ewVtmjZi6vi+BGnCpoiIxyhYSPUqLoBZ90KzMBh0/2kP91PyXm5813mm4n/j+9KqSSM3FCkiIqdKkzelei15GfZugqu/AF+/0xpq0eZMbnlvFeEt/fno1j46UyEiUgMoWEj12bcVFj4PcRdB9LDTGmr+xgzGf7CayCB/Pr61Ly38tcCYiEhN4NKlEGPMcGPMRmNMsjHmoQqe/5Mx5jdjzC/GmMXGmDj3lyq1mrUw637wagDDnzmtoX5MymD8+6uJCg7gfwoVIiI1ykmDhTHGG5gEjADigKsqCA4fW2u7WWvPAP4FvODuQqWWS/wKtvwA5zwGTdqc8jDfr9/DbR+splPrQD6+tQ/NFSpERGoUVy6F9AaSrbVbAYwxU4ExwPojO1hrD5bZ3x+w7ixSarnD2TDnIWhzBpx5yykPMzcxnQkf/0xcmya8f1Mfmvo1cF+NIrXY6tWrW/n4+EwBuqJJ+VL1HMC64uLiW+Lj4zPKP+lKsAgFUss8TgP6lN/JGPMX4F7AFxha0UDGmPHAeIAOHdRuud6Y9xTkZsJVU8HL+5SG+Oa33dzxvzV0a9eU927qTZNGChUiR/j4+Exp3bp1bHBw8H4vLy/9YidVyuFwmMzMzLj09PQpwOjyz7st2VprJ1lrOwIPAo8eZ5/J1toEa21CcHCwu15aarKdP8OKN+HMWyG01ykNMevX3Uz43xp6tG/G+woVIhXpGhwcfFChQqqDl5eXDQ4OzsZ5huzY510YYyfQvszjdqXbjmcqcJGrBUodVlIMM++GgBAY+sgpDTF97S7unLqGXh2a8d5NvQlUqBCpiJdChVSn0j9vFWYIV4LFSiDaGBNhjPEFxgLTy+5gjIku83AUsPkUa5W6ZOUU2L0Whj8NjZpW+vCv1+zk7qlriA9rzrs39iagoe6OFhGp6U4aLKy1xcAEYC6wAfjUWptojJlojDlybWWCMSbRGPMLznkW11dVwVJLHNzlnFsRNQy6XFzpwz9fncY9n/5Cn4iWvHvjmfgrVIjUKX5+fj0BUlJSGgwfPjyyon169+7daeHChSfspDdx4sRWhw4dOvpZNnjw4Ki9e/ee2mQuNypfV3lXXnll2OrVq11qFRwbGxu3ZMmSxgBFRUX4+fn1fPXVV4+utNilS5fYxYsXH/M+/frrrw0HDx4cFRYW1jUuLi525MiRkampqVX+j6lLcyystbOttTHW2o7W2n+UbnvcWju99Pu7rLVdrLVnWGvPttYmVmXRUgvMeQgcRTDyuUovMvbpylT++vlaBnQM4u0bzsTPV6FCpK4KDw8vmjNnztZTPf6NN94IycnJOfpZtmDBguSgoKAS91R36srXVVZxcTGffPLJ9vj4+MOujHXmmWfmLFy4MABg2bJlfuHh4YeXLFkSAHDw4EGvHTt2NOzbt29e2WPy8vLMhRdeGH3bbbdlbt++fd369es33H777Znp6eku/YNaVFR0wscnotuSxP02fQvrp8FZ90OLCn8ROa7/rdjBA1/8ysCoIKZcn0BjX4//4iEiJ3H77beHPv3000dn5N97771tH3/88ZDs7Gyvfv36xcTFxcXGxMTEffjhh83KH7tx40bf6OjoLgA5OTnmggsuiIyMjOxy7rnndjx8+PDR30quvvrqDl27do2Niorqcs8997QFeOqpp1plZGQ0GDx4cEyfPn1iAEJDQ7vt3r3bB+DJJ58MiY6O7hIdHd1l4sSJrY68XmRkZJexY8eGRUVFdRkwYEB0Tk7OMb/9XHrppeFXX311hx49enRu165dt5kzZwZefvnl4ZGRkV0uvfTS8CP7ffnll03OOOOMznFxcbEjRoyIzM7O9qqoLj8/v5633npru06dOsX98MMPAWXPxnz++edN4uLiYjt16hTXr1+/mPK19O/fP2fp0qUBAAsXLgy46aabMhMTE/0AFixY4N+1a9c8H58/5oXJkye36NWrV864ceOyj2y74IILDp155pmH8/LyzGWXXRYeExMTFxsbGzdjxoxAgJdffrnl0KFDo/r27RvTv3//TuUfn+SPwVH6VVDcqzAPZt8PQZ2g/12VOvTDZdt59Ot1DOkUzOvXxNOogUKFSGX99fO17TelHzq9hXjKiWkdmPfcZT1Sj/f81Vdfve/uu+/u8PDDD2cCTJs2rfncuXM3+fn5OWbNmpXcokULx+7du3369OnTedy4cQe8vCr+nfb5559v1bhxY8fWrVsTly9f3njAgAFHmzG+8MILO0NCQkqKi4vp379/p+XLlzd+9NFHM1577bWQBQsWbGrTpk1x2bEWLVrk9/HHH7dcvXr1Bmst8fHxseecc86hoKCgkh07djT68MMPt/bv33/7yJEjI99///3mt99++77y9WRnZ/usWbMm6eOPP242duzYqHnz5iXFx8fnd+/ePXbJkiWNIyIiiv75z3+2Wbhw4aYmTZo4HnnkkdZ///vfQ55//vnd5evKz8/36tOnT+6bb76ZBvDYY48BsGvXLp8JEyaEz58/P6lz586Fe/bsOeYfvqFDh+Y89dRToQDLli3znzhx4u7PPvusxf79+70WL17s37t375zyx6xbt65xr1698spvB3j22WdbGWPYtGnT+jVr1jQaOXJk9JYtW9YBJCYm+v3666+JISEhJS+//HLLso8r/r9/LAULca+Fz8GB7XDDLPBxvSvm+0tTeHxaIud0bsWr1/SioY9ChUhtMWDAgPysrCyflJSUBrt37/Zp2rRpSVRUVFFBQYG5++672y1btizAy8uLjIwM37S0NJ8OHToUVzTO4sWLA+68884MgD59+uTHxMQc/WB87733Wrz77rtBxcXFJjMzs8HatWsb9enTJ/94Nc2fPz9g5MiRB5o0aeIAGDVq1P4ff/wx8PLLLz8QGhpa0L9//3yAnj175qWkpFS4guGoUaMOeHl50atXr7yWLVsW9e7dOx8gJiYmf8uWLQ23b9/uu2XLlka9e/fuDFBUVGTi4+OP+ZAH8Pb25oYbbthfQZ3+vXv3PtS5c+dCgIo+wGNiYgqLiorMjh07fJKTkxv36NHjcM+ePfMWLFjgv3z58oAJEyYc06TqRJYsWRJwxx13ZJT+/Ifbtm1b+NtvvzUCGDRo0MGyNZR/7AoFC3GfjA3O1UvPuBrCB7p82Ds/beNvM9ZzblwIr4zrqVAhchpOdGahKo0ePXr/hx9+2Dw9Pb3BJZdcsg/gjTfeaJGVleXz22+/bWjYsKENDQ3tlp+fX+lL8ElJSb6vvPJKyOrVqzcEBweXXHrppeGHDx8+5Uv5vr6+R2/N9fb2tserqVGjRrZ0nz8c4+XlRXFxsfH29rYDBw48OGPGjG0uvKaj/OWKyoiPj8/54IMPWrRq1arQy8uLfv365SxatChg7dq1/meffXbuvHnz/G+//fYwgMcee2xnly5dDh+Zl1EZfn5+jhM9doXmWIh7OBww815oGAjnTnT5sCmLtvK3GesZ3qU1k8bpTIVIbXXNNdfs++KLL1rMnDmz+bXXXrsfIDs72zsoKKioYcOGdsaMGYG7du064WnMgQMH5nz00UctAFauXNlo06ZNfgD79+/3bty4saNFixYlqampPvPnzz96/7q/v39Jdnb2MZ9lZ599ds7s2bObHTp0yOvgwYNes2fPbn722WcfcufPPGTIkNxVq1YFrFu3riE4J1L++uuvDU9UV0VjrFixIjApKckXoKJLIQB9+/bNef3111v17t0798hxn376acvg4OCili1blgwdOjQ3KSlpfVJS0vqrr746+9Zbb81avXp1wNSpU4++V998803AypUrGw0YMCDnww8/bAHOO0d2797t2717d5cmkrpCwULcY+3HsGOJM1T4B7l0yBsLtvDUrA2M6taG/47ria+P/jiK1FYJCQmHc3NzvUJCQgrDwsKKAG655ZZ9a9eu9Y+JiYl77733WkZERJzww+v+++/PyM3N9Y6MjOzyyCOPhMbFxeUC9OvXL79r1655HTt27HrFFVdElr3ccP311+8dPnz40UmSRwwcODBv3LhxWb169YqNj4+PvfbaazMHDBhw3Esnp6Jt27bFb7zxRsrYsWMjY2Ji4hISEjofuaRwvLoqGuPll19Oufjii6M6deoUd/HFF1c4433w4ME5aWlpDQcMGJADEBYWVuRwOExCQkJuRfsHBATYadOmJU+aNKlVWFhY144dO3aZNGlSq9atWxc/8MADGQ6Hw8TExMRdeeWVHd94442Uxo0bu63BmrHWM83aEhIS7KpVqzzy2uJGJUWw6m344e/QuivcMBuOMzGrrEk/JvPc3I1c2KMtL17RAx9vhQoRVxhjVltrE8puW7t2bUqPHj32eqomqZ/Wrl0b1KNHj/Dy2zXHQk5d8vcw5/9g70aIPBtGv+xSqPjvD5v593ebGHNGW/59uUKFiEhdomAhlbd3M8x9BDbPdfapuGoqxAx3qRHWS99v4qXvN3NJz1Ceu7wH3l6Va54lIiI1m4LFKVq6JYs1qfu5PL49wYEV3qlU9+QfgAX/ghVvQAM/OO8p6H2bS7eVWmt58btNvDwvmcvj2/HMpd0VKkRE6iAFi1OQV1jMXVPXkHGogJe+28yYM9py86AIOrdu4unSqoajBFa/Cz/+A/L2Qa/rYOhjEBB80kPBGSqem7uRV+dvYeyZ7fnnxd3wUqgQEamTFCxOwduLt5FxqIAXr+zBz9sP8NnqVD5bncbAqCBuHhjB4JjguvPBuXUBzHkYMhIhbKBzpdI23V0+3FrLM98k8cbCrYzr04GnxnStO++NiIgcQ8GikvbmFPD6gq2c3yWEi3u24+Ke7bjvvBg+XrGD95akcOO7K+kY7M9NAyO4pGe72rvWxb5t8O2jkDQTmnWAK96H2NGVWlDMWss/Zm1gyuJtXNs3jIljumAquSCZiIjULpqOX0n//WEz+UUlPDC889Ftzfx8uX1IFIseGMpLV55BY19vHvlqHf2f+YHn5iax56Db+o5UvYJD8N0TMKk3bPnRecnjLyshbkylQ8XEmeuZsngbN/QPV6gQqcP27t3r/cwzz7h2bbSc6l7mvD4vZ15dFCwqIWVvLh8t38HYM9vTMfjYTqm+Pl5c1DOUGRMG8sn4vpwZ3oJX529h4LPzuPeTX1i3M7uCUWsIhwN+/gBe7gU/vQRdL4M7VjtXKG3g0t+xo6y1PDE9kXd+SuHmgRE8cWGcQoVIHZaVleX91ltvtarouZMtt13dy5zX5+XMq4uCRSU8N3cjvj5e3DUs+oT7GWPoE9mSydclMP/+IVzdJ4w5ielc8N/FjJ28lO/W78Hh8ExjsgptXwpvDoHpE6B5ONw6Dy5+DZq0qfRQDofl0a/X8f7S7dx2ViSPjopVqBCp4+677752qampDTt37hx32223tZs5c2ZgfHx8p6FDh0ZFR0d3BRg2bFjHLl26xEZFRXV5/vnnj7bnPbLMuZYzr/rlzKtLnTn1UtXW7NjPrN92c9c50bQKdP03+LCW/jw5ugv3nBvDJyt38O5PKdz6/ioigvy5cUA4l8W3w8/XQ/8bDqTCd49D4pfQJBQumQLdLqvUJY+yHA7L/331G1NXpvLnIR154PxOChUi1e3rv7QnY71bl02nVVweF0067uJm//73v9MuuOCCxklJSesBZs6cGbh+/Xq/NWvWJB5ZtfOjjz5KCQkJKcnJyTE9e/aMu+aaa/a3bt36D2cqtJx51S5nXl0ULFxgreXpb5IICmjIrWdV2Mb9pJo2bsD4szpy44AI5qxLZ8ribTw+LZHn525kXJ8wru8fRpumjd1c+XEU5sLil5wrkWJg8EMw4E7w9T/lIUscloe++JXPVqdxx9Ao7j03RqFCpB7r3r177pFQAfDss8+GzJo1qxlAenp6g8TExEatW7f+wzoXWs68apczry4KFi6Yl5TBim37+PtFXQloeHpvWQNvLy7s0ZYLurfh5x37eWvxNiYv3MKURVsZ2a0NtwyKoHu7Zu4pvDyHA377DL5/Eg7tcs6jGPYkNGt/SsMdLiohOSOHjemHmJOYznfr93DXOdHcPSxaoULEU05wZqE6lV1ue+bMmYELFiwIXLVqVVJgYKCjd+/enSpaqlzLmR+fO5Yzry4uvZvGmOHAfwBvYIq19plyz98L3AIUA5nATdba7W6u1SOKSxw8800SkUH+jD3z1D6AK2KMIT6sBfFhLUjdl8e7S1L4ZGUq09fu4szw5tw8MJJz40Lc150ybTXMeRDSVkKbM+Dyd6BDX5cOdTgsqfvzSEo/xMbSr6T0g6Rk5VFSOlfE18eLv57fib+cHeWeekWk1mjatGlJbm7ucefsHThwwLtp06YlgYGBjjVr1jRau3btqZ8edcGQIUNy77vvvg7r1q1r2LVr14KDBw96paSkNOjevXvBkeXM27Q58RyyIUOG5N57771hSUlJvkcuhVR0huDIcuZjx47NOnLco48+2q78cuZH9s/JyTEvvvhi66lTpzYdO3ZsNjiXMw8KCio+spz56NGjD5Vdznz58uXuvbRVxU4aLIwx3sAk4FwgDVhpjJlurV1fZrc1QIK1Ns8Y82fgX8CVVVFwdft8dRqbM3J4/ZpeNKiixbLat/DjsQviuHtYNJ+sTOXdJSn86cPVdGjhxw39w7nizPanfqbk4C74/m/w61QICIExr0KPq467WFhWTkFpcCgNEHsOsXnPIfIKf//71KGFH51aBzKyWxs6tQ6kc+tAwlv6azExkXqqdevWJfHx8TnR0dFdhg4dmn3hhRf+4Ra4Sy+9NHvy5MnBkZGRXSIjIw/36NGjwqW+3aXscuaFhYUG4IknntjZvXv3giPLmYeEhBQuX75804nGOLKcucPhoGXLlkVLlizZXH6/wYMH5zzxxBPtK7uc+Z133tn+wQcfbO/j42NjY2PzX3vttR0PPPBAxnXXXRcWExMT5+3tjbuXM68uJ1023RjTD3jSWnt+6eOHAay1Tx9n/57AK9baAScatzYsm55XWMzZz88ntFljvvhz/2o7vV9c4uC79XuYsngbq7fvJ7ChD2N7t+f6/uG0a+5icC3KhyWvwOIXnC25+/0FBt0LDQMByC8sYXPGoXJnIQ6xN6fg6BAt/H3pFBJ4NDx0ah1ITEgg/qd5OUhETp2WTZea4nSWTQ8Fyl6zSwP6nGD/m4FvKnrCGDMeGA/QoUMHF17as95evI09BwuYNK5Xtc4Z8PH2YkS3Nozo1oZfUg/w1uJtvP1TCm//lMLwLq25eVAEvTo0r/hga2H91/Dt45C9Axs7mtSEh0nMa07Swt1sTN/Exj2HSMnK5UimbNTAi5iQQIZ0Cj4aIDq1DiQ4oKHmSoiISKW49VdPY8w1QAIwuKLnrbWTgcngPGPhztd2t6zS1t3nxYWQEN7i5AdUkTPaN+O/V/XkoRGdeX9JCh+v2MGs33bTs0MzbhkYyfldQvDx9sJay/4tq/Ce+zBNM1eyq2FH/tvkab5aF8HhNduAbXgZCG/pT+fWgYw5o21piGhChxZ+WmlURETcwpVgsRMoO2uxXem2PzDGDAMeAQZbawvKP1/b/Hde8jGtuz0ptFljHh4Zy53nRPPZqlTeWZLCXz7+mdBmjenWtIDzM95kjGMe+wng4eKbmedzPjFNm3FNzJFLGU2IDgmgUYNaunaJiJyIw+FwGC8vrxr9C5vUHQ6HwwAV3pniSrBYCUQbYyJwBoqxwLiyO5TOq3gDGG6trdR9uzVRyt5cPly2nSvPbE9Uq0rfFVSl/Bv6cMOACK7tG8bS5T+RufR/nJ/xJQ1tIevDriG33738tUM7nvb39XSpIlJ91mVmZsYFBwdnK1xIVXM4HCYzM7MpsK6i508aLKy1xcaYCcBcnLebvm2tTTTGTARWWWunA88BAcBnpdfkd1hrR7vrh6huz33rbN1990lad1c7a2HXGtgwA+8N0xmYlQwY6DQCzv07XYN0q6dIfVRcXHxLenr6lPT09K5oqQapeg5gXXFx8S0VPenSHAtr7Wxgdrltj5f5ftjpVFiT/JJ6gFm/7ubOSrburjKOEkhdARumw4YZkJ0KxhsiBkHfP0PnCyCwtaerFBEPio+PzwBq7S9zUrfovsEyrLU8PXsDQQG+jD/F1t1uUVIEKYucQWLDTMjNAO+G0HEoDHnYeYbCz3MTSkVERI5HwaKMeUkZLN+2j7+P6XLarbsrregwbP0R1k+HjbPh8AFo4A/R50LcaIg+72gPChERkZpKwaLUkdbdEUH+jO1dTT02CnJg87fOMxObv4XCHGjUFDqNhNgLnWcoGlTTwmQiIiJuoGBR6oufna27X7u66lp3A5C/HzbOcc6ZSP4BSgrAP9i5XHnsaAgfBD66o0NERGonBQuc7a1f+G4TPTs0Y3jXKpgIeWgPbJzlvMyRsggcxdCkHSTc5Dwz0aEveKm/hIiI1H4KFsDbPzlbd7/iztbdB1JLJ1/OgB1LAQstIqHfBOeciba9QO2yRUSkjqn3wSIrp4DX5m/h3LgQzjzd1t17k0tvC53u7DcBENIVhjzkvMzRKlZhQkRE6rR6HyyOtO5+8FRad1sLexJ/7zGRUbqSfGg8DPub8zJHy47uLVhERKQGq9fBYntWLh8t384VCafQunv9NPj+Sdi3FYwXdOgPw5+F2AugabsqqVdERKSmq9fB4rm5G/Hx8uKeyrbuztoCX/0JWnSEC/8DnUZBQHDVFCkiIlKL1NtgsTb1ADN/3c2dQ6No1aQSrbsdJfD17eDdAK7+FJq0rboiRUREapl6GSystfxz9gZa+vsyfnAl50AsnQSpy+DiyQoVIiIi5dTLVfB+3Ohs3X3XsOjKte7OSIJ5TzkX/up+RdUVKCIiUkvVu2BR4rBHW3dfVZnW3SVF8PWfoGEAXPCSbhsVERGpQL0LFl+sTmPTnhweOL9T5Vp3L37R2Zvighc1UVNEROQ46lWwOOXW3bt/hQXPQrfLIW5M1RUoIiJSy9WrYPH2T9tIP3iYh0fEut66u7jAeWupXxCM+FfVFigiIlLL1Zu7QvblFvL6/C0Miw2hd0QlWnfPfwYyEmHcZ+B3mi2/RURE6rh6c8biv/M2k1tYzEMjOrl+UNoq+Okl6HktxJxXZbWJiIjUFS4FC2PMcGPMRmNMsjHmoQqeP8sY87MxptgYc5n7yzw9O7Ly+HDZdq48sz1RrQJdO6gwD766DZqEwvn/rNoCRURE6oiTBgtjjDcwCRgBxAFXGWPiyu22A7gB+NjdBbrDc986W3ffPSzG9YPm/R2ykmHMK9CoSdUVJyIiUoe4csaiN5Bsrd1qrS0EpgJ/uDXCWptirf0VcFRBjadlbeoBZqzdxS2DIghxtXV3ymJY9hr0Hg+RQ6q0PhERkbrElWARCqSWeZxWuq3SjDHjjTGrjDGrMjMzT2WISrHW8vQ3pa27z4p07aCCQ861QFpEwLAnq7Q+ERGRuqZaJ29aaydbaxOstQnBwVXfZGr+xkyWbXW27g5s1MC1g759DA7sgIteA1//qi1QRESkjnElWOwE2pd53K50W412pHV3eEs/11t3J38Pq9+B/ndAh75VW6CIiEgd5EqwWAlEG2MijDG+wFhgetWWdfq++DmNjXsO8cDwzq617s4/ANPugODOcPYjVV6fiIhIXXTST1xrbTEwAZgLbAA+tdYmGmMmGmNGAxhjzjTGpAGXA28YYxKrsuiTOVxUwgvfbuKM9s0Y4Wrr7jkPQc4e5yWQBi5O8hQREZE/cKnzprV2NjC73LbHy3y/EuclkhrhSOvu/4w9w7XW3UmzYO3/YPCDENqr6gsUERGpo+pc5819uYW89uMWhsW2ok9ky5MfkJsFM+6C1t1g0P1VX6CIiEgdVufWCnllXjK5hcU8OLyzawfMutc5v+K6aeDjW6W1iYiI1HV16ozFjqw8PliWwhUJ7YkOcaF197ovYP3XcPbDENKlyusTERGp6+pUsHj+2414exnuOdeF1t2H0mHWfRCaAP3vqvriRERE6oE6Eyx+TTvA9LW7uHVQ5Mlbd1vrnFdRlA8Xvw7ede6KkIiIiEfUiU9Uay1Pz06ihautu3/5GDbNgfOfhqDoqi9QRESknqgTZyzmb8pk6dYs7jrHhdbdB1KdPSvCBkKfP1VPgSIiIvVErQ8WJQ7Ls98kEeZK625rYfoEcJTARZPAq9b/+CIiIjVKrf9k/fLnNJLSD/HA+Z3x9TnJj7PqLdg6H85/CpqHV0d5IiIi9UqtDhaHi0p44btN9GjfjJHdTtK6e99W58qlHYdC/I3VU6CIiEg9U6uDxTs/pbA7+zAPj+h84tbdjhL4+i/g1QBGvwKutPkWERGRSqu1d4Xszy3k1fnJnNO5FX1P1rp72WuwYwlc9Do0Da2eAkVEROqhWnvG4pUfk8ktKObBESdp3Z25EX6YCJ1GQY+x1VOciIhIPVUrg0XqvjzeX+ps3R1zotbdJcXw1Z/A1x8ufEmXQERERKpYrbwU4nLr7p9ehF0/w+XvQkCraqlNRESkPqt1Zyx+S8tm2i+7uGXgSVp3p/8G85+FrpdCl4urr0AREZF6rNYFi9Xb9xEc2JDbBp+gdXdxofMSiF8LGPl89RUnIiJSz9W6SyE3DIjg8oT2+Dc8QekLnoU96+Cqqc5wISIiItWi1p2xAE4cKtJWw+IX4IxroNOI6itKREREXAsWxpjhxpiNxphkY8xDFTzf0BjzSenzy40x4W6v1BVF+fD1nyCwLQz/p0dKEBERqc9OGiyMMd7AJGAEEAdcZYyJK7fbzcB+a20U8CLwrLsLdcm8p2DvJhjzCjRq6pESRERE6jNXzlj0BpKttVuttYXAVGBMuX3GAO+Vfv85cI45YY/tKrB9CSydBGfeAh3PrtaXFhERESdXgkUokFrmcVrptgr3sdYWA9nAMX22jTHjjTGrjDGrMjMzT63iihTkwNd/huZhMOxv7htXREREKqVaJ29aaydbaxOstQnBwcHuG/i7x2H/drjoNWgY4L5xRUREpFJcCRY7gfZlHrcr3VbhPsYYH6ApkOWOAk9qyzxY9Rb0+wuE9a+WlxQREZGKuRIsVgLRxpgIY4wvMBaYXm6f6cD1pd9fBsyz1lr3lXkch7Nh2gQIioGhj1b5y4mIiMiJnbRBlrW22BgzAZgLeANvW2sTjTETgVXW2unAW8AHxphkYB/O8FH15jwMh9Lh5u+gQeNqeUkRERE5Ppc6b1prZwOzy217vMz3h4HL3VvaSSTNhl8+gkH3Q7v4an1pERERqVit7LxJ3j6YcReEdIXBD3q6GhERESlV69YKAWDWfZC/H679Enx8PV2NiIiIlKp9ZyzWfQmJX8KQB6F1N09XIyIiImXUvmDRqCl0GgUD7vF0JSIiIlJO7bsUEnWO80tERERqnNp3xkJERERqLAULERERcRsFCxEREXEbBQsRERFxGwULERERcRsFCxEREXEbBQsRERFxGwULERERcRtjrfXMCxuTCWw/xcODgL1uLKe20/vxR3o/fqf34o/qwvsRZq0N9nQRIsfjsWBxOowxq6y1CZ6uo6bQ+/FHej9+p/fij/R+iFQ9XQoRERERt1GwEBEREbeprcFisqcLqGH0fvyR3o/f6b34I70fIlWsVs6xEBERkZqptp6xEBERkRqo1gULY8xwY8xGY0yyMeYhT9fjKcaY9saYH40x640xicaYuzxdU01gjPE2xqwxxsz0dC2eZoxpZoz53BiTZIzZYIzp5+maPMUYc0/p35N1xpj/GWMaebomkbqqVgULY4w3MAkYAcQBVxlj4jxblccUA/dZa+OAvsBf6vF7UdZdwAZPF1FD/AeYY63tDPSgnr4vxphQ4E4gwVrbFfAGxnq2KpG6q1YFC6A3kGyt3WqtLQSmAmM8XJNHWGt3W2t/Lv3+EM4PjVDPVuVZxph2wChgiqdr8TRjTFPgLOAtAGttobX2gEeL8iwfoLExxgfwA3Z5uB6ROqu2BYtQILXM4zTq+YcpgDEmHOgJLPdwKZ72EvAA4PBwHTVBBJAJvFN6aWiKMcbf00V5grV2J/A8sAPYDWRba7/1bFUidVdtCxZSjjEmAPgCuNtae9DT9XiKMeYCIMNau9rTtdQQPkAv4DVrbU8gF6iXc5KMMc1xntmMANoC/saYazxblUjdVduCxU6gfZnH7Uq31UvGmAY4Q8VH1tovPV2Phw0ARhtjUnBeIhtqjPnQsyV5VBqQZq09chbrc5xBoz4aBmyz1mZaa4uAL4H+Hq5JpM6qbcFiJRBtjIkwxvjinIA13cM1eYQxxuC8fr7BWvuCp+vxNGvtw9badtbacJx/LuZZa+vtb6XW2nQg1RjTqXTTOcB6D5bkSTuAvsYYv9K/N+dQTyeyilQHH08XUBnW2mJjzARgLs6Z3W9baxM9XJanDACuBX4zxvxSuu3/rLWzPVeS1DB3AB+VhvCtwI0erscjrLXLjTGfAz/jvJtqDerAKVJl1HlTRERE3Ka2XQoRERGRGkzBQkRERNxGwUJERETcRsFCRERE3EbBQkRERNxGwUJERETcRsFCRERE3EbBQkRERNzm/wEbiDd7xpyHxAAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":"# create predictions on training set\npredictions = model.predict(train_generator)\ny_true = []\nfor x,y in train_generator: y_true.append(y)\ny_true = np.squeeze(np.concatenate(y_true), axis=-1)\n\n\nprint(predictions.shape, y_true.shape)\nassert predictions.shape == y_true.shape, f\"{predictions.shape}, {y_true.shape}\"\n\n# Evaluate predictions on validation set\nprint(\"Window Size: \", WINDOW_SIZE)\nprint(\"Prediction length: \", prediction_length)\nprint(\"Epochs: \", EPOCHS)\n\nprint('---------------------')\nprint('Asset:    Corr. coef.')\nprint('---------------------')\nasset_w_corr = []\nasset_corrs = []\nasset_mae = []\ny_true = np.squeeze(y_true)\ny_pred = np.squeeze(predictions)\nreal_target_ind = np.argwhere(y_true!=0)\n# asset_id = list(assets_order.keys())[i]\n# asset_name = assets[assets.Asset_ID == asset_id]['Asset_Name'].item()\nmae_asset = mae(y_true, y_pred)\nasset_corr = np.corrcoef(np.nan_to_num(y_pred.flatten()), np.nan_to_num(y_true.flatten()))[0,1]\n\nprint(f\"corr: {asset_corr:.4f}\")\nprint(f\"mae: {mae_asset:.4f}\")\nprint(\"\\n\")\nprint(f\"Predictions Min: {predictions.min()}, Predictions Max: {predictions.max()}, Predictions mean:{predictions.mean()}, Predictions var:{predictions.var()}\") \nprint(f\"y_true Min: {y_true.min()}, y_true Max: {y_true.max()}, y_true mean:{y_true.mean()}, y_true var:{y_true.var()}\") ","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:55:14.347082Z","iopub.execute_input":"2022-02-16T23:55:14.347826Z","iopub.status.idle":"2022-02-16T23:55:29.274659Z","shell.execute_reply.started":"2022-02-16T23:55:14.347761Z","shell.execute_reply":"2022-02-16T23:55:29.273749Z"},"trusted":true},"execution_count":31,"outputs":[{"name":"stdout","text":"(28480, 1) (28480, 1)\nWindow Size:  64\nPrediction length:  1\nEpochs:  10\n---------------------\nAsset:    Corr. coef.\n---------------------\ncorr: 0.3798\nmae: 0.0085\n\n\nPredictions Min: -0.82203608751297, Predictions Max: 0.6891940236091614, Predictions mean:-0.00030043613514862955, Predictions var:0.0006620517815463245\ny_true Min: -0.06692692637443542, y_true Max: 0.12924709916114807, y_true mean:9.333968773717061e-05, y_true var:2.5452485715504736e-05\n","output_type":"stream"}]},{"cell_type":"code","source":"# create predictions on validation set\npredictions = model.predict(val_generator)\ny_true = []\nfor x,y in val_generator: y_true.append(y)\ny_true = np.squeeze(np.concatenate(y_true), axis=-1)\n\n\nprint(predictions.shape, y_true.shape)\nassert predictions.shape == y_true.shape, f\"{predictions.shape}, {y_true.shape}\"\n\n# Evaluate predictions on validation set\nprint(\"Window Size: \", WINDOW_SIZE)\nprint(\"Prediction length: \", prediction_length)\nprint(\"Epochs: \", EPOCHS)\n\nprint('---------------------')\nprint('Asset:    Corr. coef.')\nprint('---------------------')\nasset_w_corr = []\nasset_corrs = []\nasset_mae = []\ny_true = np.squeeze(y_true)\ny_pred = np.squeeze(predictions)\nreal_target_ind = np.argwhere(y_true!=0)\n# asset_id = list(assets_order.keys())[i]\n# asset_name = assets[assets.Asset_ID == asset_id]['Asset_Name'].item()\nmae_asset = mae(y_true, y_pred)\nasset_corr = np.corrcoef(np.nan_to_num(y_pred.flatten()), np.nan_to_num(y_true.flatten()))[0,1]\n\nprint(f\"corr: {asset_corr:.4f}\")\nprint(f\"mae: {mae_asset:.4f}\")\nprint(\"\\n\")\nprint(f\"Predictions Min: {predictions.min()}, Predictions Max: {predictions.max()}, Predictions mean:{predictions.mean()}, Predictions var:{predictions.var()}\") \nprint(f\"y_true Min: {y_true.min()}, y_true Max: {y_true.max()}, y_true mean:{y_true.mean()}, y_true var:{y_true.var()}\") ","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:55:29.276432Z","iopub.execute_input":"2022-02-16T23:55:29.276750Z","iopub.status.idle":"2022-02-16T23:55:30.682831Z","shell.execute_reply.started":"2022-02-16T23:55:29.276708Z","shell.execute_reply":"2022-02-16T23:55:30.681744Z"},"trusted":true},"execution_count":32,"outputs":[{"name":"stdout","text":"(3456, 1) (3456, 1)\nWindow Size:  64\nPrediction length:  1\nEpochs:  10\n---------------------\nAsset:    Corr. coef.\n---------------------\ncorr: 0.6892\nmae: 0.0047\n\n\nPredictions Min: -0.030504096299409866, Predictions Max: 0.039875004440546036, Predictions mean:-0.0010224042925983667, Predictions var:5.996235631755553e-05\ny_true Min: -0.00931723602116108, y_true Max: 0.010553549975156784, y_true mean:-0.00014779226330574602, y_true var:5.8813193390960805e-06\n","output_type":"stream"}]},{"cell_type":"code","source":"# create predictions on validation set\npredictions = model.predict(test_generator)\ny_true = []\nfor x,y in test_generator: y_true.append(y)\ny_true = np.squeeze(np.concatenate(y_true), axis=-1)\n\n\nprint(predictions.shape, y_true.shape)\nassert predictions.shape == y_true.shape, f\"{predictions.shape}, {y_true.shape}\"\n\n# Evaluate predictions on validation set\nprint(\"Window Size: \", WINDOW_SIZE)\nprint(\"Prediction length: \", prediction_length)\nprint(\"Epochs: \", EPOCHS)\n\nasset_w_corr = []\nasset_corrs = []\nasset_mae = []\ny_true = np.squeeze(y_true)\npredictions = np.squeeze(predictions)\nreal_target_ind = np.argwhere(y_true!=0)\n# asset_id = list(assets_order.keys())[i]\n# asset_name = assets[assets.Asset_ID == asset_id]['Asset_Name'].item()\nmae_asset = mae(y_true, predictions)\nasset_corr = np.corrcoef(np.nan_to_num(y_pred.flatten()), np.nan_to_num(y_true.flatten()))[0,1]\n\nprint('---------------------')\nprint('Asset:    Corr. coef.')\nprint('---------------------')\nprint(f\"corr: {asset_corr:.4f}\")\nprint(f\"mae: {mae_asset:.4f}\")\nprint(\"\\n\")\nprint(f\"Predictions Min: {predictions.min()}, Predictions Max: {predictions.max()}, Predictions mean:{predictions.mean()}, Predictions var:{predictions.var()}\") \nprint(f\"y_true Min: {y_true.min()}, y_true Max: {y_true.max()}, y_true mean:{y_true.mean()}, y_true var:{y_true.var()}\") \n\n","metadata":{"execution":{"iopub.status.busy":"2022-02-16T23:55:30.684775Z","iopub.execute_input":"2022-02-16T23:55:30.685144Z","iopub.status.idle":"2022-02-16T23:55:32.078635Z","shell.execute_reply.started":"2022-02-16T23:55:30.685100Z","shell.execute_reply":"2022-02-16T23:55:32.077778Z"},"trusted":true},"execution_count":33,"outputs":[{"name":"stdout","text":"(3456, 1) (3456, 1)\nWindow Size:  64\nPrediction length:  1\nEpochs:  10\n---------------------\nAsset:    Corr. coef.\n---------------------\ncorr: -0.0540\nmae: 0.0039\n\n\nPredictions Min: -0.024076171219348907, Predictions Max: 0.04401088133454323, Predictions mean:-0.001201344421133399, Predictions var:4.027350223623216e-05\ny_true Min: -0.009781436994671822, y_true Max: 0.016066623851656914, y_true mean:2.7860420232173055e-05, y_true var:5.2875429901177995e-06\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}