src/Full Bayesian Model Training and Predictions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.training_models import (\n",
    "    infer, cv_train_model,\n",
    "    lr_cv, nn_cv, rf_cv, lgb_cv, xgb_cv\n",
    ")\n",
    "\n",
    "from modules.training_utils import (\n",
    "    get_indiv_important_cols, round_float_to, get_round_num,\n",
    "    get_opt_val_seeds, make_country_sub, timing\n",
    ")\n",
    "\n",
    "from modules.training_optimizers import (\n",
    "    get_optimized_weighted_preds_for\n",
    ")\n",
    "\n",
    "import logging\n",
    "\n",
    "logging.basicConfig(filename='output.logs', format='%(asctime)s: %(message)s', level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "import os\n",
    "\n",
    "import cPickle\n",
    "import glob\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from collections import Counter\n",
    "from bayes_opt import BayesianOptimization\n",
    "from hashlib import md5\n",
    "\n",
    "\n",
    "def vprint(x):\n",
    "    if log_verbose:\n",
    "        logging.info(x)\n",
    "\n",
    "\n",
    "def prepare_test_data(X_hhold_test=None, X_indiv_cat_test=None, X_train=None, X_val=None, fill='mean'):\n",
    "    # http://fastml.com/how-to-use-pd-dot-get-dummies-with-the-test-set/\n",
    "    '''\n",
    "    X_hhold_test: equivalent to <country_code>_hhold_test\n",
    "    X_indiv_cat_test: equivalent to <country_code>_indiv_cat_test\n",
    "    X_train: corresponding training dataset\n",
    "    '''\n",
    "\n",
    "    if X_val is None:\n",
    "        test_index = X_hhold_test.index\n",
    "        X_test = pd.concat([X_hhold_test, X_indiv_cat_test], axis=1)\n",
    "        X_test = X_test.loc[test_index]\n",
    "    else:\n",
    "        X_test = X_val\n",
    "\n",
    "    for col in X_train.columns.difference(X_test.columns):\n",
    "        if fill == 'mean':\n",
    "            X_test[col] = X_train[col].mean()\n",
    "        elif fill == None:\n",
    "            X_test[col] = np.nan\n",
    "        else:\n",
    "            raise ValueError('Unknow fill method!')\n",
    "\n",
    "    X_test = X_test[X_train.columns]\n",
    "\n",
    "    if fill == 'mean':\n",
    "        X_test = X_test.fillna(X_train.mean())\n",
    "    elif fill == None:\n",
    "        # Do nothing since missing data is NaN already.\n",
    "        pass\n",
    "    else:\n",
    "        raise ValueError('Unknow fill method!')\n",
    "\n",
    "    return X_test\n",
    "\n",
    "\n",
    "def get_train_X_y(indiv_corr_thresh):\n",
    "    '''\n",
    "    hhold_train, indiv_train, indiv_cat_train, and country_code are global variables!\n",
    "    '''\n",
    "    indiv_sample_cols = get_indiv_important_cols(indiv_train, indiv_cat_train, country_code, min_corr_val=indiv_corr_thresh)\n",
    "\n",
    "    X_train = hhold_train.loc[indiv_cat_train.index].drop('poor', axis=1)\n",
    "    y_train = np.ravel(hhold_train.loc[indiv_cat_train.index].poor)\n",
    "\n",
    "    X_train = pd.concat([X_train, indiv_cat_train[indiv_sample_cols]], axis=1)\n",
    "\n",
    "    return X_train, y_train\n",
    "\n",
    "\n",
    "def build_data_subset(indiv_corr_thresh, opt_val_seed=1029):\n",
    "    if indiv_corr_thresh >= 0:\n",
    "        X_train, y_train = get_train_X_y(indiv_corr_thresh=indiv_corr_thresh)\n",
    "        X_train, X_opt_val, y_train, y_opt_val = train_test_split(\n",
    "           X_train, y_train, test_size=0.05, random_state=opt_val_seed, stratify=y_train\n",
    "        )\n",
    "    else:\n",
    "        X_train, y_train = get_train_X_y(indiv_corr_thresh=0)\n",
    "        X_train, X_opt_val, y_train, y_opt_val = train_test_split(\n",
    "           X_train, y_train, test_size=0.05, random_state=42, stratify=y_train\n",
    "        )\n",
    "        X_train, X_feat_sel, y_train, y_feat_sel = train_test_split(\n",
    "            X_train, y_train, test_size=0.05, random_state=opt_val_seed, stratify=y_train\n",
    "        )\n",
    "\n",
    "        feat_C = 10. ** (round(abs(indiv_corr_thresh), 1) * 10)\n",
    "\n",
    "        br = LogisticRegression(C=feat_C, penalty='l1', random_state=42)\n",
    "        br.fit(X_feat_sel, y_feat_sel)\n",
    "\n",
    "        important_feats = X_feat_sel.columns[br.coef_[0] > 0]\n",
    "        X_train = X_train[important_feats]\n",
    "        \n",
    "    X_train = X_train[X_train.columns[X_train.std() != 0]]\n",
    "    vprint('Using {} features :: opt_val_seed {}...'.format(X_train.shape[1], opt_val_seed))\n",
    "    \n",
    "    return (X_train, y_train), (X_opt_val, y_opt_val)\n",
    "\n",
    "\n",
    "def bayesian_optimize_model(country_code, model_type, tunable_params=None, num_iter=50, init_points=0):\n",
    "    global round_num\n",
    "    global is_training\n",
    "\n",
    "    round_num = 0\n",
    "    is_training = True\n",
    "\n",
    "    store_fname = 'xgbBO_{}_res_{}_optimization_{}.dict'.format(country_code, model_type, datetime.now())\n",
    "    logging.info('Bayesian optimization results will be stored in {} after training...'.format(store_fname))\n",
    "\n",
    "    if tunable_params is None:\n",
    "        if model_type == 'xgb':\n",
    "            tunable_params = {\n",
    "                'indiv_corr_thresh': (0, 0.3),\n",
    "                'colsample_bytree': (0.2, 1),\n",
    "                'max_depth': (2, 6),\n",
    "                'subsample': (0.2, 1),\n",
    "                'gamma': (0, 2),\n",
    "                'scale_pos_weight': (0, 1),\n",
    "            }\n",
    "        elif model_type == 'lr':\n",
    "            tunable_params = {\n",
    "                'indiv_corr_thresh': (-0.9, 0.3),\n",
    "                'C': (0.001, 0.6),\n",
    "            }\n",
    "        elif model_type == 'lgb':\n",
    "            tunable_params = {\n",
    "                'indiv_corr_thresh': (0, 0.4),\n",
    "                'num_leaves': (4, 64),\n",
    "                'colsample_bytree' : (0.2, 1),\n",
    "                'subsample' : (0.2, 1),\n",
    "                'min_child_samples': (2, 120),\n",
    "                'scale_pos_weight': (0, 1),\n",
    "            }\n",
    "        elif model_type == 'nn':\n",
    "            tunable_params = {\n",
    "                'indiv_corr_thresh': (-0.9, 0.3),\n",
    "                'l1_num': (0, 100),\n",
    "                'l2_num' : (0, 100),\n",
    "                'l3_num' : (0, 100),\n",
    "                'alpha' : (0.005, 0.1),\n",
    "            }\n",
    "\n",
    "        elif model_type == 'rf':\n",
    "            tunable_params = {\n",
    "                'indiv_corr_thresh': (0, 0.4),\n",
    "                'max_depth': (2, 20),\n",
    "                'min_samples_split' : (2, 20),\n",
    "                'min_samples_leaf' : (2, 20),\n",
    "            }\n",
    "\n",
    "    if model_type == 'xgb':\n",
    "        model_predict = xgb_predict\n",
    "    elif model_type == 'lr':\n",
    "        model_predict = lr_predict\n",
    "    elif model_type == 'lgb':\n",
    "        model_predict = lgb_predict\n",
    "    elif model_type == 'nn':\n",
    "        model_predict = nn_predict\n",
    "    elif model_type == 'rf':\n",
    "        model_predict = rf_predict\n",
    "\n",
    "    modelBO = BayesianOptimization(\n",
    "        model_predict, tunable_params, verbose=bayes_opt_verbose\n",
    "    )\n",
    "\n",
    "    modelBO.maximize(init_points=init_points, n_iter=num_iter, acq=\"poi\", xi=0.1)\n",
    "\n",
    "    with open('./bayesian-opts-res/{:0.5}'.format(-modelBO.res['max']['max_val']) + '-' + store_fname, 'w') as fl:\n",
    "        cPickle.dump(modelBO.res, fl)\n",
    "\n",
    "    return modelBO\n",
    "\n",
    "\n",
    "def infer_test_val(\n",
    "    model_id, res_name, X_train, y_oof_preds,\n",
    "    X_hhold_test, X_indiv_cat_test,\n",
    "    X_val, fill_type, classifiers\n",
    "):\n",
    "    X_test = prepare_test_data(\n",
    "        X_hhold_test=X_hhold_test, X_indiv_cat_test=X_indiv_cat_test,\n",
    "        X_train=X_train, fill=fill_type\n",
    "    )\n",
    "\n",
    "    test_payload = infer(model_id, res_name, X_test, fill_type, classifiers)\n",
    "\n",
    "    payload = {\n",
    "        'train': {\n",
    "            '{}_{}'.format(model_id, res_name): y_oof_preds,\n",
    "            'index': X_train.index\n",
    "        },\n",
    "        'test': test_payload\n",
    "    }\n",
    "\n",
    "    if X_val is not None:\n",
    "        X_val = prepare_test_data(\n",
    "            X_train=X_train, X_val=X_val, fill=fill_type\n",
    "        )\n",
    "        val_payload = infer(model_id, res_name, X_val, fill_type, classifiers)\n",
    "\n",
    "        payload['val'] = val_payload\n",
    "\n",
    "    return payload\n",
    "\n",
    "\n",
    "def lr_predict(\n",
    "    indiv_corr_thresh,\n",
    "    C,\n",
    "    class_weight=None,\n",
    "    X_test=None,\n",
    "    X_val=None,\n",
    "    res_name=None,\n",
    "    opt_val_seed=None,\n",
    "    model_id='lr',\n",
    "    cv_func=lr_cv,\n",
    "    fill_type='mean'\n",
    "):\n",
    "    indiv_corr_thresh = round(indiv_corr_thresh, 2)\n",
    "\n",
    "    if opt_val_seed is None:\n",
    "        global round_num\n",
    "        global opt_val_seeds\n",
    "        opt_val_seed = opt_val_seeds[round_num]\n",
    "\n",
    "    (X_train, y_train), (X_opt_val, y_opt_val) = build_data_subset(\n",
    "        indiv_corr_thresh=indiv_corr_thresh, opt_val_seed=opt_val_seed\n",
    "    )\n",
    "\n",
    "    X_opt_val = prepare_test_data(\n",
    "        X_train=X_train, X_val=X_opt_val, fill=fill_type\n",
    "    )\n",
    "\n",
    "    params = {}\n",
    "    params['C'] = max(round(C, 2), 0.001)\n",
    "    if class_weight is not None:\n",
    "        params['class_weight'] = {\n",
    "            0: 1.0,\n",
    "            1: round(class_weight, 2),\n",
    "        } if class_weight < 0.5 else None\n",
    "\n",
    "    params['penalty'] = 'l1'\n",
    "\n",
    "    y_oof_preds, classifiers, opt_val_loss = cv_train_model(\n",
    "        X_train, y_train, X_opt_val, y_opt_val, params,\n",
    "        model_id=model_id, res_name=res_name, cv_func=cv_func,\n",
    "        fill_type=fill_type, cv_split=10\n",
    "    )\n",
    "\n",
    "    if X_test is not None:\n",
    "        payload = infer_test_val(\n",
    "            model_id=model_id, res_name=res_name,\n",
    "            X_train=X_train, y_oof_preds=y_oof_preds,\n",
    "            X_hhold_test=hhold_test, X_indiv_cat_test=indiv_cat_test,\n",
    "            X_val=X_val,\n",
    "            fill_type=fill_type,\n",
    "            classifiers=classifiers\n",
    "        )\n",
    "\n",
    "        if is_training:\n",
    "            round_num += 1\n",
    "        return payload\n",
    "    else:\n",
    "        if is_training:\n",
    "            round_num += 1\n",
    "        return -opt_val_loss\n",
    "\n",
    "\n",
    "def nn_predict(\n",
    "    indiv_corr_thresh,\n",
    "    l1_num,\n",
    "    l2_num,\n",
    "    alpha,\n",
    "    l3_num=1,\n",
    "    X_test=None,\n",
    "    X_val=None,\n",
    "    res_name=None,\n",
    "    opt_val_seed=None,\n",
    "    model_id='nn',\n",
    "    cv_func=nn_cv,\n",
    "    fill_type='mean'\n",
    "):\n",
    "    indiv_corr_thresh = round(indiv_corr_thresh, 2)\n",
    "\n",
    "    if opt_val_seed is None:\n",
    "        global round_num\n",
    "        global opt_val_seeds\n",
    "        opt_val_seed = opt_val_seeds[round_num]\n",
    "\n",
    "    (X_train, y_train), (X_opt_val, y_opt_val) = build_data_subset(\n",
    "        indiv_corr_thresh=indiv_corr_thresh, opt_val_seed=opt_val_seed\n",
    "    )\n",
    "\n",
    "    X_opt_val = prepare_test_data(\n",
    "        X_train=X_train, X_val=X_opt_val, fill=fill_type\n",
    "    )\n",
    "\n",
    "    params = {\n",
    "        'hidden_layer_sizes': (50, 20, 1),\n",
    "        'alpha': 0.01,\n",
    "        'max_iter': 1000,\n",
    "        'early_stopping': True,\n",
    "        'random_state': 1029\n",
    "    }\n",
    "\n",
    "    params['alpha'] = round(alpha, 2)\n",
    "    params['hidden_layer_sizes'] = (get_round_num(l1_num, 10), get_round_num(l2_num, 10), get_round_num(l3_num, 5))\n",
    "\n",
    "    y_oof_preds, classifiers, opt_val_loss = cv_train_model(\n",
    "        X_train, y_train, X_opt_val, y_opt_val, params,\n",
    "        model_id=model_id, res_name=res_name, cv_func=cv_func,\n",
    "        fill_type=fill_type, cv_split=10\n",
    "    )\n",
    "\n",
    "    if X_test is not None:\n",
    "        payload = infer_test_val(\n",
    "            model_id=model_id, res_name=res_name,\n",
    "            X_train=X_train, y_oof_preds=y_oof_preds,\n",
    "            X_hhold_test=hhold_test, X_indiv_cat_test=indiv_cat_test,\n",
    "            X_val=X_val,\n",
    "            fill_type=fill_type,\n",
    "            classifiers=classifiers\n",
    "        )\n",
    "\n",
    "        if is_training:\n",
    "            round_num += 1\n",
    "        return payload\n",
    "    else:\n",
    "        if is_training:\n",
    "            round_num += 1\n",
    "        return -opt_val_loss\n",
    "\n",
    "\n",
    "def rf_predict(\n",
    "    indiv_corr_thresh,\n",
    "    max_depth,\n",
    "    min_samples_split,\n",
    "    min_samples_leaf,\n",
    "    X_test=None,\n",
    "    X_val=None,\n",
    "    res_name=None,\n",
    "    opt_val_seed=None,\n",
    "    model_id='rf',\n",
    "    cv_func=rf_cv,\n",
    "    fill_type='mean'\n",
    "):\n",
    "    indiv_corr_thresh = round(indiv_corr_thresh, 2)\n",
    "\n",
    "    if opt_val_seed is None:\n",
    "        global round_num\n",
    "        global opt_val_seeds\n",
    "        opt_val_seed = opt_val_seeds[round_num]\n",
    "\n",
    "    (X_train, y_train), (X_opt_val, y_opt_val) = build_data_subset(\n",
    "        indiv_corr_thresh=indiv_corr_thresh, opt_val_seed=opt_val_seed\n",
    "    )\n",
    "\n",
    "    X_opt_val = prepare_test_data(\n",
    "        X_train=X_train, X_val=X_opt_val, fill=fill_type\n",
    "    )\n",
    "\n",
    "    params = dict(\n",
    "        max_depth=20, min_samples_split=20, min_samples_leaf=10, n_jobs=7, random_state=1029\n",
    "    )\n",
    "\n",
    "    params['max_depth'] = int(max_depth)\n",
    "    params['min_samples_split'] = get_round_num(min_samples_split, 2)\n",
    "    params['min_samples_leaf'] = get_round_num(min_samples_leaf, 2)\n",
    "\n",
    "    y_oof_preds, classifiers, opt_val_loss = cv_train_model(\n",
    "        X_train, y_train, X_opt_val, y_opt_val, params,\n",
    "        model_id=model_id, res_name=res_name, cv_func=cv_func,\n",
    "        fill_type=fill_type, cv_split=10\n",
    "    )\n",
    "\n",
    "    if X_test is not None:\n",
    "        payload = infer_test_val(\n",
    "            model_id=model_id, res_name=res_name,\n",
    "            X_train=X_train, y_oof_preds=y_oof_preds,\n",
    "            X_hhold_test=hhold_test, X_indiv_cat_test=indiv_cat_test,\n",
    "            X_val=X_val,\n",
    "            fill_type=fill_type,\n",
    "            classifiers=classifiers\n",
    "        )\n",
    "\n",
    "        if is_training:\n",
    "            round_num += 1\n",
    "        return payload\n",
    "    else:\n",
    "        if is_training:\n",
    "            round_num += 1\n",
    "        return -opt_val_loss\n",
    "\n",
    "\n",
    "def lgb_predict(\n",
    "    indiv_corr_thresh,\n",
    "    num_leaves,\n",
    "    colsample_bytree,\n",
    "    subsample,\n",
    "    min_child_samples,\n",
    "    scale_pos_weight,\n",
    "    subsample_for_bin=None,\n",
    "    X_test=None,\n",
    "    X_val=None,\n",
    "    res_name=None,\n",
    "    opt_val_seed=None,\n",
    "    model_id='lgb',\n",
    "    cv_func=lgb_cv,\n",
    "    fill_type='mean'\n",
    "):\n",
    "    indiv_corr_thresh = round(indiv_corr_thresh, 2)\n",
    "\n",
    "    if opt_val_seed is None:\n",
    "        global round_num\n",
    "        global opt_val_seeds\n",
    "        opt_val_seed = opt_val_seeds[round_num]\n",
    "\n",
    "    (X_train, y_train), (X_opt_val, y_opt_val) = build_data_subset(\n",
    "        indiv_corr_thresh=indiv_corr_thresh, opt_val_seed=opt_val_seed\n",
    "    )\n",
    "\n",
    "    X_opt_val = prepare_test_data(\n",
    "        X_train=X_train, X_val=X_opt_val, fill=fill_type\n",
    "    )\n",
    "\n",
    "    params = {\n",
    "        'boosting_type': 'gbdt',\n",
    "        'max_depth' : -1,\n",
    "        'objective': 'binary',\n",
    "        'learning_rate': 0.05,\n",
    "        'subsample_freq': 1,\n",
    "        'scale_pos_weight': 1,\n",
    "        'metric' : 'binary_error'\n",
    "    }\n",
    "\n",
    "    params['colsample_bytree'] = max(min(round(colsample_bytree, 2), 1), 0)\n",
    "    params['subsample'] = max(min(round(subsample, 2), 1), 0)\n",
    "    params['num_leaves'] = int(num_leaves)\n",
    "    params['min_child_samples'] = int(min_child_samples)\n",
    "    params['subsample_for_bin'] = int(subsample_for_bin) if subsample_for_bin is not None else 200000\n",
    "\n",
    "    params['learning_rate'] = 0.1\n",
    "    params['n_estimators'] = 10000\n",
    "    params['objective'] = 'binary'\n",
    "    params['random_state'] = 1029\n",
    "    params['n_jobs'] = 7\n",
    "    params['silent'] = True\n",
    "\n",
    "    scale_pos_weight = 1 if scale_pos_weight > 0.5 else 1.0 * sum(y_train < 0.5) / sum(y_train > 0.5)\n",
    "    params['scale_pos_weight'] = scale_pos_weight\n",
    "\n",
    "    y_oof_preds, classifiers, opt_val_loss = cv_train_model(\n",
    "        X_train, y_train, X_opt_val, y_opt_val, params,\n",
    "        model_id=model_id, res_name=res_name, cv_func=cv_func,\n",
    "        fill_type=fill_type, cv_split=10\n",
    "    )\n",
    "\n",
    "    if X_test is not None:\n",
    "        payload = infer_test_val(\n",
    "            model_id=model_id, res_name=res_name,\n",
    "            X_train=X_train, y_oof_preds=y_oof_preds,\n",
    "            X_hhold_test=hhold_test, X_indiv_cat_test=indiv_cat_test,\n",
    "            X_val=X_val,\n",
    "            fill_type=fill_type,\n",
    "            classifiers=classifiers\n",
    "        )\n",
    "\n",
    "        if is_training:\n",
    "            round_num += 1\n",
    "        return payload\n",
    "    else:\n",
    "        if is_training:\n",
    "            round_num += 1\n",
    "        return -opt_val_loss\n",
    "\n",
    "\n",
    "def xgb_predict(\n",
    "    indiv_corr_thresh,\n",
    "    colsample_bytree,\n",
    "    max_depth,\n",
    "    subsample,\n",
    "    gamma,\n",
    "    scale_pos_weight,\n",
    "    X_test=None,\n",
    "    X_val=None,\n",
    "    res_name=None,\n",
    "    opt_val_seed=None,\n",
    "    model_id='xgb',\n",
    "    cv_func=xgb_cv,\n",
    "    fill_type=None\n",
    "):\n",
    "    '''\n",
    "    DMatrix params\n",
    "    - weight\n",
    "    - missing (-999)\n",
    "\n",
    "    xgboost params\n",
    "    max_depth\n",
    "    min_child_weight\n",
    "    gamma -> minimum loss reduction required to make a further partition on a leaf node of the tree. The larger, the more conservative the algorithm will be.\n",
    "    subsample -> proportion of data\n",
    "    colsample_bytree -> proportion of columns used in each tree\n",
    "    max_delta_step -> for imbalanced use finite value, e.g., 1\n",
    "\n",
    "    Decrease eta and increase nrounds if overfitting is observed.\n",
    "    '''\n",
    "    indiv_corr_thresh = round(indiv_corr_thresh, 2)\n",
    "\n",
    "    if opt_val_seed is None:\n",
    "        global round_num\n",
    "        global opt_val_seeds\n",
    "        opt_val_seed = opt_val_seeds[round_num]\n",
    "\n",
    "    (X_train, y_train), (X_opt_val, y_opt_val) = build_data_subset(\n",
    "        indiv_corr_thresh=indiv_corr_thresh, opt_val_seed=opt_val_seed\n",
    "    )\n",
    "\n",
    "    X_opt_val = prepare_test_data(\n",
    "        X_train=X_train, X_val=X_opt_val, fill=fill_type\n",
    "    )\n",
    "    \n",
    "    params = {}\n",
    "    params['colsample_bytree'] = max(min(round(colsample_bytree, 2), 1), 0)\n",
    "    params['max_depth'] = int(max_depth)\n",
    "    params['subsample'] = max(min(round(subsample, 2), 1), 0)\n",
    "    params['gamma'] = max(round(gamma, 2), 0)\n",
    "    params['learning_rate'] = 0.2\n",
    "    params['n_estimators'] = 10000\n",
    "    params['objective'] = 'binary:logistic'\n",
    "    params['random_state'] = 1029\n",
    "    params['n_jobs'] = 7\n",
    "    params['silent'] = True\n",
    "\n",
    "    scale_pos_weight = 1 if scale_pos_weight > 0.5 else 1.0 * sum(y_train < 0.5) / sum(y_train > 0.5)\n",
    "    params['scale_pos_weight'] = scale_pos_weight\n",
    "\n",
    "    y_oof_preds, classifiers, opt_val_loss = cv_train_model(\n",
    "        X_train, y_train, X_opt_val, y_opt_val, params,\n",
    "        model_id=model_id, res_name=res_name, cv_func=cv_func,\n",
    "        fill_type=fill_type, cv_split=10\n",
    "    )\n",
    "\n",
    "    if X_test is not None:\n",
    "        payload = infer_test_val(\n",
    "            model_id=model_id, res_name=res_name,\n",
    "            X_train=X_train, y_oof_preds=y_oof_preds,\n",
    "            X_hhold_test=hhold_test, X_indiv_cat_test=indiv_cat_test,\n",
    "            X_val=X_val,\n",
    "            fill_type=fill_type,\n",
    "            classifiers=classifiers\n",
    "        )\n",
    "\n",
    "        if is_training:\n",
    "            round_num += 1\n",
    "        return payload\n",
    "    else:\n",
    "        if is_training:\n",
    "            round_num += 1\n",
    "        return -opt_val_loss\n",
    "\n",
    "\n",
    "def predict_from_opt(country_code, model_type, num_params=10, use_latest=False, default_dir='./bayesian-opts-res/'):\n",
    "    global is_training\n",
    "    is_training = False\n",
    "\n",
    "    candidate_fnames = sorted(glob.glob(os.path.join(default_dir, '*-*_{}_res_{}_*_*'.format(country_code, model_type))))\n",
    "\n",
    "    if use_latest:\n",
    "        fname = max(candidate_fnames, key=os.path.getctime)\n",
    "    else:\n",
    "        fname = candidate_fnames[0]\n",
    "\n",
    "    vprint(fname)\n",
    "\n",
    "    with open(fname) as fl:\n",
    "        res = cPickle.load(fl)\n",
    "\n",
    "    if model_type == 'xgb':\n",
    "        model_predict = xgb_predict\n",
    "    elif model_type == 'lr':\n",
    "        model_predict = lr_predict\n",
    "    elif model_type == 'lgb':\n",
    "        model_predict = lgb_predict\n",
    "    elif model_type == 'nn':\n",
    "        model_predict = nn_predict\n",
    "    elif model_type == 'rf':\n",
    "        model_predict = rf_predict\n",
    "\n",
    "    res_df = pd.DataFrame(res['all'])\n",
    "    top_res = res_df['values'].argsort().iloc[-num_params:]\n",
    "    num_round_index = top_res.values[::-1]\n",
    "    params = res_df.loc[top_res.values[::-1]]['params']\n",
    "\n",
    "    model_test_preds = pd.DataFrame()\n",
    "    model_train_preds = pd.DataFrame()\n",
    "\n",
    "    for res_name, (nm_rnd, param) in enumerate(zip(num_round_index, params)):\n",
    "        opt_val_seed = opt_val_seeds[nm_rnd]\n",
    "        r = model_predict(X_test=True, opt_val_seed=opt_val_seed, res_name=res_name, **param)\n",
    "\n",
    "        train_preds = pd.DataFrame(index=r['train']['index'])\n",
    "        train_preds['{}_{}'.format(model_type, res_name)] = r['train']['{}_{}'.format(model_type, res_name)]\n",
    "\n",
    "        test_preds = r['test']['{}_{}'.format(model_type, res_name)]\n",
    "\n",
    "        if model_test_preds.empty:\n",
    "            model_test_preds = test_preds\n",
    "        else:\n",
    "            model_test_preds = pd.concat([model_test_preds, test_preds], axis=1)\n",
    "\n",
    "        if model_train_preds.empty:\n",
    "            model_train_preds = train_preds\n",
    "        else:\n",
    "            model_train_preds = pd.concat([model_train_preds, train_preds], axis=1)\n",
    "\n",
    "    return model_test_preds, model_train_preds\n",
    "\n",
    "\n",
    "def load_data(country_code, data_part='train'):\n",
    "    hhold = os.path.join(DATA_DIR, '{}_hhold_{}.csv'.format(country_code, data_part))\n",
    "    indiv = os.path.join(DATA_DIR, '{}_indiv_{}.csv'.format(country_code, data_part))\n",
    "\n",
    "    hhold = pd.read_csv(hhold, index_col='id')\n",
    "    indiv = pd.read_csv(indiv, index_col=['id', 'iid'])\n",
    "\n",
    "    return hhold, indiv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data directories\n",
    "DATA_DIR = os.path.join('..', 'data')\n",
    "SUBMISSION_DIR = os.path.join('..', 'submission')\n",
    "BAYESIAN_OPTS_DIR = './bayesian-opts-res'\n",
    "BAYESIAN_OPTS_TEST_DIR = os.path.join(BAYESIAN_OPTS_DIR, 'bayesian-opt-test-preds')\n",
    "\n",
    "# Setup directories\n",
    "if not os.path.isdir(BAYESIAN_OPTS_DIR):\n",
    "    logging.info('Creating {} directory...'.format(BAYESIAN_OPTS_DIR))\n",
    "    os.mkdir(BAYESIAN_OPTS_DIR)\n",
    "\n",
    "if not os.path.isdir(BAYESIAN_OPTS_TEST_DIR):\n",
    "    logging.info('Creating {} directory...'.format(BAYESIAN_OPTS_TEST_DIR))\n",
    "    os.mkdir(BAYESIAN_OPTS_TEST_DIR)\n",
    "\n",
    "if not os.path.isdir(SUBMISSION_DIR):\n",
    "    logging.info('Creating {} directory...'.format(SUBMISSION_DIR))\n",
    "    os.mkdir(SUBMISSION_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define and initialize variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "bayes_opt_verbose = False  # Flag for bayes-opt logging\n",
    "log_verbose = False  # Flag for miscellaneous log logging\n",
    "init_points = 2  # Number of random initialization rounds (30 used in winning submission)\n",
    "opt_iter = 2  # Number of bayesian optimization rounds (70 used in winning submission)\n",
    "\n",
    "num_gen_params =  2  # Number of top meta predictions to use for blending (20 used in winning submission)\n",
    "\n",
    "round_num = None  # Global variable to keep track of the bayesian optimization round\n",
    "is_training = None  # Global variable to indicate training or testing periods\n",
    "\n",
    "session = datetime.now()\n",
    "opt_val_seeds = get_opt_val_seeds(init_points * opt_iter)  # Generation of seeds for each bayesian optimization round\n",
    "\n",
    "model_set = ['lgb', 'lr', 'nn', 'xgb', 'rf']\n",
    "\n",
    "opt_res = {\n",
    "    'A': {},\n",
    "    'B': {},\n",
    "    'C': {}\n",
    "}\n",
    "\n",
    "country_preds_dict = {\n",
    "    'A': {},\n",
    "    'B': {},\n",
    "    'C': {},\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Execute training and meta prediction inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fragment [1. Training bayesian opts for lgb model.] done in 45.14 s\n",
      "\n",
      "Fragment [2. Training bayesian opts for lr model.] done in 44.74 s\n",
      "\n",
      "Fragment [3. Training bayesian opts for nn model.] done in 52.82 s\n",
      "\n",
      "Fragment [4. Training bayesian opts for xgb model.] done in 118.75 s\n",
      "\n",
      "Fragment [5. Training bayesian opts for rf model.] done in 33.48 s\n",
      "\n",
      "Fragment [Start bayesian optimization and generation of optimal sub-models for country A] done in 294.95 s\n",
      "\n",
      "Fragment [1. Generating meta predictions for lgb model.] done in 22.11 s\n",
      "\n",
      "Fragment [2. Generating meta predictions for lr model.] done in 18.12 s\n",
      "\n",
      "Fragment [3. Generating meta predictions for nn model.] done in 20.62 s\n",
      "\n",
      "Fragment [4. Generating meta predictions for xgb model.] done in 46.99 s\n",
      "\n",
      "Fragment [5. Generating meta predictions for rf model.] done in 18.61 s\n",
      "\n",
      "Fragment [Generate inference using the top N models based on bayesian OOF score for country A] done in 126.45 s\n",
      "\n",
      "Fragment [Performing training and meta predictions for country A.] done in 422.84 s\n",
      "\n",
      "Fragment [1. Training bayesian opts for lgb model.] done in 56.64 s\n",
      "\n",
      "Fragment [2. Training bayesian opts for lr model.] done in 31.71 s\n",
      "\n",
      "Fragment [3. Training bayesian opts for nn model.] done in 44.81 s\n",
      "\n",
      "Fragment [4. Training bayesian opts for xgb model.] done in 66.52 s\n",
      "\n",
      "Fragment [5. Training bayesian opts for rf model.] done in 25.53 s\n",
      "\n",
      "Fragment [Start bayesian optimization and generation of optimal sub-models for country B] done in 225.21 s\n",
      "\n",
      "Fragment [1. Generating meta predictions for lgb model.] done in 44.25 s\n",
      "\n",
      "Fragment [2. Generating meta predictions for lr model.] done in 20.24 s\n",
      "\n",
      "Fragment [3. Generating meta predictions for nn model.] done in 17.66 s\n",
      "\n",
      "Fragment [4. Generating meta predictions for xgb model.] done in 34.60 s\n",
      "\n",
      "Fragment [5. Generating meta predictions for rf model.] done in 15.20 s\n",
      "\n",
      "Fragment [Generate inference using the top N models based on bayesian OOF score for country B] done in 131.96 s\n",
      "\n",
      "Fragment [Performing training and meta predictions for country B.] done in 358.45 s\n",
      "\n",
      "Fragment [1. Training bayesian opts for lgb model.] done in 30.38 s\n",
      "\n",
      "Fragment [2. Training bayesian opts for lr model.] done in 30.61 s\n",
      "\n",
      "Fragment [3. Training bayesian opts for nn model.] done in 55.36 s\n",
      "\n",
      "Fragment [4. Training bayesian opts for xgb model.] done in 64.98 s\n",
      "\n",
      "Fragment [5. Training bayesian opts for rf model.] done in 22.72 s\n",
      "\n",
      "Fragment [Start bayesian optimization and generation of optimal sub-models for country C] done in 204.05 s\n",
      "\n",
      "Fragment [1. Generating meta predictions for lgb model.] done in 14.01 s\n",
      "\n",
      "Fragment [2. Generating meta predictions for lr model.] done in 18.39 s\n",
      "\n",
      "Fragment [3. Generating meta predictions for nn model.] done in 13.55 s\n",
      "\n",
      "Fragment [4. Generating meta predictions for xgb model.] done in 27.63 s\n",
      "\n",
      "Fragment [5. Generating meta predictions for rf model.] done in 13.40 s\n",
      "\n",
      "Fragment [Generate inference using the top N models based on bayesian OOF score for country C] done in 87.00 s\n",
      "\n",
      "Fragment [Performing training and meta predictions for country C.] done in 292.00 s\n",
      "\n",
      "Fragment [Full model for all countries.] done in 1073.29 s\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with timing('Full model for all countries.'):\n",
    "    for country_code in ['A', 'B', 'C']:\n",
    "        with timing('Performing training and meta predictions for country {}.'.format(country_code)):\n",
    "            x_model_train_preds = pd.DataFrame()\n",
    "            x_model_test_preds = pd.DataFrame()\n",
    "\n",
    "            _, indiv_train = load_data(country_code, data_part='train')\n",
    "\n",
    "            indiv_cat_train = pd.read_hdf(os.path.join(DATA_DIR, 'indiv_cat_train.hdf'), '{}_indiv_cat_train'.format(country_code))\n",
    "            indiv_cat_test = pd.read_hdf(os.path.join(DATA_DIR, 'indiv_cat_test.hdf'), '{}_indiv_cat_test'.format(country_code))\n",
    "\n",
    "            hhold_train = pd.read_csv(os.path.join(DATA_DIR, '{}-hhold-transformed-train.csv'.format(country_code)), index_col='id')\n",
    "            hhold_test = pd.read_csv(os.path.join(DATA_DIR, '{}-hhold-transformed-test.csv'.format(country_code)), index_col='id')\n",
    "\n",
    "            y_train = np.ravel(hhold_train.loc[indiv_cat_train.index].poor)\n",
    "\n",
    "            with timing('Start bayesian optimization and generation of optimal sub-models for country {}'.format(country_code)):\n",
    "                for ix, model_type in enumerate(model_set):\n",
    "                    with timing('{}. Training bayesian opts for {} model.'.format(ix + 1, model_type)):\n",
    "                        modelBO = bayesian_optimize_model(country_code=country_code, model_type=model_type, tunable_params=None, num_iter=opt_iter, init_points=init_points)\n",
    "                    opt_res[country_code][model_type] = modelBO.res\n",
    "\n",
    "            # Store optimization results\n",
    "            fname = os.path.join(BAYESIAN_OPTS_DIR, 'bayesian_opt_res-{}.pkl'.format(session))\n",
    "            vprint(fname)\n",
    "\n",
    "            with open(fname, 'w') as fl:\n",
    "                cPickle.dump(opt_res, fl)\n",
    "\n",
    "            with timing('Generate inference using the top N models based on bayesian OOF score for country {}'.format(country_code)):\n",
    "                for ix, model_type in enumerate(model_set):\n",
    "                    with timing('{}. Generating meta predictions for {} model.'.format(ix + 1, model_type)):\n",
    "                        model_test_preds, model_train_preds = predict_from_opt(\n",
    "                            country_code, model_type, num_params=num_gen_params,\n",
    "                            use_latest=True, default_dir=BAYESIAN_OPTS_DIR\n",
    "                        )\n",
    "\n",
    "                        model_test_preds.to_hdf(\n",
    "                            os.path.join(BAYESIAN_OPTS_TEST_DIR, 'model_test_preds-{}.hdf'.format(session)),\n",
    "                            'model_test_preds_{}_{}'.format(country_code, model_type)\n",
    "                        )\n",
    "\n",
    "                        model_train_preds.to_hdf(\n",
    "                            os.path.join(BAYESIAN_OPTS_TEST_DIR, 'model_train_preds-{}.hdf'.format(session)),\n",
    "                            'model_train_preds{}_{}'.format(country_code, model_type)\n",
    "                        )\n",
    "\n",
    "                        # Collect meta predictions\n",
    "                        x_model_train_preds = pd.concat([x_model_train_preds, model_train_preds], axis=1)\n",
    "                        x_model_test_preds = pd.concat([x_model_test_preds, model_test_preds], axis=1)\n",
    "\n",
    "            country_preds_dict[country_code]['train'] = x_model_train_preds\n",
    "            country_preds_dict[country_code]['test'] = x_model_test_preds\n",
    "            country_preds_dict[country_code]['y_train'] = hhold_train.loc[x_model_train_preds.index].poor.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform OOF optimized blending"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "median_weight_optimized_preds_sub = []\n",
    "mean_weight_optimized_preds_sub = []\n",
    "\n",
    "for country_code in ['A', 'B', 'C']:\n",
    "    with timing('Optimizing meta prediction weights for country {}.'.format(country_code)):\n",
    "        x_test_median_preds, x_test_mean_preds, x_optimized_weights_preds, x_coeffs = get_optimized_weighted_preds_for(country_preds_dict, country_code)\n",
    "\n",
    "    median_weight_optimized_preds_sub.append(make_country_sub(x_test_median_preds, x_test_median_preds, country_code))\n",
    "    mean_weight_optimized_preds_sub.append(make_country_sub(x_test_mean_preds, x_test_mean_preds, country_code))\n",
    "\n",
    "median_weight_optimized_preds_sub = pd.concat(median_weight_optimized_preds_sub)\n",
    "mean_weight_optimized_preds_sub = pd.concat(mean_weight_optimized_preds_sub)\n",
    "\n",
    "\n",
    "date_now = datetime.now()\n",
    "median_sub_fname = os.path.join(SUBMISSION_DIR, 'submission-test-median-agg-models-{}.csv'.format(date_now))\n",
    "mean_sub_fname = os.path.join(SUBMISSION_DIR, 'submission-test-mean-agg-models-{}.csv'.format(date_now))\n",
    "\n",
    "logging.info(median_sub_fname)\n",
    "logging.info(mean_sub_fname)\n",
    "\n",
    "median_weight_optimized_preds_sub.to_csv(median_sub_fname)\n",
    "mean_weight_optimized_preds_sub.to_csv(mean_sub_fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "median_weight_optimized_preds_sub.groupby('country').hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_weight_optimized_preds_sub.groupby('country').hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}