KickstarterModels.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import sklearn\n",
    "import xgboost as xgb\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import linear_model\n",
    "%matplotlib inline\n",
    "import plotly\n",
    "import plotly.offline as py\n",
    "plotly.tools.set_credentials_file(username='soniakt23', api_key='FoAfF3pFQAZGkRqJHf43')\n",
    "py.init_notebook_mode(connected=True)\n",
    "import plotly.graph_objs as go\n",
    "import plotly.tools as tls\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import numpy as np  \n",
    "import locale\n",
    "from locale import atof\n",
    "from xgboost import plot_importance\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# Going to use these 5 base models for the stacking\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.cross_validation import KFold;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load in the train and test datasets\n",
    "data = pd.read_csv('SAGAR_FINAL_DATA_10_CSV.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Make non numerical datatypes into float\n",
    "data['backed'] = data['backed'].str.replace( ',','').astype(float)\n",
    "data['comments'] = data['comments'].str.replace( ',', '').astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(37064, 305)\n",
      "(37064, 305)\n"
     ]
    }
   ],
   "source": [
    "#Randomize data\n",
    "df = shuffle(df)\n",
    "\n",
    "#get rid of uneccessary rows in training and testing\n",
    "df = df[df.state_canceled != 1]\n",
    "df = df[df.state_suspended !=1]\n",
    "df = df[df.state_live !=1]\n",
    "\n",
    "\n",
    "#get rid of uneccessary columns in training and testing\n",
    "drop_cols = ['user_profiles', 'state_failed', 'state_suspended', 'state_canceled', 'Unnamed: 0', 'pledged', \n",
    "            'usd_pledged', 'converted_pledged_amount', 'state_live']\n",
    "df = df.drop(drop_cols, axis = 1)\n",
    "\n",
    "print df.shape\n",
    "#Drop na values\n",
    "#df = df.dropna()\n",
    "\n",
    "print df.shape\n",
    "\n",
    "for column in df.columns:\n",
    "    df[column] = df[column].fillna(df[column].median())\n",
    "\n",
    "y = df['state_successful']\n",
    "X = df.drop('state_successful', axis =1)\n",
    "\n",
    "\n",
    "\n",
    "#Split dataset into training and testing datasets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=42)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4021, 305)\n"
     ]
    }
   ],
   "source": [
    "print df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>goal</th>\n",
       "      <th>deadline</th>\n",
       "      <th>created_at</th>\n",
       "      <th>launched_at</th>\n",
       "      <th>backed</th>\n",
       "      <th>comments</th>\n",
       "      <th>created</th>\n",
       "      <th>total_time</th>\n",
       "      <th>adventure</th>\n",
       "      <th>...</th>\n",
       "      <th>subcategory_spaces</th>\n",
       "      <th>subcategory_stationery</th>\n",
       "      <th>subcategory_taxidermy</th>\n",
       "      <th>subcategory_textiles</th>\n",
       "      <th>subcategory_typography</th>\n",
       "      <th>subcategory_video art</th>\n",
       "      <th>subcategory_weaving</th>\n",
       "      <th>subcategory_webcomics</th>\n",
       "      <th>subcategory_woodworking</th>\n",
       "      <th>subcategory_workshops</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4.021000e+03</td>\n",
       "      <td>4.021000e+03</td>\n",
       "      <td>4.021000e+03</td>\n",
       "      <td>4.021000e+03</td>\n",
       "      <td>4.021000e+03</td>\n",
       "      <td>4021.000000</td>\n",
       "      <td>4021.000000</td>\n",
       "      <td>4021.000000</td>\n",
       "      <td>4.021000e+03</td>\n",
       "      <td>4021.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4021.0</td>\n",
       "      <td>4021.0</td>\n",
       "      <td>4021.0</td>\n",
       "      <td>4021.0</td>\n",
       "      <td>4021.0</td>\n",
       "      <td>4021.0</td>\n",
       "      <td>4021.0</td>\n",
       "      <td>4021.000000</td>\n",
       "      <td>4021.0</td>\n",
       "      <td>4021.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.081489e+09</td>\n",
       "      <td>3.562879e+04</td>\n",
       "      <td>1.410730e+09</td>\n",
       "      <td>1.403932e+09</td>\n",
       "      <td>1.407991e+09</td>\n",
       "      <td>11.822542</td>\n",
       "      <td>50.372793</td>\n",
       "      <td>2.498722</td>\n",
       "      <td>2.739050e+06</td>\n",
       "      <td>0.013927</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.107187</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6.212697e+08</td>\n",
       "      <td>1.577630e+06</td>\n",
       "      <td>4.541847e+07</td>\n",
       "      <td>4.613587e+07</td>\n",
       "      <td>4.550755e+07</td>\n",
       "      <td>43.476713</td>\n",
       "      <td>510.000895</td>\n",
       "      <td>3.538418</td>\n",
       "      <td>9.583753e+05</td>\n",
       "      <td>0.119306</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.309390</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>5.349160e+05</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.246421e+09</td>\n",
       "      <td>1.242847e+09</td>\n",
       "      <td>1.243123e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.640000e+04</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>5.302994e+08</td>\n",
       "      <td>8.500000e+02</td>\n",
       "      <td>1.383336e+09</td>\n",
       "      <td>1.374514e+09</td>\n",
       "      <td>1.380629e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.588400e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.081741e+09</td>\n",
       "      <td>3.000000e+03</td>\n",
       "      <td>1.408311e+09</td>\n",
       "      <td>1.402911e+09</td>\n",
       "      <td>1.405564e+09</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.592000e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.621427e+09</td>\n",
       "      <td>1.000000e+04</td>\n",
       "      <td>1.440963e+09</td>\n",
       "      <td>1.434447e+09</td>\n",
       "      <td>1.438371e+09</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>2.110000</td>\n",
       "      <td>2.821754e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.146831e+09</td>\n",
       "      <td>1.000000e+08</td>\n",
       "      <td>1.508080e+09</td>\n",
       "      <td>1.505415e+09</td>\n",
       "      <td>1.505863e+09</td>\n",
       "      <td>1721.000000</td>\n",
       "      <td>18062.000000</td>\n",
       "      <td>74.000000</td>\n",
       "      <td>7.776000e+06</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 302 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id          goal      deadline    created_at   launched_at  \\\n",
       "count  4.021000e+03  4.021000e+03  4.021000e+03  4.021000e+03  4.021000e+03   \n",
       "mean   1.081489e+09  3.562879e+04  1.410730e+09  1.403932e+09  1.407991e+09   \n",
       "std    6.212697e+08  1.577630e+06  4.541847e+07  4.613587e+07  4.550755e+07   \n",
       "min    5.349160e+05  1.000000e+00  1.246421e+09  1.242847e+09  1.243123e+09   \n",
       "25%    5.302994e+08  8.500000e+02  1.383336e+09  1.374514e+09  1.380629e+09   \n",
       "50%    1.081741e+09  3.000000e+03  1.408311e+09  1.402911e+09  1.405564e+09   \n",
       "75%    1.621427e+09  1.000000e+04  1.440963e+09  1.434447e+09  1.438371e+09   \n",
       "max    2.146831e+09  1.000000e+08  1.508080e+09  1.505415e+09  1.505863e+09   \n",
       "\n",
       "            backed      comments      created    total_time    adventure  \\\n",
       "count  4021.000000   4021.000000  4021.000000  4.021000e+03  4021.000000   \n",
       "mean     11.822542     50.372793     2.498722  2.739050e+06     0.013927   \n",
       "std      43.476713    510.000895     3.538418  9.583753e+05     0.119306   \n",
       "min       0.000000      0.000000     1.000000  8.640000e+04     0.000000   \n",
       "25%       0.000000      0.000000     1.000000  2.588400e+06     0.000000   \n",
       "50%       2.000000      1.000000     1.000000  2.592000e+06     0.000000   \n",
       "75%       9.000000     18.000000     2.110000  2.821754e+06     0.000000   \n",
       "max    1721.000000  18062.000000    74.000000  7.776000e+06     2.000000   \n",
       "\n",
       "               ...            subcategory_spaces  subcategory_stationery  \\\n",
       "count          ...                        4021.0                  4021.0   \n",
       "mean           ...                           0.0                     0.0   \n",
       "std            ...                           0.0                     0.0   \n",
       "min            ...                           0.0                     0.0   \n",
       "25%            ...                           0.0                     0.0   \n",
       "50%            ...                           0.0                     0.0   \n",
       "75%            ...                           0.0                     0.0   \n",
       "max            ...                           0.0                     0.0   \n",
       "\n",
       "       subcategory_taxidermy  subcategory_textiles  subcategory_typography  \\\n",
       "count                 4021.0                4021.0                  4021.0   \n",
       "mean                     0.0                   0.0                     0.0   \n",
       "std                      0.0                   0.0                     0.0   \n",
       "min                      0.0                   0.0                     0.0   \n",
       "25%                      0.0                   0.0                     0.0   \n",
       "50%                      0.0                   0.0                     0.0   \n",
       "75%                      0.0                   0.0                     0.0   \n",
       "max                      0.0                   0.0                     0.0   \n",
       "\n",
       "       subcategory_video art  subcategory_weaving  subcategory_webcomics  \\\n",
       "count                 4021.0               4021.0            4021.000000   \n",
       "mean                     0.0                  0.0               0.107187   \n",
       "std                      0.0                  0.0               0.309390   \n",
       "min                      0.0                  0.0               0.000000   \n",
       "25%                      0.0                  0.0               0.000000   \n",
       "50%                      0.0                  0.0               0.000000   \n",
       "75%                      0.0                  0.0               0.000000   \n",
       "max                      0.0                  0.0               1.000000   \n",
       "\n",
       "       subcategory_woodworking  subcategory_workshops  \n",
       "count                   4021.0                 4021.0  \n",
       "mean                       0.0                    0.0  \n",
       "std                        0.0                    0.0  \n",
       "min                        0.0                    0.0  \n",
       "25%                        0.0                    0.0  \n",
       "50%                        0.0                    0.0  \n",
       "75%                        0.0                    0.0  \n",
       "max                        0.0                    0.0  \n",
       "\n",
       "[8 rows x 302 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n"
     ]
    }
   ],
   "source": [
    "X_train.shape\n",
    "print type(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4021, 305)\n"
     ]
    }
   ],
   "source": [
    "print df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Some useful parameters which will come in handy later on\n",
    "ntrain = X_train.shape[0]\n",
    "ntest = X_test.shape[0]\n",
    "SEED = 0 # for reproducibility\n",
    "NFOLDS = 5 # set folds for out-of-fold prediction\n",
    "kf = KFold(ntrain, n_folds= NFOLDS, random_state=SEED)\n",
    "\n",
    "# Class to extend the Sklearn classifier\n",
    "class SklearnHelper(object):\n",
    "    def __init__(self, clf, seed=0, params=None):\n",
    "        params['random_state'] = seed\n",
    "        self.clf = clf(**params)\n",
    "\n",
    "    def train(self, x_train, y_train):\n",
    "        self.clf.fit(x_train, y_train)\n",
    "\n",
    "    def predict(self, x):\n",
    "        return self.clf.predict_proba(x)\n",
    "    \n",
    "    def fit(self,x,y):\n",
    "        return self.clf.fit(x,y)\n",
    "    \n",
    "    def feature_importances(self,x,y):\n",
    "        return(self.clf.fit(x,y).feature_importances_)\n",
    "    \n",
    "    def estimators(self,x,y):\n",
    "        return(self.clf.fit(x,y).estimators_)\n",
    "    \n",
    "# Class to extend XGboost classifer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_oof(clf, x_train, y_train, x_test, isXGB):\n",
    "    oof_train = np.zeros((ntrain,))\n",
    "    oof_test = np.zeros((ntest,))\n",
    "    oof_test_skf = np.empty((NFOLDS, ntest))\n",
    "\n",
    "    for i, (train_index, test_index) in enumerate(kf):\n",
    "        x_tr = x_train[train_index]\n",
    "        y_tr = y_train[train_index]\n",
    "        x_te = x_train[test_index]\n",
    "        \n",
    "        if(isXGB):\n",
    "            clf.fit(x_tr, y_tr)\n",
    "            oof_train[test_index] = clf.predict_proba(x_te)\n",
    "            temp = clf.predict_proba(x_test)\n",
    "        else:\n",
    "            clf.train(x_tr, y_tr)\n",
    "            oof_train[test_index] = clf.predict(x_te)\n",
    "            temp = clf.predict(x_test)\n",
    "\n",
    "       \n",
    "        \n",
    "        oof_test_skf[i, :] = temp[:,1]\n",
    "        \n",
    "       \n",
    "\n",
    "    oof_test[:] = oof_test_skf.mean(axis=0)\n",
    "    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Put in our parameters for said classifiers\n",
    "# Random Forest parameters\n",
    "rf_params = {\n",
    "    'n_jobs': -1,\n",
    "    #'n_estimators': 500,\n",
    "     'warm_start': True, \n",
    "     #'max_features': 0.2,\n",
    "    'max_depth': 6,\n",
    "    'min_samples_leaf': 2,\n",
    "    'max_features' : 'sqrt',\n",
    "    'verbose': 0\n",
    "}\n",
    "\n",
    "# Extra Trees Parameters\n",
    "et_params = {\n",
    "    'n_jobs': -1,\n",
    "    #'n_estimators':500,\n",
    "    #'max_features': 0.5,\n",
    "    'max_depth': 8,\n",
    "    'min_samples_leaf': 2,\n",
    "    'verbose': 0\n",
    "}\n",
    "\n",
    "# AdaBoost parameters\n",
    "ada_params = {\n",
    "   # 'n_estimators': 500,\n",
    "    'learning_rate' : 0.75\n",
    "}\n",
    "\n",
    "# Gradient Boosting parameters\n",
    "gb_params = {\n",
    "    #'n_estimators': 500,\n",
    "     #'max_features': 0.2,\n",
    "    'max_depth': 5,\n",
    "    'min_samples_leaf': 2,\n",
    "    'verbose': 0\n",
    "}\n",
    "\n",
    "# Support Vector Classifier parameters \n",
    "svc_params = {\n",
    "    'kernel' : 'linear',\n",
    "    'C' : 0.025\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create 5 objects that represent our 4 models\n",
    "from xgboost import XGBClassifier\n",
    "xgb = XGBClassifier()\n",
    "rf = SklearnHelper(clf=RandomForestClassifier, seed=SEED, params=rf_params)\n",
    "et = SklearnHelper(clf=ExtraTreesClassifier, seed=SEED, params=et_params)\n",
    "ada = SklearnHelper(clf=AdaBoostClassifier, seed=SEED, params=ada_params)\n",
    "gb = SklearnHelper(clf=GradientBoostingClassifier, seed=SEED, params=gb_params)\n",
    "svc = SklearnHelper(clf=SVC, seed=SEED, params=svc_params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train=X_train\n",
    "y = y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = y_train.ravel()\n",
    "X_train = X_train.values\n",
    "X_test = X_test.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3216, 304)\n"
     ]
    }
   ],
   "source": [
    "print X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training is complete\n"
     ]
    }
   ],
   "source": [
    " #Create our OOF train and test predictions. These base results will be used as new features\n",
    "et_oof_train, et_oof_test = get_oof(et, X_train, y_train, x_test) # Extra Trees\n",
    "rf_oof_train, rf_oof_test = get_oof(rf,X_train, y_train, X_test, False) # Random Forest\n",
    "ada_oof_train, ada_oof_test = get_oof(ada, X_train, y_train, X_test, False) # AdaBoost \n",
    "gb_oof_train, gb_oof_test = get_oof(gb,X_train, y_train, X_test, False) # Gradient Boost\n",
    "xgb_oof_train, xgb_oof_test = get_oof(xgb, X_train, y_train, X_test, True) # xgb\n",
    "svc_oof_train, svc_oof_test = get_oof(svc,X_train, y_train, X_test, False) # Support Vector Classifier\n",
    "\n",
    "print(\"Training is complete\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ada_feature = ada.feature_importances(X_train, y_train)\n",
    "gb_feature = gb.feature_importances(X_train,y_train)\n",
    "rf_feature = rf.feature_importances(X_train, y_train)\n",
    "xgb_feature = xgb.feature_importances_\n",
    "svc_feature = svc.feature_importances(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ada_features=list(ada_feature)\n",
    "gb_features = list(gb_feature)\n",
    "rf_features = list(rf_feature)\n",
    "xgb_features=  list(xgb_feature)\n",
    "svc_features = list(svc_feature)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Feature Importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimators = ada.estimators(X_train, y_train)\n",
    "print type(estimators)\n",
    "std = np.std([ada_feature for tree in estimators],\n",
    "             axis=0)\n",
    "indices = np.argsort(ada_feature)[::-1]\n",
    "x_axis = []\n",
    "for index in indices:\n",
    "    x_axis.append(x_train.columns[index])\n",
    "    \n",
    "x_axis = x_axis[:10]\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(x_train.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], ada_feature[indices[f]]))\n",
    "\n",
    "y_pos = np.arange(len(x_axis))\n",
    "# Plot the feature importances of the forest\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature ranking:\n",
      "1. feature 1 (0.100000)\n",
      "2. feature 4 (0.100000)\n",
      "3. feature 3 (0.060000)\n",
      "4. feature 6 (0.060000)\n",
      "5. feature 0 (0.040000)\n",
      "6. feature 5 (0.040000)\n",
      "7. feature 7 (0.040000)\n",
      "8. feature 8 (0.040000)\n",
      "9. feature 36 (0.020000)\n",
      "10. feature 276 (0.020000)\n",
      "11. feature 274 (0.020000)\n",
      "12. feature 27 (0.020000)\n",
      "13. feature 68 (0.020000)\n",
      "14. feature 31 (0.020000)\n",
      "15. feature 192 (0.020000)\n",
      "16. feature 102 (0.020000)\n",
      "17. feature 254 (0.020000)\n",
      "18. feature 189 (0.020000)\n",
      "19. feature 41 (0.020000)\n",
      "20. feature 56 (0.020000)\n",
      "21. feature 278 (0.020000)\n",
      "22. feature 107 (0.020000)\n",
      "23. feature 45 (0.020000)\n",
      "24. feature 54 (0.020000)\n",
      "25. feature 200 (0.020000)\n",
      "26. feature 199 (0.020000)\n",
      "27. feature 245 (0.020000)\n",
      "28. feature 2 (0.020000)\n",
      "29. feature 144 (0.020000)\n",
      "30. feature 81 (0.020000)\n",
      "31. feature 13 (0.020000)\n",
      "32. feature 288 (0.020000)\n",
      "33. feature 15 (0.020000)\n",
      "34. feature 21 (0.020000)\n",
      "35. feature 103 (0.000000)\n",
      "36. feature 109 (0.000000)\n",
      "37. feature 108 (0.000000)\n",
      "38. feature 87 (0.000000)\n",
      "39. feature 88 (0.000000)\n",
      "40. feature 106 (0.000000)\n",
      "41. feature 105 (0.000000)\n",
      "42. feature 104 (0.000000)\n",
      "43. feature 95 (0.000000)\n",
      "44. feature 90 (0.000000)\n",
      "45. feature 101 (0.000000)\n",
      "46. feature 100 (0.000000)\n",
      "47. feature 91 (0.000000)\n",
      "48. feature 99 (0.000000)\n",
      "49. feature 98 (0.000000)\n",
      "50. feature 92 (0.000000)\n",
      "51. feature 93 (0.000000)\n",
      "52. feature 97 (0.000000)\n",
      "53. feature 96 (0.000000)\n",
      "54. feature 94 (0.000000)\n",
      "55. feature 110 (0.000000)\n",
      "56. feature 89 (0.000000)\n",
      "57. feature 119 (0.000000)\n",
      "58. feature 111 (0.000000)\n",
      "59. feature 136 (0.000000)\n",
      "60. feature 130 (0.000000)\n",
      "61. feature 131 (0.000000)\n",
      "62. feature 132 (0.000000)\n",
      "63. feature 133 (0.000000)\n",
      "64. feature 134 (0.000000)\n",
      "65. feature 135 (0.000000)\n",
      "66. feature 137 (0.000000)\n",
      "67. feature 112 (0.000000)\n",
      "68. feature 138 (0.000000)\n",
      "69. feature 139 (0.000000)\n",
      "70. feature 140 (0.000000)\n",
      "71. feature 141 (0.000000)\n",
      "72. feature 142 (0.000000)\n",
      "73. feature 143 (0.000000)\n",
      "74. feature 129 (0.000000)\n",
      "75. feature 128 (0.000000)\n",
      "76. feature 127 (0.000000)\n",
      "77. feature 126 (0.000000)\n",
      "78. feature 125 (0.000000)\n",
      "79. feature 124 (0.000000)\n",
      "80. feature 123 (0.000000)\n",
      "81. feature 122 (0.000000)\n",
      "82. feature 121 (0.000000)\n",
      "83. feature 120 (0.000000)\n",
      "84. feature 85 (0.000000)\n",
      "85. feature 118 (0.000000)\n",
      "86. feature 117 (0.000000)\n",
      "87. feature 116 (0.000000)\n",
      "88. feature 115 (0.000000)\n",
      "89. feature 114 (0.000000)\n",
      "90. feature 113 (0.000000)\n",
      "91. feature 86 (0.000000)\n",
      "92. feature 71 (0.000000)\n",
      "93. feature 84 (0.000000)\n",
      "94. feature 47 (0.000000)\n",
      "95. feature 44 (0.000000)\n",
      "96. feature 43 (0.000000)\n",
      "97. feature 42 (0.000000)\n",
      "98. feature 40 (0.000000)\n",
      "99. feature 39 (0.000000)\n",
      "100. feature 38 (0.000000)\n",
      "101. feature 37 (0.000000)\n",
      "102. feature 35 (0.000000)\n",
      "103. feature 34 (0.000000)\n",
      "104. feature 33 (0.000000)\n",
      "105. feature 32 (0.000000)\n",
      "106. feature 30 (0.000000)\n",
      "107. feature 29 (0.000000)\n",
      "108. feature 28 (0.000000)\n",
      "109. feature 26 (0.000000)\n",
      "110. feature 25 (0.000000)\n",
      "111. feature 24 (0.000000)\n",
      "112. feature 23 (0.000000)\n",
      "113. feature 22 (0.000000)\n",
      "114. feature 20 (0.000000)\n",
      "115. feature 19 (0.000000)\n",
      "116. feature 18 (0.000000)\n",
      "117. feature 17 (0.000000)\n",
      "118. feature 16 (0.000000)\n",
      "119. feature 14 (0.000000)\n",
      "120. feature 12 (0.000000)\n",
      "121. feature 11 (0.000000)\n",
      "122. feature 10 (0.000000)\n",
      "123. feature 9 (0.000000)\n",
      "124. feature 46 (0.000000)\n",
      "125. feature 48 (0.000000)\n",
      "126. feature 83 (0.000000)\n",
      "127. feature 49 (0.000000)\n",
      "128. feature 82 (0.000000)\n",
      "129. feature 80 (0.000000)\n",
      "130. feature 79 (0.000000)\n",
      "131. feature 78 (0.000000)\n",
      "132. feature 77 (0.000000)\n",
      "133. feature 76 (0.000000)\n",
      "134. feature 75 (0.000000)\n",
      "135. feature 74 (0.000000)\n",
      "136. feature 73 (0.000000)\n",
      "137. feature 72 (0.000000)\n",
      "138. feature 146 (0.000000)\n",
      "139. feature 70 (0.000000)\n",
      "140. feature 69 (0.000000)\n",
      "141. feature 67 (0.000000)\n",
      "142. feature 66 (0.000000)\n",
      "143. feature 65 (0.000000)\n",
      "144. feature 64 (0.000000)\n",
      "145. feature 63 (0.000000)\n",
      "146. feature 62 (0.000000)\n",
      "147. feature 61 (0.000000)\n",
      "148. feature 60 (0.000000)\n",
      "149. feature 59 (0.000000)\n",
      "150. feature 58 (0.000000)\n",
      "151. feature 57 (0.000000)\n",
      "152. feature 55 (0.000000)\n",
      "153. feature 53 (0.000000)\n",
      "154. feature 52 (0.000000)\n",
      "155. feature 51 (0.000000)\n",
      "156. feature 50 (0.000000)\n",
      "157. feature 145 (0.000000)\n",
      "158. feature 303 (0.000000)\n",
      "159. feature 147 (0.000000)\n",
      "160. feature 262 (0.000000)\n",
      "161. feature 244 (0.000000)\n",
      "162. feature 246 (0.000000)\n",
      "163. feature 247 (0.000000)\n",
      "164. feature 248 (0.000000)\n",
      "165. feature 249 (0.000000)\n",
      "166. feature 250 (0.000000)\n",
      "167. feature 251 (0.000000)\n",
      "168. feature 252 (0.000000)\n",
      "169. feature 253 (0.000000)\n",
      "170. feature 255 (0.000000)\n",
      "171. feature 256 (0.000000)\n",
      "172. feature 257 (0.000000)\n",
      "173. feature 258 (0.000000)\n",
      "174. feature 259 (0.000000)\n",
      "175. feature 260 (0.000000)\n",
      "176. feature 243 (0.000000)\n",
      "177. feature 242 (0.000000)\n",
      "178. feature 241 (0.000000)\n",
      "179. feature 232 (0.000000)\n",
      "180. feature 226 (0.000000)\n",
      "181. feature 227 (0.000000)\n",
      "182. feature 228 (0.000000)\n",
      "183. feature 229 (0.000000)\n",
      "184. feature 230 (0.000000)\n",
      "185. feature 231 (0.000000)\n",
      "186. feature 233 (0.000000)\n",
      "187. feature 240 (0.000000)\n",
      "188. feature 234 (0.000000)\n",
      "189. feature 235 (0.000000)\n",
      "190. feature 236 (0.000000)\n",
      "191. feature 237 (0.000000)\n",
      "192. feature 238 (0.000000)\n",
      "193. feature 239 (0.000000)\n",
      "194. feature 261 (0.000000)\n",
      "195. feature 263 (0.000000)\n",
      "196. feature 148 (0.000000)\n",
      "197. feature 264 (0.000000)\n",
      "198. feature 286 (0.000000)\n",
      "199. feature 287 (0.000000)\n",
      "200. feature 289 (0.000000)\n",
      "201. feature 290 (0.000000)\n",
      "202. feature 291 (0.000000)\n",
      "203. feature 292 (0.000000)\n",
      "204. feature 293 (0.000000)\n",
      "205. feature 294 (0.000000)\n",
      "206. feature 295 (0.000000)\n",
      "207. feature 296 (0.000000)\n",
      "208. feature 297 (0.000000)\n",
      "209. feature 298 (0.000000)\n",
      "210. feature 299 (0.000000)\n",
      "211. feature 300 (0.000000)\n",
      "212. feature 301 (0.000000)\n",
      "213. feature 285 (0.000000)\n",
      "214. feature 284 (0.000000)\n",
      "215. feature 283 (0.000000)\n",
      "216. feature 271 (0.000000)\n",
      "217. feature 265 (0.000000)\n",
      "218. feature 266 (0.000000)\n",
      "219. feature 267 (0.000000)\n",
      "220. feature 268 (0.000000)\n",
      "221. feature 269 (0.000000)\n",
      "222. feature 270 (0.000000)\n",
      "223. feature 272 (0.000000)\n",
      "224. feature 282 (0.000000)\n",
      "225. feature 273 (0.000000)\n",
      "226. feature 275 (0.000000)\n",
      "227. feature 277 (0.000000)\n",
      "228. feature 279 (0.000000)\n",
      "229. feature 280 (0.000000)\n",
      "230. feature 281 (0.000000)\n",
      "231. feature 225 (0.000000)\n",
      "232. feature 224 (0.000000)\n",
      "233. feature 223 (0.000000)\n",
      "234. feature 222 (0.000000)\n",
      "235. feature 167 (0.000000)\n",
      "236. feature 168 (0.000000)\n",
      "237. feature 169 (0.000000)\n",
      "238. feature 170 (0.000000)\n",
      "239. feature 171 (0.000000)\n",
      "240. feature 172 (0.000000)\n",
      "241. feature 173 (0.000000)\n",
      "242. feature 174 (0.000000)\n",
      "243. feature 175 (0.000000)\n",
      "244. feature 176 (0.000000)\n",
      "245. feature 177 (0.000000)\n",
      "246. feature 178 (0.000000)\n",
      "247. feature 179 (0.000000)\n",
      "248. feature 180 (0.000000)\n",
      "249. feature 181 (0.000000)\n",
      "250. feature 166 (0.000000)\n",
      "251. feature 165 (0.000000)\n",
      "252. feature 164 (0.000000)\n",
      "253. feature 155 (0.000000)\n",
      "254. feature 149 (0.000000)\n",
      "255. feature 150 (0.000000)\n",
      "256. feature 302 (0.000000)\n",
      "257. feature 152 (0.000000)\n",
      "258. feature 153 (0.000000)\n",
      "259. feature 154 (0.000000)\n",
      "260. feature 156 (0.000000)\n",
      "261. feature 163 (0.000000)\n",
      "262. feature 157 (0.000000)\n",
      "263. feature 158 (0.000000)\n",
      "264. feature 159 (0.000000)\n",
      "265. feature 160 (0.000000)\n",
      "266. feature 161 (0.000000)\n",
      "267. feature 162 (0.000000)\n",
      "268. feature 182 (0.000000)\n",
      "269. feature 183 (0.000000)\n",
      "270. feature 184 (0.000000)\n",
      "271. feature 214 (0.000000)\n",
      "272. feature 208 (0.000000)\n",
      "273. feature 209 (0.000000)\n",
      "274. feature 210 (0.000000)\n",
      "275. feature 211 (0.000000)\n",
      "276. feature 212 (0.000000)\n",
      "277. feature 213 (0.000000)\n",
      "278. feature 215 (0.000000)\n",
      "279. feature 206 (0.000000)\n",
      "280. feature 216 (0.000000)\n",
      "281. feature 217 (0.000000)\n",
      "282. feature 218 (0.000000)\n",
      "283. feature 219 (0.000000)\n",
      "284. feature 220 (0.000000)\n",
      "285. feature 221 (0.000000)\n",
      "286. feature 207 (0.000000)\n",
      "287. feature 205 (0.000000)\n",
      "288. feature 185 (0.000000)\n",
      "289. feature 194 (0.000000)\n",
      "290. feature 186 (0.000000)\n",
      "291. feature 187 (0.000000)\n",
      "292. feature 188 (0.000000)\n",
      "293. feature 190 (0.000000)\n",
      "294. feature 191 (0.000000)\n",
      "295. feature 193 (0.000000)\n",
      "296. feature 195 (0.000000)\n",
      "297. feature 204 (0.000000)\n",
      "298. feature 196 (0.000000)\n",
      "299. feature 197 (0.000000)\n",
      "300. feature 198 (0.000000)\n",
      "301. feature 201 (0.000000)\n",
      "302. feature 202 (0.000000)\n",
      "303. feature 203 (0.000000)\n",
      "304. feature 151 (0.000000)\n"
     ]
    }
   ],
   "source": [
    "indices = np.argsort(ada_feature)[::-1]\n",
    "x_axis = []\n",
    "for index in indices:\n",
    "    x_axis.append(x_train.columns[index])\n",
    "    \n",
    "x_axis = x_axis[:10]\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(x_train.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], ada_feature[indices[f]]))\n",
    "\n",
    "y_pos = np.arange(len(x_axis))\n",
    "# Plot the feature importances of the for"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAc0AAAEHCAYAAADMNh3PAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcHFW5//FPm4U1LCGDigIRhS8QxGuIkSUhJCAKIiDL\nFRHZUeEicAM/uCpLEBFQEQQXVJRVyDUgy481siSAIAQQNC4PsgQCBBIEUdaQZO4f54xpxllqerqn\nM1Pf9+uV19RU1ak6T9Wknz7nVPeptLa2YmZmZt17R7MrYGZm1l84aZqZmRXkpGlmZlaQk6aZmVlB\nTppmZmYFOWmamZkVNLjZFTDrryT9Blg5Ij7UxT7jgEsjYmQ3x5oD7BMRd9Wzjh2c5+PAnyPiqQ62\ntQKPAYuqVj8ZER+v8VzLAZ+JiItrqmyxc7QCa0fE0406RyfnfSfw0Yi4ti/Pa83npGlWA0mbAC8D\nL0raIiLuaXadCvpv4BvAvyXNbJs6JqAPA/sCDUuaTTQR2A5w0iwZJ02z2uwHTAPeICWGfyVNSccD\nXwReoOpFVdKKwAXAfwBDgSsj4piqY06SdC4wArgoIo7P5fYETiL9f30WOCQiHpO0PHA26QV8CXAD\ncGxELJZ0OPBfQAX4B3AAsBewLbCRpGMj4n+LBivpvcCPAOVVR0bEjXnbwcDRuX7zgM/n63IVsIqk\nO/O6RyNicC4zsu13SfsDOwOrAg9ExLGSvgBMBpbP1/bAiHi9mzrOAc7Msb4HODTH+wlgAbBDRLyU\nW6dHAgcCawEnRsR5+RhHAF8iDV0FcHBELJB0IfAiKVH+Avh/wGBJK0fEXh1dg4h4Msf2SdI9GE9q\nxe8ZEX+UNIL09zAKeAU4JiKmS1oNOBf4aD7eKRFxQa7fN4A9Sff1aVLvxLNdXRerL49pmvWQpEHA\nbsCVwDXAjpKG5m0bk17sx+R/m1YVPRQYBmwIjAb2z923bTbLZTYDDpP0IUnrAD8Fdo2IDYHrgR/n\n/Y8C1ia96I4mvSh/VtIw4BRgbC7zbeCTEXEC8AzwuZ4kzOwi4KGI2ADYEbhU0hqS1gS+D3wsItYH\nHgVOiIjnga8A90TE+ALH3x74Uk6Y43P9J+Vu7Zfz70VsEhGj8/6XkN7YfID0Wrdb1X7rR8R/kK7Z\n2TmWzUnJcJt83Z4CTqsqsy3pmp6RY74iJ8wOr0FVuR2BH+ZrdzvpvgGcDvwpItYjvQm7PHdpn0l6\nE7QhKXGeLGkTSaOA/8wxbkB6U7JdwetideKkadZzHwdmRcQ/IuI1YAbwqbxta2BmRDwfEYuBS9sK\nRcSZwC4R0RoRLwF/BNarOu4vImJxRMwHZgJbAB8Dbo+IR/M+5wMTJQ0mtWB+EhGLcivsF6Tk8wbQ\nChwk6Z0RMS0ivlUwthmS/lL176eSViK1Zs/KcTwK3ElKxPOBVaq6dO9sF1NRj0TEX/Pyp4D/rWpB\nncfbE15Xrs4//wC8HhEzIqKVdK3Xqtrv5zmWILUox5Ku5xU5JkjXevuqMrdGxBvtT1jgGvwpIh7I\nyw8C6+TlHYHL8zF+B4yMiDdz/N+LiCURsQD4VY7/70AL8DlJq0fEuY0cL7aOuXvWrOf2J7Uu/55/\nHwysTmp5Die1jNq81LYgaX3gu5I2BBaTWokXVO27oGr55XzM1upjRMTLkiqkLtyW6m15ec2IeEvS\ntsBXSa2U3wOHRcQfCsT2b2OaktYidQfeLbX1zrIycFtudX9d0s7AIFJL+pEC52nvxarl1YBPS2pL\nWO8gdWcX8c/8czGpy5Oq3wd1cr6XSNe6hdT9Xb1+zU7K/EuBa1D991BdjxGkRAhARLTVfTXgl5La\nHshaAZgWEc9I2g04BjhX0h2k1vncjupljeGkadYDklYHtgGGR8TCvG4w8LSktiS2alWRlqrlHwAP\nkLpaF+enb6sNr1penfQivZDU4qw+/xLSeOnzwBpVZdbI69paLnvmbuNjSa21rWoIGWA+6cV+TERU\nJyIk7U0aj9w6Il6QdAjwuQ6OsRh4h6RKbvmt3sX5niWN6R7TxT69NQJ4Mi8PJ13rTq9nNz5DsWvQ\n3gu5HnPgX+O8z5Di3zUiZrcvEBG3A7fn1v93SF28Rc5ldeLuWbOe2Qu4rS1hAkTEIuBm4LOkh1bG\nSWrJLZB9qsquCfwuJ8yPAeuTWmz/Orakd+QxsvGkbr5fA1tLauvu+xIwPZ/zOlIX7KD8Ivp54HpJ\nH5Q0TdLQXM/7SS1WgLdILZnC8rmuz+dG0oqSfi5p7RzTnJws1iCNubXF9BbpQaAKKUEsBj6Yt+3b\nxSmvBXbLb0KQtIuk43pS5wI+m4+9Eek+3EuKcbccB6SHua7vpHz1dezqGnTlWlKvRdtY+IOkhsw1\nLL3WgyWdJWm0pO0l/UDSOyLiVeBhlt5X6yNOmmY9sx9Lx82qXQXsGxEPkVp1D5JaldWfu/wGcKak\n2cAE4GRS92lbC3AWcB8pyZ0VEX/KXaUHA9dI+gtpzPSLef9zgbmk8br7SUl0GjAbeAL4o6Q/AlNI\nT4sCXAFMlTS5h3EfCkzIdXgQeDx3C14OrCHp0bx8PLC2pDNz7GuRWk4LSU8A3yTpfuChzk4UEQ8C\n3ySNr/6Z9GDVNT2sb3fmS3oIuAM4IiJeioj7SC23O3OcqwFf66T8dNLTzrPo+hp05Tjgvfmp3/8F\n9s5j0ycAq0oK0r0dBPw+13VF4JF8Xz8DnFhT9FaziufTNLMyadYXItjA4JammZlZQU6aZmZmBbl7\n1szMrCC3NM3MzAry5zQHsEWLFre+9NJrza5G06y++oo4/nLGX+bYwfH3Nv6WlmGVzra5pTmADR48\nqPudBjDHX974yxw7OP5Gxu+kaWZmVpCTppmZWUFOmmZmZgU5aZqZmRXkpGlmZlaQk6aZmVlBTppm\nZmYFOWmamZkV5G8EGsgqFVqaXYdm8vcqm1mduaVpZmZWkJOmmZlZQU6aZmZmBTlpmpmZFVSXpClp\njqSVe1F+a0lr1qMuvajD/pI+LWkbSVfkdS/UcJw9erDvOpLG5uWzJb2vp+czM7O+s6y0NA8Empo0\nI+LCiLiqN8eQNBSY3IMik4Cx+fxHRcQTvTm/mZk1VpcfOZG0DnApsDjvewswLCKOyS3L2RExMu/+\nVUnjgUXAp4FXgYuAdYE3gH2BfwKXASsBKwJfBlYFdgVGSdodGAMcnY9zf0QcLWlV4ApgBeAG4JCI\neJ+kbYBvAm8BT5OS72eBHYC1gL8Av42In+V4/gSMj4i/dRDrFOAFYHYH22YAh0fEbEmHAyOAs4Bf\nAsvlf/8FHAR8UNIPgfuq6rFXjmkssDxwHnANMAV4S9JTpGR7ODAXuBBYDRgCHBERD0p6NJfZEvg7\n8MmIWPJvN83MzBqmu89p7gH8OiJOkTQa2B4Y1sm+v4+Ir0r6DvB54HXguYjYW9JewM7ArcD5EXG1\npEnAcRGxu6SHSAnjReB4YIuIeFPSLyVtBYwG/hQRR0o6DGibVfs84GMRMVfS94G9gVZgHVJyGQV8\nF/iZpI2BxztKmDXaFng6Ig6StB6wAfBt4KMRcZik/avqsRwwJyImS1oBeCwizpd0IfBCRFwrqa2F\neiQp0Z8haQwpOU8A1gMuym8ifgtsCjxUp1gGrJaWzv5cy6HM8Zc5dnD8jYq/u6Q5HbhK0mqklt5z\npFZWR27PP+8DtiZ1/d4KEBFTAXKL8QRJx5ASyavtjjGKlGhulgSpFbousBEwI+9zLXCspOFAa0TM\nrTr/BOBBYFZEtAKzJa0mqQXYBfhFN/H2xD3ANySdB/wqIm6SNLLdPm31eEPScEl3Awuhy+8cGAOc\nChAR90v6QF7/j4j4fV5+mnRtrBsLFvyz2VVompaWYaWNv8yxg+PvbfxdJdwuxzQjYjbwIeBO4DRS\nK67NkHa7t7ZbXtzB8Y8CnomIccChHZxyIfBARGyT/304Ii4jtSzbuiJbq35WqsoOrdpnYdX6y4Dd\nSC3DazqKs4B/izsi5pGuza+AQyWd2Ek8SJpAGr+cEBHbAG92c67quAbln4va7VfBzMz6VJdJM3er\nbhIRV5O6TY8B3p03j2u3+/j8c3Pgz8AsUqJA0k6SvkpqpT6W9/s0KdFBSnaDgQA2anuSVtLJkt6T\ny4zJ++4AEBEvAa153BVSK/P+DsK4HDgAmBcRr3UVbxf+wdK4t8p12w7YLiKmk8Zmx1TF0d4IYG5E\nvCVpZ2BQfmioo/1nARPzOTangzFWMzNrju6enn0E+L6k24CTgH0A5QdjNmRpyw7Sgzy3kMbaLgWm\nAitJmklqYV4EXAxMljQduBd4l6QDgJmk7t/35X1vkPQbYA3gWdKDMePzed9JasUCHAJcltcPyed8\nm4h4HniF1OKs1U+AH0i6PtcH4FHga/ncF5PGM+cBQyVNa1f+FmD9fC3eD1wH/IjUxXuspM9V7fs9\nYLN8zU8njXGamdkyoNLaD77UWtK6wIYRcbOkLYCTI2L7gmVHADcBY0v3tGmlsuzf3EZqbfW4Tknj\nL3Ps4PjrMKbZ6fBXf5nl5GVSC/VE0ljeEUUKSdoVOBmY3JYwJf0KGN7++BGxSx3ra2ZmA1C/aGla\njdzS9LvtksZf5tjB8bulabUpe9JodgXMbMBZVr5Gz8zMbJnnpGlmZlaQk6aZmVlBHtMcyCqVco/r\n+SE3M6sztzTNzMwKctI0MzMryEnTzMysICdNMzOzgpw0G0jSypLmNLseZmZWH06aZmZmBfkjJ3Um\naRXgSmB54K687q/ADcB84ALgZ6S5RBcDB0fEU5KOBvYgvZG5ISJOljSFNBfnB4D1SHOaHgiMBHaM\niMf7LjIzM3PSrL99gNkR8d+SPgN8ljTX540RcZOknwFnRsQtknYETiDNCwppYu8lwOOSzsrrhkfE\nJySdCuyXl08BdgbO7svA+qOWlmHNrkJTlTn+MscOjr9R8Ttp1t/GpEm1AWZUrb8v/9ySNJH38cAg\nYEFe/1out4jUuhzertw8oO3T+s+TJui2bpT6C+tLPNNFmWMHx1+HWU463eakWX8VUmsR3j5mvLDq\n554RMa9tQ55kezLw4Yh4RdLsqnKLOlnudOoaMzNrDD8IVH8BjMnLEzvYfi+wK4CkSZL2JrUs5+eE\nORpYlzTmaWZmyxAnzfq7GNhc0q2AWNql2mYKsKukO4CTgHuAh4BXJP0G+AzwY+CHfVZjMzMrpNLq\nL7UeuCqVct/csk/CXeJxrTLHDo6/DmOanQ5/uaVpZmZWkJOmmZlZQU6aZmZmBfkjJwNZ2cf0ml0B\nMxtw3NI0MzMryEnTzMysICdNMzOzgjymOZBVKqUf1yt1/P4MtlnduaVpZmZWkJOmmZlZQU6aZmZm\nBTlpdkLS7l1s21lSp7OQSLpQ0k5dbN9U0gZ5eaqkFXpXWzMz6wtOmh2QNBL4bBe7TKZ3U3ftBmwA\nEBF7RcTrvTiWmZn1ET8927EfAGMlnQT8B7AaMAQ4AhgFbA7cKGlb4HRgLLA8cF5EnN/VgSV9EPgS\nsEDSfOCXwCbA94H5wGakhz7PAA4gzbU5AXgF+AmwXq7LiRFxWx1jNjOzbril2bFvAzNJc2H+NiIm\nAkcBZ0XEJcBzwA6k6zcnIsYB44Gvd3fgiPgDcBPwlYi4r93mRRGxLfAHYMuI2C4vTwT2BubluuwK\nnN37MM3MrCfc0uzaGOBUgIi4X9IHqjdGxBuShku6G1hI7z8W2JZE5wF/ycvPA6uSWrfjJY3L61eQ\nNDQiFvbynDaAtbQMa3YVmqbMsYPjb1T8TppdawWqJyMdVL1R0gRgEjAhIt6S9Eovz7eok+UKKSmf\nGhGX9/IcViJl/cJ+T8Ls+Hs5CXWn29w927ElpDcUs0hdo0jaHJjdbvsIYG5OmDsDg7p6qraD4/fE\nvcAuuS5rSvpmD8ubmVkvOWl27M/AaGANYDNJt5Ee+Dkyb58B3AXcD6wvaSbwfuA64EcFjn8ncE5+\nkKioXwKv5K7g/5+PYWZmfajS6u+nHLgqFd/cMivxfKrunnT8veyerXS2zWOaDSJpHeDiDjbNjIiT\n+ro+ZmbWe06aDRIRTwHbNLseZmZWPx7TNDMzK8gtzYGsxGNa4HGdUs8latYgbmmamZkV5KRpZmZW\nkJOmmZlZQR7THMgqldKPa5U6fn8G26zu3NI0MzMryEnTzMysICdNMzOzgpw0zczMCnLSXEZI2rng\ntGJI2kTSjAZXyczM2nHSXHZMBgolTTMzaw5/5KQXJA0BLgLWBd4AbgM2BdYC9gJ2BfYmTTp9dUSc\nKem9wCX5EEOA/YAtgc2BG/Mcm4d0Um4a8CbwcN9EaGZm1TyfZi9IOhjYOCImS9oLWB3Yl5QERwI/\nBybl3X9DSqTvBFaOiNslHQiMioijJc0BNiF9tLCjcocDz0TE9yQdB+wQEdt0WUHPp1lu/r9tVivP\np9kgo4FbASJiqqT9gVkR0SppLLA+cHvedxgpkT4BnCPpZFKSfaDdMTsrtzGppQkwA9ih7tHYgFPW\nL6wv/Zf1O/7eTkLd6TYnzd5ZzL+PCy+s+nl9RHyxeqOkC4CbI+I8SXsAO3VQvqNyx5G6a+ngnGZm\n1gecNHtnFqkbdZqknUhjmW0eAM6QtCLwOnA28D/ACOAxSRVgF2BQ3n8J6X50Vi6AMXn7xAbHZWZm\nHXCLpXemAitJmgkcRVU/eEQ8RUp4dwC/BZ6LiNeBHwPnAjfm8hMkbU/qcr0LeK2Tct8DDpR0M6lb\n18zM+pgfBBrI/CBQuZV4EnKP6Tn+Xo5pdvogkFuaZmZmBTlpmpmZFeQHgQayEnfPgbuoSj2XqFmD\nuKVpZmZWkJOmmZlZQU6aZmZmBXlMcyCrVEo/rlXq+P1xMrO6c0vTzMysICdNMzOzgpw0zczMCnLS\nNDMzK8hJsw4k7S/pO31ZVtLhkqbUck4zM6uNk6aZmVlB/shJ/bxP0g3A2sBZwJvAl0kTVf8xIr4g\naQhwEbAu8Aawb/UBJJ0GvAqcBvwEWA8YApwYEbdJ2pY0bdhzwDzg8b4IzMzMEifN+tkAGA2sAjwM\nnAJ8IiL+LukOSR8EPkqaH3NvSXsBO5MmmkbSnsDaEbGPpM8D8yLiIEkjgNuATUnJdJ+IeDgnaCdN\n61JLy7BmV6Fpyhw7OP5Gxe+kWT93RcRbwN8k/QP4G3CNJICNgDVISfVWgIiYCmlMExgF7AZsnI+1\nJTBe0rj8+wqShgIjI+LhvG4msEKjg7L+raxfWF/6L+t3/L2dT7PTbU6a9dP+61cuJ7Ucn5N0XV63\nmI7HkUcCfwT2AC4FFgKnRsTl1TtJWlL1q8ejzcz6mJNm/WwhaRAwnDSuOT8nzLWBMcBQYBYwCZgm\naSdSl+uzwPXAGcBdkn4N3AvsAlwuaU3gqIj4KvCMUtP1EWAb4J6+DNDMrOycNOvnL8A04APAocB2\nkmaRxje/RXo4aHRePxN4C9gP+BhARCyQdBLwI+A/gUmS7gYGAVPyOb4GXAE8Ccztm7DMzKxNpdVf\n6jxwVSq+uWVW4knIPabn+Hs5plnpbJvHxczMzApy0jQzMyvIY5oDWYm758BdVKWeS9SsQdzSNDMz\nK8hJ08zMrCAnTTMzs4I8pjmQVSqlH9cqdfz+OJlZ3bmlaWZmVpCTppmZWUFOmmZmZgU5aZqZmRXk\npNkPSPqEpEPbrbtf0sgmVcnMrJT89Gw/EBE3NbsOZmbmpNkvSNof2IQ0J+cWQORlMzPrQ06a/cd6\nwDrAWOA9wKPNrY71By0tw5pdhaYpc+zg+BsVv5Nm/7EpcHNELAHmSnq82RWyZV9Zv7C+9F/W7/h7\nO59mp9v8IFD/UQGWVP3ue2dm1sfc0uw/HgY2k1QhddO+r8n1MTMrHSfN/uNxYAFwD/AI8FBzq2Nm\nVj5Omv1ARFzY7DqYmZnHxczMzApz0jQzMyvI3bMDWWurHzsvc/zNroDZAOSWppmZWUFOmmZmZgU5\naZqZmRXkMc2BrFIp/biW4y+p1tZm18AGKLc0zczMCnLSNDMzK8hJ08zMrCAnTTMzs4KcNJdxkvZo\ndh3MzCxx0lyGSRoKTG52PczMLPFHTjJJQ4CLgHWBN4ADgSnAesBywIkRMV3SY8BPgT2AR4EHgD2B\nv0bE5yRdCMwHNiM98X8GcAAwApgAvAL8JB93SD7ubZJmALcAE/O+nwKOAz4o6YfA6cClwGLSfdsn\nIp5s3BUxM7P2nDSX2g94LiL2lrQXsD/wRkRMkLQWMAPYABgEPEhKhk8BV0bEWElPSVotH2tRRGwr\n6RfAlhGxnaRLSAlxGDAvIg6SNAK4Ddg0l3s5lzsd2A34NvDRiDhM0mTg1xFxiqTRwLsBJ02zTrS0\nDGt2FZrK8TcmfifNpUYDtwJExFRJ55ASJRHxrKQ3JQ3P+94XEa2Sngd+l9fNB1Zt255/zgP+kpef\nz9s3B8ZLGpfXr5C7YQHuzD+fBtZoV7/pwFU5MV8REff0KlqzAa7UX9Zf9skKehl/VwnXY5pLLebt\n16MVqFT9PhRYkpcXVa2vXq4U2L4QODUitsn/1o+IhV0cC4CImA18iJRYT5O0b/chmZlZPTlpLjUL\nmAQgaSfgb6TuVCStDSyJiL/X4Tz3Arvk464p6Ztd7LuE3BuQu4w3iYirgeOBMXWoi5mZ9YC7Z5ea\nCmwnaSbwFnAQcIKk20mtzC/W6Ty/BCZJups0Pjqli33nAUMlTQNOA86T9AqpVXxEnepjZmYFVVr9\nxcYDV6Xim2vl5AnYHX/vxjQrnW1z96yZmVlBTppmZmYFeUxzIHMXleMvafylnUfUGs4tTTMzs4Kc\nNM3MzApy0jQzMyvIY5oDWaVS+rEdx19S/iidNYhbmmZmZgU5aZqZmRXkpGlmZlaQk6aZmVlBTpp1\nJmnnqvkxu9t3E0kzajjH1pLW7HHlzMysV5w0628yaVaURjoQcNI0M+tj/sgJIGkIcBGwLvAGcBuw\nKbAWsBewK7A3aX7LqyPiTEnvBS7JhxgC7AdsCWwO3ChpW+CQTspNA94EHu6mXqsAlwErASsCXwZW\nzfUZJWn3iHiqLhfBzMy65ZZmsh/wXERsBfwUeBFYB9ia1GrcAxiXf99d0jrAu4GvR8RE4OfAYRFx\nCfAcsAPwnk7KHQFMjYhtgGe7qde7gPPzOb4CHBcRvwYeAg5wwjQz61tuaSajgVsBImKqpP2BWRHR\nKmkssD5we953GDASeAI4R9LJwOrAA+2O2Vm5jUktTYAZpATbmedJE2EfAywHvFpTdGYl1NIyrNlV\naCrH35j4nTSTxfx7q3th1c/rI+KL1RslXQDcHBHnSdoD2KmD8h2VO47UXUsH52zvKOCZiPi8pDHA\ndwpFY2alneEFyj3DDdRlEupOtzlpJrOAScA0STuRxjLbPACcIWlF4HXgbOB/gBHAY5IqwC7AoLz/\nEtJ17axcAGPy9ond1GsE8Pu8/GmWPmDUdg4zM+tDHtNMpgIrSZpJat1V2jbkccOzgTuA35LGPl8H\nfgycC9yYy0+QtD2py/Uu4LVOyn0POFDSzaRu3a5cDEyWNB24F3iXpAOAmcAVkkbVIXYzMyuo0uov\nNh64KhXfXCsnT8Du+HvXPVvpbJu7+JYBkn5IekCovR1y69TMzJYBTprLgIg4rNl1MDOz7nlM08zM\nrCC3NAcyj+s4/pLGX9rJt63h3NI0MzMryEnTzMysICdNMzOzgjymOZBVKqUf23H85VXm2KHk8Tfw\n+wfc0jQzMyvISdPMzKwgJ00zM7OCnDTNzMwKKk3SlLS/pIbMRylpiqTDayi3sqQ5NZTbVNIGPS1n\nZma9U5qkOcDsBjhpmpn1sdJ95ETSd4GxwPLAeRFxvqQLgSsi4ro8CfUewBTgIuBxYFPgdxFxsKR1\n8/pBwJPAfvnQm0i6DlgfODIibpK0G3A0sAi4PyKOlrQKcGU+/13d1HVwPtd7gZVynZ4EvgQskDQ/\nIu6rw2UxM7MCytjSnBMR44DxwNe72Xcz4CvAR4AdJa0GnAp8NyLGA88CY/K+IyJiJ+AI4EuSVgaO\nByZFxARgbUlbAfsAs3P5h7o5/3Bgei7/n8DJEfEH4CbgK06YZmYda2kZVvO/rpSupQkMl3Q3sJDu\nP//7aEQ8ByDpWWBVYDRwJEBEHJu37cDSVuMzeb9RwDrAzZLI69YlzZs5M+87o5vzvwR8RNIXgCXA\nGoUiNDMruV5OQt3ptrIlzc1I3aoTIuItSa/k9dVfHzGkanlRu/IVYDEdt9AXtdtvIfBARHy8eqfc\n2lySf+2upb83qbU5Pv+8v5v9zcysgcrWPTsSmJsT5s7AIElDgX8A7877jOvmGLOASQCSvi5pu072\nC2AjSWvmfU+W9J68vq1Ld2I35xoBPBERS0gP/wzN65dQvjc8ZmZNV7akeTWwvqSZwPuB64AfAZcA\nx0i6CXirm2OcBBySj/E+4PaOdoqI14CjgBsk/YbUtfoscDGwuaRbAfH2Vm57VwKfyvu+Cjwt6UTg\nTuAcSdsWiNnMzOqk0trAL7a1JqtUfHPNrHxaW3s7plnpbJu7+JYBufU4qYNNB0TEE31dHzMz65hb\nmgOZW5pmVkZuaVpNevmH09+1tAxz/CWNv8yxg+Nv5FyiZXsQyMzMrGZOmmZmZgU5aZqZmRXkMc2B\nrFJpaN9+f+D4y6vMsUPJ42/gA65uaZqZmRXkpGlmZlaQk6aZmVlBTppmZmYFOWn2U5Lm5Imuzcys\njzhpmpmZFeSPnDSJpFWBK4AVgBuAQ4ADgG+Spid7GjgQWA64DFgJWBH4ckTc14w6m5mVnZNm8+wL\n/CkijpR0GFABzgM+FhFzJX0f2Bv4DXB+RFwtaRJwHLB702ptZtYPtLQMa8hxnTSbZyNgRl6+Fjgd\neCYi5uZ1twMTgF8BJ0g6htTqfLWP62lm1u/0cpaTTrd5TLN5KsCSvNya/1VPRzM0bz+KlEzHAYf2\naQ3NzOxtnDSb5zFgTF7eAXgJaJW0Tl43AbgfGJH3Bfg0KZmamVkTOGk2z4XAeEkzgHcCi0kPA12W\n1w0BpgIXA5MlTQfuBd4l6YBmVNjMrOwqrQ38YlvrnKR1gQ0j4mZJWwAnR8T2dT1JpeKba2bl09ra\n2zHNSmdOBttJAAAEEklEQVTb/CBQ87xMakGeSBrLPKLJ9TEzs264pTmQuaVpZmXklqbVpJd/OP1d\nS8swx1/S+MscOzj+Rs4l6geBzMzMCnLSNDMzK8hJ08zMrCAnTTMzs4KcNM3MzApy0jQzMyvISdPM\nzKwgJ00zM7OCnDTNzMwK8tfomZmZFeSWppmZWUFOmmZmZgU5aZqZmRXkpGlmZlaQk6aZmVlBTppm\nZmYFOWmamZkVNLjZFbDaSToL2BxoBY6MiFlV27YDvgksBm6IiFO6K9Of1Bj7t4DxpL/70yLiV31e\n8TqpJf68bQVgNnBKRFzYp5Wuoxrv/+eAY4FFwIkRcX2fV7xOehq/pJWBi4HVgeWAkyPi5r6vee91\nE/vywI+BURExpkiZnnJLs5+SNAFYPyK2AA4Czmm3yznA7sBWwPaSNi5Qpl+oMfaJwCa5zCeAs/uy\nzvVUS/xV244HXuyTijZIjfd/DeAkYBywE7BLH1a5rmq8//sDERETgT2A7/VdjeunQOzfBh7qYZke\ncdLsv7YFrgaIiD8Dq0taBUDSesCLETE3IpYAN+T9Oy3Tz9QS+x3Anrn834GVJA3q85rXRy3xI2lD\nYGOg37awslri3w64JSL+GRHzIuILTap7PdQS/wvAGrn86vn3/qi717CvAlf1sEyPOGn2X+8CFlT9\nviCv62jbfODd3ZTpT3oce0QsjohX87qDSN1Wixte08ao5d4DnAlMbnjtGq+W+EcCK0q6VtKdkrbt\ni4o2SC1//1OBdSQ9SnoDeUxfVLQBunwNi4h/9rRMTzlpDhyVGrZ1VaY/KRy7pF1ISfPwhtaob3Ub\nv6R9gXsi4om+qVKfKnL/K6SW1m6krsoLJJXm71/SPsBTEfEBYBLw/b6oWB+o5R726r47afZfz/L2\nd0trAfM62faevK6rMv1JLbEj6ePA14AdIuLlPqhno9QS/yeBXST9FjgYOCE/MNIf1RL/88DdEbEo\nIh4D/gm09EFdG6GW+LcCbgaIiIeBtfrp8EQtr2F1fd1z0uy/ppMG9JE0Gni2rWsiIuYAq0gaKWkw\n6cGH6V2V6Wd6HLukVUkPCewUEf36QRhqiD8iPhMRH4mIzYHzSU/P3tKc6vdarX/7kyS9Iz8UtDL9\nd1yvlvgfBT6ay6wLvNJPhydqeQ2r6+uepwbrxySdDmwNLAH+C/gw8HJEXCVpa+CMvOuVEfGdjsrk\nd539Tk9jl/QFYArwSNVh9o2Ip/qw2nVTy72vKjsFmNPPP3JSy9/+F0ld8wDfiIhr+7jadVPD3//K\nwM+Bd5I+cnVCRNzWhKr3WjexTwPWBkYBDwA/iYjL6vm656RpZmZWkLtnzczMCnLSNDMzK8hJ08zM\nrCAnTTMzs4KcNM3MzApy0jQzMyvISdPMzKyg/wNFUg/HP1jEpAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb7dbfe9090>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "indices = indices[:10]\n",
    "plt.title(\"Adaboost Feature Importances\")\n",
    "plt.barh(range(len(x_axis)), ada_feature[indices],\n",
    "       color=\"r\", yerr=std[indices], align=\"center\")\n",
    "plt.yticks(range(len(x_axis)), x_axis)\n",
    "#plt.xlim([-1, y_pos])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, colsample_bylevel=1, colsample_bytree=1,\n",
       "       gamma=0, learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "       min_child_weight=1, missing=None, n_estimators=100, nthread=-1,\n",
       "       objective='binary:logistic', reg_alpha=0, reg_lambda=1,\n",
       "       scale_pos_weight=1, seed=0, silent=True, subsample=1)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = XGBClassifier()\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<type 'list'>\n",
      "Feature ranking:\n",
      "1. feature 250 (0.171950)\n",
      "2. feature 254 (0.148393)\n",
      "3. feature 3 (0.144511)\n",
      "4. feature 5 (0.128775)\n",
      "5. feature 4 (0.063022)\n",
      "6. feature 274 (0.047912)\n",
      "7. feature 6 (0.029963)\n",
      "8. feature 1 (0.025357)\n",
      "9. feature 276 (0.022903)\n",
      "10. feature 34 (0.022151)\n",
      "11. feature 301 (0.019139)\n",
      "12. feature 288 (0.017893)\n",
      "13. feature 7 (0.014105)\n",
      "14. feature 268 (0.011970)\n",
      "15. feature 278 (0.008611)\n",
      "16. feature 2 (0.008233)\n",
      "17. feature 200 (0.007637)\n",
      "18. feature 10 (0.007467)\n",
      "19. feature 8 (0.007051)\n",
      "20. feature 81 (0.006252)\n",
      "21. feature 139 (0.005427)\n",
      "22. feature 134 (0.005395)\n",
      "23. feature 27 (0.005286)\n",
      "24. feature 45 (0.004810)\n",
      "25. feature 31 (0.004717)\n",
      "26. feature 95 (0.004416)\n",
      "27. feature 62 (0.003968)\n",
      "28. feature 35 (0.003943)\n",
      "29. feature 116 (0.003778)\n",
      "30. feature 22 (0.002574)\n",
      "31. feature 247 (0.002374)\n",
      "32. feature 25 (0.002230)\n",
      "33. feature 50 (0.001869)\n",
      "34. feature 235 (0.001829)\n",
      "35. feature 233 (0.001731)\n",
      "36. feature 87 (0.001681)\n",
      "37. feature 0 (0.001489)\n",
      "38. feature 107 (0.001446)\n",
      "39. feature 124 (0.001410)\n",
      "40. feature 113 (0.001259)\n",
      "41. feature 33 (0.001197)\n",
      "42. feature 12 (0.001122)\n",
      "43. feature 138 (0.001094)\n",
      "44. feature 231 (0.001061)\n",
      "45. feature 21 (0.001060)\n",
      "46. feature 28 (0.001011)\n",
      "47. feature 11 (0.000977)\n",
      "48. feature 78 (0.000934)\n",
      "49. feature 245 (0.000928)\n",
      "50. feature 186 (0.000839)\n",
      "51. feature 65 (0.000737)\n",
      "52. feature 94 (0.000718)\n",
      "53. feature 192 (0.000710)\n",
      "54. feature 84 (0.000701)\n",
      "55. feature 184 (0.000698)\n",
      "56. feature 18 (0.000683)\n",
      "57. feature 92 (0.000669)\n",
      "58. feature 68 (0.000661)\n",
      "59. feature 112 (0.000644)\n",
      "60. feature 51 (0.000641)\n",
      "61. feature 178 (0.000629)\n",
      "62. feature 38 (0.000611)\n",
      "63. feature 182 (0.000605)\n",
      "64. feature 30 (0.000586)\n",
      "65. feature 121 (0.000576)\n",
      "66. feature 172 (0.000543)\n",
      "67. feature 58 (0.000458)\n",
      "68. feature 98 (0.000436)\n",
      "69. feature 77 (0.000415)\n",
      "70. feature 210 (0.000389)\n",
      "71. feature 129 (0.000363)\n",
      "72. feature 32 (0.000283)\n",
      "73. feature 42 (0.000275)\n",
      "74. feature 238 (0.000235)\n",
      "75. feature 141 (0.000211)\n",
      "76. feature 64 (0.000184)\n",
      "77. feature 75 (0.000162)\n",
      "78. feature 70 (0.000159)\n",
      "79. feature 69 (0.000132)\n",
      "80. feature 63 (0.000123)\n",
      "81. feature 43 (0.000097)\n",
      "82. feature 228 (0.000096)\n",
      "83. feature 239 (0.000088)\n",
      "84. feature 44 (0.000087)\n",
      "85. feature 17 (0.000073)\n",
      "86. feature 131 (0.000046)\n",
      "87. feature 149 (0.000042)\n",
      "88. feature 80 (0.000033)\n",
      "89. feature 153 (0.000026)\n",
      "90. feature 190 (0.000020)\n",
      "91. feature 106 (0.000001)\n",
      "92. feature 89 (0.000000)\n",
      "93. feature 108 (0.000000)\n",
      "94. feature 128 (0.000000)\n",
      "95. feature 117 (0.000000)\n",
      "96. feature 133 (0.000000)\n",
      "97. feature 123 (0.000000)\n",
      "98. feature 132 (0.000000)\n",
      "99. feature 110 (0.000000)\n",
      "100. feature 130 (0.000000)\n",
      "101. feature 109 (0.000000)\n",
      "102. feature 99 (0.000000)\n",
      "103. feature 100 (0.000000)\n",
      "104. feature 119 (0.000000)\n",
      "105. feature 127 (0.000000)\n",
      "106. feature 101 (0.000000)\n",
      "107. feature 125 (0.000000)\n",
      "108. feature 120 (0.000000)\n",
      "109. feature 102 (0.000000)\n",
      "110. feature 103 (0.000000)\n",
      "111. feature 111 (0.000000)\n",
      "112. feature 104 (0.000000)\n",
      "113. feature 126 (0.000000)\n",
      "114. feature 115 (0.000000)\n",
      "115. feature 105 (0.000000)\n",
      "116. feature 122 (0.000000)\n",
      "117. feature 118 (0.000000)\n",
      "118. feature 114 (0.000000)\n",
      "119. feature 97 (0.000000)\n",
      "120. feature 66 (0.000000)\n",
      "121. feature 96 (0.000000)\n",
      "122. feature 29 (0.000000)\n",
      "123. feature 49 (0.000000)\n",
      "124. feature 48 (0.000000)\n",
      "125. feature 47 (0.000000)\n",
      "126. feature 46 (0.000000)\n",
      "127. feature 41 (0.000000)\n",
      "128. feature 40 (0.000000)\n",
      "129. feature 39 (0.000000)\n",
      "130. feature 37 (0.000000)\n",
      "131. feature 36 (0.000000)\n",
      "132. feature 26 (0.000000)\n",
      "133. feature 53 (0.000000)\n",
      "134. feature 24 (0.000000)\n",
      "135. feature 23 (0.000000)\n",
      "136. feature 20 (0.000000)\n",
      "137. feature 19 (0.000000)\n",
      "138. feature 16 (0.000000)\n",
      "139. feature 15 (0.000000)\n",
      "140. feature 14 (0.000000)\n",
      "141. feature 13 (0.000000)\n",
      "142. feature 9 (0.000000)\n",
      "143. feature 52 (0.000000)\n",
      "144. feature 54 (0.000000)\n",
      "145. feature 93 (0.000000)\n",
      "146. feature 74 (0.000000)\n",
      "147. feature 91 (0.000000)\n",
      "148. feature 90 (0.000000)\n",
      "149. feature 88 (0.000000)\n",
      "150. feature 86 (0.000000)\n",
      "151. feature 85 (0.000000)\n",
      "152. feature 83 (0.000000)\n",
      "153. feature 82 (0.000000)\n",
      "154. feature 79 (0.000000)\n",
      "155. feature 76 (0.000000)\n",
      "156. feature 73 (0.000000)\n",
      "157. feature 55 (0.000000)\n",
      "158. feature 72 (0.000000)\n",
      "159. feature 71 (0.000000)\n",
      "160. feature 67 (0.000000)\n",
      "161. feature 136 (0.000000)\n",
      "162. feature 61 (0.000000)\n",
      "163. feature 60 (0.000000)\n",
      "164. feature 59 (0.000000)\n",
      "165. feature 57 (0.000000)\n",
      "166. feature 56 (0.000000)\n",
      "167. feature 135 (0.000000)\n",
      "168. feature 303 (0.000000)\n",
      "169. feature 137 (0.000000)\n",
      "170. feature 253 (0.000000)\n",
      "171. feature 244 (0.000000)\n",
      "172. feature 246 (0.000000)\n",
      "173. feature 248 (0.000000)\n",
      "174. feature 249 (0.000000)\n",
      "175. feature 251 (0.000000)\n",
      "176. feature 252 (0.000000)\n",
      "177. feature 255 (0.000000)\n",
      "178. feature 140 (0.000000)\n",
      "179. feature 256 (0.000000)\n",
      "180. feature 257 (0.000000)\n",
      "181. feature 258 (0.000000)\n",
      "182. feature 259 (0.000000)\n",
      "183. feature 260 (0.000000)\n",
      "184. feature 261 (0.000000)\n",
      "185. feature 243 (0.000000)\n",
      "186. feature 242 (0.000000)\n",
      "187. feature 241 (0.000000)\n",
      "188. feature 240 (0.000000)\n",
      "189. feature 237 (0.000000)\n",
      "190. feature 236 (0.000000)\n",
      "191. feature 234 (0.000000)\n",
      "192. feature 232 (0.000000)\n",
      "193. feature 230 (0.000000)\n",
      "194. feature 229 (0.000000)\n",
      "195. feature 227 (0.000000)\n",
      "196. feature 226 (0.000000)\n",
      "197. feature 225 (0.000000)\n",
      "198. feature 224 (0.000000)\n",
      "199. feature 223 (0.000000)\n",
      "200. feature 222 (0.000000)\n",
      "201. feature 221 (0.000000)\n",
      "202. feature 262 (0.000000)\n",
      "203. feature 263 (0.000000)\n",
      "204. feature 264 (0.000000)\n",
      "205. feature 285 (0.000000)\n",
      "206. feature 300 (0.000000)\n",
      "207. feature 299 (0.000000)\n",
      "208. feature 298 (0.000000)\n",
      "209. feature 297 (0.000000)\n",
      "210. feature 296 (0.000000)\n",
      "211. feature 295 (0.000000)\n",
      "212. feature 294 (0.000000)\n",
      "213. feature 293 (0.000000)\n",
      "214. feature 292 (0.000000)\n",
      "215. feature 291 (0.000000)\n",
      "216. feature 290 (0.000000)\n",
      "217. feature 289 (0.000000)\n",
      "218. feature 287 (0.000000)\n",
      "219. feature 286 (0.000000)\n",
      "220. feature 284 (0.000000)\n",
      "221. feature 265 (0.000000)\n",
      "222. feature 283 (0.000000)\n",
      "223. feature 282 (0.000000)\n",
      "224. feature 281 (0.000000)\n",
      "225. feature 280 (0.000000)\n",
      "226. feature 279 (0.000000)\n",
      "227. feature 277 (0.000000)\n",
      "228. feature 275 (0.000000)\n",
      "229. feature 273 (0.000000)\n",
      "230. feature 272 (0.000000)\n",
      "231. feature 271 (0.000000)\n",
      "232. feature 270 (0.000000)\n",
      "233. feature 269 (0.000000)\n",
      "234. feature 267 (0.000000)\n",
      "235. feature 266 (0.000000)\n",
      "236. feature 220 (0.000000)\n",
      "237. feature 219 (0.000000)\n",
      "238. feature 218 (0.000000)\n",
      "239. feature 159 (0.000000)\n",
      "240. feature 174 (0.000000)\n",
      "241. feature 173 (0.000000)\n",
      "242. feature 171 (0.000000)\n",
      "243. feature 170 (0.000000)\n",
      "244. feature 169 (0.000000)\n",
      "245. feature 168 (0.000000)\n",
      "246. feature 167 (0.000000)\n",
      "247. feature 166 (0.000000)\n",
      "248. feature 165 (0.000000)\n",
      "249. feature 164 (0.000000)\n",
      "250. feature 163 (0.000000)\n",
      "251. feature 162 (0.000000)\n",
      "252. feature 161 (0.000000)\n",
      "253. feature 160 (0.000000)\n",
      "254. feature 158 (0.000000)\n",
      "255. feature 176 (0.000000)\n",
      "256. feature 157 (0.000000)\n",
      "257. feature 156 (0.000000)\n",
      "258. feature 155 (0.000000)\n",
      "259. feature 154 (0.000000)\n",
      "260. feature 152 (0.000000)\n",
      "261. feature 302 (0.000000)\n",
      "262. feature 150 (0.000000)\n",
      "263. feature 148 (0.000000)\n",
      "264. feature 147 (0.000000)\n",
      "265. feature 146 (0.000000)\n",
      "266. feature 145 (0.000000)\n",
      "267. feature 144 (0.000000)\n",
      "268. feature 143 (0.000000)\n",
      "269. feature 142 (0.000000)\n",
      "270. feature 175 (0.000000)\n",
      "271. feature 177 (0.000000)\n",
      "272. feature 217 (0.000000)\n",
      "273. feature 201 (0.000000)\n",
      "274. feature 216 (0.000000)\n",
      "275. feature 215 (0.000000)\n",
      "276. feature 214 (0.000000)\n",
      "277. feature 213 (0.000000)\n",
      "278. feature 212 (0.000000)\n",
      "279. feature 211 (0.000000)\n",
      "280. feature 209 (0.000000)\n",
      "281. feature 208 (0.000000)\n",
      "282. feature 207 (0.000000)\n",
      "283. feature 206 (0.000000)\n",
      "284. feature 205 (0.000000)\n",
      "285. feature 204 (0.000000)\n",
      "286. feature 203 (0.000000)\n",
      "287. feature 202 (0.000000)\n",
      "288. feature 199 (0.000000)\n",
      "289. feature 179 (0.000000)\n",
      "290. feature 198 (0.000000)\n",
      "291. feature 197 (0.000000)\n",
      "292. feature 196 (0.000000)\n",
      "293. feature 195 (0.000000)\n",
      "294. feature 194 (0.000000)\n",
      "295. feature 193 (0.000000)\n",
      "296. feature 191 (0.000000)\n",
      "297. feature 189 (0.000000)\n",
      "298. feature 188 (0.000000)\n",
      "299. feature 187 (0.000000)\n",
      "300. feature 185 (0.000000)\n",
      "301. feature 183 (0.000000)\n",
      "302. feature 181 (0.000000)\n",
      "303. feature 180 (0.000000)\n",
      "304. feature 151 (0.000000)\n"
     ]
    }
   ],
   "source": [
    "estimators = rf.estimators(X_train, y_train)\n",
    "print type(estimators)\n",
    "std = np.std([rf_feature for tree in estimators],\n",
    "             axis=0)\n",
    "indices = np.argsort(rf_feature)[::-1]\n",
    "x_axis = []\n",
    "for index in indices:\n",
    "    x_axis.append(x_train.columns[index])\n",
    "    \n",
    "x_axis = x_axis[:10]\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(x_train.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], rf_feature[indices[f]]))\n",
    "\n",
    "y_pos = np.arange(len(x_axis))\n",
    "# Plot the feature importances of the forest\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAEHCAYAAACdoi6sAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYXEW9//F3mwUhxISQYdUEUPwQgnJvyEW2kIVNrkjY\nVAhrQFSQzcADVy4gEQVB3LioqKBshsi+iewksigEEDUsX5BFEiCAsoclIenfH1Xzo9OZpWcyM6c7\n83k9T57uPl1V53vOnMy3q+r0VKlcLmNmZmbF+VDRAZiZmfV2TsZmZmYFczI2MzMrmJOxmZlZwZyM\nzczMCuZkbGZmVrC+RQdg1ugklYEngffzpr7ATOCIiJjfRfv4KDAnIkpd0V5V2+OAm4Gnqt66LCJO\n7Or9Ve1bwOoR8ccW3psBfBJ4o+qtsRHxYif3d3BE/KozdWtsfwZwbkRc3F37aGPf3Xps1r2cjM26\nxriImAsgaQVgOnA88L+FRlW7ZyNigwL2uyvp99BSyTg7tqsSm6Q+wPeB5S5hSVoDOJbl8Nh6Cydj\nsy4WEe9JuhHYGUDSSsBvgP8A+gNXRMQx+b0ZwLXAbsC6pKQ0KSLKkg4EvkXqGf62uX1JHwJOAXbP\nm/4MfD0i5uf2bgQmAp8ATgZWAfYBFgOfi4inO3I8uef8Q2Al4PW8r/slHZCPcRDwQEQcK+krwBTg\nw8CfgAMj4h1JY4Ef5e0l4CTgXeCbwAJJq0TE0R2IaQVSYv0s6Zz+MiJOze9tDpwNDMjHfERE3Arc\nAgyS9BiwI3AHsE9E3JXrPZPP01zgHuB3wKiIGCtpS+DHpHP5L9LPqHokoTrG84HngC2AjUiJ8ing\nSGAg8IWImJV/ZjPysaxLuh6+FhGLaj33wB7AR/OxfRrYpKVzIGkd0s/lNOBgYAgwJSJ+J6kE/ID0\nAWkh8KuI+H7efiKwN+nnd3Wus0jSF0jXaJ9c54iImNHWebGWec7YrItJWgWYRPqFDnAI6ZfvBsAo\n4ABJW1VU+TywHWlIdgKwRW7jLOCzEfEpYK2K8l8kJZNNgJHAYOAbFe9vDYwBJgNnAHNzr/cR4MAO\nHsvKwGXA4bmNM4Bp+QMBwPakxHGspDGkDwkTImIdUvI4JZc7E/hGRGxISiK7RsR1wFXATzqSiLNj\ngQ2BT5HOwR6Sdsrv/RL4fo73e8A5efuBwKKI2KCGDyRDgYdyIh4IXAccHxGfAH4CXFpjnDsCOwHj\nc8xN+ed5OXBEVbkJpGS8NbBTR859PrZn87EtaOMcNB/b4hzHUcB38va9gU1J1+Fo4HBJm5I+oHwx\nv/fx/O+QXOdnpA94I4BDyR9AreOcjM26xgxJj0l6CngauA04HSAifgBMjIhyRLwKPAysV1H38oh4\nJ88vPw4MAz4DPBERj+YyF1SU/xxwQUTMj4hFpF739hXvXxcR7wN/J/WoLs/b/86SSb3SsBx/5b+D\ncxxzI+LufCxXkH6Zr5PrPR4RT+Tnnwd+FxHP59fnkHr8AC8B+0naICKeiIhJrcRR7YyqmP5asa+f\nRcR7+bxdWLGv/+CDZHknS57rWvUjfVCA9MFmbkTcAhARlwCfkDSshnZuyfE9TPp9e13eXv2zmB4R\nb0fE26SRjS3o2Lmv1tY56Eu6ZgAeJF1vAP9NuhYXRsQbwAhgFulc/zoiXs/X1bks+XP9mqThEXFX\nREyp4ZxYCzxMbdY1xkXEXElDSQn1d/kXF5LWB34oaQNgEfAxPvhlCKkH2WwRachvSNX2VyueN1W9\nfhVYreL1mxVtERFvVbXdkhbnjCXtWbUvgNcq9vdKxfbBwK6Smj8YfIg0hAyp53YCcKukd4BvRsTl\ntK+1OePBwI8knZpfrwDcl5/vDRyRe7R9SMPiHbUoJ6TmfX08DwE3e4/0c3i2nXbeBMjTDouB1n4W\nlefxVVKirv45Q+vnvlpb52BRxY2FlXEMze2TY54PIGkwcEyegoCUN17Oz3cm/VwfkDQHOCoiZrYR\nl7XCydisC0XEvySdRRpSnJg3/5Q0r7dLnme7u4amXiXNBzZrqnj+IrBqxetV87busMS+8vzhkLy9\nOnk/T+qxH1PdSL77+XDS0Of2wJV5Xr2zngfOjIjrKzdKWps0N/uZiHgofxB6vJU2qhPiKm3s69GI\nGL0M8bZnaMXzIaRE25FzT0W5jpyDSv+qjEPS6sA7pOO/NiLOrq4QEU8Ck/PQ+X7ANGDtGvZlVTxM\nbdb1fkCa9x2bX68G/CUn4u2A9YGV22njftI3f9bPr/eveO96YB9JK0nqCxwE/L7rwl/CfcAa+aYo\ngD1JNzg900LZa4HdJDUBSJoo6ThJ/STNkLRmLvcA6WafxflxcCfiugb4sqQ+kkqSTpD0WdKHlvnA\nY/ncfCXHsnLe14dybxHgBWDj/P6XSDcnteReYE1Jn8ll15N0UU6OXWVXSStIGkCaP76Tjp37hcDK\n+ZjbOgdtuRbYqyKOu0g3nl0D7JtvRETSVyXtL6lJ0i2SPhIRi0k3EnoZwE5yMjbrYhHxJummmTPz\nL+zvAD+QNBsYC0wFpuY7dFtr42XgaNKw7mwgKt6+HLiBlNRmA3NIN3t1x7HMJ928c3Yepj0U2DMi\nlvqlGxEPAqeS5s8fJd1VfU1ELCTNM94m6RHSd7APz/Oj15HmHGsZsq70U+CfpLnYx0jzm3cBfyWd\nm8dJdw1fR0oSM0nJ9y7gWUlbkG4um5LP7wjSDW4tnYN3SHcr/18+rqtI38HuysRzD+nu7mfy4x86\ncu6Bv5F60/NIoyqtnYO2/A64CXgC+AtwXkTcQ7p7+jrgwRzHzsBN+Rq9EZiVf67TSR8MrRNKXs/Y\nzKw4KvAPhVj9cM/YzMysYE7GZmZmBfMwtZmZWcHcMzYzMyuYv2dsNXv//UXlV199u+gwOmyVVVai\nEeOGxo3dcfe8Ro29N8Td1DSw3a/BuWdsNevbt7U/3lTfGjVuaNzYHXfPa9TYHXfiZGxmZlYwJ2Mz\nM7OCORmbmZkVzMnYzMysYE7GZmZmBXMyNjMzK5iTsZmZWcGcjM3MzArmv8BltSuVaCo6hna8/NIb\nRYdgZtZh7hmbmZkVzMnYzMysYE7GZmZmBXMyNjMzK5iTcYOQtLKkZzpQ/j8kTe2+iMzMrKv4burl\nVEQ8BDxUdBxmZtY+J+M6JukjwBXAh4G78rYxwKnAQmAOcDCwInApsEL+93XgI8BhEbGHpOOAvYCn\ngH7AD4BxwGBAwHrAURHxh546NjMz+0CpXC4XHYO1QtKhwPoR8Q1JXwJOB14FtomIVySdAfwVeAf4\nXEQcJGk94JPAu8BhwFdymU+SEvQTwM6kZLxRTtafBb4WEbu0GVCpVP8Xi69nM6s/pfYKuGdc3zYE\nZubnM4DVgaHAlZIABgD/Ai4CviPpHODKiLhR0rhc7xPA3yPiHeAdSfdVtH9XfpwLDOrG4+gxL7/8\n5lLbmpoGtri9ETRq7I675zVq7L0h7qamge2WcTKubyVgcX7+IWABMC8ixlUXlLQxMB44RNJmwB9b\naAOgsuv4ftW+zMysAE7G9S2A0aR54/GkIWokbRgRj0g6nNRzXg3oFxF/kPQI8DM+SMbPABtJ6kea\nIx7ds4dgZmbtcTKubxcCV0m6jTSkXAYOAn4jaQHwPPBL4A3g4nyj1mLgW0AfgIh4UdI04D7g0fy4\nqKcPxMzMWudkXMci4jVSj7jZt/LjZ6qKPgNs1UITM/Lj48DJpGHpvwNPR8TJFfuZTbqhy8zMCuBk\n3DusAdwLvAf8NiLmFhyPmZlVcDLuBSLie8D3io7DzMxa5mRstSuXG/IrCGZm9c5/m9rMzKxgTsZm\nZmYFczI2MzMrmOeMrXalEk1Fx1CDl196o+gQzMw6xD1jMzOzgjkZm5mZFczJ2MzMrGBOxmZmZgWr\ny2Qs6RlJKy9D/a0lrdaVMXUihgMk7SppnKTL87Z/daKdPTpQdpikTfPzH0tat6P7MzOznleXybgL\nHEhaVrAwEXF+RFy1LG1I6g9M6UCVCcCmef9HRcTTy7J/MzPrGT361SZJw4CLSUv49QVuBQZGxDG5\nJzw7ItbJxY+XNIa00tCuwHzgAmA48C6wH/AmMA0YAKwEHA4MAnYBRkranbR+79G5nfsj4mhJg4DL\ngRWBG4CDI2JdSeOAU4GFwFxSUt8L2BFYC3gM+HNEnJeP5xFgTET8u4VjPRn4FzC7hfdmAIdFxGxJ\nhwFDgR8BlwIr5H9fJy2X+ClJPyMtfdgcx575mDYFPgycA1xDWplpoaRnSUn8MGAOcD5pLeN+wBER\n8aCkf+Q6WwCvAZ+LiMVL/dDMzKzb9fT3jPcAbomIUySNArYHBrZS9m8RcbykM4F9gXeAeRExSdKe\nwM7AbcC5EXG1pAnAcRGxu6SHSInoFeAEYPOIeE/SpZK2BEYBj0TEkZIOBUp5n+cA20XEHElnA5NI\nawgPIyWtkcAPgfMkbQg81VIi7qRtgLkRcZCk9YBPAt8HPhMRh0o6oCKOFYBnImKKpBWBJyPiXEnn\nA/+KiGslNfeojyR9gDhd0mhS0h8LrAdckD+c/Bn4NPBQFx1LoZqalr6kWtrWKBo1dsfd8xo1dsfd\n88n4ZuAqSYNJPdN5pF5hS+7Ij/cBW5OG1G8DiIjpALmHe6KkY0gJan5VGyNJCewmSZB6zcOBEXyw\n1u+1wLGShgDliJhTsf+xwIPArIgoA7MlDZbUBEwEftuJc9CaPwHfkXQOcGVE3ChpnaoyzXG8K2mI\npHuABdDm3+IYDXwXICLul/SJvP2NiPhbfj6XdG6WC9WLWTQ1DWzYBS4aNXbH3fMaNfbeEHctSbtH\n54zzIvYbA3cCp5F6nc36VRUvVz1fxNLxHgU8FxFbAYe0sMsFwAMRMS7/+8+ImEbqCTcPyZYrHksV\ndftXlFlQsX0asBupJ3tNS8dZg6WOOyJeIJ2bK4FDJJ3UyvEgaSxpfnhsRIwjrVPc1r4qj6tPfny/\nqlwJMzMrRI8m4zy8vFFEXE0aPj4GWDO/vVVV8TH5cTPgUWAWKQEhaSdJx5N61U/mcruSEiikJNoX\nCGBE853VkqZKWjvXGZ3L7ggQEa8C5TyvDalXfH8Lh3EJMBl4ISLe7tAJ+MAbfHDcW+bYtgW2jYib\nSXPfoyuOo9pQYE5ELJS0M9An3+zVUvlZwPi8j81oYQ7bzMyK1dN3Uz8OnC3pduBbwD6A8g1NG/BB\nTxTSDVi3kuYyLwamAwMkzST1iC8ALgSmSLoZuBdYQ9JkYCZpGHzdXPYGSXcDqwLPk25oGpP3uzqp\n1w1wMDAtb++X97mEiHgReIvUQ+6sXwI/lfT7HA/AP4D/zfu+kDRf/ALQX9JlVfVvBdbP5+LjwPXA\nz0lD3cdK2rui7E+ATfI5/x5pDtnMzOpIqVwut19qOSNpOLBBRNwkaXNgakRsX2PdocCNwKa97u7j\nUqkhLpbqhSIadU4KGjd2x93zGjX23hB3U9PAdqcBe+uqTa+TetQnkeZKj6ilkqRdgKnAlOZELOlK\nYEh1+xExsQvjNTOz5VivTMYR8RqwQyfqXQ1cXbVtt66Ky8zMeqfl9S9wmZmZNYxe2TO2TiqXG3Ju\nx8ys3rlnbGZmVjAnYzMzs4I5GZuZmRXMc8ZWu1KpzT+CXdeqvntsZlZP3DM2MzMrmJOxmZlZwZyM\nzczMCuZkbEh6RtLKRcdhZtZbORmbmZkVzHdTLwckDSItGbkicANpKcjJwKnAQmAucCCwAmnpxwHA\nSsDhEXFfETGbmdkH3DNePuwHPBIRWwGvkVaiOgf4UkSMBV4FJgFrAOdGxHjgm8BxBcVrZmYV3DNe\nPowAZuTn1wLfA56LiDl52x3AWOBK4ERJx5B6yfN7OM7CNDUNLDqETmvU2B13z2vU2B23k/HyogQs\nzs/L+V/lYtb98/tHkZL0vpJGA2f2aJQFatQFLnrDwuv1pFHjhsaNvTfEXUvS9jD18uFJYHR+viNp\nWLosaVjeNha4HxiaywLsSkrSZmZWMCfj5cP5wBhJM4DVgUWkm7im5W39gOnAhcAUSTcD9wJrSJpc\nRMBmZvYBD1MvHwYA346ImyRtDoyNiLuArarKzSLNLze7Nj/+pgdiNDOzVjgZLx9eJ/V4TyLNFR9R\ncDxmZtYBTsbLgYh4Ddih6DjMzKxzPGdsZmZWMPeMrXblcsN+BYEGjNvMeg/3jM3MzArmZGxmZlYw\nJ2MzM7OCec7Yalcq0VR0DJ3UXtwvv/RGj8RhZtYS94zNzMwK5mRsZmZWMCdjMzOzgjkZm5mZFczJ\nuJeRtEfRMZiZ2ZKcjHsRSf2BKUXHYWZmS/JXm7qJpH7ABcBw4F3gQOBkYD1gBeCkiLhZ0pPAr4A9\ngH8ADwBfAJ6IiL0lnQ+8BGxC+obO6cBkYCgwFngL+GVut19u9/a8jvGtwPhc9vPAccCnJP0M+B5w\nMWnt477APhHxz+47I2Zm1hon4+6zPzAvIiZJ2hM4AHg3IsZKWguYAXwS6AM8SEqyzwJXRMSmkp6V\nNDi39X5EbCPpt8AWEbGtpItIiXYg8EJEHCRpKHA78Olc7/Vc73vAbsD3gc9ExKGSpgC3RMQpkkYB\nawK9Nhk3NQ0sOoRW1XNsbXHcPa9RY3fcTsbdaRRwG0BETJd0FikBExHPS3pP0pBc9r6IKEt6EfhL\n3vYSMKj5/fz4AvBYfv5ifn8zYIykrfL2FfNwNMCd+XEusGpVfDcDV+WEf3lE/GmZjrbB1esCGE1N\nA+s2trY47p7XqLH3hrhrSdqeM+4+i1jy/JaBUsXr/sDi/Pz9iu2Vz0s1vL8A+G5EjMv/1o+IBW20\nBUBEzAY2JiXs0yTt1/4hmZlZd3Ay7j6zgAkAknYC/k0aVkbSx4DFEfFaF+znXmBibnc1Sae2UXYx\neTQkD51vFBFXAycAo7sgFjMz6wQPU3ef6cC2kmYCC4GDgBMl3UHqFX+1i/ZzKTBB0j2k+eeT2yj7\nAtBf0mXAacA5kt4i9eKP6KJ4zMysg0rlcrnoGKxRlErL7cVSrwtF9Ib5tHrSqHFD48beG+JuahpY\naq+Mh6nNzMwK5mRsZmZWMM8ZW+3K5eV+OMnMrAjuGZuZmRXMydjMzKxgTsZmZmYF85yx1a5Uoqno\nGDqpUeKu169YmVn3cs/YzMysYE7GZmZmBXMyNjMzK5iTsZmZWcG6LBlLekbSystQf2tJq3VVPPVA\n0vl5xabKbWtI+kUn2rpf0jodrHNNR/djZmY9r57upj4QOBN4qehAulNEzKPrVmxqb18Te2I/Zma2\nbNpMxpKGAReTltjrC+xDWpN3o4g4JveEZ0fEOrnK8ZLGkBa13xWYD1wADAfeBfYD3gSmAQOAlYDD\ngUHALsBISbuT1tY9Ordzf0QcLWkQcDmwInADcHBErCtpHHAqaZnCuaSkvhewI7AW8Bjw54g4Lx/T\nI8CYiPh3C8f7UeAyYAHwx1xunKQngAeBm4F/AqfkMq8CXwS2AI4D3svHenlEfDc3O17SYcAwYO9c\n5/KIGC1puxz7ImB6RPy4Kp6zgM2BIC27iKS1gPPy60XAlyPi2Vx2NGkZxZ9HxPmS/hURQyVtC/wY\nmJfbehmYARwGlIENckxTq8+JmZl1v/Z6xnsAt0TEKZJGAWu2U/5vEXG8pDOBfYF3gHkRMSkvZr8z\ncBtwbkRcLWkCcFxE7C7pIVJyeIW02P3mEfGepEslbQmMAh6JiCMlHQo0L0l1DrBdRMyRdDYwiZRg\nhpGS5Ejgh8B5kjYEnmopEWffAC6NiB9JOqNi+3rALhHxsKQvAJMi4mlJFwI7kD5gjAbWJX2AeEzS\nObluOSI+K+mrwP6kpIikEvCzHOMrwDWSfhER7+T3N8zvbQqsDfwjt3cK8IOIuFXSf5PWSD4O+FxE\nfFxSP+CAquM6Pf88/gbcSfpQQW57A9J0xTOAk3HBmpoG1rStETjunteosTvu9pPxzcBVkgaTek5/\nkrRBG+XvyI/3AVuTfsnfBhAR0wFyD/dESccAK5B6z5VGkhLpTZIg9ZqHAyNIvTmAa4FjJQ0hJbs5\nFfsfS+rFzoqIMjBb0mBJTcBE4LdtxD8C+F3FPjbNz+dHxMP5+cvAuZL6kpL07aRkfG9EvJWPcTbw\n8Vz+rvz4HLBZxb6agHcj4uX8eom5ZWDD3OZiYI6kp/L2LdIudAKpF/xyRLwi6fE8R3wZcGFVW8Mj\n4i85thv44Of+YES8nbe3cVqsp1QvaNGoi1w47p7XqLH3hrhrSdptJuOImC1pY2B74DRJvyb1Opv1\nq6pSrnq+iKVvEjsKeC4i9pU0mjRPXGkB8EBE7FC5MQ9/L67aT5kPesiQhm6byyyo2D4N2A3YhtQ7\nb02phX1Ut/VrUi/00dwTb1Z5nKWK+u9XbW/W0rlpLZbK9hcAX4iIFyoLR8SOefRiEmk6YPtW2q08\nrvdbKWNmZj2ozbup89DyRhFxNWnoeDTwBh8MV29VVWVMftwMeBSYBUzIbe0k6XhgKPBkLrcreS6U\nlHj6kuY0RzTfWS1pqqS1c53RueyOABHxKlDOc9uQesX3t3AolwCTgReae4KtWGofLRgEPJtHC8ZX\nxD9K0kqSPkzq1T7Rxn7IQ+V9JK0tqSTp+tzm/y8CbJLfG04aAge4lzS/jqQJkiZJWkfSERHxYEQc\nA6xatbt5kjaQ1IfWk7SZmRWkva82PQ6cLel24FvAz0nDzpI0gzTfWNl7GynpVuDTpBu/pgMDJM0k\n9YgvIA2hTpF0MymxrCFpMjCTdIPWurnsDZLuJiWW54HzgTF5v6uTepYABwPT8vZ+eZ9LiIgXgbdI\nPeS2/AT4aj6GUsU+Kv0UuBv4JXAG8E3Sh5NHSL3me4BzIuK1dvYFcGg+5nuA2yrrRMTfgL8DfyLN\nEz+U3zoZ2EXSH0k/kz+Rzs8Wku6RdEeOo9IJwJWkofdHWzkuMzMrSKlcLrdfqg7k3uEGEXGTpM2B\nqRFRUy9P0lDgRmDTPAfbWrmRwOCIuFvSXsD4iPhKDe2PAw6LiD1qiaenSdoeeDwinsnfcZ4ZEe19\nMFlaqdQYF0sDq14oojfMp9WTRo0bGjf23hB3U9PAUntl6ul7xu15ndSjPonUaz2ilkqSdiHdJTyl\nORFLuhIY0kL7hwO/kFQm9fgnd1HsRSuRbsR7E3iR1Bs3M7M60TA9Y6sD7hl3O/eMi9WocUPjxt4b\n4l7eesZWtHJ5uf9PY2ZWBC8UYWZmVjAnYzMzs4I5GZuZmRXMc8ZWu1KJpqJj6KRGjRu6N/bqG8bM\nrBjuGZuZmRXMydjMzKxgTsZmZmYFczI2MzMrmJNxJ0k6QFL18o9d1fbJkg7rRL2VJT3TiXqflvTJ\njtYzM7Ou4WRskNZ6djI2MyuIv9q0jCT9ENgU+DBp6cRzJZ0PXB4R10vaCdiDtPThBcBTpCUm/xIR\nX86rUV0A9AH+Ceyfm95I0vXA+sCREXGjpN2Ao4H3gfsj4mhJHwGuyPu/q51Y++Z9fRQYkGP6J/A1\n4GVJL0XEfV1wWszMrAOcjJfdMxExRdKKwJPAuW2U3QT4EvASMFfSYOC7wA8j4lpJZwCjc9mhEbGT\npB2AQyTdRVqXePOIeE/SpZK2BDYGZkfENyR9Cdirjf0PAW6OiAskrQdcFhGbSLqR9OHBibiXaWoa\n2JBtd6dGjRsaN3bH7WTcFYZIugdYQPt/n+EfETEPQNLzwCBgFHAkQEQcm9/bkQ96uc/lciOBYcBN\nksjbhgMbAjNz2Rnt7P9V4L8kfYW0ROSqNR2hLbe6awGNRl2co1HjhsaNvTfEXUvSdjJeNpuQhpfH\nRsRCSW/l7ZVLDfareP5+Vf0SsIiW5+7fryq3AHggInaoLJR7x4vzy/buAZhE6h2PyY/3t1PezMx6\ngG/gWjbrAHNyIt4Z6COpP/AGsGYus1U7bcwCJgBI+rakbVspF8AISavlslMlrZ23Nw9tj29nX0OB\npyNiMemmrf55+2L8wczMrDBOxsvmamB9STOBjwPXAz8HLgKOyXOxC9tp41vAwbmNdYE7WioUEW8D\nRwE3SLqbNMT8PHAhsJmk2wCxZK+82hXA53PZ+aR565OAO4GzJG1TwzGbmVkXK5XLbf3uNqtQKvli\nWc5010IRvWEesN40auy9Ie6mpoGl9sp4aHI5lHu7E1p4a3JEPN3T8ZiZWducjJdDEfFt4NtFx2Fm\nZrVxMrbalcvL/XBSvWnk2M2sdr6By8zMrGBOxmZmZgVzMjYzMyuY54ytdqVSu3/vs141atzQuLE7\n7p7XWuzd9RU26zruGZuZmRXMydjMzKxgTsZmZmYFczI2MzMrmJNxnZF0gKQze7KupMMkndyZfZqZ\n2bJzMjYzMyuYv9pUn9aVdAPwMeBHwHvA4cAi4OGI+IqkfsAFwHDgXWC/ygYknUZaJvE04JfAekA/\n4KSIuD0vl/hjYB7wAvBUTxyYmZktzcm4Pn0SGAV8BPgrcArw2Yh4TdIfJX0K+AwwLyImSdoT2Bl4\nB0DSF4CPRcQ+kvYFXoiIgyQNBW4HPk1K0vtExF9z4ncyNltONTUNLDqENtV7fK3pyridjOvTXRGx\nEPi3pDeAfwPXSAIYAaxKSta3AUTEdEhzxsBIYDdgw9zWFsAYSVvl1ytK6g+sExF/zdtmAit290GZ\nWTHqebGRRl0MpYPrGbdbxsm4PpWrXl9C6unOk3R93raIluf81wEeBvYALgYWAN+NiEsqC0laXPHS\n9w6YmRXIybg+bS6pDzCENG/8Uk7EHwNGA/2BWcAE4DJJO5GGnp8Hfg+cDtwl6RbgXmAicImk1YCj\nIuJ44DmlrvbjwDjgTz15gGZm9gEn4/r0GHAZ8AngEGBbSbNI88dnkG7qGpW3zwQWAvsD2wFExMuS\nvgX8HPgiMEHSPUAf4OS8j/8FLgf+CczpmcMyM7OWlMrl6hFRs1aUSr5YzBpQPS8U0UvmjEvtlfFc\noZmZWcGcjM3MzArmOWOrXbm83A8n1ZtGjd1x97xGjt3cMzYzMyuck7GZmVnBnIzNzMwK5jljq12p\nRFPRMXRSo8YNjRu74+559RR7PX+dqh65Z2xmZlYwJ2MzM7OCORmbmZkVzMnYzMysYE7GdU7Sznn9\n4VrKbiRoeWrxAAAN7ElEQVRpRif2sXVe0cnMzArgZFz/ppCWTOxOBwJOxmZmBfFXm7qBpH7ABcBw\n4F3gdtJ6w2sBewK7AJOAxcDVEfEDSR8FLspN9CMtibgFsBnwB0nbAAe3Uu8y4D3SEottxfURYBow\nAFgJOBwYlOMZKWn3iHi2S06CmZnVzD3j7rE/MC8itgR+BbwCDAO2JvVy9wC2yq93lzQMWBP4dkSM\nB34NHBoRFwHzgB2BtVupdwQwPSLGAc+3E9cawLl5H98EjouIW4CHgMlOxGZmxXDPuHuMAm4DiIjp\nkg4AZkVEWdKmwPrAHbnsQGAd4GngLElTgVWAB6rabK3ehqSeMcAMUuJuzYvAiZKOAVYA5nfq6MzM\n2tHUNLBbytaTrozbybh7LGLpUYcFFY+/j4ivVr4p6TfATRFxjqQ9gJ1aqN9SveNIw9a0sM9qRwHP\nRcS+kkYDZ9Z0NGZmHVTrClKNutpUR+KuJWk7GXePWcAE4DJJO5Hmips9AJwuaSXgHeDHwP8AQ4En\nJZWAiUCfXH4x6efUWr0ARuf3x7cT11Dgb/n5rnxwY1jzPszMrACeM+4e04EBkmaSeqOl5jfyvOyP\ngT8CfybNLb8D/AL4P+APuf5YSduThp7vAt5upd5PgAMl3UQa3m7LhcAUSTcD9wJrSJoMzAQulzSy\nC47dzMw6qFQul4uOwRpFqeSLxcxqUutCEb1kmLrUXhkPTS6HJP2MdGNXtR1zb9rMzOqIk/FyKCIO\nLToGMzOrneeMzczMCuaesdWuXF7u53bqTaPG7rh7XiPHbu4Zm5mZFc7J2MzMrGBOxmZmZgXznLHV\nrlSiqegYOqlR44bGjd1x97xGib3W7yD3Ju4Zm5mZFczJ2MzMrGBOxmZmZgVzMjYzMyuYk3ELJG0t\nabWi42gmaSNJMzpY57OSDummkMzMrAv5buqWHQicCbxUdCCdFRE3Fh2DmZnVplclY0n9gAuA4cC7\npKT7U2AAsBJwODAI2AUYKWl3YDRwNPA+cH9EHC1pEHA5sCJwA3BwRKwraRxwKrAQmJvb3wvYEVgL\neAz4c0Scl+N5BBgTEf9uIdaPApcB7wF/rdi+WwvxDAMuBhaRfqb7AOOBjSLiGElnAVsADwMC9gRO\nBl4ARgHDgL0j4sHOnlszM+u8XpWMgf2BeRExSdKepKR7bkRcLWkCcFxE7C7pIeAw4BXgBGDziHhP\n0qWStiQlsEci4khJhwLNa1WeA2wXEXMknQ1MAsqkZLcFMBL4IXCepA2Bp1pKxNkRwPSI+Imk44CN\nJa3cSjyfAW6JiFMkjQLWbG5E0qeArUgfKkYCf6nYR/+I2EHS14D9ACdjM+t2TU0D23zdKLoy7t6W\njEcBtwFExPTcwz1b0jHACsD8qvIjSYn0JkmQes3DgRHAjFzmWuBYSUOAckTMydvvAMaSEtysiCgD\nsyUNltQETAR+20asG5J6xuR97dhGPDcDV0kaDFweEX+StEGuO4LUG18M/F3SMxX7uDM/ziUldDOz\nble5oEWjLnDRkbhrSdq9LRkvYsmb1o4CnouIfSWNJs0TV1oAPBARO1RulDQGWJxfliseSxXF+leU\nWVCxfRqwG7ANsHMbsZYq6jfH3GI8OaaNge2B0yT9upV2KuOFNNRdWc7MzArQ2+6mngVMAJC0E2nI\n98n83q6kBAopefUFAhjRfGe1pKmS1s51RueyOwJExKtAOc/fQuoV399CDJcAk4EXIuLtNmKNin2M\nr9i2VDx5yH2jiLg6H9PoinaeBDaRVJI0gtSTNjOzOtLbkvF0YICkmaRe8bbAFEk3A/cCa0iaDMwk\n3aC1bi53g6S7gVWB54HzgTH560ark3rcAAcD0/L2fnl/S4iIF4G3SD3ktvwEOFDSTcAque7brcTz\nOGm4/XbgW8DPK/Z3f37/3lz3kYp4zcysDpTK5XL7pWwJkoYDG0TETZI2B6ZGxPY11h0K3Ahsmudx\nu5WkFYAvRcSFkgaQ7uheNyLeb6fq0kolXyxmtswqF4roJXPG7U4D9rY5467yOqlHfRJprvWIWipJ\n2gWYCkxpTsSSrgSGVLcfERO7ItB81/V/STqCNPx+YqcSsZmZdRv3jK127hmbWRdwz3hp7hlb7crl\n5f4/Tb1p1Ngdd89r5Nit993AZWZmVnecjM3MzArmZGxmZlYwzxlb7UolmoqOoZMaNW5o3Ngdd89r\n1NjrPe7KG866i3vGZmZmBXMyNjMzK5iTsZmZWcGcjM3MzArmZNyNJG3dvMJSPZP0aUmfLDoOM7Pe\nysm4ex0I1H0yJq2v7GRsZlYQf7WpEyT1Ay4grQ38Linp/hQYAKwEHA4MAnYBRkranbTG8NHA+8D9\nEXG0pEGkpRpXBG4ADo6IdSWNA04FFgJzc/t7kdZOXou08tKfI+K8HM8jwJiI+HcLsX4UuCi/7Afs\nHxFPSnoCeBC4B/ga8LKklyLivi47UWZmVhP3jDtnf2BeRGwJ/IqUdM+NiPHAN4HjIuIW4CFgMvAK\ncAIwISLGAh+TtCWwH/BIRGwFvEZaAQrgHNKyh2OBV4FJefswYGvSWsdfApC0IfBUS4k4WxP4do7t\n18Cheft6eftPSEs6ftOJ2MxsaU1NA5f619r21sq2xz3jzhkF3AYQEdNzD/dsSccAKwDzq8qPJCXS\nmyRB6jUPB0YAM3KZa4FjJQ0ByhExJ2+/AxhL6sXOiogyMFvSYElNwETgt23EOg84S9JUYBXggbx9\nfkQ83JmDNzPrTVpagKODqza1W8Y9485ZxJLn7ijgudzDPaSF8guAByJiXP73nxExjdQTXpzLlCse\nK5fb6l9RZkHF9mmkud5tgGvaiPXbwE0RsTVpLeXKmMzMrA44GXfOLGACgKSdSEPQT+b3diUlUEhJ\ntC8QwIjmO6slTZW0dq4zOpfdESAiXgXKkobl7WOB+1uI4RLSEPgLEfF2G7EOBZ6UVCL1ovu3UKY5\nTjMzK4CTcedMBwZImknqFW8LTJF0M3AvsIakycBM0g1a6+ZyN0i6G1gVeB44HxgjaQawOqnHDXAw\nMC1v75f3t4SIeBF4i9RDbssvgP8D/pDbGStp+6oyd5KGsrep8fjNzKwLlcrlcvulrFtIGg5sEBE3\nSdocmBoR1YmytbpDSTdebRoRi9sr3yVKJV8sZtbrtLRQRAfnjEvtlfHQZLFeJ/WoTyLNEx9RSyVJ\nu5Dmf6c0J2JJVwJDqtuPiIldGK+ZmXUD94ytdu4Zm1kv5J6x1ZdyueaLr5505D9NvWnU2B13z2vU\n2Bs17q7mG7jMzMwK5mRsZmZWMCdjMzOzgjkZm5mZFczJ2MzMrGBOxmZmZgVzMjYzMyuYk7GZmVnB\nnIzNzMwK5j+HaWZmVjD3jM3MzArmZGxmZlYwJ2MzM7OCORmbmZkVzMnYzMysYE7GZmZmBXMyNjMz\nK1jfogOwYkn6EbAZUAaOjIhZFe9tC5wKLAJuiIhTWqsj6WPARUAf4AVg34h4r87iPgMYQ7ruT4uI\nKyWdD2wC/DtX/35E/L5e4pY0DrgMeDgX+3tEHN7T57uTsR8E7FvRxOiIWLnOzvmHgV8AIyNidFt1\n6uwaby3uwq/xzsReL9d5J+LusmvcybgXkzQWWD8iNpc0Avg1sHlFkbOAHYDngJmSrgCaWqnzbeCn\nEXGZpFOBA4Gf11HcqwMb5TqrAn8BrszlvxkR13dHrF0QN8DMiNijqrkeO9+djT0izgPOq6j/xYry\n9XLOvw88BIysoU49XeMtxT2egq/xzsaeFXqddyburrzGPUzdu20DXA0QEY8Cq0j6CICk9YBXImJO\nRCwGbsjlW6szDrg2t3sdsG2dxf1H4Au5/mvAAEl9ujHGroq7NePoufMNyx77ScAp3RxjS1qNOzse\nuKrGOuOog2s8aynuerjGoXOxt2Yc9X3OKy3TNe5k3LutAbxc8frlvK2l914C1myjzoCK4aPmst2l\nw3FHxKKImJ+3HUQaSl2UXx8m6XZJ0yUNrae48/MNJV0r6S5J2+VtPXm+W4qv1tiR9F/AnIiYV1Gm\nHs45EfFmB+rUyzXeYtx1co1D5845FH+ddzbuLrnGnYytUqkT77W0va12ukPNcUuaSPpFdVjedBHw\nPxExgTQEdXJ3BFhLbK289wQwFZgI7A+cJ6l/B9rpLh25Vr4MnF/xul7PeUfq1NM1voQ6u8ahttjr\n8TrvyP6W+Rr3nHHv9jwVn/yAtUg3SbT03tp524JW6rwlacWIeKeibHfpTNxI2gH4X+CzEfE6QETc\nVlH2Wrpx3rWF2NqNOyKeA36Xtz0paV5+ryfPd0vx1XTOs3HA4c0v6uicd7ROvVzjraqDaxw6EXud\nXOedOufZOJbxGnfPuHe7GdgDQNIo0i//NwEi4hngI5LWkdQX2CmXb63OrcDuud3dgRvrKW5Jg0g3\nYOwUEa80NyTpijznCek/1Ow6i3tvScfkOmuQbkR7jp49352KPZddC3grIhY0N1Qv57wTderiGm9N\nnVzj0LnY6+E678y10mXXuJdQ7OUkfQ/YGlgMfB34T+D1iLhK0tbA6bnoFRFxZkt1IuKvktYELgQ+\nDPwTmBwRC+slbklfIQ0VPV7RzH7Ax4EzgLeBt3LcL9VR3AOBacBgoD8wNSJu6Onz3ZnYc51NgO9E\nxI4V7Yynfs75ZcDHSHfIPgD8MiKmNcA1vlTcwMrUwTXeydivow6u805eK11yjTsZm5mZFczD1GZm\nZgVzMjYzMyuYk7GZmVnBnIzNzMwK5mRsZmZWMCdjMzOzgjkZm5mZFez/AQULlHydfUd/AAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb7dbcd8650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "indices = indices[:10]\n",
    "plt.title(\"Random Forest Feature Importances\")\n",
    "plt.barh(range(len(x_axis)), rf_feature[indices],\n",
    "       color=\"r\", yerr=std[indices], align=\"center\")\n",
    "plt.yticks(range(len(x_axis)), x_axis)\n",
    "#plt.xlim([-1, y_pos])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cols = x_train.columns.values\n",
    "# Create a dataframe with features\n",
    "feature_dataframe = pd.DataFrame( {'features': cols,\n",
    "     'Random Forest feature importances': rf_features,\n",
    "     #'Extra Trees  feature importances': et_features,\n",
    "      'AdaBoost feature importances': ada_features,\n",
    "   # 'Gradient Boost feature importances': gb_features,\n",
    "    'XGB Boost feature importances': xgb_features\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AdaBoost feature importances</th>\n",
       "      <th>Random Forest feature importances</th>\n",
       "      <th>XGB Boost feature importances</th>\n",
       "      <th>features</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000403</td>\n",
       "      <td>0.015544</td>\n",
       "      <td>id</td>\n",
       "      <td>0.005316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0.017638</td>\n",
       "      <td>0.107081</td>\n",
       "      <td>goal</td>\n",
       "      <td>0.074906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.032068</td>\n",
       "      <td>0.084629</td>\n",
       "      <td>deadline</td>\n",
       "      <td>0.038899</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   AdaBoost feature importances  Random Forest feature importances  \\\n",
       "0                           0.0                           0.000403   \n",
       "1                           0.1                           0.017638   \n",
       "2                           0.0                           0.032068   \n",
       "\n",
       "   XGB Boost feature importances  features      mean  \n",
       "0                       0.015544        id  0.005316  \n",
       "1                       0.107081      goal  0.074906  \n",
       "2                       0.084629  deadline  0.038899  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create the new column containing the average of values\n",
    "\n",
    "feature_dataframe['mean'] = feature_dataframe.mean(axis= 1) # axis = 1 computes the mean row-wise\n",
    "feature_dataframe.head(3)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Stacked Model DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AdaBoost</th>\n",
       "      <th>RandomForest</th>\n",
       "      <th>XGB Boost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.338189</td>\n",
       "      <td>0.389801</td>\n",
       "      <td>0.028347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.661811</td>\n",
       "      <td>0.610199</td>\n",
       "      <td>0.971653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.501012</td>\n",
       "      <td>0.594387</td>\n",
       "      <td>0.607857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.498988</td>\n",
       "      <td>0.405613</td>\n",
       "      <td>0.392143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.532609</td>\n",
       "      <td>0.604508</td>\n",
       "      <td>0.907953</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   AdaBoost  RandomForest  XGB Boost\n",
       "0  0.338189      0.389801   0.028347\n",
       "1  0.661811      0.610199   0.971653\n",
       "2  0.501012      0.594387   0.607857\n",
       "3  0.498988      0.405613   0.392143\n",
       "4  0.532609      0.604508   0.907953"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_predictions_train = pd.DataFrame( {'RandomForest': rf_oof_train.ravel(),\n",
    "    # 'ExtraTrees': et_oof_train.ravel(),\n",
    "     'AdaBoost': ada_oof_train.ravel(),\n",
    "      'XGB Boost': xgb_oof_train.ravel()\n",
    "    })\n",
    "base_predictions_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Determine Accuracies of Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC AUC Score :\n",
      "0.870187952115\n",
      "0.898676013705\n",
      "0.919245796228\n",
      "0.497146385475\n"
     ]
    }
   ],
   "source": [
    "#Print Scoring\n",
    "print (\"ROC AUC Score :\"  )\n",
    "print roc_auc_score(y_test, rf_oof_test)\n",
    "print roc_auc_score(y_test, ada_oof_test)\n",
    "print roc_auc_score(y_test, xgb_oof_test)\n",
    "print roc_auc_score(y_test, predictions[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = np.concatenate(( ada_oof_train, xgb_oof_train, rf_oof_train), axis=1)\n",
    "x_test = np.concatenate(( ada_oof_test, xgb_oof_test, rf_oof_test), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.33818949  0.02834666  0.38980149]\n",
      " [ 0.66181051  0.97165334  0.61019851]\n",
      " [ 0.50101157  0.60785699  0.59438664]\n",
      " ..., \n",
      " [ 0.50904184  0.71265376  0.54659868]\n",
      " [ 0.52014695  0.9226886   0.61290761]\n",
      " [ 0.47985305  0.07731141  0.38709239]]\n"
     ]
    }
   ],
   "source": [
    "print x_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 0 0 ..., 0 1 0]\n"
     ]
    }
   ],
   "source": [
    "print y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Overlay XGBoost Model\n",
    "gbm = XGBClassifier(\n",
    "    #learning_rate = 0.02,\n",
    " n_estimators= 2000,\n",
    " max_depth= 4,\n",
    " min_child_weight= 2,\n",
    " #gamma=1,\n",
    " gamma=0.9,                        \n",
    " subsample=0.8,\n",
    " colsample_bytree=0.8,\n",
    " objective= 'binary:logistic',\n",
    " nthread= -1,\n",
    " scale_pos_weight=1).fit(x_train, y_train)\n",
    "predictions = gbm.predict_proba(x_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4763, 3)\n"
     ]
    }
   ],
   "source": [
    "print x_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "id = []\n",
    "for i in range(49999,99999):\n",
    "    id.append(i)\n",
    "print predictions.shape\n",
    "print len(id)\n",
    "\n",
    "StackingSubmission = pd.DataFrame({ \n",
    "                            'Y': predictions[:,1], 'id': id })\n",
    "StackingSubmission.to_csv(\"StackingSubmissionXGB.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}