try_lightGBM_single_thread_basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This file is the very first try of lightGBM, compare with xgboost\n",
    "# using single thread, wihout cross validation\n",
    "\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "from pandas import Series, DataFrame \n",
    "import lightgbm as lgb \n",
    "import xgboost as xgb \n",
    "from datetime import datetime \n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital_Status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital_gain</th>\n",
       "      <th>capital_loss</th>\n",
       "      <th>hours_per_week</th>\n",
       "      <th>native_country</th>\n",
       "      <th>Income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>77516</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>83311</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>215646</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>Private</td>\n",
       "      <td>234721</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>Private</td>\n",
       "      <td>338409</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Wife</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>Cuba</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age          workclass  fnlwgt   education  education-num  \\\n",
       "0   39          State-gov   77516   Bachelors             13   \n",
       "1   50   Self-emp-not-inc   83311   Bachelors             13   \n",
       "2   38            Private  215646     HS-grad              9   \n",
       "3   53            Private  234721        11th              7   \n",
       "4   28            Private  338409   Bachelors             13   \n",
       "\n",
       "        marital_Status          occupation    relationship    race      sex  \\\n",
       "0        Never-married        Adm-clerical   Not-in-family   White     Male   \n",
       "1   Married-civ-spouse     Exec-managerial         Husband   White     Male   \n",
       "2             Divorced   Handlers-cleaners   Not-in-family   White     Male   \n",
       "3   Married-civ-spouse   Handlers-cleaners         Husband   Black     Male   \n",
       "4   Married-civ-spouse      Prof-specialty            Wife   Black   Female   \n",
       "\n",
       "   capital_gain  capital_loss  hours_per_week  native_country  Income  \n",
       "0          2174             0              40   United-States   <=50K  \n",
       "1             0             0              13   United-States   <=50K  \n",
       "2             0             0              40   United-States   <=50K  \n",
       "3             0             0              40   United-States   <=50K  \n",
       "4             0             0              40            Cuba   <=50K  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=pd.read_csv('adult.csv',header=None) \n",
    "# assign column names to the data\n",
    "data.columns=['age','workclass','fnlwgt','education','education-num','marital_Status','occupation','relationship','race','sex','capital_gain','capital_loss','hours_per_week','native_country','Income'] \n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder,OneHotEncoder\n",
    "\n",
    "# encode label\n",
    "l=LabelEncoder() \n",
    "l.fit(data.Income) \n",
    "l.classes_ \n",
    "data.Income=Series(l.transform(data.Income))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    24720\n",
       "1     7841\n",
       "Name: Income, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.Income.value_counts() # label has been encoded as 0, 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# convert categorical data into one-hot, and drop original categorical data\n",
    "one_hot_workclass=pd.get_dummies(data.workclass) \n",
    "one_hot_education=pd.get_dummies(data.education) \n",
    "one_hot_marital_Status=pd.get_dummies(data.marital_Status) \n",
    "one_hot_occupation=pd.get_dummies(data.occupation)\n",
    "one_hot_relationship=pd.get_dummies(data.relationship) \n",
    "one_hot_race=pd.get_dummies(data.race) \n",
    "one_hot_sex=pd.get_dummies(data.sex) \n",
    "one_hot_native_country=pd.get_dummies(data.native_country) \n",
    "\n",
    "data.drop(['workclass','education','marital_Status','occupation','relationship','race','sex','native_country'],axis=1,inplace=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data=pd.concat([data,one_hot_workclass,one_hot_education,one_hot_marital_Status,one_hot_occupation,one_hot_relationship,one_hot_race,one_hot_sex,one_hot_native_country],axis=1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 16,  17,  18,  19,  20,  21,  22,   7,  40,  60,  41,  61,  23,\n",
       "        24,  25,  62,  68,  69,  70,  71,  42,  72,  32,  26,  73,  74,\n",
       "        75,  76,  43,  44,   8,  65,  77,  78,  79,  80,  27,  81,  45,\n",
       "        82,  83,  84,  85,  54,  86,  87,  88,  89,  90,  91,  92,   9,\n",
       "        46,  66,  33,  34,  35,  28,  93,  36,  10,  94,  55,  63,  56,\n",
       "        47,  95,  57,  96,  97,  98,  99,  29,  48,  11,  30,  49,  50,\n",
       "       100,  51, 101,  12,  13,  37,  31, 102,  14, 103,  52, 104,  53,\n",
       "       105, 106,  58, 107,  64,  38,  59,  15, 108,   6,   0,   3,   4,\n",
       "         2,   1,   5])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#removing dulpicate columns \n",
    "i = np.unique(data.columns, return_index=True) \n",
    "i[1]  # index of unique columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>10th</th>\n",
       "      <th>11th</th>\n",
       "      <th>12th</th>\n",
       "      <th>1st-4th</th>\n",
       "      <th>5th-6th</th>\n",
       "      <th>7th-8th</th>\n",
       "      <th>9th</th>\n",
       "      <th>?</th>\n",
       "      <th>Adm-clerical</th>\n",
       "      <th>Amer-Indian-Eskimo</th>\n",
       "      <th>...</th>\n",
       "      <th>Wife</th>\n",
       "      <th>Without-pay</th>\n",
       "      <th>Yugoslavia</th>\n",
       "      <th>Income</th>\n",
       "      <th>age</th>\n",
       "      <th>capital_gain</th>\n",
       "      <th>capital_loss</th>\n",
       "      <th>education-num</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>hours_per_week</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>77516</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>83311</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>215646</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>53</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>234721</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>338409</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 107 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    10th   11th   12th   1st-4th   5th-6th   7th-8th   9th   ?   Adm-clerical  \\\n",
       "0      0      0      0         0         0         0     0   0              1   \n",
       "1      0      0      0         0         0         0     0   0              0   \n",
       "2      0      0      0         0         0         0     0   0              0   \n",
       "3      0      1      0         0         0         0     0   0              0   \n",
       "4      0      0      0         0         0         0     0   0              0   \n",
       "\n",
       "    Amer-Indian-Eskimo       ...         Wife   Without-pay   Yugoslavia  \\\n",
       "0                    0       ...            0             0            0   \n",
       "1                    0       ...            0             0            0   \n",
       "2                    0       ...            0             0            0   \n",
       "3                    0       ...            0             0            0   \n",
       "4                    0       ...            1             0            0   \n",
       "\n",
       "   Income  age  capital_gain  capital_loss  education-num  fnlwgt  \\\n",
       "0       0   39          2174             0             13   77516   \n",
       "1       0   50             0             0             13   83311   \n",
       "2       0   38             0             0              9  215646   \n",
       "3       0   53             0             0              7  234721   \n",
       "4       0   28             0             0             13  338409   \n",
       "\n",
       "   hours_per_week  \n",
       "0              40  \n",
       "1              13  \n",
       "2              40  \n",
       "3              40  \n",
       "4              40  \n",
       "\n",
       "[5 rows x 107 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=data.iloc[:, i[1]]  # use the index of unique columns\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = data.drop('Income',axis=1) \n",
    "label = data.Income "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "label.mode()[0]\n",
    "label.fillna(label.mode()[0],inplace=True)  # impute missing data with mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    24720\n",
       "1     7841\n",
       "Name: Income, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Now splitting our dataset into test and train \n",
    "from sklearn.model_selection import train_test_split \n",
    "\n",
    "features_train,features_test,label_train,label_test=train_test_split(features,label,test_size=.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19851    0\n",
       "15010    0\n",
       "5949     0\n",
       "668      0\n",
       "30677    0\n",
       "Name: Income, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_train.head()\n",
    "features_test.head()\n",
    "label_train.head()\n",
    "label_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# method 1 - XGBOOST, single thread\n",
    "dtrain=xgb.DMatrix(features_train,label=label_train)\n",
    "dtest=xgb.DMatrix(features_test)\n",
    "## xgboost booster params\n",
    "parameters={'max_depth':7, 'eta':1, 'silent':1,'objective':'binary:logistic','eval_metric':'auc','learning_rate':.05}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:00:05.880084\n"
     ]
    }
   ],
   "source": [
    "num_round=50    # number of boosting iterations\n",
    "start = datetime.now() \n",
    "xg=xgb.train(parameters,dtrain,num_round)  # train the model\n",
    "stop = datetime.now()\n",
    "execution_time_xgb = stop-start \n",
    "print(execution_time_xgb)   # 0:00:05.880084"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.05520488  0.44592044  0.12861401 ...,  0.04373964  0.04373964\n",
      "  0.32243791]\n"
     ]
    }
   ],
   "source": [
    "ypred=xg.predict(dtest) \n",
    "print(ypred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Converting probabilities into 1 or 0  \n",
    "for i in range(0,9769): \n",
    "    if ypred[i]>=.5:       # setting threshold to .5 \n",
    "       ypred[i]=1 \n",
    "    else: \n",
    "       ypred[i]=0  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.87030402292967546"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_xgb = accuracy_score(label_test,ypred) \n",
    "accuracy_xgb  # 0.87030402292967546"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.777341026858\n"
     ]
    }
   ],
   "source": [
    "cm = confusion_matrix(label_test, ypred)\n",
    "TP = cm[0][0]\n",
    "FP = cm[0][1]\n",
    "FN = cm[1][0]\n",
    "TN = cm[1][1]\n",
    "\n",
    "accuracy_xgb = (TP + TN)/(TP+FP+FN+TN)   # accuracy: 0.87030402293\n",
    "auc_score_xgb = roc_auc_score(label_test, ypred)   # AUC: 0.777341026858\n",
    "precision_xgb = TP/(TP+FP)   # precision: 0.952047413793\n",
    "specificity_xgb = TN/(TN+FP)  # specificity: 0.797612279704\n",
    "recall_xgb = TP/(TP+FN)   # recall: 0.885017421603\n",
    "print(auc_score_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# method 2 - Light GBM, single thread (make -j)\n",
    "\n",
    "dtrain = lgb.Dataset(features_train,label=label_train)\n",
    "param = {'num_leaves':150, 'objective':'binary','max_depth':7,'learning_rate':.05,'max_bin':200}\n",
    "param['metric'] = ['auc', 'binary_logloss']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:00:01.107647\n"
     ]
    }
   ],
   "source": [
    "# train the model\n",
    "\n",
    "num_round=50\n",
    "start=datetime.now()\n",
    "lgbm=lgb.train(param,dtrain,num_round)\n",
    "stop=datetime.now()\n",
    "execution_time_lgbm = stop-start\n",
    "print(execution_time_lgbm)    # 0:00:01.107647"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.0568377 ,  0.44797529,  0.13431321,  0.04504189,  0.06630088])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predict \n",
    "\n",
    "ypred2=lgbm.predict(features_test)\n",
    "ypred2[0:5]  # showing first 5 predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Converting probabilities into 1 or 0  \n",
    "for i in range(0,9769): \n",
    "    if ypred2[i]>=.5:       # setting threshold to .5 \n",
    "       ypred2[i]=1 \n",
    "    else: \n",
    "       ypred2[i]=0  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.87081584604360729"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_lgb = accuracy_score(label_test,ypred2) \n",
    "accuracy_lgb  # 0.87081584604360729"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.88470617468\n"
     ]
    }
   ],
   "source": [
    "cm = confusion_matrix(label_test, ypred2)\n",
    "TP = cm[0][0]\n",
    "FP = cm[0][1]\n",
    "FN = cm[1][0]\n",
    "TN = cm[1][1]\n",
    "\n",
    "accuracy_lgb = (TP + TN)/(TP+FP+FN+TN)   # accuracy: 0.870815846044\n",
    "auc_score_lgb = roc_auc_score(label_test, ypred2)   # AUC: 0.777080344354\n",
    "precision_lgb = TP/(TP+FP)   # precision: 0.955197853789\n",
    "specificity_lgb = TN/(TN+FP)  # specificity: 0.805813953488\n",
    "recall_lgb = TP/(TP+FN)   # recall: 0.88470617468\n",
    "print(recall_lgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy score</th>\n",
       "      <th>auc score</th>\n",
       "      <th>execution time</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>specificity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LightGBM</th>\n",
       "      <td>0.870816</td>\n",
       "      <td>0.777080</td>\n",
       "      <td>00:00:01.107647</td>\n",
       "      <td>0.955198</td>\n",
       "      <td>0.884706</td>\n",
       "      <td>0.805814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgboost</th>\n",
       "      <td>0.870304</td>\n",
       "      <td>0.777341</td>\n",
       "      <td>00:00:05.880084</td>\n",
       "      <td>0.953991</td>\n",
       "      <td>0.885017</td>\n",
       "      <td>0.802077</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          accuracy score  auc score  execution time  precision    recall  \\\n",
       "LightGBM        0.870816   0.777080 00:00:01.107647   0.955198  0.884706   \n",
       "xgboost         0.870304   0.777341 00:00:05.880084   0.953991  0.885017   \n",
       "\n",
       "          specificity  \n",
       "LightGBM     0.805814  \n",
       "xgboost      0.802077  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compare single thread light GBM and single thread xgboost\n",
    "comparison_dict = {'accuracy score':(accuracy_lgb,accuracy_xgb),'auc score':(auc_score_lgb,auc_score_xgb),'execution time':(execution_time_lgbm,execution_time_xgb),\n",
    "                  'precision':(precision_lgb, precision_xgb), 'specificity':(specificity_lgb, specificity_xgb), 'recall':(recall_lgb, recall_xgb)}\n",
    "comparison_df = DataFrame(comparison_dict) \n",
    "comparison_df.index= ['LightGBM','xgboost'] \n",
    "comparison_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "NOTE: when you are using LightGBM/xgboost train() method, it does not give seed, \n",
    "therefore, if you run it multiple times, you will get different results\n",
    "'''"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}