model_gradient_boosting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Boosting Model\n",
    "This notebook aims to build an XGBoost model with parameters selected by GridSearchCV. This file was executed on AWS."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import packages\n",
    "from xgboost import XGBRegressor # pip install xgboost\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing of Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import dataset\n",
    "df = pd.read_csv('data/features_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into training and test sets (30% held out for testing)\n",
    "y = df.loc[:, ['viewCount']] # dependent variable\n",
    "X = df.loc[:, ['titleLen', 'subscriberCount', 'avgViewCount', 'humanCount', 'HOW TO & STYLE', 'SPORTS', 'TRAVEL', 'Negative', 'titleINTJ']] # selected independant variables\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=123)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performing Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "XGBModel = XGBRegressor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting parameters for Grid Search\n",
    "parameters = {'nthread':[4], #when use hyperthread, xgboost may become slower\n",
    "              'objective':['reg:linear'],\n",
    "              'learning_rate': [.03, 0.05, .07], #so called `eta` value\n",
    "              'max_depth': [5, 6, 7],\n",
    "              'min_child_weight': [4],\n",
    "              'subsample': [0.7],\n",
    "              'colsample_bytree': [0.7],\n",
    "              'n_estimators': [500]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 9 candidates, totalling 18 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  18 out of  18 | elapsed: 19.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[17:58:24] WARNING: ../src/objective/regression_obj.cu:174: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[18:01:42] WARNING: ../src/objective/regression_obj.cu:174: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=2, error_score=nan,\n",
       "             estimator=XGBRegressor(base_score=None, booster=None,\n",
       "                                    colsample_bylevel=None,\n",
       "                                    colsample_bynode=None,\n",
       "                                    colsample_bytree=None, gamma=None,\n",
       "                                    gpu_id=None, importance_type='gain',\n",
       "                                    interaction_constraints=None,\n",
       "                                    learning_rate=None, max_delta_step=None,\n",
       "                                    max_depth=None, min_child_weight=None,\n",
       "                                    missing=nan, monotone_constraints=None,\n",
       "                                    n_estima...\n",
       "                                    subsample=None, tree_method=None,\n",
       "                                    validate_parameters=None, verbosity=None),\n",
       "             iid='deprecated', n_jobs=5,\n",
       "             param_grid={'colsample_bytree': [0.7],\n",
       "                         'learning_rate': [0.03, 0.05, 0.07],\n",
       "                         'max_depth': [5, 6, 7], 'min_child_weight': [4],\n",
       "                         'n_estimators': [500], 'nthread': [4],\n",
       "                         'objective': ['reg:linear'], 'subsample': [0.7]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=True)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Run Grid Search\n",
    "grid_result = GridSearchCV(XGBModel, parameters, cv=2, n_jobs=5, verbose=True)\n",
    "grid_result.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.073854 using {'colsample_bytree': 0.7, 'learning_rate': 0.03, 'max_depth': 5, 'min_child_weight': 4, 'n_estimators': 500, 'nthread': 4, 'objective': 'reg:linear', 'subsample': 0.7}\n"
     ]
    }
   ],
   "source": [
    "# Print Best Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print All Results\n",
    "# means = grid_result.cv_results_['mean_test_score']\n",
    "# stds = grid_result.cv_results_['std_test_score']\n",
    "# params = grid_result.cv_results_['params']\n",
    "# for mean, stdev, param in zip(means, stds, params):\n",
    "#     print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "XGBModel = XGBRegressor(nthread=4, objective='reg:linear', learning_rate=0.03, max_depth=5, min_child_weight=4, subsample=0.7, colsample_bytree=0.7, n_estimators=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[18:01:43] WARNING: ../src/objective/regression_obj.cu:174: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[18:04:21] WARNING: ../src/objective/regression_obj.cu:174: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "             colsample_bynode=1, colsample_bytree=0.7, gamma=0, gpu_id=-1,\n",
       "             importance_type='gain', interaction_constraints='',\n",
       "             learning_rate=0.03, max_delta_step=0, max_depth=5,\n",
       "             min_child_weight=4, missing=nan, monotone_constraints='()',\n",
       "             n_estimators=500, n_jobs=4, nthread=4, num_parallel_tree=1,\n",
       "             objective='reg:linear', random_state=0, reg_alpha=0, reg_lambda=1,\n",
       "             scale_pos_weight=1, subsample=0.7, tree_method='exact',\n",
       "             validate_parameters=1, verbosity=None)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "XGBModel.fit(X_train, y_train, verbose=0) #set verbose=1 to see training progress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the mean absolute error on the validation data :\n",
    "XGBpredictions = XGBModel.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting Best Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBoost validation MAE =  2363008.4038565527\n"
     ]
    }
   ],
   "source": [
    "MAE = mean_absolute_error(y_test, XGBpredictions)\n",
    "print('XGBoost validation MAE = ', MAE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBoost validation MSE =  890408446899621.0\n"
     ]
    }
   ],
   "source": [
    "MSE = mean_squared_error(y_test, XGBpredictions)\n",
    "print('XGBoost validation MSE = ', MSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBoost validation RMSE =  29839712.580713995\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "print('XGBoost validation RMSE = ', math.sqrt(MSE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}