TD_C with outliers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sns \n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     CAST_NO  GRADE   BLOW_DUR  MIXER_C  MIXER_MN  MIXER_S  MIXER_P  MIXER_SI  \\\n",
      "0     M44919    980  18.766667     4.48    0.0460   0.0590   0.1940    0.6450   \n",
      "1     M44923    980  18.183333     4.48    0.0460   0.0500   0.1870    0.6790   \n",
      "2     M44925    980  18.166667     4.48    0.0420   0.0780   0.1900    0.8090   \n",
      "3     M44927    980  17.616667     4.48    0.0470   0.0570   0.1980    0.7550   \n",
      "4     M44931    980  18.350000     4.48    0.0440   0.0640   0.1980    0.7920   \n",
      "...      ...    ...        ...      ...       ...      ...      ...       ...   \n",
      "8143  M60321    980  16.333333     4.65    0.0529   0.0250   0.1704    0.9286   \n",
      "8144  M60325    909  16.683333     4.29    0.0438   0.0400   0.1679    0.5499   \n",
      "8145  M60330    909  15.850000     4.48    0.0434   0.0368   0.1736    0.7030   \n",
      "8146  M60344    980  16.850000     4.55    0.0431   0.0510   0.1573    0.6491   \n",
      "8147  M60348    980  16.350000     4.62    0.0431   0.0410   0.1773    0.7617   \n",
      "\n",
      "      MIXER_TI  HM_CHARGED    LIME  IRON_ORE   OXY  SCRAP  SCRAP_HVY_MELT  \\\n",
      "0       0.0655      140.00   8.498     2.906  7897  21.10            3.10   \n",
      "1       0.0656      140.22   8.590     3.000  7354  21.50            3.50   \n",
      "2       0.0700      144.96   8.807     3.165  7624  20.00            0.00   \n",
      "3       0.0729      140.00   9.000     4.500  7200  20.00            5.00   \n",
      "4       0.0679      135.00   8.217     3.248  7230  20.97            5.97   \n",
      "...        ...         ...     ...       ...   ...    ...             ...   \n",
      "8143    0.0995      144.15   8.187     7.589  7188  15.16            5.16   \n",
      "8144    0.0634      146.63   8.773     5.846  7540  15.81            5.81   \n",
      "8145    0.0709      152.37   8.629     8.269  6724   0.00            0.00   \n",
      "8146    0.0631      158.69  10.217    11.905  7234   0.00            0.00   \n",
      "8147    0.0723      155.95   9.029     8.645  6867   0.00            0.00   \n",
      "\n",
      "      SCRAP_WRP_OH_GRADE_FINES   TD_P   TD_C  \n",
      "0                         18.0  0.010  0.042  \n",
      "1                         18.0  0.021  0.050  \n",
      "2                         20.0  0.022  0.053  \n",
      "3                          5.0  0.015  0.046  \n",
      "4                         15.0  0.019  0.054  \n",
      "...                        ...    ...    ...  \n",
      "8143                       5.0  0.015  0.052  \n",
      "8144                       5.0  0.020  0.048  \n",
      "8145                       0.0  0.016  0.057  \n",
      "8146                       0.0  0.020  0.074  \n",
      "8147                       0.0  0.018  0.075  \n",
      "\n",
      "[8148 rows x 18 columns]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "data = pd.read_excel (r'heat data.xlsx')\n",
    "df =data.copy()\n",
    "print (df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CAST_NO</th>\n",
       "      <th>GRADE</th>\n",
       "      <th>BLOW_DUR</th>\n",
       "      <th>MIXER_C</th>\n",
       "      <th>MIXER_MN</th>\n",
       "      <th>MIXER_S</th>\n",
       "      <th>MIXER_P</th>\n",
       "      <th>MIXER_SI</th>\n",
       "      <th>MIXER_TI</th>\n",
       "      <th>HM_CHARGED</th>\n",
       "      <th>LIME</th>\n",
       "      <th>IRON_ORE</th>\n",
       "      <th>OXY</th>\n",
       "      <th>SCRAP</th>\n",
       "      <th>SCRAP_HVY_MELT</th>\n",
       "      <th>SCRAP_WRP_OH_GRADE_FINES</th>\n",
       "      <th>TD_P</th>\n",
       "      <th>TD_C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M44919</td>\n",
       "      <td>980</td>\n",
       "      <td>18.766667</td>\n",
       "      <td>4.48</td>\n",
       "      <td>0.046</td>\n",
       "      <td>0.059</td>\n",
       "      <td>0.194</td>\n",
       "      <td>0.645</td>\n",
       "      <td>0.0655</td>\n",
       "      <td>140.00</td>\n",
       "      <td>8.498</td>\n",
       "      <td>2.906</td>\n",
       "      <td>7897</td>\n",
       "      <td>21.10</td>\n",
       "      <td>3.10</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.010</td>\n",
       "      <td>0.042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M44923</td>\n",
       "      <td>980</td>\n",
       "      <td>18.183333</td>\n",
       "      <td>4.48</td>\n",
       "      <td>0.046</td>\n",
       "      <td>0.050</td>\n",
       "      <td>0.187</td>\n",
       "      <td>0.679</td>\n",
       "      <td>0.0656</td>\n",
       "      <td>140.22</td>\n",
       "      <td>8.590</td>\n",
       "      <td>3.000</td>\n",
       "      <td>7354</td>\n",
       "      <td>21.50</td>\n",
       "      <td>3.50</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.021</td>\n",
       "      <td>0.050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M44925</td>\n",
       "      <td>980</td>\n",
       "      <td>18.166667</td>\n",
       "      <td>4.48</td>\n",
       "      <td>0.042</td>\n",
       "      <td>0.078</td>\n",
       "      <td>0.190</td>\n",
       "      <td>0.809</td>\n",
       "      <td>0.0700</td>\n",
       "      <td>144.96</td>\n",
       "      <td>8.807</td>\n",
       "      <td>3.165</td>\n",
       "      <td>7624</td>\n",
       "      <td>20.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.022</td>\n",
       "      <td>0.053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M44927</td>\n",
       "      <td>980</td>\n",
       "      <td>17.616667</td>\n",
       "      <td>4.48</td>\n",
       "      <td>0.047</td>\n",
       "      <td>0.057</td>\n",
       "      <td>0.198</td>\n",
       "      <td>0.755</td>\n",
       "      <td>0.0729</td>\n",
       "      <td>140.00</td>\n",
       "      <td>9.000</td>\n",
       "      <td>4.500</td>\n",
       "      <td>7200</td>\n",
       "      <td>20.00</td>\n",
       "      <td>5.00</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M44931</td>\n",
       "      <td>980</td>\n",
       "      <td>18.350000</td>\n",
       "      <td>4.48</td>\n",
       "      <td>0.044</td>\n",
       "      <td>0.064</td>\n",
       "      <td>0.198</td>\n",
       "      <td>0.792</td>\n",
       "      <td>0.0679</td>\n",
       "      <td>135.00</td>\n",
       "      <td>8.217</td>\n",
       "      <td>3.248</td>\n",
       "      <td>7230</td>\n",
       "      <td>20.97</td>\n",
       "      <td>5.97</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.019</td>\n",
       "      <td>0.054</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  CAST_NO  GRADE   BLOW_DUR  MIXER_C  MIXER_MN  MIXER_S  MIXER_P  MIXER_SI  \\\n",
       "0  M44919    980  18.766667     4.48     0.046    0.059    0.194     0.645   \n",
       "1  M44923    980  18.183333     4.48     0.046    0.050    0.187     0.679   \n",
       "2  M44925    980  18.166667     4.48     0.042    0.078    0.190     0.809   \n",
       "3  M44927    980  17.616667     4.48     0.047    0.057    0.198     0.755   \n",
       "4  M44931    980  18.350000     4.48     0.044    0.064    0.198     0.792   \n",
       "\n",
       "   MIXER_TI  HM_CHARGED   LIME  IRON_ORE   OXY  SCRAP  SCRAP_HVY_MELT  \\\n",
       "0    0.0655      140.00  8.498     2.906  7897  21.10            3.10   \n",
       "1    0.0656      140.22  8.590     3.000  7354  21.50            3.50   \n",
       "2    0.0700      144.96  8.807     3.165  7624  20.00            0.00   \n",
       "3    0.0729      140.00  9.000     4.500  7200  20.00            5.00   \n",
       "4    0.0679      135.00  8.217     3.248  7230  20.97            5.97   \n",
       "\n",
       "   SCRAP_WRP_OH_GRADE_FINES   TD_P   TD_C  \n",
       "0                      18.0  0.010  0.042  \n",
       "1                      18.0  0.021  0.050  \n",
       "2                      20.0  0.022  0.053  \n",
       "3                       5.0  0.015  0.046  \n",
       "4                      15.0  0.019  0.054  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.019109793814433115\n",
      "0.04967668139420729\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(data['TD_P']))\n",
    "print(np.mean(data['TD_C']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8148 entries, 0 to 8147\n",
      "Data columns (total 18 columns):\n",
      " #   Column                    Non-Null Count  Dtype  \n",
      "---  ------                    --------------  -----  \n",
      " 0   CAST_NO                   8148 non-null   object \n",
      " 1   GRADE                     8148 non-null   int64  \n",
      " 2   BLOW_DUR                  8148 non-null   float64\n",
      " 3   MIXER_C                   8148 non-null   float64\n",
      " 4   MIXER_MN                  8138 non-null   float64\n",
      " 5   MIXER_S                   8148 non-null   float64\n",
      " 6   MIXER_P                   8138 non-null   float64\n",
      " 7   MIXER_SI                  8138 non-null   float64\n",
      " 8   MIXER_TI                  8138 non-null   float64\n",
      " 9   HM_CHARGED                8148 non-null   float64\n",
      " 10  LIME                      8148 non-null   float64\n",
      " 11  IRON_ORE                  8148 non-null   float64\n",
      " 12  OXY                       8148 non-null   int64  \n",
      " 13  SCRAP                     8148 non-null   float64\n",
      " 14  SCRAP_HVY_MELT            8148 non-null   float64\n",
      " 15  SCRAP_WRP_OH_GRADE_FINES  8148 non-null   float64\n",
      " 16  TD_P                      8148 non-null   float64\n",
      " 17  TD_C                      8148 non-null   float64\n",
      "dtypes: float64(15), int64(2), object(1)\n",
      "memory usage: 1.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DATA PRE-PROCESSING\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GRADE                       0\n",
       "BLOW_DUR                    0\n",
       "MIXER_C                     0\n",
       "MIXER_MN                    0\n",
       "MIXER_S                     0\n",
       "MIXER_P                     0\n",
       "MIXER_SI                    0\n",
       "MIXER_TI                    0\n",
       "HM_CHARGED                  0\n",
       "LIME                        0\n",
       "IRON_ORE                    0\n",
       "OXY                         0\n",
       "SCRAP                       0\n",
       "SCRAP_HVY_MELT              0\n",
       "SCRAP_WRP_OH_GRADE_FINES    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=df.drop('TD_P',axis=1)\n",
    "df=df.drop('TD_C',axis=1)\n",
    "df=df.drop('CAST_NO',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We see that there are some values in the dataset. Since there might be some outliers also,we will try to impute it with median.df[\"MIXER_MN\"].fillna(df[\"MIXER_MN\"].median(), inplace=True)\n",
    "df[\"MIXER_P\"].fillna(df[\"MIXER_P\"].median(), inplace=True)\n",
    "df[\"MIXER_SI\"].fillna(df[\"MIXER_SI\"].median(), inplace=True)\n",
    "df[\"MIXER_TI\"].fillna(df[\"MIXER_TI\"].median(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of Feature Matrix is - (8148, 15)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "X_std = StandardScaler().fit_transform(df)\n",
    "\n",
    "print (\"The shape of Feature Matrix is -\",X_std.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=X_std\n",
    "y=data['TD_C']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn import model_selection\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import r2_score\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,\n",
    "                                                    test_size=0.2,\n",
    "                                                    random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Applying the Random forest regression along with hyperparameter tuning using RandomsearchCV. Below are the hypermaters that can be used for traing from which the best ones are to be chosen.\n",
    "\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "rf = RandomForestRegressor(random_state = 42)\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "max_depth.append(None)\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "# Create the random grid\n",
    "random_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 100 candidates, totalling 300 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  25 tasks      | elapsed: 13.5min\n",
      "[Parallel(n_jobs=-1)]: Done 146 tasks      | elapsed: 42.3min\n",
      "[Parallel(n_jobs=-1)]: Done 300 out of 300 | elapsed: 79.6min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=3, error_score=nan,\n",
       "                   estimator=RandomForestRegressor(bootstrap=True,\n",
       "                                                   ccp_alpha=0.0,\n",
       "                                                   criterion='mse',\n",
       "                                                   max_depth=None,\n",
       "                                                   max_features='auto',\n",
       "                                                   max_leaf_nodes=None,\n",
       "                                                   max_samples=None,\n",
       "                                                   min_impurity_decrease=0.0,\n",
       "                                                   min_impurity_split=None,\n",
       "                                                   min_samples_leaf=1,\n",
       "                                                   min_samples_split=2,\n",
       "                                                   min_weight_fraction_leaf=0.0,\n",
       "                                                   n_estimators=100,\n",
       "                                                   n_jobs=None, oob_score=Fals...\n",
       "                   param_distributions={'bootstrap': [True, False],\n",
       "                                        'max_depth': [10, 20, 30, 40, 50, 60,\n",
       "                                                      70, 80, 90, 100, 110,\n",
       "                                                      None],\n",
       "                                        'max_features': ['auto', 'sqrt'],\n",
       "                                        'min_samples_leaf': [1, 2, 4],\n",
       "                                        'min_samples_split': [2, 5, 10],\n",
       "                                        'n_estimators': [200, 400, 600, 800,\n",
       "                                                         1000, 1200, 1400, 1600,\n",
       "                                                         1800, 2000]},\n",
       "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
       "                   return_train_score=False, scoring=None, verbose=2)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf = RandomForestRegressor()\n",
    "# Random search of parameters, using 3 fold cross validation, \n",
    "# search across 100 different combinations, and use all available cores\n",
    "rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1)\n",
    "# Fit the random search model\n",
    "rf_random.fit(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Performance\n",
      "Average Error: 0.0096 degrees.\n",
      "Accuracy = 81.32%.\n",
      "R2: 0.01\n"
     ]
    }
   ],
   "source": [
    "#evaluation of the model\n",
    "def evaluate(model, test_features, test_labels):\n",
    "    predictions = model.predict(test_features)\n",
    "    errors = abs(predictions - test_labels)\n",
    "    mape = 100 * np.mean(errors / test_labels)\n",
    "    accuracy = 100 - mape\n",
    "    print('Model Performance')\n",
    "    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))\n",
    "    print('Accuracy = {:0.2f}%.'.format(accuracy))\n",
    "    \n",
    "    return accuracy\n",
    "model2=rf_random.best_estimator_\n",
    "best_random =model2\n",
    "random_accuracy = evaluate(best_random, X_train,y_train)\n",
    "y_pred = best_random.predict(X_test)\n",
    "\n",
    "\n",
    "from sklearn.metrics import r2_score\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "print(\"R2: %.2f\" % r2 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2: -31.98\n"
     ]
    }
   ],
   "source": [
    "#checking with decision tree model\n",
    "from sklearn import tree\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "dt = DecisionTreeRegressor()\n",
    "dt.fit(X_train, y_train)\n",
    "y_pred = dt.predict(X_test)\n",
    "from sklearn.metrics import r2_score\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "print(\"R2: %.2f\" % r2 )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1.4462880329920484\n"
     ]
    }
   ],
   "source": [
    "#checking with the ridge regression\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "# load the diabetes datasets\n",
    "\n",
    "# prepare a range of alpha values to test\n",
    "alphas = np.array([1,0.1,0.01,0.001,0.0001,0])\n",
    "# create and fit a ridge regression model, testing each alpha\n",
    "model = Ridge()\n",
    "grid = GridSearchCV(estimator=model, param_grid=dict(alpha=alphas),scoring='r2')\n",
    "grid.fit(X_train,y_train)\n",
    "\n",
    "# summarize the results of the grid search\n",
    "print(grid.best_score_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Performance\n",
      "Average Error: 0.0139 degrees.\n",
      "Accuracy = 72.34%.\n",
      "R2: -0.01\n"
     ]
    }
   ],
   "source": [
    "def evaluate(model, test_features, test_labels):\n",
    "    predictions = model.predict(test_features)\n",
    "    errors = abs(predictions - test_labels)\n",
    "    mape = 100 * np.mean(errors / test_labels)\n",
    "    accuracy = 100 - mape\n",
    "    print('Model Performance')\n",
    "    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))\n",
    "    print('Accuracy = {:0.2f}%.'.format(accuracy))\n",
    "    \n",
    "    return accuracy\n",
    "best_random = grid.best_estimator_\n",
    "random_accuracy = evaluate(best_random, X_train,y_train)\n",
    "y_pred = best_random.predict(X_test)\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "from sklearn.metrics import r2_score\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "print(\"R2: %.2f\" % r2 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE: 23.42\n",
      "RS: -61828.12\n"
     ]
    }
   ],
   "source": [
    "#applying SGDRegressor\n",
    "from sklearn.linear_model import SGDRegressor\n",
    "sgd_reg = SGDRegressor(alpha=0.0001, epsilon=0.1, eta0=0.01, fit_intercept=True,\n",
    "       l1_ratio=0.15, learning_rate='invscaling', loss='squared_loss', penalty='l2', power_t=0.25, random_state=None,\n",
    "       shuffle=False, verbose=0, warm_start=False)\n",
    "sgd_reg.fit(X_train, y_train)\n",
    "y_pred=sgd_reg.predict(X_test)\n",
    "from sklearn.metrics import mean_squared_error\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "print(\"MSE: %.2f\" % mse)\n",
    "from sklearn.metrics import r2_score\n",
    "rs = r2_score(y_test, y_pred)\n",
    "print(\"RS: %.2f\" % rs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Performance\n",
      "Average Error: 7.9232 degrees.\n",
      "Accuracy = -18480.21%.\n"
     ]
    }
   ],
   "source": [
    "def evaluate(model, test_features, test_labels):\n",
    "    predictions = model.predict(test_features)\n",
    "    errors = abs(predictions - test_labels)\n",
    "    mape = 100 * np.mean(errors / test_labels)\n",
    "    accuracy = 100 - mape\n",
    "    print('Model Performance')\n",
    "    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))\n",
    "    print('Accuracy = {:0.2f}%.'.format(accuracy))\n",
    "    \n",
    "    return accuracy\n",
    "best_random = sgd_reg\n",
    "random_accuracy = evaluate(best_random, X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score=nan,\n",
       "             estimator=XGBRegressor(base_score=0.5, booster='gbtree',\n",
       "                                    colsample_bylevel=1, colsample_bynode=1,\n",
       "                                    colsample_bytree=1, gamma=0,\n",
       "                                    importance_type='gain', learning_rate=0.1,\n",
       "                                    max_delta_step=0, max_depth=3,\n",
       "                                    min_child_weight=1, missing=None,\n",
       "                                    n_estimators=100, n_jobs=1, nthread=None,\n",
       "                                    objective='reg:linear', random_state=0,\n",
       "                                    reg_alpha=0, reg...\n",
       "                                    scale_pos_weight=1, seed=None, silent=None,\n",
       "                                    subsample=1, verbosity=1),\n",
       "             iid='deprecated', n_jobs=5,\n",
       "             param_grid={'colsample_bytree': [0.7],\n",
       "                         'learning_rate': [0.03, 0.05, 0.07],\n",
       "                         'max_depth': [5, 6, 7], 'min_child_weight': [4],\n",
       "                         'n_estimators': [500], 'nthread': [4],\n",
       "                         'objective': ['reg:linear'], 'silent': [1],\n",
       "                         'subsample': [0.7]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#applying XGBRegressor\n",
    "from xgboost import XGBRegressor\n",
    "xgb1 = XGBRegressor()\n",
    "parameters = {'nthread':[4], #when use hyperthread, xgboost may become slower\n",
    "              'objective':['reg:linear'],\n",
    "              'learning_rate': [.03, 0.05, .07], #so called `eta` value\n",
    "              'max_depth': [5, 6, 7],\n",
    "              'min_child_weight': [4],\n",
    "              'silent': [1],\n",
    "              'subsample': [0.7],\n",
    "              'colsample_bytree': [0.7],\n",
    "              'n_estimators': [500]}\n",
    "\n",
    "xgb_grid = GridSearchCV(xgb1,\n",
    "                        parameters,\n",
    "                        cv = 10,\n",
    "                        n_jobs = 5,\n",
    "                        verbose=0)\n",
    "\n",
    "xgb_grid.fit(X_train,\n",
    "         y_train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.7,\n",
       " 'learning_rate': 0.03,\n",
       " 'max_depth': 5,\n",
       " 'min_child_weight': 4,\n",
       " 'n_estimators': 500,\n",
       " 'nthread': 4,\n",
       " 'objective': 'reg:linear',\n",
       " 'silent': 1,\n",
       " 'subsample': 0.7}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Performance\n",
      "Average Error: 0.0117 degrees.\n",
      "Accuracy = 76.09%.\n",
      "R2: -0.69\n"
     ]
    }
   ],
   "source": [
    "def evaluate(model, test_features, test_labels):\n",
    "    predictions = model.predict(test_features)\n",
    "    errors = abs(predictions - test_labels)\n",
    "    mape = 100 * np.mean(errors / test_labels)\n",
    "    accuracy = 100 - mape\n",
    "    print('Model Performance')\n",
    "    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))\n",
    "    print('Accuracy = {:0.2f}%.'.format(accuracy))\n",
    "    \n",
    "    return accuracy\n",
    "best_random = xgb_grid.best_estimator_\n",
    "random_accuracy = evaluate(best_random, X_train,y_train)\n",
    "y_pred = best_random.predict(X_test)\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "from sklearn.metrics import r2_score\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "print(\"R2: %.2f\" % r2 )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Here we can see that the score is not that good in different algorithms. However among all these Randomforest regressor is the best."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving the model\n",
    "import pickle\n",
    "filename = 'TD_C with outliers_model2.sav'\n",
    "pickle.dump(model2, open(filename, 'wb'))\n",
    " \n",
    "# some time later...\n",
    " \n",
    "# load the model from disk\n",
    "loaded_model = pickle.load(open(filename, 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.005460703354758789\n"
     ]
    }
   ],
   "source": [
    "loaded_model = pickle.load(open(filename, 'rb'))\n",
    "result = loaded_model.score(X_test, y_test)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}