Data-Sphere Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.utils import resample\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_validate, KFold\n",
    "\n",
    "from sklearn.metrics import recall_score, roc_auc_score, f1_score\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score, \\\n",
    "                            classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read Data which was cleaned and pre-processed data.\n",
    "df_selected = pd.read_csv('./dataset/Cleaned_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>...</th>\n",
       "      <th>fico_range_high</th>\n",
       "      <th>open_acc</th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>application_type</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>issue_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.078824e+06</td>\n",
       "      <td>1.441997e+04</td>\n",
       "      <td>2.412581e-01</td>\n",
       "      <td>1.323962e+01</td>\n",
       "      <td>4.380755e+02</td>\n",
       "      <td>1.745809e+00</td>\n",
       "      <td>1.069302e+01</td>\n",
       "      <td>6.052432e+00</td>\n",
       "      <td>2.911998e+00</td>\n",
       "      <td>7.624764e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>7.001852e+02</td>\n",
       "      <td>1.159352e+01</td>\n",
       "      <td>2.152760e-01</td>\n",
       "      <td>1.624811e+04</td>\n",
       "      <td>5.181002e+01</td>\n",
       "      <td>2.498084e+01</td>\n",
       "      <td>5.827727e-01</td>\n",
       "      <td>1.917774e-02</td>\n",
       "      <td>5.128929e-03</td>\n",
       "      <td>5.531292e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6.841849e+05</td>\n",
       "      <td>8.717051e+03</td>\n",
       "      <td>4.278467e-01</td>\n",
       "      <td>4.768716e+00</td>\n",
       "      <td>2.615126e+02</td>\n",
       "      <td>1.295071e+00</td>\n",
       "      <td>6.449912e+00</td>\n",
       "      <td>3.562249e+00</td>\n",
       "      <td>1.915676e+00</td>\n",
       "      <td>6.992510e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>3.185316e+01</td>\n",
       "      <td>5.473789e+00</td>\n",
       "      <td>6.018647e-01</td>\n",
       "      <td>2.232817e+04</td>\n",
       "      <td>2.451326e+01</td>\n",
       "      <td>1.199847e+01</td>\n",
       "      <td>4.931013e-01</td>\n",
       "      <td>1.371494e-01</td>\n",
       "      <td>7.143268e-02</td>\n",
       "      <td>3.412514e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.000000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.310000e+00</td>\n",
       "      <td>4.930000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>6.290000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.783052e+05</td>\n",
       "      <td>8.000000e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>9.750000e+00</td>\n",
       "      <td>2.484800e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>4.578000e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>6.740000e+02</td>\n",
       "      <td>8.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.943000e+03</td>\n",
       "      <td>3.350000e+01</td>\n",
       "      <td>1.600000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.118210e+06</td>\n",
       "      <td>1.200000e+04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.274000e+01</td>\n",
       "      <td>3.754300e+02</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>6.500000e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>6.940000e+02</td>\n",
       "      <td>1.100000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.113400e+04</td>\n",
       "      <td>5.210000e+01</td>\n",
       "      <td>2.300000e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.755354e+06</td>\n",
       "      <td>2.000000e+04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.599000e+01</td>\n",
       "      <td>5.807300e+02</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.400000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>9.000000e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>7.140000e+02</td>\n",
       "      <td>1.400000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.975575e+04</td>\n",
       "      <td>7.070000e+01</td>\n",
       "      <td>3.200000e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>9.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.260697e+06</td>\n",
       "      <td>4.000000e+04</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>3.099000e+01</td>\n",
       "      <td>1.719830e+03</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>3.400000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>1.099920e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>8.500000e+02</td>\n",
       "      <td>9.000000e+01</td>\n",
       "      <td>8.600000e+01</td>\n",
       "      <td>2.904836e+06</td>\n",
       "      <td>8.923000e+02</td>\n",
       "      <td>1.760000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.100000e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Unnamed: 0     loan_amnt          term      int_rate   installment  \\\n",
       "count  1.345310e+06  1.345310e+06  1.345310e+06  1.345310e+06  1.345310e+06   \n",
       "mean   1.078824e+06  1.441997e+04  2.412581e-01  1.323962e+01  4.380755e+02   \n",
       "std    6.841849e+05  8.717051e+03  4.278467e-01  4.768716e+00  2.615126e+02   \n",
       "min    0.000000e+00  5.000000e+02  0.000000e+00  5.310000e+00  4.930000e+00   \n",
       "25%    3.783052e+05  8.000000e+03  0.000000e+00  9.750000e+00  2.484800e+02   \n",
       "50%    1.118210e+06  1.200000e+04  0.000000e+00  1.274000e+01  3.754300e+02   \n",
       "75%    1.755354e+06  2.000000e+04  0.000000e+00  1.599000e+01  5.807300e+02   \n",
       "max    2.260697e+06  4.000000e+04  1.000000e+00  3.099000e+01  1.719830e+03   \n",
       "\n",
       "              grade     sub_grade    emp_length  home_ownership    annual_inc  \\\n",
       "count  1.345310e+06  1.345310e+06  1.345310e+06    1.345310e+06  1.345310e+06   \n",
       "mean   1.745809e+00  1.069302e+01  6.052432e+00    2.911998e+00  7.624764e+04   \n",
       "std    1.295071e+00  6.449912e+00  3.562249e+00    1.915676e+00  6.992510e+04   \n",
       "min    0.000000e+00  0.000000e+00  1.000000e+00    0.000000e+00  0.000000e+00   \n",
       "25%    1.000000e+00  6.000000e+00  2.000000e+00    1.000000e+00  4.578000e+04   \n",
       "50%    2.000000e+00  1.000000e+01  6.000000e+00    4.000000e+00  6.500000e+04   \n",
       "75%    2.000000e+00  1.400000e+01  1.000000e+01    5.000000e+00  9.000000e+04   \n",
       "max    6.000000e+00  3.400000e+01  1.000000e+01    5.000000e+00  1.099920e+07   \n",
       "\n",
       "       ...  fico_range_high      open_acc       pub_rec     revol_bal  \\\n",
       "count  ...     1.345310e+06  1.345310e+06  1.345310e+06  1.345310e+06   \n",
       "mean   ...     7.001852e+02  1.159352e+01  2.152760e-01  1.624811e+04   \n",
       "std    ...     3.185316e+01  5.473789e+00  6.018647e-01  2.232817e+04   \n",
       "min    ...     6.290000e+02  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "25%    ...     6.740000e+02  8.000000e+00  0.000000e+00  5.943000e+03   \n",
       "50%    ...     6.940000e+02  1.100000e+01  0.000000e+00  1.113400e+04   \n",
       "75%    ...     7.140000e+02  1.400000e+01  0.000000e+00  1.975575e+04   \n",
       "max    ...     8.500000e+02  9.000000e+01  8.600000e+01  2.904836e+06   \n",
       "\n",
       "         revol_util     total_acc  initial_list_status  application_type  \\\n",
       "count  1.345310e+06  1.345310e+06         1.345310e+06      1.345310e+06   \n",
       "mean   5.181002e+01  2.498084e+01         5.827727e-01      1.917774e-02   \n",
       "std    2.451326e+01  1.199847e+01         4.931013e-01      1.371494e-01   \n",
       "min    0.000000e+00  2.000000e+00         0.000000e+00      0.000000e+00   \n",
       "25%    3.350000e+01  1.600000e+01         0.000000e+00      0.000000e+00   \n",
       "50%    5.210000e+01  2.300000e+01         1.000000e+00      0.000000e+00   \n",
       "75%    7.070000e+01  3.200000e+01         1.000000e+00      0.000000e+00   \n",
       "max    8.923000e+02  1.760000e+02         1.000000e+00      1.000000e+00   \n",
       "\n",
       "       disbursement_method   issue_month  \n",
       "count         1.345310e+06  1.345310e+06  \n",
       "mean          5.128929e-03  5.531292e+00  \n",
       "std           7.143268e-02  3.412514e+00  \n",
       "min           0.000000e+00  0.000000e+00  \n",
       "25%           0.000000e+00  3.000000e+00  \n",
       "50%           0.000000e+00  5.000000e+00  \n",
       "75%           0.000000e+00  9.000000e+00  \n",
       "max           1.000000e+00  1.100000e+01  \n",
       "\n",
       "[8 rows x 25 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_selected.describe(include = 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_selected=df_selected.drop(df_selected.columns[0], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>...</th>\n",
       "      <th>fico_range_high</th>\n",
       "      <th>open_acc</th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>application_type</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>issue_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "      <td>1.345310e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.441997e+04</td>\n",
       "      <td>2.412581e-01</td>\n",
       "      <td>1.323962e+01</td>\n",
       "      <td>4.380755e+02</td>\n",
       "      <td>1.745809e+00</td>\n",
       "      <td>1.069302e+01</td>\n",
       "      <td>6.052432e+00</td>\n",
       "      <td>2.911998e+00</td>\n",
       "      <td>7.624764e+04</td>\n",
       "      <td>3.015669e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>7.001852e+02</td>\n",
       "      <td>1.159352e+01</td>\n",
       "      <td>2.152760e-01</td>\n",
       "      <td>1.624811e+04</td>\n",
       "      <td>5.181002e+01</td>\n",
       "      <td>2.498084e+01</td>\n",
       "      <td>5.827727e-01</td>\n",
       "      <td>1.917774e-02</td>\n",
       "      <td>5.128929e-03</td>\n",
       "      <td>5.531292e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>8.717051e+03</td>\n",
       "      <td>4.278467e-01</td>\n",
       "      <td>4.768716e+00</td>\n",
       "      <td>2.615126e+02</td>\n",
       "      <td>1.295071e+00</td>\n",
       "      <td>6.449912e+00</td>\n",
       "      <td>3.562249e+00</td>\n",
       "      <td>1.915676e+00</td>\n",
       "      <td>6.992510e+04</td>\n",
       "      <td>4.589384e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>3.185316e+01</td>\n",
       "      <td>5.473789e+00</td>\n",
       "      <td>6.018647e-01</td>\n",
       "      <td>2.232817e+04</td>\n",
       "      <td>2.451326e+01</td>\n",
       "      <td>1.199847e+01</td>\n",
       "      <td>4.931013e-01</td>\n",
       "      <td>1.371494e-01</td>\n",
       "      <td>7.143268e-02</td>\n",
       "      <td>3.412514e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>5.000000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.310000e+00</td>\n",
       "      <td>4.930000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>6.290000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8.000000e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>9.750000e+00</td>\n",
       "      <td>2.484800e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>4.578000e+04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>6.740000e+02</td>\n",
       "      <td>8.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.943000e+03</td>\n",
       "      <td>3.350000e+01</td>\n",
       "      <td>1.600000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.200000e+04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.274000e+01</td>\n",
       "      <td>3.754300e+02</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>6.500000e+04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>6.940000e+02</td>\n",
       "      <td>1.100000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.113400e+04</td>\n",
       "      <td>5.210000e+01</td>\n",
       "      <td>2.300000e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.000000e+04</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.599000e+01</td>\n",
       "      <td>5.807300e+02</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.400000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>9.000000e+04</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>7.140000e+02</td>\n",
       "      <td>1.400000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.975575e+04</td>\n",
       "      <td>7.070000e+01</td>\n",
       "      <td>3.200000e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>9.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>4.000000e+04</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>3.099000e+01</td>\n",
       "      <td>1.719830e+03</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>3.400000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>1.099920e+07</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>8.500000e+02</td>\n",
       "      <td>9.000000e+01</td>\n",
       "      <td>8.600000e+01</td>\n",
       "      <td>2.904836e+06</td>\n",
       "      <td>8.923000e+02</td>\n",
       "      <td>1.760000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.100000e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          loan_amnt          term      int_rate   installment         grade  \\\n",
       "count  1.345310e+06  1.345310e+06  1.345310e+06  1.345310e+06  1.345310e+06   \n",
       "mean   1.441997e+04  2.412581e-01  1.323962e+01  4.380755e+02  1.745809e+00   \n",
       "std    8.717051e+03  4.278467e-01  4.768716e+00  2.615126e+02  1.295071e+00   \n",
       "min    5.000000e+02  0.000000e+00  5.310000e+00  4.930000e+00  0.000000e+00   \n",
       "25%    8.000000e+03  0.000000e+00  9.750000e+00  2.484800e+02  1.000000e+00   \n",
       "50%    1.200000e+04  0.000000e+00  1.274000e+01  3.754300e+02  2.000000e+00   \n",
       "75%    2.000000e+04  0.000000e+00  1.599000e+01  5.807300e+02  2.000000e+00   \n",
       "max    4.000000e+04  1.000000e+00  3.099000e+01  1.719830e+03  6.000000e+00   \n",
       "\n",
       "          sub_grade    emp_length  home_ownership    annual_inc  \\\n",
       "count  1.345310e+06  1.345310e+06    1.345310e+06  1.345310e+06   \n",
       "mean   1.069302e+01  6.052432e+00    2.911998e+00  7.624764e+04   \n",
       "std    6.449912e+00  3.562249e+00    1.915676e+00  6.992510e+04   \n",
       "min    0.000000e+00  1.000000e+00    0.000000e+00  0.000000e+00   \n",
       "25%    6.000000e+00  2.000000e+00    1.000000e+00  4.578000e+04   \n",
       "50%    1.000000e+01  6.000000e+00    4.000000e+00  6.500000e+04   \n",
       "75%    1.400000e+01  1.000000e+01    5.000000e+00  9.000000e+04   \n",
       "max    3.400000e+01  1.000000e+01    5.000000e+00  1.099920e+07   \n",
       "\n",
       "       verification_status  ...  fico_range_high      open_acc       pub_rec  \\\n",
       "count         1.345310e+06  ...     1.345310e+06  1.345310e+06  1.345310e+06   \n",
       "mean          3.015669e-01  ...     7.001852e+02  1.159352e+01  2.152760e-01   \n",
       "std           4.589384e-01  ...     3.185316e+01  5.473789e+00  6.018647e-01   \n",
       "min           0.000000e+00  ...     6.290000e+02  0.000000e+00  0.000000e+00   \n",
       "25%           0.000000e+00  ...     6.740000e+02  8.000000e+00  0.000000e+00   \n",
       "50%           0.000000e+00  ...     6.940000e+02  1.100000e+01  0.000000e+00   \n",
       "75%           1.000000e+00  ...     7.140000e+02  1.400000e+01  0.000000e+00   \n",
       "max           1.000000e+00  ...     8.500000e+02  9.000000e+01  8.600000e+01   \n",
       "\n",
       "          revol_bal    revol_util     total_acc  initial_list_status  \\\n",
       "count  1.345310e+06  1.345310e+06  1.345310e+06         1.345310e+06   \n",
       "mean   1.624811e+04  5.181002e+01  2.498084e+01         5.827727e-01   \n",
       "std    2.232817e+04  2.451326e+01  1.199847e+01         4.931013e-01   \n",
       "min    0.000000e+00  0.000000e+00  2.000000e+00         0.000000e+00   \n",
       "25%    5.943000e+03  3.350000e+01  1.600000e+01         0.000000e+00   \n",
       "50%    1.113400e+04  5.210000e+01  2.300000e+01         1.000000e+00   \n",
       "75%    1.975575e+04  7.070000e+01  3.200000e+01         1.000000e+00   \n",
       "max    2.904836e+06  8.923000e+02  1.760000e+02         1.000000e+00   \n",
       "\n",
       "       application_type  disbursement_method   issue_month  \n",
       "count      1.345310e+06         1.345310e+06  1.345310e+06  \n",
       "mean       1.917774e-02         5.128929e-03  5.531292e+00  \n",
       "std        1.371494e-01         7.143268e-02  3.412514e+00  \n",
       "min        0.000000e+00         0.000000e+00  0.000000e+00  \n",
       "25%        0.000000e+00         0.000000e+00  3.000000e+00  \n",
       "50%        0.000000e+00         0.000000e+00  5.000000e+00  \n",
       "75%        0.000000e+00         0.000000e+00  9.000000e+00  \n",
       "max        1.000000e+00         1.000000e+00  1.100000e+01  \n",
       "\n",
       "[8 rows x 24 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_selected.describe(include = 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    0.800374\n",
       "0    0.199626\n",
       "Name: loan_status, dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Class Balance is checked and we see that both classes are unbalanced\n",
    "#To solve the problem of unbalanced class and here we solve it using upsampling minority.It improves the model's generalization\n",
    "# on unseen data\n",
    "df_selected.loan_status.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    1076751\n",
       "0     268559\n",
       "Name: loan_status, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_selected.loan_status.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#One of the popular techniques for dealing with highly unbalanced data sets is called resampling. \n",
    "#Although the technique has proven to be effective in many cases to solve the unbalanced class issue, however, \n",
    "#these techniques also have their weaknesses. For example, over-sampling records from the minority class, which can lead \n",
    "#to overfitting while removing random records from the majority class, which can cause loss of information.\n",
    "\n",
    "df_major = df_selected[df_selected.loan_status == 1]\n",
    "df_minor = df_selected[df_selected.loan_status == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_minor_upsmapled = resample(df_minor, replace = True, n_samples = 1076751, random_state = 2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_minor_upsmapled = pd.concat([df_minor_upsmapled, df_major])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    1076751\n",
       "0    1076751\n",
       "Name: loan_status, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_minor_upsmapled.loan_status.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#To see the performance of the unknown data, we use a function named as \"evaluate_model\" which prints \n",
    "#different evaluation criteria: 1) accuracy, 2) ROC-AUC score, 3) confusion matrix and 4) detailed classification report.\n",
    "\n",
    "def evaluate_model(ytest, ypred, ypred_proba = None):\n",
    "    if ypred_proba is not None:\n",
    "        print('ROC-AUC score of the model: {}'.format(roc_auc_score(ytest, ypred_proba[:, 1])))\n",
    "    print('Accuracy of the model: {}\\n'.format(accuracy_score(ytest, ypred)))\n",
    "    print('Classification report: \\n{}\\n'.format(classification_report(ytest, ypred)))\n",
    "    print('Confusion matrix: \\n{}\\n'.format(confusion_matrix(ytest, ypred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Standardize the data.\n",
    "X = df_minor_upsmapled.drop('loan_status', axis = 1)\n",
    "Y = df_minor_upsmapled.loan_status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "xtrain, xtest, ytrain, ytest = train_test_split(X, Y, test_size=0.25, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "mms = StandardScaler()\n",
    "mms.fit(xtrain)\n",
    "xtrain_scaled = mms.transform(xtrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2153502, 24)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(df_minor_upsmapled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Logistic Regression Model\n",
    "logisticRegr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logisticRegr.fit(xtrain_scaled, ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "xtest_scaled = mms.transform(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_pred = logisticRegr.predict(xtest_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the model: 0.6495348975437241\n",
      "\n",
      "Classification report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.65      0.64      0.64    269305\n",
      "           1       0.65      0.66      0.65    269071\n",
      "\n",
      "    accuracy                           0.65    538376\n",
      "   macro avg       0.65      0.65      0.65    538376\n",
      "weighted avg       0.65      0.65      0.65    538376\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[171262  98043]\n",
      " [ 90639 178432]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(ytest, lr_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random Forest Model: We use this model here because we have a lot of categorical variables which work best with\n",
    "# decision trees. So, this model may give better accuracy than Logistic Regression.\n",
    "\n",
    "def random_forest(xtrain, xtest, ytrain):\n",
    "    rf_params = {\n",
    "        'n_estimators': 126, \n",
    "        'max_depth': 14\n",
    "    }\n",
    "\n",
    "    rf = RandomForestClassifier(**rf_params)\n",
    "    rf.fit(xtrain, ytrain)\n",
    "    rfpred = rf.predict(xtest)\n",
    "    rfpred_proba = rf.predict_proba(xtest)\n",
    "    \n",
    "    return rfpred, rfpred_proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfpred, rfpred_proba = random_forest(xtrain_scaled, xtest_scaled, ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC-AUC score of the model: 0.755971549095154\n",
      "Accuracy of the model: 0.6877869741593235\n",
      "\n",
      "Classification report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.68      0.72      0.70    269305\n",
      "           1       0.70      0.65      0.68    269071\n",
      "\n",
      "    accuracy                           0.69    538376\n",
      "   macro avg       0.69      0.69      0.69    538376\n",
      "weighted avg       0.69      0.69      0.69    538376\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[194675  74630]\n",
      " [ 93458 175613]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(ytest, rfpred, rfpred_proba)\n",
    "\n",
    "#Here, we see that there is almost 10% increase in accuracy in the RF model compared to LR model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#To improve the model genralization on testing data, we try gradient-boosted tree based alogorithm.\n",
    "# Here , we use LightGBM model to determine accuracy.\n",
    "\n",
    "import lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "lbg_params = {\n",
    "    'n_estimators': 8000,\n",
    "    'max_depth': 100,\n",
    "    'objective': 'binary',\n",
    "    'learning_rate' : 0.02,\n",
    "    'num_leaves' : 250,\n",
    "    'feature_fraction': 0.64, \n",
    "    'bagging_fraction': 0.8, \n",
    "    'bagging_freq': 1,\n",
    "    'boosting_type' : 'gbdt'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb = lightgbm.LGBMClassifier(**lbg_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] feature_fraction is set=0.64, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.64\n",
      "[LightGBM] [Warning] bagging_fraction is set=0.8, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(bagging_fraction=0.8, bagging_freq=1, feature_fraction=0.64,\n",
       "               learning_rate=0.02, max_depth=100, n_estimators=8000,\n",
       "               num_leaves=250, objective='binary')"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgb.fit(xtrain_scaled, ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb_pred = lgb.predict(xtest_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb_pred_proba = lgb.predict_proba(xtest_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC-AUC score of the model: 0.889605784002013\n",
      "Accuracy of the model: 0.8085130094952226\n",
      "\n",
      "Classification report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.86      0.82    269305\n",
      "           1       0.84      0.76      0.80    269071\n",
      "\n",
      "    accuracy                           0.81    538376\n",
      "   macro avg       0.81      0.81      0.81    538376\n",
      "weighted avg       0.81      0.81      0.81    538376\n",
      "\n",
      "\n",
      "Confusion matrix: \n",
      "[[230658  38647]\n",
      " [ 64445 204626]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model(ytest, lgb_pred, lgb_pred_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#As we can see that the accuracy is now almost 89% which increased by 24% when compared to LR model and an increase of\n",
    "# 14% compared to RF model."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}