loan behavior test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import datetime as dt\n",
    "import statsmodels.api as st\n",
    "from statistics import mean \n",
    "from dateutil import relativedelta\n",
    "import time\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import f1_score\n",
    "from collections import Counter\n",
    "\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data_pandas(filename, sep=','):\n",
    "    \n",
    "    df = pd.read_csv(filename, sep=sep)\n",
    "    cols = df.columns.tolist()\n",
    "    cols = [col.lower().replace(' ', '_') for col in cols]\n",
    "    df.columns = cols\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = read_data_pandas('loan.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record_date</th>\n",
       "      <th>bank_code</th>\n",
       "      <th>branch_code</th>\n",
       "      <th>currency</th>\n",
       "      <th>customer_number</th>\n",
       "      <th>account_number</th>\n",
       "      <th>loan_type</th>\n",
       "      <th>prepayment_indicator</th>\n",
       "      <th>gl_account_number</th>\n",
       "      <th>indicator_for_prime/hibor_based_mortgage</th>\n",
       "      <th>effective_date_of_the_loan</th>\n",
       "      <th>maturity_date_of_the_loan</th>\n",
       "      <th>loan_age</th>\n",
       "      <th>time_to_maturity</th>\n",
       "      <th>scheduled_payment</th>\n",
       "      <th>local_currency_equivalent_scheduled_payment</th>\n",
       "      <th>scheduled_payment_frequency</th>\n",
       "      <th>last_payment</th>\n",
       "      <th>local_currency_equivalent_amount_of_last_payment_received</th>\n",
       "      <th>outstanding_balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011/8/31</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>148.54</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2011/9/30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>143.75</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011/10/31</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>148.54</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2011/11/30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30134.16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2011/5/31</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>110039000 000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2000/1/31</td>\n",
       "      <td>2015/2/1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12976.28</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>543168.55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  record_date  bank_code  branch_code  currency  customer_number  \\\n",
       "0   2011/8/31        NaN          NaN       NaN              NaN   \n",
       "1   2011/9/30        NaN          NaN       NaN              NaN   \n",
       "2  2011/10/31        NaN          NaN       NaN              NaN   \n",
       "3  2011/11/30        NaN          NaN       NaN              NaN   \n",
       "4   2011/5/31        NaN          NaN       NaN              NaN   \n",
       "\n",
       "   account_number  loan_type  prepayment_indicator  gl_account_number  \\\n",
       "0  1080084173 000        NaN                   NaN                NaN   \n",
       "1  1080084173 000        NaN                   NaN                NaN   \n",
       "2  1080084173 000        NaN                   NaN                NaN   \n",
       "3  1080084173 000        NaN                   NaN                NaN   \n",
       "4   110039000 000        NaN                   NaN                NaN   \n",
       "\n",
       "   indicator_for_prime/hibor_based_mortgage effective_date_of_the_loan  \\\n",
       "0                                       NaN                   2011/8/9   \n",
       "1                                       NaN                   2011/8/9   \n",
       "2                                       NaN                   2011/8/9   \n",
       "3                                       NaN                   2011/8/9   \n",
       "4                                       NaN                  2000/1/31   \n",
       "\n",
       "  maturity_date_of_the_loan  loan_age  time_to_maturity  scheduled_payment  \\\n",
       "0                 2011/12/7       NaN               NaN                NaN   \n",
       "1                 2011/12/7       NaN               NaN                NaN   \n",
       "2                 2011/12/7       NaN               NaN                NaN   \n",
       "3                 2011/12/7       NaN               NaN                NaN   \n",
       "4                  2015/2/1       NaN               NaN                NaN   \n",
       "\n",
       "   local_currency_equivalent_scheduled_payment  scheduled_payment_frequency  \\\n",
       "0                                       148.54                          NaN   \n",
       "1                                       143.75                          NaN   \n",
       "2                                       148.54                          NaN   \n",
       "3                                     30134.16                          NaN   \n",
       "4                                     12976.28                          NaN   \n",
       "\n",
       "   last_payment  local_currency_equivalent_amount_of_last_payment_received  \\\n",
       "0           NaN                                                NaN           \n",
       "1           NaN                                                NaN           \n",
       "2           NaN                                                NaN           \n",
       "3           NaN                                                NaN           \n",
       "4           NaN                                                NaN           \n",
       "\n",
       "   outstanding_balance  \n",
       "0             30000.00  \n",
       "1             30000.00  \n",
       "2             30000.00  \n",
       "3             30000.00  \n",
       "4            543168.55  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop nan columns\n",
    "new = df.dropna(axis=1, how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record_date</th>\n",
       "      <th>account_number</th>\n",
       "      <th>effective_date_of_the_loan</th>\n",
       "      <th>maturity_date_of_the_loan</th>\n",
       "      <th>local_currency_equivalent_scheduled_payment</th>\n",
       "      <th>outstanding_balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011/8/31</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>148.54</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2011/9/30</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>143.75</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011/10/31</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>148.54</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2011/11/30</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>30134.16</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2011/5/31</td>\n",
       "      <td>110039000 000</td>\n",
       "      <td>2000/1/31</td>\n",
       "      <td>2015/2/1</td>\n",
       "      <td>12976.28</td>\n",
       "      <td>543168.55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  record_date  account_number effective_date_of_the_loan  \\\n",
       "0   2011/8/31  1080084173 000                   2011/8/9   \n",
       "1   2011/9/30  1080084173 000                   2011/8/9   \n",
       "2  2011/10/31  1080084173 000                   2011/8/9   \n",
       "3  2011/11/30  1080084173 000                   2011/8/9   \n",
       "4   2011/5/31   110039000 000                  2000/1/31   \n",
       "\n",
       "  maturity_date_of_the_loan  local_currency_equivalent_scheduled_payment  \\\n",
       "0                 2011/12/7                                       148.54   \n",
       "1                 2011/12/7                                       143.75   \n",
       "2                 2011/12/7                                       148.54   \n",
       "3                 2011/12/7                                     30134.16   \n",
       "4                  2015/2/1                                     12976.28   \n",
       "\n",
       "   outstanding_balance  \n",
       "0             30000.00  \n",
       "1             30000.00  \n",
       "2             30000.00  \n",
       "3             30000.00  \n",
       "4            543168.55  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record_date</th>\n",
       "      <th>account_number</th>\n",
       "      <th>effective_date</th>\n",
       "      <th>maturity_date</th>\n",
       "      <th>sch_payment</th>\n",
       "      <th>out_balance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011/8/31</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>148.54</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2011/9/30</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>143.75</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011/10/31</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>148.54</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2011/11/30</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011/8/9</td>\n",
       "      <td>2011/12/7</td>\n",
       "      <td>30134.16</td>\n",
       "      <td>30000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2011/5/31</td>\n",
       "      <td>110039000 000</td>\n",
       "      <td>2000/1/31</td>\n",
       "      <td>2015/2/1</td>\n",
       "      <td>12976.28</td>\n",
       "      <td>543168.55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  record_date  account_number effective_date maturity_date  sch_payment  \\\n",
       "0   2011/8/31  1080084173 000       2011/8/9     2011/12/7       148.54   \n",
       "1   2011/9/30  1080084173 000       2011/8/9     2011/12/7       143.75   \n",
       "2  2011/10/31  1080084173 000       2011/8/9     2011/12/7       148.54   \n",
       "3  2011/11/30  1080084173 000       2011/8/9     2011/12/7     30134.16   \n",
       "4   2011/5/31   110039000 000      2000/1/31      2015/2/1     12976.28   \n",
       "\n",
       "   out_balance  \n",
       "0     30000.00  \n",
       "1     30000.00  \n",
       "2     30000.00  \n",
       "3     30000.00  \n",
       "4    543168.55  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#rename columns to a readable format\n",
    "new.columns = ['record_date','account_number','effective_date','maturity_date','sch_payment','out_balance']\n",
    "new.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "new[['record_date','effective_date','maturity_date']] = \\\n",
    "new[['record_date','effective_date','maturity_date']].apply(pd.to_datetime,format='%Y/%m/%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# difference in years\n",
    "new['time_to_maturity'] = round((new['maturity_date'] - new['record_date']).dt.days/365)\n",
    "new['loan_age'] = round((new['record_date'] - new['effective_date']).dt.days/365)\n",
    "new['loan_age_m'] = round(((new['record_date'] - new['effective_date']).dt.days/365)*12,0)\n",
    "new['tenor'] = new['time_to_maturity'] + new['loan_age']\n",
    "new['tenor_m'] = round((new['time_to_maturity'] + new['loan_age'])*12,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record_date</th>\n",
       "      <th>account_number</th>\n",
       "      <th>effective_date</th>\n",
       "      <th>maturity_date</th>\n",
       "      <th>sch_payment</th>\n",
       "      <th>out_balance</th>\n",
       "      <th>time_to_maturity</th>\n",
       "      <th>loan_age</th>\n",
       "      <th>tenor</th>\n",
       "      <th>interest_rate</th>\n",
       "      <th>month</th>\n",
       "      <th>loan_age_m</th>\n",
       "      <th>tenor_m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011-08-31</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011-08-09</td>\n",
       "      <td>2011-12-07</td>\n",
       "      <td>148.54</td>\n",
       "      <td>30000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.27</td>\n",
       "      <td>8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2011-09-30</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011-08-09</td>\n",
       "      <td>2011-12-07</td>\n",
       "      <td>143.75</td>\n",
       "      <td>30000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.56</td>\n",
       "      <td>9</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011-10-31</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011-08-09</td>\n",
       "      <td>2011-12-07</td>\n",
       "      <td>148.54</td>\n",
       "      <td>30000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.48</td>\n",
       "      <td>10</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2011-11-30</td>\n",
       "      <td>1080084173 000</td>\n",
       "      <td>2011-08-09</td>\n",
       "      <td>2011-12-07</td>\n",
       "      <td>30134.16</td>\n",
       "      <td>30000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.36</td>\n",
       "      <td>11</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2011-05-31</td>\n",
       "      <td>110039000 000</td>\n",
       "      <td>2000-01-31</td>\n",
       "      <td>2015-02-01</td>\n",
       "      <td>12976.28</td>\n",
       "      <td>543168.55</td>\n",
       "      <td>4.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>4.40</td>\n",
       "      <td>5</td>\n",
       "      <td>136.0</td>\n",
       "      <td>180.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  record_date  account_number effective_date maturity_date  sch_payment  \\\n",
       "0  2011-08-31  1080084173 000     2011-08-09    2011-12-07       148.54   \n",
       "1  2011-09-30  1080084173 000     2011-08-09    2011-12-07       143.75   \n",
       "2  2011-10-31  1080084173 000     2011-08-09    2011-12-07       148.54   \n",
       "3  2011-11-30  1080084173 000     2011-08-09    2011-12-07     30134.16   \n",
       "4  2011-05-31   110039000 000     2000-01-31    2015-02-01     12976.28   \n",
       "\n",
       "   out_balance  time_to_maturity  loan_age  tenor  interest_rate month  \\\n",
       "0     30000.00               0.0       0.0    0.0           9.27     8   \n",
       "1     30000.00               0.0       0.0    0.0           5.56     9   \n",
       "2     30000.00               0.0       0.0    0.0           0.48    10   \n",
       "3     30000.00               0.0       0.0    0.0           1.36    11   \n",
       "4    543168.55               4.0      11.0   15.0           4.40     5   \n",
       "\n",
       "   loan_age_m  tenor_m  \n",
       "0         1.0      0.0  \n",
       "1         2.0      0.0  \n",
       "2         3.0      0.0  \n",
       "3         4.0      0.0  \n",
       "4       136.0    180.0  "
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create some extra random columns\n",
    "new['interest_rate'] = np.random.uniform(0, 20,size = [16231,1])\n",
    "decimals = 2    \n",
    "new['interest_rate'] = new['interest_rate'].apply(lambda x: round(x, decimals))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "new['month'] = new['record_date'].dt.month.astype(object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 16231 entries, 0 to 16230\n",
      "Data columns (total 13 columns):\n",
      "record_date         16231 non-null datetime64[ns]\n",
      "account_number      16231 non-null object\n",
      "effective_date      16231 non-null datetime64[ns]\n",
      "maturity_date       16231 non-null datetime64[ns]\n",
      "sch_payment         16231 non-null float64\n",
      "out_balance         16231 non-null float64\n",
      "time_to_maturity    16231 non-null float64\n",
      "loan_age            16231 non-null float64\n",
      "tenor               16231 non-null float64\n",
      "interest_rate       16231 non-null float64\n",
      "month               16231 non-null object\n",
      "loan_age_m          16231 non-null float64\n",
      "tenor_m             16231 non-null float64\n",
      "dtypes: datetime64[ns](3), float64(8), object(2)\n",
      "memory usage: 1.6+ MB\n"
     ]
    }
   ],
   "source": [
    "new.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "account_num = list(set(new['account_number']))\n",
    "latest_observation = max(new['record_date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas._libs.tslibs.timestamps.Timestamp"
      ]
     },
     "execution_count": 393,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(latest_observation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the position where the rows of elements of list == 1 (把只有一期的帳號刪掉)\n",
    "def del_one_acc(df):\n",
    "    indexes_to_drop = []\n",
    "    for acc in account_num:\n",
    "        tempDF = df[df['account_number']== acc]\n",
    "        for index in tempDF.index: \n",
    "            if len(tempDF)==1: \n",
    "                indexes_to_drop.append(index)\n",
    "    return indexes_to_drop\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*speed test*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.142238616943359\n"
     ]
    }
   ],
   "source": [
    "st=time.time()\n",
    "\n",
    "indexes_to_drop = []\n",
    "for acc in account_num:\n",
    "    tempDF = new[new['account_number']== acc]\n",
    "    for index, row in tempDF.iterrows(): \n",
    "        if len(tempDF)==1: \n",
    "            indexes_to_drop.append(index)\n",
    "print(time.time()-st)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.582808017730713\n"
     ]
    }
   ],
   "source": [
    "st=time.time()\n",
    "\n",
    "indexes_to_drop = []\n",
    "for acc in account_num:\n",
    "    tempDF = new[new['account_number']== acc]\n",
    "    for index in tempDF.index: \n",
    "        if len(tempDF)==1: \n",
    "            indexes_to_drop.append(index)\n",
    "print(time.time() - st)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes_to_drop = del_one_acc(new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "604"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(indexes_to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16231"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15627"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new) - len(indexes_to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexes_to_keep = set(range(new.shape[0])) - set(indexes_to_drop)\n",
    "df_sliced = new.take(list(indexes_to_keep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15627"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_sliced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sliced = df_sliced.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sliced = df_sliced.drop(df_sliced.columns[0],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prepay_amt(df):\n",
    "    \"\"\"\n",
    "    Calculate prepayment amount\n",
    "    Data Description: :\n",
    "        account_number : Account number\n",
    "        prepay_amt : prepayment amount\n",
    "        out_balance : Local Currency Equivalent Outstanding Balance of the Loan The outstanding balance of the mortgage loan in local currency\n",
    "        sch_payment: The scheduled payment for the mortgage loan\n",
    "        record_date: Record Date The date when the records are observed and recorded\n",
    "        maturity_date: Date of the Loan the date of maturity for the mortgage loan\n",
    "    Argument:\n",
    "        df : desired dataframe\n",
    "        account_num : a unique list of dataframe's account number \n",
    "        latest_observation : the latest observation date within required time frame\n",
    "\n",
    "    Return:\n",
    "        new dataframe after appending \"prepay_amt\"  \n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    newDF = pd.DataFrame()\n",
    "\n",
    "    for acc in account_num:\n",
    "        tempDF = df[df['account_number']== acc].reset_index()\n",
    "        tempDF = tempDF.drop(['index'],axis=1)\n",
    "        for index, row in tempDF.iterrows() : #enumerate into each account\n",
    "            if index == len(tempDF)-1:    # if it belongs to the last tranction of that account\n",
    "                if row['record_date'] == latest_observation:   # if its record date is tantamount to lastest observation date \n",
    "                    if row['out_balance'] != 0:   # if outstanding balance is not tantamount to zero \n",
    "                        tempDF.loc[index,'prepay_amt'] = np.nan    # prepayment is set to NA \n",
    "                    elif (row['maturity_date']-row['record_date']) > dt.timedelta(days=0):  # if outstanding bal is tantamount to zero and its Maturity date > Record date  \n",
    "                        tempDF.loc[index,'prepay_amt'] = max((tempDF.loc[index,'out_balance']- tempDF.loc[index-1,'out_balance'])*(-1) - tempDF.loc[index,'sch_payment'],0)\n",
    "                        # prepayment amount is calculated by its formula\n",
    "                    else:  # if outstanding balance is tantamount to zero and Maturiy date <= Record date \n",
    "                        tempDF.loc[index,'prepay_amt'] = 0\n",
    "                else : # if last tranction of that account is not tantamount to lastest observation date \n",
    "                    if row['out_balance'] != 0: # and its outsanding bal is not tantamount to zero\n",
    "                        tempDF.loc[index,'prepay_amt'] = tempDF.loc[index-1,'out_balance']  # then set prepayment as last index's outstanding balance\n",
    "                    elif (row['maturity_date']- row['record_date']) <= dt.timedelta(days=0) :  # if its outsanding balance is tantamount to zero and its Maturity date < Record date\n",
    "                        tempDF.loc[index,'prepay_amt'] = 0  #set prepayment amount to zero\n",
    "                    else:\n",
    "                        tempDF.loc[index,'prepay_amt'] = max((tempDF.loc[index,'out_balance']- tempDF.loc[index-1,'out_balance'])*(-1) - tempDF.loc[index,'sch_payment'],0)\n",
    "            else:    \n",
    "                if index > 0:\n",
    "                    tempDF.loc[index,'prepay_amt'] =  max((tempDF.loc[index,'out_balance']- tempDF.loc[index-1,'out_balance'])*(-1) - tempDF.loc[index,'sch_payment'],0)\n",
    "                \n",
    "        newDF = pd.concat([newDF, tempDF], axis = 0, ignore_index=True)\n",
    "        \n",
    "        #newDF = newDF.drop(['index'],axis=1)\n",
    "        \n",
    "    return newDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {},
   "outputs": [],
   "source": [
    "pp_amt_df = get_prepay_amt(df_sliced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>account_number</th>\n",
       "      <th>effective_date</th>\n",
       "      <th>interest_rate</th>\n",
       "      <th>loan_age</th>\n",
       "      <th>loan_age_m</th>\n",
       "      <th>maturity_date</th>\n",
       "      <th>month</th>\n",
       "      <th>out_balance</th>\n",
       "      <th>prepay_amt</th>\n",
       "      <th>record_date</th>\n",
       "      <th>sch_payment</th>\n",
       "      <th>tenor</th>\n",
       "      <th>tenor_m</th>\n",
       "      <th>time_to_maturity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>15.46</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>12</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>2770.83</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>4.75</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>1</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-01-31</td>\n",
       "      <td>4520.83</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>18.11</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>2</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-02-28</td>\n",
       "      <td>4083.33</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>12.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>3</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-03-29</td>\n",
       "      <td>4520.84</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>12.57</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>4</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-04-30</td>\n",
       "      <td>4375.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>5.42</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>5</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-05-31</td>\n",
       "      <td>4520.83</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>2.49</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>6</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-06-28</td>\n",
       "      <td>4375.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>6.59</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>7</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>2013-07-31</td>\n",
       "      <td>4520.83</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>180105070 000</td>\n",
       "      <td>2013-12-30</td>\n",
       "      <td>5.02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-04-29</td>\n",
       "      <td>12</td>\n",
       "      <td>80000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-12-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>180105070 000</td>\n",
       "      <td>2013-12-30</td>\n",
       "      <td>12.77</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2014-04-29</td>\n",
       "      <td>1</td>\n",
       "      <td>80000.00</td>\n",
       "      <td>80000.0</td>\n",
       "      <td>2014-01-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>180101031 000</td>\n",
       "      <td>2012-12-05</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-04-04</td>\n",
       "      <td>12</td>\n",
       "      <td>8000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>37.88</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>180101031 000</td>\n",
       "      <td>2012-12-05</td>\n",
       "      <td>18.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2013-04-04</td>\n",
       "      <td>1</td>\n",
       "      <td>8000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-01-31</td>\n",
       "      <td>37.89</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>180101031 000</td>\n",
       "      <td>2012-12-05</td>\n",
       "      <td>14.52</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2013-04-04</td>\n",
       "      <td>2</td>\n",
       "      <td>8000.00</td>\n",
       "      <td>8000.0</td>\n",
       "      <td>2013-02-28</td>\n",
       "      <td>34.22</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>180106320 000</td>\n",
       "      <td>2015-08-26</td>\n",
       "      <td>10.46</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-12-24</td>\n",
       "      <td>8</td>\n",
       "      <td>170000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-08-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>180106320 000</td>\n",
       "      <td>2015-08-26</td>\n",
       "      <td>14.49</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-12-24</td>\n",
       "      <td>9</td>\n",
       "      <td>170000.00</td>\n",
       "      <td>170000.0</td>\n",
       "      <td>2015-09-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>180090144 000</td>\n",
       "      <td>2012-12-12</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-04-11</td>\n",
       "      <td>12</td>\n",
       "      <td>200000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>818.05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>180090144 000</td>\n",
       "      <td>2012-12-12</td>\n",
       "      <td>12.57</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2013-04-11</td>\n",
       "      <td>1</td>\n",
       "      <td>40000.00</td>\n",
       "      <td>200000.0</td>\n",
       "      <td>2013-01-31</td>\n",
       "      <td>818.06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>180090706 000</td>\n",
       "      <td>2011-04-27</td>\n",
       "      <td>3.68</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2011-08-25</td>\n",
       "      <td>5</td>\n",
       "      <td>60000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011-05-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>180090706 000</td>\n",
       "      <td>2011-04-27</td>\n",
       "      <td>10.19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2011-08-25</td>\n",
       "      <td>6</td>\n",
       "      <td>60000.00</td>\n",
       "      <td>60000.0</td>\n",
       "      <td>2011-06-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>180090834 000</td>\n",
       "      <td>2011-10-18</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-02-15</td>\n",
       "      <td>10</td>\n",
       "      <td>125000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011-10-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>180090834 000</td>\n",
       "      <td>2011-10-18</td>\n",
       "      <td>5.66</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-02-15</td>\n",
       "      <td>11</td>\n",
       "      <td>125000.00</td>\n",
       "      <td>125000.0</td>\n",
       "      <td>2011-11-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>180105127 000</td>\n",
       "      <td>2014-03-25</td>\n",
       "      <td>14.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-07-23</td>\n",
       "      <td>3</td>\n",
       "      <td>232000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014-03-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>180105127 000</td>\n",
       "      <td>2014-03-25</td>\n",
       "      <td>19.75</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2014-07-23</td>\n",
       "      <td>4</td>\n",
       "      <td>232000.00</td>\n",
       "      <td>232000.0</td>\n",
       "      <td>2014-04-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>180090123 000</td>\n",
       "      <td>2012-11-13</td>\n",
       "      <td>15.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-03-13</td>\n",
       "      <td>11</td>\n",
       "      <td>135000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>674.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>180090123 000</td>\n",
       "      <td>2012-11-13</td>\n",
       "      <td>3.40</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2013-03-13</td>\n",
       "      <td>12</td>\n",
       "      <td>135000.00</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>542.82</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>210193000 000</td>\n",
       "      <td>2015-04-15</td>\n",
       "      <td>11.58</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-03-30</td>\n",
       "      <td>4</td>\n",
       "      <td>4184981.30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-04-30</td>\n",
       "      <td>21143.69</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>210193000 000</td>\n",
       "      <td>2015-04-15</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2022-03-30</td>\n",
       "      <td>5</td>\n",
       "      <td>4184981.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-05-29</td>\n",
       "      <td>21143.69</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>210193000 000</td>\n",
       "      <td>2015-04-15</td>\n",
       "      <td>16.60</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2022-03-30</td>\n",
       "      <td>6</td>\n",
       "      <td>4167487.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-06-30</td>\n",
       "      <td>21143.69</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>210193000 000</td>\n",
       "      <td>2015-04-15</td>\n",
       "      <td>10.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2022-03-30</td>\n",
       "      <td>7</td>\n",
       "      <td>4158499.47</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>21143.69</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>210193000 000</td>\n",
       "      <td>2015-04-15</td>\n",
       "      <td>18.55</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2022-03-30</td>\n",
       "      <td>8</td>\n",
       "      <td>4149889.04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-08-31</td>\n",
       "      <td>21143.69</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15597</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>9.03</td>\n",
       "      <td>3.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>12</td>\n",
       "      <td>2040744.63</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-12-31</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15598</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>9.64</td>\n",
       "      <td>3.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>1</td>\n",
       "      <td>2036502.45</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-01-29</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15599</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>4.15</td>\n",
       "      <td>3.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>2</td>\n",
       "      <td>2031705.51</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-02-29</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15600</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>7.54</td>\n",
       "      <td>3.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>3</td>\n",
       "      <td>2027428.03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-03-31</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15601</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>15.86</td>\n",
       "      <td>3.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>4</td>\n",
       "      <td>2022863.85</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15602</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>4.45</td>\n",
       "      <td>3.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>5</td>\n",
       "      <td>2018549.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-05-31</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15603</th>\n",
       "      <td>180091015 000</td>\n",
       "      <td>2013-08-28</td>\n",
       "      <td>8.39</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-12-26</td>\n",
       "      <td>8</td>\n",
       "      <td>140000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-08-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15604</th>\n",
       "      <td>180091015 000</td>\n",
       "      <td>2013-08-28</td>\n",
       "      <td>7.95</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-12-26</td>\n",
       "      <td>9</td>\n",
       "      <td>140000.00</td>\n",
       "      <td>140000.0</td>\n",
       "      <td>2013-09-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15605</th>\n",
       "      <td>180101013 000</td>\n",
       "      <td>2012-09-25</td>\n",
       "      <td>16.59</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>9</td>\n",
       "      <td>25000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-09-28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15606</th>\n",
       "      <td>180101013 000</td>\n",
       "      <td>2012-09-25</td>\n",
       "      <td>8.42</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>10</td>\n",
       "      <td>25000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-10-31</td>\n",
       "      <td>25000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15607</th>\n",
       "      <td>180101013 000</td>\n",
       "      <td>2012-09-25</td>\n",
       "      <td>5.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>11</td>\n",
       "      <td>25000.00</td>\n",
       "      <td>25000.0</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>25129.86</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15608</th>\n",
       "      <td>180090716 000</td>\n",
       "      <td>2011-05-09</td>\n",
       "      <td>7.42</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2011-09-06</td>\n",
       "      <td>5</td>\n",
       "      <td>55000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011-05-31</td>\n",
       "      <td>284.16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15609</th>\n",
       "      <td>180090716 000</td>\n",
       "      <td>2011-05-09</td>\n",
       "      <td>10.03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2011-09-06</td>\n",
       "      <td>6</td>\n",
       "      <td>55000.00</td>\n",
       "      <td>55000.0</td>\n",
       "      <td>2011-06-30</td>\n",
       "      <td>275.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15610</th>\n",
       "      <td>180106497 000</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>9.55</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-08-27</td>\n",
       "      <td>4</td>\n",
       "      <td>55000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15611</th>\n",
       "      <td>180106497 000</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>10.16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-08-27</td>\n",
       "      <td>5</td>\n",
       "      <td>55000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-05-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15612</th>\n",
       "      <td>180106138 000</td>\n",
       "      <td>2014-11-26</td>\n",
       "      <td>17.29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-03-26</td>\n",
       "      <td>11</td>\n",
       "      <td>85000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014-11-28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15613</th>\n",
       "      <td>180106138 000</td>\n",
       "      <td>2014-11-26</td>\n",
       "      <td>2.56</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-03-26</td>\n",
       "      <td>12</td>\n",
       "      <td>85000.00</td>\n",
       "      <td>85000.0</td>\n",
       "      <td>2014-12-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15614</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10.02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-10-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15615</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>11</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15616</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>14.38</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>12</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15617</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>11.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2013-01-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15618</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>14.23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2013-02-28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15619</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>3</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2013-03-29</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15620</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>2.74</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>4</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2013-04-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15621</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>5.38</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2013-05-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15622</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>15.96</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>6</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2013-06-28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15623</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>16.81</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>7</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2013-07-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15624</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>17.37</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>8</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2013-08-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15625</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>11.07</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>9</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>2013-09-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15626</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>4.33</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-10-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15627 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      account_number effective_date  interest_rate  loan_age  loan_age_m  \\\n",
       "0      230112000 000     2012-12-26          15.46       0.0         0.0   \n",
       "1      230112000 000     2012-12-26           4.75       0.0         1.0   \n",
       "2      230112000 000     2012-12-26          18.11       0.0         2.0   \n",
       "3      230112000 000     2012-12-26          12.07       0.0         3.0   \n",
       "4      230112000 000     2012-12-26          12.57       0.0         4.0   \n",
       "5      230112000 000     2012-12-26           5.42       0.0         5.0   \n",
       "6      230112000 000     2012-12-26           2.49       1.0         6.0   \n",
       "7      230112000 000     2012-12-26           6.59       1.0         7.0   \n",
       "8      180105070 000     2013-12-30           5.02       0.0         0.0   \n",
       "9      180105070 000     2013-12-30          12.77       0.0         1.0   \n",
       "10     180101031 000     2012-12-05           0.16       0.0         1.0   \n",
       "11     180101031 000     2012-12-05          18.93       0.0         2.0   \n",
       "12     180101031 000     2012-12-05          14.52       0.0         3.0   \n",
       "13     180106320 000     2015-08-26          10.46       0.0         0.0   \n",
       "14     180106320 000     2015-08-26          14.49       0.0         1.0   \n",
       "15     180090144 000     2012-12-12           0.92       0.0         1.0   \n",
       "16     180090144 000     2012-12-12          12.57       0.0         2.0   \n",
       "17     180090706 000     2011-04-27           3.68       0.0         1.0   \n",
       "18     180090706 000     2011-04-27          10.19       0.0         2.0   \n",
       "19     180090834 000     2011-10-18           0.97       0.0         0.0   \n",
       "20     180090834 000     2011-10-18           5.66       0.0         1.0   \n",
       "21     180105127 000     2014-03-25          14.31       0.0         0.0   \n",
       "22     180105127 000     2014-03-25          19.75       0.0         1.0   \n",
       "23     180090123 000     2012-11-13          15.30       0.0         1.0   \n",
       "24     180090123 000     2012-11-13           3.40       0.0         2.0   \n",
       "25     210193000 000     2015-04-15          11.58       0.0         0.0   \n",
       "26     210193000 000     2015-04-15           0.10       0.0         1.0   \n",
       "27     210193000 000     2015-04-15          16.60       0.0         2.0   \n",
       "28     210193000 000     2015-04-15          10.93       0.0         4.0   \n",
       "29     210193000 000     2015-04-15          18.55       0.0         5.0   \n",
       "...              ...            ...            ...       ...         ...   \n",
       "15597  110303000 000     2013-01-07           9.03       3.0        36.0   \n",
       "15598  110303000 000     2013-01-07           9.64       3.0        37.0   \n",
       "15599  110303000 000     2013-01-07           4.15       3.0        38.0   \n",
       "15600  110303000 000     2013-01-07           7.54       3.0        39.0   \n",
       "15601  110303000 000     2013-01-07          15.86       3.0        40.0   \n",
       "15602  110303000 000     2013-01-07           4.45       3.0        41.0   \n",
       "15603  180091015 000     2013-08-28           8.39       0.0         0.0   \n",
       "15604  180091015 000     2013-08-28           7.95       0.0         1.0   \n",
       "15605  180101013 000     2012-09-25          16.59       0.0         0.0   \n",
       "15606  180101013 000     2012-09-25           8.42       0.0         1.0   \n",
       "15607  180101013 000     2012-09-25           5.30       0.0         2.0   \n",
       "15608  180090716 000     2011-05-09           7.42       0.0         1.0   \n",
       "15609  180090716 000     2011-05-09          10.03       0.0         2.0   \n",
       "15610  180106497 000     2016-04-29           9.55       0.0         0.0   \n",
       "15611  180106497 000     2016-04-29          10.16       0.0         1.0   \n",
       "15612  180106138 000     2014-11-26          17.29       0.0         0.0   \n",
       "15613  180106138 000     2014-11-26           2.56       0.0         1.0   \n",
       "15614  230109000 000     2012-10-25          10.02       0.0         0.0   \n",
       "15615  230109000 000     2012-10-25           0.43       0.0         1.0   \n",
       "15616  230109000 000     2012-10-25          14.38       0.0         2.0   \n",
       "15617  230109000 000     2012-10-25          11.30       0.0         3.0   \n",
       "15618  230109000 000     2012-10-25          14.23       0.0         4.0   \n",
       "15619  230109000 000     2012-10-25           2.31       0.0         5.0   \n",
       "15620  230109000 000     2012-10-25           2.74       1.0         6.0   \n",
       "15621  230109000 000     2012-10-25           5.38       1.0         7.0   \n",
       "15622  230109000 000     2012-10-25          15.96       1.0         8.0   \n",
       "15623  230109000 000     2012-10-25          16.81       1.0         9.0   \n",
       "15624  230109000 000     2012-10-25          17.37       1.0        10.0   \n",
       "15625  230109000 000     2012-10-25          11.07       1.0        11.0   \n",
       "15626  230109000 000     2012-10-25           4.33       1.0        12.0   \n",
       "\n",
       "      maturity_date month  out_balance  prepay_amt record_date  sch_payment  \\\n",
       "0        2013-12-14    12   1000000.00         NaN  2012-12-31      2770.83   \n",
       "1        2013-12-14     1   1000000.00         0.0  2013-01-31      4520.83   \n",
       "2        2013-12-14     2   1000000.00         0.0  2013-02-28      4083.33   \n",
       "3        2013-12-14     3   1000000.00         0.0  2013-03-29      4520.84   \n",
       "4        2013-12-14     4   1000000.00         0.0  2013-04-30      4375.00   \n",
       "5        2013-12-14     5   1000000.00         0.0  2013-05-31      4520.83   \n",
       "6        2013-12-14     6   1000000.00         0.0  2013-06-28      4375.00   \n",
       "7        2013-12-14     7   1000000.00   1000000.0  2013-07-31      4520.83   \n",
       "8        2014-04-29    12     80000.00         NaN  2013-12-31         0.00   \n",
       "9        2014-04-29     1     80000.00     80000.0  2014-01-31         0.00   \n",
       "10       2013-04-04    12      8000.00         NaN  2012-12-31        37.88   \n",
       "11       2013-04-04     1      8000.00         0.0  2013-01-31        37.89   \n",
       "12       2013-04-04     2      8000.00      8000.0  2013-02-28        34.22   \n",
       "13       2015-12-24     8    170000.00         NaN  2015-08-31         0.00   \n",
       "14       2015-12-24     9    170000.00    170000.0  2015-09-30         0.00   \n",
       "15       2013-04-11    12    200000.00         NaN  2012-12-31       818.05   \n",
       "16       2013-04-11     1     40000.00    200000.0  2013-01-31       818.06   \n",
       "17       2011-08-25     5     60000.00         NaN  2011-05-31         0.00   \n",
       "18       2011-08-25     6     60000.00     60000.0  2011-06-30         0.00   \n",
       "19       2012-02-15    10    125000.00         NaN  2011-10-31         0.00   \n",
       "20       2012-02-15    11    125000.00    125000.0  2011-11-30         0.00   \n",
       "21       2014-07-23     3    232000.00         NaN  2014-03-31         0.00   \n",
       "22       2014-07-23     4    232000.00    232000.0  2014-04-30         0.00   \n",
       "23       2013-03-13    11    135000.00         NaN  2012-11-30       674.99   \n",
       "24       2013-03-13    12    135000.00    135000.0  2012-12-31       542.82   \n",
       "25       2022-03-30     4   4184981.30         NaN  2015-04-30     21143.69   \n",
       "26       2022-03-30     5   4184981.30         0.0  2015-05-29     21143.69   \n",
       "27       2022-03-30     6   4167487.99         0.0  2015-06-30     21143.69   \n",
       "28       2022-03-30     7   4158499.47         0.0  2015-07-31     21143.69   \n",
       "29       2022-03-30     8   4149889.04         0.0  2015-08-31     21143.69   \n",
       "...             ...   ...          ...         ...         ...          ...   \n",
       "15597    2019-12-31    12   2040744.63         0.0  2015-12-31     12587.72   \n",
       "15598    2019-12-31     1   2036502.45         0.0  2016-01-29     12587.72   \n",
       "15599    2019-12-31     2   2031705.51         0.0  2016-02-29     12587.72   \n",
       "15600    2019-12-31     3   2027428.03         0.0  2016-03-31     12587.72   \n",
       "15601    2019-12-31     4   2022863.85         0.0  2016-04-29     12587.72   \n",
       "15602    2019-12-31     5   2018549.00         NaN  2016-05-31     12587.72   \n",
       "15603    2013-12-26     8    140000.00         NaN  2013-08-30         0.00   \n",
       "15604    2013-12-26     9    140000.00    140000.0  2013-09-30         0.00   \n",
       "15605    2012-11-30     9     25000.00         NaN  2012-09-28         0.00   \n",
       "15606    2012-11-30    10     25000.00         0.0  2012-10-31     25000.00   \n",
       "15607    2012-11-30    11     25000.00     25000.0  2012-11-30     25129.86   \n",
       "15608    2011-09-06     5     55000.00         NaN  2011-05-31       284.16   \n",
       "15609    2011-09-06     6     55000.00     55000.0  2011-06-30       275.00   \n",
       "15610    2016-08-27     4     55000.00         NaN  2016-04-29         0.00   \n",
       "15611    2016-08-27     5     55000.00         NaN  2016-05-31         0.00   \n",
       "15612    2015-03-26    11     85000.00         NaN  2014-11-28         0.00   \n",
       "15613    2015-03-26    12     85000.00     85000.0  2014-12-31         0.00   \n",
       "15614    2013-10-15    10         0.00         NaN  2012-10-31         0.00   \n",
       "15615    2013-10-15    11         0.00        -0.0  2012-11-30         0.00   \n",
       "15616    2013-10-15    12         0.00        -0.0  2012-12-31         0.00   \n",
       "15617    2013-10-15     1         0.00        -0.0  2013-01-31         0.00   \n",
       "15618    2013-10-15     2         0.00        -0.0  2013-02-28         0.00   \n",
       "15619    2013-10-15     3         0.00        -0.0  2013-03-29         0.00   \n",
       "15620    2013-10-15     4         0.00        -0.0  2013-04-30         0.00   \n",
       "15621    2013-10-15     5         0.00        -0.0  2013-05-31         0.00   \n",
       "15622    2013-10-15     6         0.00        -0.0  2013-06-28         0.00   \n",
       "15623    2013-10-15     7         0.00        -0.0  2013-07-31         0.00   \n",
       "15624    2013-10-15     8         0.00        -0.0  2013-08-30         0.00   \n",
       "15625    2013-10-15     9         0.00        -0.0  2013-09-30         0.00   \n",
       "15626    2013-10-15    10         0.00         0.0  2013-10-31         0.00   \n",
       "\n",
       "       tenor  tenor_m  time_to_maturity  \n",
       "0        1.0     12.0               1.0  \n",
       "1        1.0     12.0               1.0  \n",
       "2        1.0     12.0               1.0  \n",
       "3        1.0     12.0               1.0  \n",
       "4        1.0     12.0               1.0  \n",
       "5        1.0     12.0               1.0  \n",
       "6        1.0     12.0               0.0  \n",
       "7        1.0     12.0               0.0  \n",
       "8        0.0      0.0               0.0  \n",
       "9        0.0      0.0               0.0  \n",
       "10       0.0      0.0               0.0  \n",
       "11       0.0      0.0               0.0  \n",
       "12       0.0      0.0               0.0  \n",
       "13       0.0      0.0               0.0  \n",
       "14       0.0      0.0               0.0  \n",
       "15       0.0      0.0               0.0  \n",
       "16       0.0      0.0               0.0  \n",
       "17       0.0      0.0               0.0  \n",
       "18       0.0      0.0               0.0  \n",
       "19       0.0      0.0               0.0  \n",
       "20       0.0      0.0               0.0  \n",
       "21       0.0      0.0               0.0  \n",
       "22       0.0      0.0               0.0  \n",
       "23       0.0      0.0               0.0  \n",
       "24       0.0      0.0               0.0  \n",
       "25       7.0     84.0               7.0  \n",
       "26       7.0     84.0               7.0  \n",
       "27       7.0     84.0               7.0  \n",
       "28       7.0     84.0               7.0  \n",
       "29       7.0     84.0               7.0  \n",
       "...      ...      ...               ...  \n",
       "15597    7.0     84.0               4.0  \n",
       "15598    7.0     84.0               4.0  \n",
       "15599    7.0     84.0               4.0  \n",
       "15600    7.0     84.0               4.0  \n",
       "15601    7.0     84.0               4.0  \n",
       "15602    7.0     84.0               4.0  \n",
       "15603    0.0      0.0               0.0  \n",
       "15604    0.0      0.0               0.0  \n",
       "15605    0.0      0.0               0.0  \n",
       "15606    0.0      0.0               0.0  \n",
       "15607    0.0      0.0               0.0  \n",
       "15608    0.0      0.0               0.0  \n",
       "15609    0.0      0.0               0.0  \n",
       "15610    0.0      0.0               0.0  \n",
       "15611    0.0      0.0               0.0  \n",
       "15612    0.0      0.0               0.0  \n",
       "15613    0.0      0.0               0.0  \n",
       "15614    1.0     12.0               1.0  \n",
       "15615    1.0     12.0               1.0  \n",
       "15616    1.0     12.0               1.0  \n",
       "15617    1.0     12.0               1.0  \n",
       "15618    1.0     12.0               1.0  \n",
       "15619    1.0     12.0               1.0  \n",
       "15620    1.0     12.0               0.0  \n",
       "15621    1.0     12.0               0.0  \n",
       "15622    1.0     12.0               0.0  \n",
       "15623    1.0     12.0               0.0  \n",
       "15624    1.0     12.0               0.0  \n",
       "15625    1.0     12.0               0.0  \n",
       "15626    1.0     12.0              -0.0  \n",
       "\n",
       "[15627 rows x 14 columns]"
      ]
     },
     "execution_count": 406,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pp_amt_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prepay_rate(df):\n",
    "   \n",
    "    '''\n",
    "    RREPAY_RATIO: The ratio of the prepayment amount to the current outstanding balance\n",
    "    BAL_LCE : Local Currency Equivalent Outstanding Balance of the Loan The outstanding balance of \n",
    "             the mortgage loan in local currency\n",
    "    ''' \n",
    "    newDF = pd.DataFrame()\n",
    "    for acc in account_num:\n",
    "        tempDF = df[df['account_number']== acc].reset_index()\n",
    "        for index in tempDF.index:\n",
    "            if index == 0 :\n",
    "                continue\n",
    "            tempDF.loc[index,'prepay_ratio'] = tempDF.loc[index,'prepay_amt'] / tempDF.loc[index-1,'out_balance']\n",
    "        \n",
    "        newDF = pd.concat([newDF,tempDF],ignore_index=True)\n",
    "        #print(f'this is newDF: \\n {newDF}')   \n",
    "            \n",
    "    return newDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*soeed test*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30.318856239318848\n"
     ]
    }
   ],
   "source": [
    "st=time.time()\n",
    "newDF = pd.DataFrame()\n",
    "for acc in account_num:\n",
    "    tempDF = pp_amt_df[pp_amt_df['account_number']== acc].reset_index()\n",
    "    for index, row in tempDF.iterrows():\n",
    "        if index == 0 :\n",
    "            continue\n",
    "        tempDF.loc[index,'prepay_ratio'] = tempDF.loc[index,'prepay_amt'] / tempDF.loc[index-1,'out_balance']\n",
    "\n",
    "    newDF = pd.concat([newDF,tempDF],ignore_index=True)\n",
    "print(time.time()-st)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23.003831386566162\n"
     ]
    }
   ],
   "source": [
    "st=time.time()\n",
    "newDF = pd.DataFrame()\n",
    "for acc in account_num:\n",
    "    tempDF = pp_amt_df[pp_amt_df['account_number']== acc].reset_index()\n",
    "    for index in tempDF.index:\n",
    "        if index == 0 :\n",
    "            continue\n",
    "        tempDF.loc[index,'prepay_ratio'] = tempDF.loc[index,'prepay_amt'] / tempDF.loc[index-1,'out_balance']\n",
    "\n",
    "    newDF = pd.concat([newDF,tempDF],ignore_index=True)\n",
    "print(time.time()-st)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "metadata": {},
   "outputs": [],
   "source": [
    "pp_rate_df = get_prepay_rate(pp_amt_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>account_number</th>\n",
       "      <th>effective_date</th>\n",
       "      <th>index</th>\n",
       "      <th>interest_rate</th>\n",
       "      <th>level_0</th>\n",
       "      <th>loan_age</th>\n",
       "      <th>loan_age_m</th>\n",
       "      <th>maturity_date</th>\n",
       "      <th>month</th>\n",
       "      <th>out_balance</th>\n",
       "      <th>prepay_amt</th>\n",
       "      <th>prepay_ratio</th>\n",
       "      <th>record_date</th>\n",
       "      <th>sch_payment</th>\n",
       "      <th>tenor</th>\n",
       "      <th>tenor_m</th>\n",
       "      <th>time_to_maturity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>10892</td>\n",
       "      <td>15.46</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>12</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>2770.83</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>10893</td>\n",
       "      <td>4.75</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>1</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-01-31</td>\n",
       "      <td>4520.83</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>10894</td>\n",
       "      <td>18.11</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>2</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-02-28</td>\n",
       "      <td>4083.33</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>10895</td>\n",
       "      <td>12.07</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>3</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-03-29</td>\n",
       "      <td>4520.84</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>10896</td>\n",
       "      <td>12.57</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>4</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-04-30</td>\n",
       "      <td>4375.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>10897</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>5</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-05-31</td>\n",
       "      <td>4520.83</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>10898</td>\n",
       "      <td>2.49</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>6</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-06-28</td>\n",
       "      <td>4375.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>230112000 000</td>\n",
       "      <td>2012-12-26</td>\n",
       "      <td>10899</td>\n",
       "      <td>6.59</td>\n",
       "      <td>7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2013-12-14</td>\n",
       "      <td>7</td>\n",
       "      <td>1000000.00</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-07-31</td>\n",
       "      <td>4520.83</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>180105070 000</td>\n",
       "      <td>2013-12-30</td>\n",
       "      <td>7195</td>\n",
       "      <td>5.02</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-04-29</td>\n",
       "      <td>12</td>\n",
       "      <td>80000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-12-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>180105070 000</td>\n",
       "      <td>2013-12-30</td>\n",
       "      <td>7196</td>\n",
       "      <td>12.77</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2014-04-29</td>\n",
       "      <td>1</td>\n",
       "      <td>80000.00</td>\n",
       "      <td>80000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2014-01-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>180101031 000</td>\n",
       "      <td>2012-12-05</td>\n",
       "      <td>6870</td>\n",
       "      <td>0.16</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-04-04</td>\n",
       "      <td>12</td>\n",
       "      <td>8000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>37.88</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>180101031 000</td>\n",
       "      <td>2012-12-05</td>\n",
       "      <td>6871</td>\n",
       "      <td>18.93</td>\n",
       "      <td>11</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2013-04-04</td>\n",
       "      <td>1</td>\n",
       "      <td>8000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-01-31</td>\n",
       "      <td>37.89</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>180101031 000</td>\n",
       "      <td>2012-12-05</td>\n",
       "      <td>6872</td>\n",
       "      <td>14.52</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2013-04-04</td>\n",
       "      <td>2</td>\n",
       "      <td>8000.00</td>\n",
       "      <td>8000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-02-28</td>\n",
       "      <td>34.22</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>180106320 000</td>\n",
       "      <td>2015-08-26</td>\n",
       "      <td>7707</td>\n",
       "      <td>10.46</td>\n",
       "      <td>13</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-12-24</td>\n",
       "      <td>8</td>\n",
       "      <td>170000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-08-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>180106320 000</td>\n",
       "      <td>2015-08-26</td>\n",
       "      <td>7708</td>\n",
       "      <td>14.49</td>\n",
       "      <td>14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-12-24</td>\n",
       "      <td>9</td>\n",
       "      <td>170000.00</td>\n",
       "      <td>170000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-09-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>180090144 000</td>\n",
       "      <td>2012-12-12</td>\n",
       "      <td>5126</td>\n",
       "      <td>0.92</td>\n",
       "      <td>15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-04-11</td>\n",
       "      <td>12</td>\n",
       "      <td>200000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>818.05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>180090144 000</td>\n",
       "      <td>2012-12-12</td>\n",
       "      <td>5127</td>\n",
       "      <td>12.57</td>\n",
       "      <td>16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2013-04-11</td>\n",
       "      <td>1</td>\n",
       "      <td>40000.00</td>\n",
       "      <td>200000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-01-31</td>\n",
       "      <td>818.06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>180090706 000</td>\n",
       "      <td>2011-04-27</td>\n",
       "      <td>5408</td>\n",
       "      <td>3.68</td>\n",
       "      <td>17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2011-08-25</td>\n",
       "      <td>5</td>\n",
       "      <td>60000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011-05-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>180090706 000</td>\n",
       "      <td>2011-04-27</td>\n",
       "      <td>5409</td>\n",
       "      <td>10.19</td>\n",
       "      <td>18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2011-08-25</td>\n",
       "      <td>6</td>\n",
       "      <td>60000.00</td>\n",
       "      <td>60000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2011-06-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>180090834 000</td>\n",
       "      <td>2011-10-18</td>\n",
       "      <td>5741</td>\n",
       "      <td>0.97</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-02-15</td>\n",
       "      <td>10</td>\n",
       "      <td>125000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011-10-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>180090834 000</td>\n",
       "      <td>2011-10-18</td>\n",
       "      <td>5742</td>\n",
       "      <td>5.66</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-02-15</td>\n",
       "      <td>11</td>\n",
       "      <td>125000.00</td>\n",
       "      <td>125000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2011-11-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>180105127 000</td>\n",
       "      <td>2014-03-25</td>\n",
       "      <td>7263</td>\n",
       "      <td>14.31</td>\n",
       "      <td>21</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-07-23</td>\n",
       "      <td>3</td>\n",
       "      <td>232000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014-03-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>180105127 000</td>\n",
       "      <td>2014-03-25</td>\n",
       "      <td>7264</td>\n",
       "      <td>19.75</td>\n",
       "      <td>22</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2014-07-23</td>\n",
       "      <td>4</td>\n",
       "      <td>232000.00</td>\n",
       "      <td>232000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2014-04-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>180090123 000</td>\n",
       "      <td>2012-11-13</td>\n",
       "      <td>5098</td>\n",
       "      <td>15.30</td>\n",
       "      <td>23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-03-13</td>\n",
       "      <td>11</td>\n",
       "      <td>135000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>674.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>180090123 000</td>\n",
       "      <td>2012-11-13</td>\n",
       "      <td>5099</td>\n",
       "      <td>3.40</td>\n",
       "      <td>24</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2013-03-13</td>\n",
       "      <td>12</td>\n",
       "      <td>135000.00</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>542.82</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>210193000 000</td>\n",
       "      <td>2015-04-15</td>\n",
       "      <td>9957</td>\n",
       "      <td>11.58</td>\n",
       "      <td>25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-03-30</td>\n",
       "      <td>4</td>\n",
       "      <td>4184981.30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-04-30</td>\n",
       "      <td>21143.69</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>210193000 000</td>\n",
       "      <td>2015-04-15</td>\n",
       "      <td>9958</td>\n",
       "      <td>0.10</td>\n",
       "      <td>26</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2022-03-30</td>\n",
       "      <td>5</td>\n",
       "      <td>4184981.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-05-29</td>\n",
       "      <td>21143.69</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>210193000 000</td>\n",
       "      <td>2015-04-15</td>\n",
       "      <td>9959</td>\n",
       "      <td>16.60</td>\n",
       "      <td>27</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2022-03-30</td>\n",
       "      <td>6</td>\n",
       "      <td>4167487.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-06-30</td>\n",
       "      <td>21143.69</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>210193000 000</td>\n",
       "      <td>2015-04-15</td>\n",
       "      <td>9960</td>\n",
       "      <td>10.93</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2022-03-30</td>\n",
       "      <td>7</td>\n",
       "      <td>4158499.47</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>21143.69</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>210193000 000</td>\n",
       "      <td>2015-04-15</td>\n",
       "      <td>9961</td>\n",
       "      <td>18.55</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2022-03-30</td>\n",
       "      <td>8</td>\n",
       "      <td>4149889.04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-08-31</td>\n",
       "      <td>21143.69</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15597</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>3034</td>\n",
       "      <td>9.03</td>\n",
       "      <td>15597</td>\n",
       "      <td>3.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>12</td>\n",
       "      <td>2040744.63</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-12-31</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15598</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>3035</td>\n",
       "      <td>9.64</td>\n",
       "      <td>15598</td>\n",
       "      <td>3.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>1</td>\n",
       "      <td>2036502.45</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-01-29</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15599</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>3036</td>\n",
       "      <td>4.15</td>\n",
       "      <td>15599</td>\n",
       "      <td>3.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>2</td>\n",
       "      <td>2031705.51</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-02-29</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15600</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>3037</td>\n",
       "      <td>7.54</td>\n",
       "      <td>15600</td>\n",
       "      <td>3.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>3</td>\n",
       "      <td>2027428.03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-03-31</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15601</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>3038</td>\n",
       "      <td>15.86</td>\n",
       "      <td>15601</td>\n",
       "      <td>3.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>4</td>\n",
       "      <td>2022863.85</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15602</th>\n",
       "      <td>110303000 000</td>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>3039</td>\n",
       "      <td>4.45</td>\n",
       "      <td>15602</td>\n",
       "      <td>3.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>5</td>\n",
       "      <td>2018549.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-05-31</td>\n",
       "      <td>12587.72</td>\n",
       "      <td>7.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15603</th>\n",
       "      <td>180091015 000</td>\n",
       "      <td>2013-08-28</td>\n",
       "      <td>6147</td>\n",
       "      <td>8.39</td>\n",
       "      <td>15603</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-12-26</td>\n",
       "      <td>8</td>\n",
       "      <td>140000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-08-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15604</th>\n",
       "      <td>180091015 000</td>\n",
       "      <td>2013-08-28</td>\n",
       "      <td>6148</td>\n",
       "      <td>7.95</td>\n",
       "      <td>15604</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-12-26</td>\n",
       "      <td>9</td>\n",
       "      <td>140000.00</td>\n",
       "      <td>140000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-09-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15605</th>\n",
       "      <td>180101013 000</td>\n",
       "      <td>2012-09-25</td>\n",
       "      <td>6823</td>\n",
       "      <td>16.59</td>\n",
       "      <td>15605</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>9</td>\n",
       "      <td>25000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-09-28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15606</th>\n",
       "      <td>180101013 000</td>\n",
       "      <td>2012-09-25</td>\n",
       "      <td>6824</td>\n",
       "      <td>8.42</td>\n",
       "      <td>15606</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>10</td>\n",
       "      <td>25000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-10-31</td>\n",
       "      <td>25000.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15607</th>\n",
       "      <td>180101013 000</td>\n",
       "      <td>2012-09-25</td>\n",
       "      <td>6825</td>\n",
       "      <td>5.30</td>\n",
       "      <td>15607</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>11</td>\n",
       "      <td>25000.00</td>\n",
       "      <td>25000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>25129.86</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15608</th>\n",
       "      <td>180090716 000</td>\n",
       "      <td>2011-05-09</td>\n",
       "      <td>5428</td>\n",
       "      <td>7.42</td>\n",
       "      <td>15608</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2011-09-06</td>\n",
       "      <td>5</td>\n",
       "      <td>55000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011-05-31</td>\n",
       "      <td>284.16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15609</th>\n",
       "      <td>180090716 000</td>\n",
       "      <td>2011-05-09</td>\n",
       "      <td>5429</td>\n",
       "      <td>10.03</td>\n",
       "      <td>15609</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2011-09-06</td>\n",
       "      <td>6</td>\n",
       "      <td>55000.00</td>\n",
       "      <td>55000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2011-06-30</td>\n",
       "      <td>275.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15610</th>\n",
       "      <td>180106497 000</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>7942</td>\n",
       "      <td>9.55</td>\n",
       "      <td>15610</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-08-27</td>\n",
       "      <td>4</td>\n",
       "      <td>55000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15611</th>\n",
       "      <td>180106497 000</td>\n",
       "      <td>2016-04-29</td>\n",
       "      <td>7943</td>\n",
       "      <td>10.16</td>\n",
       "      <td>15611</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-08-27</td>\n",
       "      <td>5</td>\n",
       "      <td>55000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-05-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15612</th>\n",
       "      <td>180106138 000</td>\n",
       "      <td>2014-11-26</td>\n",
       "      <td>7494</td>\n",
       "      <td>17.29</td>\n",
       "      <td>15612</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015-03-26</td>\n",
       "      <td>11</td>\n",
       "      <td>85000.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014-11-28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15613</th>\n",
       "      <td>180106138 000</td>\n",
       "      <td>2014-11-26</td>\n",
       "      <td>7495</td>\n",
       "      <td>2.56</td>\n",
       "      <td>15613</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2015-03-26</td>\n",
       "      <td>12</td>\n",
       "      <td>85000.00</td>\n",
       "      <td>85000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2014-12-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15614</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10822</td>\n",
       "      <td>10.02</td>\n",
       "      <td>15614</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-10-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15615</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10823</td>\n",
       "      <td>0.43</td>\n",
       "      <td>15615</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>11</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-11-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15616</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10824</td>\n",
       "      <td>14.38</td>\n",
       "      <td>15616</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>12</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15617</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10825</td>\n",
       "      <td>11.30</td>\n",
       "      <td>15617</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-01-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15618</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10826</td>\n",
       "      <td>14.23</td>\n",
       "      <td>15618</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>2</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-02-28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15619</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10827</td>\n",
       "      <td>2.31</td>\n",
       "      <td>15619</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>3</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-03-29</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15620</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10828</td>\n",
       "      <td>2.74</td>\n",
       "      <td>15620</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>4</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-04-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15621</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10829</td>\n",
       "      <td>5.38</td>\n",
       "      <td>15621</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-05-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15622</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10830</td>\n",
       "      <td>15.96</td>\n",
       "      <td>15622</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>6</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-06-28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15623</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10831</td>\n",
       "      <td>16.81</td>\n",
       "      <td>15623</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>7</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-07-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15624</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10832</td>\n",
       "      <td>17.37</td>\n",
       "      <td>15624</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>8</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-08-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15625</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10833</td>\n",
       "      <td>11.07</td>\n",
       "      <td>15625</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>9</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-09-30</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15626</th>\n",
       "      <td>230109000 000</td>\n",
       "      <td>2012-10-25</td>\n",
       "      <td>10834</td>\n",
       "      <td>4.33</td>\n",
       "      <td>15626</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-10-31</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15627 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      account_number effective_date  index  interest_rate  level_0  loan_age  \\\n",
       "0      230112000 000     2012-12-26  10892          15.46        0       0.0   \n",
       "1      230112000 000     2012-12-26  10893           4.75        1       0.0   \n",
       "2      230112000 000     2012-12-26  10894          18.11        2       0.0   \n",
       "3      230112000 000     2012-12-26  10895          12.07        3       0.0   \n",
       "4      230112000 000     2012-12-26  10896          12.57        4       0.0   \n",
       "5      230112000 000     2012-12-26  10897           5.42        5       0.0   \n",
       "6      230112000 000     2012-12-26  10898           2.49        6       1.0   \n",
       "7      230112000 000     2012-12-26  10899           6.59        7       1.0   \n",
       "8      180105070 000     2013-12-30   7195           5.02        8       0.0   \n",
       "9      180105070 000     2013-12-30   7196          12.77        9       0.0   \n",
       "10     180101031 000     2012-12-05   6870           0.16       10       0.0   \n",
       "11     180101031 000     2012-12-05   6871          18.93       11       0.0   \n",
       "12     180101031 000     2012-12-05   6872          14.52       12       0.0   \n",
       "13     180106320 000     2015-08-26   7707          10.46       13       0.0   \n",
       "14     180106320 000     2015-08-26   7708          14.49       14       0.0   \n",
       "15     180090144 000     2012-12-12   5126           0.92       15       0.0   \n",
       "16     180090144 000     2012-12-12   5127          12.57       16       0.0   \n",
       "17     180090706 000     2011-04-27   5408           3.68       17       0.0   \n",
       "18     180090706 000     2011-04-27   5409          10.19       18       0.0   \n",
       "19     180090834 000     2011-10-18   5741           0.97       19       0.0   \n",
       "20     180090834 000     2011-10-18   5742           5.66       20       0.0   \n",
       "21     180105127 000     2014-03-25   7263          14.31       21       0.0   \n",
       "22     180105127 000     2014-03-25   7264          19.75       22       0.0   \n",
       "23     180090123 000     2012-11-13   5098          15.30       23       0.0   \n",
       "24     180090123 000     2012-11-13   5099           3.40       24       0.0   \n",
       "25     210193000 000     2015-04-15   9957          11.58       25       0.0   \n",
       "26     210193000 000     2015-04-15   9958           0.10       26       0.0   \n",
       "27     210193000 000     2015-04-15   9959          16.60       27       0.0   \n",
       "28     210193000 000     2015-04-15   9960          10.93       28       0.0   \n",
       "29     210193000 000     2015-04-15   9961          18.55       29       0.0   \n",
       "...              ...            ...    ...            ...      ...       ...   \n",
       "15597  110303000 000     2013-01-07   3034           9.03    15597       3.0   \n",
       "15598  110303000 000     2013-01-07   3035           9.64    15598       3.0   \n",
       "15599  110303000 000     2013-01-07   3036           4.15    15599       3.0   \n",
       "15600  110303000 000     2013-01-07   3037           7.54    15600       3.0   \n",
       "15601  110303000 000     2013-01-07   3038          15.86    15601       3.0   \n",
       "15602  110303000 000     2013-01-07   3039           4.45    15602       3.0   \n",
       "15603  180091015 000     2013-08-28   6147           8.39    15603       0.0   \n",
       "15604  180091015 000     2013-08-28   6148           7.95    15604       0.0   \n",
       "15605  180101013 000     2012-09-25   6823          16.59    15605       0.0   \n",
       "15606  180101013 000     2012-09-25   6824           8.42    15606       0.0   \n",
       "15607  180101013 000     2012-09-25   6825           5.30    15607       0.0   \n",
       "15608  180090716 000     2011-05-09   5428           7.42    15608       0.0   \n",
       "15609  180090716 000     2011-05-09   5429          10.03    15609       0.0   \n",
       "15610  180106497 000     2016-04-29   7942           9.55    15610       0.0   \n",
       "15611  180106497 000     2016-04-29   7943          10.16    15611       0.0   \n",
       "15612  180106138 000     2014-11-26   7494          17.29    15612       0.0   \n",
       "15613  180106138 000     2014-11-26   7495           2.56    15613       0.0   \n",
       "15614  230109000 000     2012-10-25  10822          10.02    15614       0.0   \n",
       "15615  230109000 000     2012-10-25  10823           0.43    15615       0.0   \n",
       "15616  230109000 000     2012-10-25  10824          14.38    15616       0.0   \n",
       "15617  230109000 000     2012-10-25  10825          11.30    15617       0.0   \n",
       "15618  230109000 000     2012-10-25  10826          14.23    15618       0.0   \n",
       "15619  230109000 000     2012-10-25  10827           2.31    15619       0.0   \n",
       "15620  230109000 000     2012-10-25  10828           2.74    15620       1.0   \n",
       "15621  230109000 000     2012-10-25  10829           5.38    15621       1.0   \n",
       "15622  230109000 000     2012-10-25  10830          15.96    15622       1.0   \n",
       "15623  230109000 000     2012-10-25  10831          16.81    15623       1.0   \n",
       "15624  230109000 000     2012-10-25  10832          17.37    15624       1.0   \n",
       "15625  230109000 000     2012-10-25  10833          11.07    15625       1.0   \n",
       "15626  230109000 000     2012-10-25  10834           4.33    15626       1.0   \n",
       "\n",
       "       loan_age_m maturity_date month  out_balance  prepay_amt  prepay_ratio  \\\n",
       "0             0.0    2013-12-14    12   1000000.00         NaN           NaN   \n",
       "1             1.0    2013-12-14     1   1000000.00         0.0           0.0   \n",
       "2             2.0    2013-12-14     2   1000000.00         0.0           0.0   \n",
       "3             3.0    2013-12-14     3   1000000.00         0.0           0.0   \n",
       "4             4.0    2013-12-14     4   1000000.00         0.0           0.0   \n",
       "5             5.0    2013-12-14     5   1000000.00         0.0           0.0   \n",
       "6             6.0    2013-12-14     6   1000000.00         0.0           0.0   \n",
       "7             7.0    2013-12-14     7   1000000.00   1000000.0           1.0   \n",
       "8             0.0    2014-04-29    12     80000.00         NaN           NaN   \n",
       "9             1.0    2014-04-29     1     80000.00     80000.0           1.0   \n",
       "10            1.0    2013-04-04    12      8000.00         NaN           NaN   \n",
       "11            2.0    2013-04-04     1      8000.00         0.0           0.0   \n",
       "12            3.0    2013-04-04     2      8000.00      8000.0           1.0   \n",
       "13            0.0    2015-12-24     8    170000.00         NaN           NaN   \n",
       "14            1.0    2015-12-24     9    170000.00    170000.0           1.0   \n",
       "15            1.0    2013-04-11    12    200000.00         NaN           NaN   \n",
       "16            2.0    2013-04-11     1     40000.00    200000.0           1.0   \n",
       "17            1.0    2011-08-25     5     60000.00         NaN           NaN   \n",
       "18            2.0    2011-08-25     6     60000.00     60000.0           1.0   \n",
       "19            0.0    2012-02-15    10    125000.00         NaN           NaN   \n",
       "20            1.0    2012-02-15    11    125000.00    125000.0           1.0   \n",
       "21            0.0    2014-07-23     3    232000.00         NaN           NaN   \n",
       "22            1.0    2014-07-23     4    232000.00    232000.0           1.0   \n",
       "23            1.0    2013-03-13    11    135000.00         NaN           NaN   \n",
       "24            2.0    2013-03-13    12    135000.00    135000.0           1.0   \n",
       "25            0.0    2022-03-30     4   4184981.30         NaN           NaN   \n",
       "26            1.0    2022-03-30     5   4184981.30         0.0           0.0   \n",
       "27            2.0    2022-03-30     6   4167487.99         0.0           0.0   \n",
       "28            4.0    2022-03-30     7   4158499.47         0.0           0.0   \n",
       "29            5.0    2022-03-30     8   4149889.04         0.0           0.0   \n",
       "...           ...           ...   ...          ...         ...           ...   \n",
       "15597        36.0    2019-12-31    12   2040744.63         0.0           0.0   \n",
       "15598        37.0    2019-12-31     1   2036502.45         0.0           0.0   \n",
       "15599        38.0    2019-12-31     2   2031705.51         0.0           0.0   \n",
       "15600        39.0    2019-12-31     3   2027428.03         0.0           0.0   \n",
       "15601        40.0    2019-12-31     4   2022863.85         0.0           0.0   \n",
       "15602        41.0    2019-12-31     5   2018549.00         NaN           NaN   \n",
       "15603         0.0    2013-12-26     8    140000.00         NaN           NaN   \n",
       "15604         1.0    2013-12-26     9    140000.00    140000.0           1.0   \n",
       "15605         0.0    2012-11-30     9     25000.00         NaN           NaN   \n",
       "15606         1.0    2012-11-30    10     25000.00         0.0           0.0   \n",
       "15607         2.0    2012-11-30    11     25000.00     25000.0           1.0   \n",
       "15608         1.0    2011-09-06     5     55000.00         NaN           NaN   \n",
       "15609         2.0    2011-09-06     6     55000.00     55000.0           1.0   \n",
       "15610         0.0    2016-08-27     4     55000.00         NaN           NaN   \n",
       "15611         1.0    2016-08-27     5     55000.00         NaN           NaN   \n",
       "15612         0.0    2015-03-26    11     85000.00         NaN           NaN   \n",
       "15613         1.0    2015-03-26    12     85000.00     85000.0           1.0   \n",
       "15614         0.0    2013-10-15    10         0.00         NaN           NaN   \n",
       "15615         1.0    2013-10-15    11         0.00        -0.0           NaN   \n",
       "15616         2.0    2013-10-15    12         0.00        -0.0           NaN   \n",
       "15617         3.0    2013-10-15     1         0.00        -0.0           NaN   \n",
       "15618         4.0    2013-10-15     2         0.00        -0.0           NaN   \n",
       "15619         5.0    2013-10-15     3         0.00        -0.0           NaN   \n",
       "15620         6.0    2013-10-15     4         0.00        -0.0           NaN   \n",
       "15621         7.0    2013-10-15     5         0.00        -0.0           NaN   \n",
       "15622         8.0    2013-10-15     6         0.00        -0.0           NaN   \n",
       "15623         9.0    2013-10-15     7         0.00        -0.0           NaN   \n",
       "15624        10.0    2013-10-15     8         0.00        -0.0           NaN   \n",
       "15625        11.0    2013-10-15     9         0.00        -0.0           NaN   \n",
       "15626        12.0    2013-10-15    10         0.00         0.0           NaN   \n",
       "\n",
       "      record_date  sch_payment  tenor  tenor_m  time_to_maturity  \n",
       "0      2012-12-31      2770.83    1.0     12.0               1.0  \n",
       "1      2013-01-31      4520.83    1.0     12.0               1.0  \n",
       "2      2013-02-28      4083.33    1.0     12.0               1.0  \n",
       "3      2013-03-29      4520.84    1.0     12.0               1.0  \n",
       "4      2013-04-30      4375.00    1.0     12.0               1.0  \n",
       "5      2013-05-31      4520.83    1.0     12.0               1.0  \n",
       "6      2013-06-28      4375.00    1.0     12.0               0.0  \n",
       "7      2013-07-31      4520.83    1.0     12.0               0.0  \n",
       "8      2013-12-31         0.00    0.0      0.0               0.0  \n",
       "9      2014-01-31         0.00    0.0      0.0               0.0  \n",
       "10     2012-12-31        37.88    0.0      0.0               0.0  \n",
       "11     2013-01-31        37.89    0.0      0.0               0.0  \n",
       "12     2013-02-28        34.22    0.0      0.0               0.0  \n",
       "13     2015-08-31         0.00    0.0      0.0               0.0  \n",
       "14     2015-09-30         0.00    0.0      0.0               0.0  \n",
       "15     2012-12-31       818.05    0.0      0.0               0.0  \n",
       "16     2013-01-31       818.06    0.0      0.0               0.0  \n",
       "17     2011-05-31         0.00    0.0      0.0               0.0  \n",
       "18     2011-06-30         0.00    0.0      0.0               0.0  \n",
       "19     2011-10-31         0.00    0.0      0.0               0.0  \n",
       "20     2011-11-30         0.00    0.0      0.0               0.0  \n",
       "21     2014-03-31         0.00    0.0      0.0               0.0  \n",
       "22     2014-04-30         0.00    0.0      0.0               0.0  \n",
       "23     2012-11-30       674.99    0.0      0.0               0.0  \n",
       "24     2012-12-31       542.82    0.0      0.0               0.0  \n",
       "25     2015-04-30     21143.69    7.0     84.0               7.0  \n",
       "26     2015-05-29     21143.69    7.0     84.0               7.0  \n",
       "27     2015-06-30     21143.69    7.0     84.0               7.0  \n",
       "28     2015-07-31     21143.69    7.0     84.0               7.0  \n",
       "29     2015-08-31     21143.69    7.0     84.0               7.0  \n",
       "...           ...          ...    ...      ...               ...  \n",
       "15597  2015-12-31     12587.72    7.0     84.0               4.0  \n",
       "15598  2016-01-29     12587.72    7.0     84.0               4.0  \n",
       "15599  2016-02-29     12587.72    7.0     84.0               4.0  \n",
       "15600  2016-03-31     12587.72    7.0     84.0               4.0  \n",
       "15601  2016-04-29     12587.72    7.0     84.0               4.0  \n",
       "15602  2016-05-31     12587.72    7.0     84.0               4.0  \n",
       "15603  2013-08-30         0.00    0.0      0.0               0.0  \n",
       "15604  2013-09-30         0.00    0.0      0.0               0.0  \n",
       "15605  2012-09-28         0.00    0.0      0.0               0.0  \n",
       "15606  2012-10-31     25000.00    0.0      0.0               0.0  \n",
       "15607  2012-11-30     25129.86    0.0      0.0               0.0  \n",
       "15608  2011-05-31       284.16    0.0      0.0               0.0  \n",
       "15609  2011-06-30       275.00    0.0      0.0               0.0  \n",
       "15610  2016-04-29         0.00    0.0      0.0               0.0  \n",
       "15611  2016-05-31         0.00    0.0      0.0               0.0  \n",
       "15612  2014-11-28         0.00    0.0      0.0               0.0  \n",
       "15613  2014-12-31         0.00    0.0      0.0               0.0  \n",
       "15614  2012-10-31         0.00    1.0     12.0               1.0  \n",
       "15615  2012-11-30         0.00    1.0     12.0               1.0  \n",
       "15616  2012-12-31         0.00    1.0     12.0               1.0  \n",
       "15617  2013-01-31         0.00    1.0     12.0               1.0  \n",
       "15618  2013-02-28         0.00    1.0     12.0               1.0  \n",
       "15619  2013-03-29         0.00    1.0     12.0               1.0  \n",
       "15620  2013-04-30         0.00    1.0     12.0               0.0  \n",
       "15621  2013-05-31         0.00    1.0     12.0               0.0  \n",
       "15622  2013-06-28         0.00    1.0     12.0               0.0  \n",
       "15623  2013-07-31         0.00    1.0     12.0               0.0  \n",
       "15624  2013-08-30         0.00    1.0     12.0               0.0  \n",
       "15625  2013-09-30         0.00    1.0     12.0               0.0  \n",
       "15626  2013-10-31         0.00    1.0     12.0              -0.0  \n",
       "\n",
       "[15627 rows x 17 columns]"
      ]
     },
     "execution_count": 399,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pp_rate_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "pp_rate_df['IS_PREPAY'] = np.where(pp_rate_df['prepay_amt'] > 0,1,0)\n",
    "pp_rate_df['IS_PREPAY'] = np.where(np.isnan(pp_rate_df['prepay_amt']),np.nan,pp_rate_df['IS_PREPAY'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prepayment_indicator(df):\n",
    "\n",
    "    df['IS_FULL'] = np.where((df['prepay_ratio']== 1) & (~np.isnan(df['prepay_ratio'])),1,0)\n",
    "    df['IS_FULL'] = np.where(np.isnan(df['prepay_ratio']),np.nan,df.IS_FULL)\n",
    "    df['IS_PARTIAL'] = np.where((df['prepay_ratio'] < 1) & (df['prepay_ratio'] > 0) & (~np.isnan(df['prepay_ratio'])),1,0)\n",
    "    df['IS_PARTIAL'] = np.where( np.isnan(df['prepay_ratio']),np.nan,df.IS_PARTIAL)\n",
    "\n",
    "    return df['IS_FULL'],df['IS_PARTIAL']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "pp_rate_df['IS_FULL'],pp_rate_df['IS_PARTIAL'] = get_prepayment_indicator(pp_rate_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 15627 entries, 0 to 12\n",
      "Data columns (total 18 columns):\n",
      "account_number      15627 non-null object\n",
      "effective_date      15627 non-null datetime64[ns]\n",
      "interest_rate       15627 non-null float64\n",
      "loan_age            15627 non-null float64\n",
      "loan_age_m          15627 non-null float64\n",
      "maturity_date       15627 non-null datetime64[ns]\n",
      "month               15627 non-null object\n",
      "out_balance         15627 non-null float64\n",
      "prepay_amt          13693 non-null float64\n",
      "prepay_ratio        12256 non-null float64\n",
      "record_date         15627 non-null datetime64[ns]\n",
      "sch_payment         15627 non-null float64\n",
      "tenor               15627 non-null float64\n",
      "tenor_m             15627 non-null float64\n",
      "time_to_maturity    15627 non-null float64\n",
      "IS_PREPAY           13693 non-null float64\n",
      "IS_FULL             12256 non-null float64\n",
      "IS_PARTIAL          12256 non-null float64\n",
      "dtypes: datetime64[ns](3), float64(13), object(2)\n",
      "memory usage: 2.3+ MB\n"
     ]
    }
   ],
   "source": [
    "pp_rate_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2456"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(pp_rate_df.IS_PARTIAL==1) + sum(pp_rate_df.IS_FULL==1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2456"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#CHECK\n",
    "sum(pp_rate_df.IS_PREPAY==1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_acc = np.unique(pp_rate_df['account_number'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1751,)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_acc.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "loan = pp_rate_df.drop(['effective_date', 'maturity_date'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "account_number         0\n",
       "interest_rate          0\n",
       "loan_age               0\n",
       "loan_age_m             0\n",
       "month                  0\n",
       "out_balance            0\n",
       "prepay_amt          1934\n",
       "prepay_ratio        3371\n",
       "record_date            0\n",
       "sch_payment            0\n",
       "tenor                  0\n",
       "tenor_m                0\n",
       "time_to_maturity       0\n",
       "IS_PREPAY           1934\n",
       "IS_FULL             3371\n",
       "IS_PARTIAL          3371\n",
       "dtype: int64"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check nan values at first\n",
    "loan.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop nan value\n",
    "loan = loan.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "account_number      0\n",
       "interest_rate       0\n",
       "loan_age            0\n",
       "loan_age_m          0\n",
       "month               0\n",
       "out_balance         0\n",
       "prepay_amt          0\n",
       "prepay_ratio        0\n",
       "record_date         0\n",
       "sch_payment         0\n",
       "tenor               0\n",
       "tenor_m             0\n",
       "time_to_maturity    0\n",
       "IS_PREPAY           0\n",
       "IS_FULL             0\n",
       "IS_PARTIAL          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loan.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "logist_loan = loan.reset_index()\n",
    "logist_loan = logist_loan.drop(['index'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save desired columns aside at first\n",
    "record_date = logist_loan['record_date']\n",
    "month = logist_loan['month']\n",
    "IS_FULL = logist_loan['IS_FULL']\n",
    "IS_PARTIAL = logist_loan['IS_PARTIAL']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [],
   "source": [
    "account_number = logist_loan['account_number']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_dummies_cols = [col for col in loan.columns if col != 'month']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p> convert \"month\" to dummy variales </p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "logist_df = pd.get_dummies(logist_loan, columns=['month'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 12256 entries, 0 to 12255\n",
      "Data columns (total 27 columns):\n",
      "account_number      12256 non-null object\n",
      "interest_rate       12256 non-null float64\n",
      "loan_age            12256 non-null float64\n",
      "loan_age_m          12256 non-null float64\n",
      "out_balance         12256 non-null float64\n",
      "prepay_amt          12256 non-null float64\n",
      "prepay_ratio        12256 non-null float64\n",
      "record_date         12256 non-null datetime64[ns]\n",
      "sch_payment         12256 non-null float64\n",
      "tenor               12256 non-null float64\n",
      "tenor_m             12256 non-null float64\n",
      "time_to_maturity    12256 non-null float64\n",
      "IS_PREPAY           12256 non-null float64\n",
      "IS_FULL             12256 non-null float64\n",
      "IS_PARTIAL          12256 non-null float64\n",
      "month_1             12256 non-null uint8\n",
      "month_2             12256 non-null uint8\n",
      "month_3             12256 non-null uint8\n",
      "month_4             12256 non-null uint8\n",
      "month_5             12256 non-null uint8\n",
      "month_6             12256 non-null uint8\n",
      "month_7             12256 non-null uint8\n",
      "month_8             12256 non-null uint8\n",
      "month_9             12256 non-null uint8\n",
      "month_10            12256 non-null uint8\n",
      "month_11            12256 non-null uint8\n",
      "month_12            12256 non-null uint8\n",
      "dtypes: datetime64[ns](1), float64(13), object(1), uint8(12)\n",
      "memory usage: 1.5+ MB\n"
     ]
    }
   ],
   "source": [
    "logist_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummies_cols = [col for col in logist_df.columns if col not in not_dummies_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummies_df = logist_df[dummies_cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model part : logistic regression \n",
    "\n",
    "\n",
    "### 1. Removing Correlated Features\n",
    "<p> The main issue of <b>RFE</b> is that it can be expensive to run — so you should do anything you can to reduce the number of features beforehand. Removing correlated features is a great way to do so because as you probably know, you don’t want highly correlated features in your dataset because they provide the same information — one is enough.</p>\n",
    "\n",
    "###  2. Initialise with Scaler\n",
    "<p> It transforms the data in such a manner that it has mean as 0 and standard deviation as 1. In short, it standardizes the data. Standardization is useful for data which has negative values. It arranges the data in a standard normal distribution. It is more useful in classification than regression. </p>\n",
    "\n",
    "### 3. Oversample data with SMOTE\n",
    "<p>Oversampling is used when the quantity of data is insufficient. It tries to balance dataset by increasing the size of rare samples. Rather than getting rid of abundant samples, new rare samples are generated by using e.g. repetition, bootstrapping or SMOTE (Synthetic Minority Over-Sampling Technique) </p>\n",
    "\n",
    "### 4. Predict with logistic regression\n",
    "<p> Logistic regression in one of the most common techniques to solve classiﬁcation problems. In our speciﬁc case, we are going to see whether a mortgage will prepay (Yi = 1) or not (Yi = 0) and $p=P(Y=1|X1,X2,…,Xt)$ is the probability of prepayment conditional on explanatory factors $X1,X2,…,Xt$.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_cor_features(data,threshold=0.8):\n",
    "    correlated_features = set()\n",
    "    correlation_matrix = data.drop(['IS_FULL','IS_PARTIAL'], axis=1).corr()\n",
    "\n",
    "    for i in range(len(correlation_matrix.columns)):\n",
    "        for j in range(i):\n",
    "            if abs(correlation_matrix.iloc[i, j]) > threshold:\n",
    "                colname = correlation_matrix.columns[i]\n",
    "                correlated_features.add(colname)\n",
    "    return correlated_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loan_age_m', 'time_to_maturity', 'tenor_m'}\n"
     ]
    }
   ],
   "source": [
    "correlated_features = remove_cor_features(logist_loan)\n",
    "print(correlated_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "#final.drop(labels=correlated_features, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### \" IS_FULL\" MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num Features:10\n",
      "Selected Features:[ True  True  True  True  True  True  True  True  True  True]\n",
      "Feature Ranking:[1 1 1 1 1 1 1 1 1 1]\n",
      "Select index: Index(['interest_rate', 'loan_age', 'loan_age_m', 'out_balance', 'prepay_amt',\n",
      "       'prepay_ratio', 'sch_payment', 'tenor', 'tenor_m', 'time_to_maturity'],\n",
      "      dtype='object')\n",
      "original distribution: Counter({0.0: 10730, 1.0: 1526})\n",
      "new distribution: Counter({0.0: 10702, 1.0: 1554})\n",
      "IS_FULL result: \n",
      "        interest_rate  loan_age  loan_age_m  out_balance  prepay_amt  \\\n",
      "0               4.75       0.0         1.0   1000000.00         0.0   \n",
      "1              18.11       0.0         2.0   1000000.00         0.0   \n",
      "2              12.07       0.0         3.0   1000000.00         0.0   \n",
      "3              12.57       0.0         4.0   1000000.00         0.0   \n",
      "4               5.42       0.0         5.0   1000000.00         0.0   \n",
      "5               2.49       1.0         6.0   1000000.00         0.0   \n",
      "6               6.59       1.0         7.0   1000000.00   1000000.0   \n",
      "7              12.77       0.0         1.0     80000.00     80000.0   \n",
      "8              18.93       0.0         2.0      8000.00         0.0   \n",
      "9              14.52       0.0         3.0      8000.00      8000.0   \n",
      "10             14.49       0.0         1.0    170000.00    170000.0   \n",
      "11             12.57       0.0         2.0     40000.00    200000.0   \n",
      "12             10.19       0.0         2.0     60000.00     60000.0   \n",
      "13              5.66       0.0         1.0    125000.00    125000.0   \n",
      "14             19.75       0.0         1.0    232000.00    232000.0   \n",
      "15              3.40       0.0         2.0    135000.00    135000.0   \n",
      "16              0.10       0.0         1.0   4184981.30         0.0   \n",
      "17             16.60       0.0         2.0   4167487.99         0.0   \n",
      "18             10.93       0.0         4.0   4158499.47         0.0   \n",
      "19             18.55       0.0         5.0   4149889.04         0.0   \n",
      "20              4.13       0.0         6.0   4141253.49         0.0   \n",
      "21              7.34       1.0         7.0   4132188.46         0.0   \n",
      "22              8.15       1.0         8.0   4123498.72         0.0   \n",
      "23             11.49       1.0         9.0   4114381.90         0.0   \n",
      "24             19.82       1.0        10.0   4114381.90         0.0   \n",
      "25             10.73       1.0        11.0   4097467.45         0.0   \n",
      "26              6.82       1.0        12.0   4088568.67         0.0   \n",
      "27              2.71       1.0        12.0   4088568.67         0.0   \n",
      "28              5.24       0.0         2.0     45000.00         0.0   \n",
      "29             18.69       0.0         3.0     45000.00     45000.0   \n",
      "...              ...       ...         ...          ...         ...   \n",
      "12226           9.04       1.0        16.0   2124674.51         0.0   \n",
      "12227           3.85       1.0        17.0   2120776.20         0.0   \n",
      "12228          12.79       1.0        18.0   2116581.68         0.0   \n",
      "12229           9.62       2.0        19.0   2112651.36         0.0   \n",
      "12230          17.59       2.0        20.0   2108703.42         0.0   \n",
      "12231          10.38       2.0        21.0   2104461.09         0.0   \n",
      "12232          14.25       2.0        22.0   2100480.64         0.0   \n",
      "12233          13.79       2.0        23.0   2096205.75         0.0   \n",
      "12234           6.00       2.0        24.0   2092190.96         0.0   \n",
      "12235          11.93       2.0        25.0   2088159.82         0.0   \n",
      "12236          13.11       2.0        26.0   2088159.82         0.0   \n",
      "12237           4.33       2.0        27.0   2079219.88         0.0   \n",
      "12238          14.12       2.0        28.0   2074861.87         0.0   \n",
      "12239          11.20       2.0        29.0   2070759.76         0.0   \n",
      "12240          13.48       2.0        30.0   2066367.17         0.0   \n",
      "12241          14.38       3.0        31.0   2062230.89         0.0   \n",
      "12242          10.30       3.0        32.0   2058076.63         0.0   \n",
      "12243           1.78       3.0        33.0   2053633.82         0.0   \n",
      "12244          12.73       3.0        34.0   2049444.86         0.0   \n",
      "12245          12.27       3.0        35.0   2044967.87         0.0   \n",
      "12246           9.03       3.0        36.0   2040744.63         0.0   \n",
      "12247           9.64       3.0        37.0   2036502.45         0.0   \n",
      "12248           4.15       3.0        38.0   2031705.51         0.0   \n",
      "12249           7.54       3.0        39.0   2027428.03         0.0   \n",
      "12250          15.86       3.0        40.0   2022863.85         0.0   \n",
      "12251           7.95       0.0         1.0    140000.00    140000.0   \n",
      "12252           8.42       0.0         1.0     25000.00         0.0   \n",
      "12253           5.30       0.0         2.0     25000.00     25000.0   \n",
      "12254          10.03       0.0         2.0     55000.00     55000.0   \n",
      "12255           2.56       0.0         1.0     85000.00     85000.0   \n",
      "\n",
      "       prepay_ratio  sch_payment  tenor  tenor_m  time_to_maturity month  \\\n",
      "0               0.0      4520.83    1.0     12.0               1.0     1   \n",
      "1               0.0      4083.33    1.0     12.0               1.0     2   \n",
      "2               0.0      4520.84    1.0     12.0               1.0     3   \n",
      "3               0.0      4375.00    1.0     12.0               1.0     4   \n",
      "4               0.0      4520.83    1.0     12.0               1.0     5   \n",
      "5               0.0      4375.00    1.0     12.0               0.0     6   \n",
      "6               1.0      4520.83    1.0     12.0               0.0     7   \n",
      "7               1.0         0.00    0.0      0.0               0.0     1   \n",
      "8               0.0        37.89    0.0      0.0               0.0     1   \n",
      "9               1.0        34.22    0.0      0.0               0.0     2   \n",
      "10              1.0         0.00    0.0      0.0               0.0     9   \n",
      "11              1.0       818.06    0.0      0.0               0.0     1   \n",
      "12              1.0         0.00    0.0      0.0               0.0     6   \n",
      "13              1.0         0.00    0.0      0.0               0.0    11   \n",
      "14              1.0         0.00    0.0      0.0               0.0     4   \n",
      "15              1.0       542.82    0.0      0.0               0.0    12   \n",
      "16              0.0     21143.69    7.0     84.0               7.0     5   \n",
      "17              0.0     21143.69    7.0     84.0               7.0     6   \n",
      "18              0.0     21143.69    7.0     84.0               7.0     7   \n",
      "19              0.0     21143.69    7.0     84.0               7.0     8   \n",
      "20              0.0     21143.69    7.0     84.0               7.0     9   \n",
      "21              0.0     21143.69    7.0     84.0               6.0    10   \n",
      "22              0.0     21143.69    7.0     84.0               6.0    11   \n",
      "23              0.0     21143.69    7.0     84.0               6.0    12   \n",
      "24              0.0     21703.36    7.0     84.0               6.0     1   \n",
      "25              0.0     21703.36    7.0     84.0               6.0     2   \n",
      "26              0.0     21703.36    7.0     84.0               6.0     3   \n",
      "27              0.0     21703.36    7.0     84.0               6.0     4   \n",
      "28              0.0       215.62    0.0      0.0               0.0     9   \n",
      "29              1.0       222.82    0.0      0.0               0.0    10   \n",
      "...             ...          ...    ...      ...               ...   ...   \n",
      "12226           0.0     12587.72    7.0     84.0               6.0     4   \n",
      "12227           0.0     12587.72    7.0     84.0               6.0     5   \n",
      "12228           0.0     12587.72    7.0     84.0               6.0     6   \n",
      "12229           0.0     12587.72    7.0     84.0               5.0     7   \n",
      "12230           0.0     12587.72    7.0     84.0               5.0     8   \n",
      "12231           0.0     12587.72    7.0     84.0               5.0     9   \n",
      "12232           0.0     12587.72    7.0     84.0               5.0    10   \n",
      "12233           0.0     12587.72    7.0     84.0               5.0    11   \n",
      "12234           0.0     12587.72    7.0     84.0               5.0    12   \n",
      "12235           0.0     12587.72    7.0     84.0               5.0     1   \n",
      "12236           0.0     12587.72    7.0     84.0               5.0     2   \n",
      "12237           0.0     12587.72    7.0     84.0               5.0     3   \n",
      "12238           0.0     12587.72    7.0     84.0               5.0     4   \n",
      "12239           0.0     12587.72    7.0     84.0               5.0     5   \n",
      "12240           0.0     12587.72    7.0     84.0               5.0     6   \n",
      "12241           0.0     12587.72    7.0     84.0               4.0     7   \n",
      "12242           0.0     12587.72    7.0     84.0               4.0     8   \n",
      "12243           0.0     12587.72    7.0     84.0               4.0     9   \n",
      "12244           0.0     12587.72    7.0     84.0               4.0    10   \n",
      "12245           0.0     12587.72    7.0     84.0               4.0    11   \n",
      "12246           0.0     12587.72    7.0     84.0               4.0    12   \n",
      "12247           0.0     12587.72    7.0     84.0               4.0     1   \n",
      "12248           0.0     12587.72    7.0     84.0               4.0     2   \n",
      "12249           0.0     12587.72    7.0     84.0               4.0     3   \n",
      "12250           0.0     12587.72    7.0     84.0               4.0     4   \n",
      "12251           1.0         0.00    0.0      0.0               0.0     9   \n",
      "12252           0.0     25000.00    0.0      0.0               0.0    10   \n",
      "12253           1.0     25129.86    0.0      0.0               0.0    11   \n",
      "12254           1.0       275.00    0.0      0.0               0.0     6   \n",
      "12255           1.0         0.00    0.0      0.0               0.0    12   \n",
      "\n",
      "       full_pred  full_prob  \n",
      "0            0.0   0.000839  \n",
      "1            0.0   0.000371  \n",
      "2            0.0   0.000292  \n",
      "3            0.0   0.000180  \n",
      "4            0.0   0.000267  \n",
      "5            0.0   0.000246  \n",
      "6            1.0   0.998347  \n",
      "7            1.0   0.999580  \n",
      "8            0.0   0.000625  \n",
      "9            1.0   0.999088  \n",
      "10           1.0   0.998421  \n",
      "11           1.0   0.999562  \n",
      "12           1.0   0.998876  \n",
      "13           1.0   0.998576  \n",
      "14           1.0   0.998127  \n",
      "15           1.0   0.997130  \n",
      "16           0.0   0.000562  \n",
      "17           0.0   0.000572  \n",
      "18           0.0   0.000481  \n",
      "19           0.0   0.000348  \n",
      "20           0.0   0.000412  \n",
      "21           0.0   0.000298  \n",
      "22           0.0   0.000346  \n",
      "23           0.0   0.000167  \n",
      "24           0.0   0.001122  \n",
      "25           0.0   0.000549  \n",
      "26           0.0   0.000427  \n",
      "27           0.0   0.000272  \n",
      "28           0.0   0.000179  \n",
      "29           1.0   0.998171  \n",
      "...          ...        ...  \n",
      "12226        0.0   0.000161  \n",
      "12227        0.0   0.000237  \n",
      "12228        0.0   0.000251  \n",
      "12229        0.0   0.000174  \n",
      "12230        0.0   0.000126  \n",
      "12231        0.0   0.000144  \n",
      "12232        0.0   0.000126  \n",
      "12233        0.0   0.000148  \n",
      "12234        0.0   0.000075  \n",
      "12235        0.0   0.000510  \n",
      "12236        0.0   0.000239  \n",
      "12237        0.0   0.000190  \n",
      "12238        0.0   0.000112  \n",
      "12239        0.0   0.000163  \n",
      "12240        0.0   0.000178  \n",
      "12241        0.0   0.000121  \n",
      "12242        0.0   0.000092  \n",
      "12243        0.0   0.000107  \n",
      "12244        0.0   0.000091  \n",
      "12245        0.0   0.000106  \n",
      "12246        0.0   0.000053  \n",
      "12247        0.0   0.000367  \n",
      "12248        0.0   0.000177  \n",
      "12249        0.0   0.000133  \n",
      "12250        0.0   0.000079  \n",
      "12251        1.0   0.998466  \n",
      "12252        0.0   0.000162  \n",
      "12253        1.0   0.998564  \n",
      "12254        1.0   0.998877  \n",
      "12255        1.0   0.997168  \n",
      "\n",
      "[12256 rows x 13 columns]\n"
     ]
    }
   ],
   "source": [
    "X_f = logist_df.drop(['record_date', 'account_number', 'IS_PREPAY', 'IS_FULL', 'IS_PARTIAL'], axis=1)\n",
    "X_f = X_f.drop(dummies_cols, axis=1)\n",
    "y_f = logist_df['IS_FULL'].values\n",
    "\n",
    "select_features = get_rfe(X_f, y_f)\n",
    "selectcols = X_f.columns[np.where(select_features)]\n",
    "print(f'Select index: {selectcols}')\n",
    "\n",
    "Xf_train = X_f[selectcols]\n",
    "Xf_train = pd.concat([Xf_train, dummies_df], axis=1) \n",
    "\n",
    "yf_pred, yf_prob = model_predict(Xf_train, y_f)\n",
    "yf_result_df = pd.DataFrame({'full_pred': yf_pred, 'full_prob': yf_prob}, columns=['full_pred', 'full_prob'])\n",
    "\n",
    "full_result = pd.concat([Xf_train, month, yf_result_df], axis=1, join_axes=[Xf_train.index])\n",
    "full_result = full_result.drop(dummies_cols, axis=1)\n",
    "\n",
    "print(f'IS_FULL result: \\n {full_result}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### \"IS_PARTIAL\" MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num Features:10\n",
      "Selected Features:[ True  True  True  True  True  True  True  True  True  True]\n",
      "Feature Ranking:[1 1 1 1 1 1 1 1 1 1]\n",
      "Select index:Index(['interest_rate', 'loan_age', 'loan_age_m', 'out_balance', 'prepay_amt',\n",
      "       'prepay_ratio', 'sch_payment', 'tenor', 'tenor_m', 'time_to_maturity'],\n",
      "      dtype='object')\n",
      "original distribution: Counter({0.0: 11326, 1.0: 930})\n",
      "new distribution: Counter({0.0: 7574, 1.0: 4682})\n",
      "IS_PARTIAL result: \n",
      "        interest_rate  loan_age  loan_age_m  out_balance  prepay_amt  \\\n",
      "0               4.75       0.0         1.0   1000000.00         0.0   \n",
      "1              18.11       0.0         2.0   1000000.00         0.0   \n",
      "2              12.07       0.0         3.0   1000000.00         0.0   \n",
      "3              12.57       0.0         4.0   1000000.00         0.0   \n",
      "4               5.42       0.0         5.0   1000000.00         0.0   \n",
      "5               2.49       1.0         6.0   1000000.00         0.0   \n",
      "6               6.59       1.0         7.0   1000000.00   1000000.0   \n",
      "7              12.77       0.0         1.0     80000.00     80000.0   \n",
      "8              18.93       0.0         2.0      8000.00         0.0   \n",
      "9              14.52       0.0         3.0      8000.00      8000.0   \n",
      "10             14.49       0.0         1.0    170000.00    170000.0   \n",
      "11             12.57       0.0         2.0     40000.00    200000.0   \n",
      "12             10.19       0.0         2.0     60000.00     60000.0   \n",
      "13              5.66       0.0         1.0    125000.00    125000.0   \n",
      "14             19.75       0.0         1.0    232000.00    232000.0   \n",
      "15              3.40       0.0         2.0    135000.00    135000.0   \n",
      "16              0.10       0.0         1.0   4184981.30         0.0   \n",
      "17             16.60       0.0         2.0   4167487.99         0.0   \n",
      "18             10.93       0.0         4.0   4158499.47         0.0   \n",
      "19             18.55       0.0         5.0   4149889.04         0.0   \n",
      "20              4.13       0.0         6.0   4141253.49         0.0   \n",
      "21              7.34       1.0         7.0   4132188.46         0.0   \n",
      "22              8.15       1.0         8.0   4123498.72         0.0   \n",
      "23             11.49       1.0         9.0   4114381.90         0.0   \n",
      "24             19.82       1.0        10.0   4114381.90         0.0   \n",
      "25             10.73       1.0        11.0   4097467.45         0.0   \n",
      "26              6.82       1.0        12.0   4088568.67         0.0   \n",
      "27              2.71       1.0        12.0   4088568.67         0.0   \n",
      "28              5.24       0.0         2.0     45000.00         0.0   \n",
      "29             18.69       0.0         3.0     45000.00     45000.0   \n",
      "...              ...       ...         ...          ...         ...   \n",
      "12226           9.04       1.0        16.0   2124674.51         0.0   \n",
      "12227           3.85       1.0        17.0   2120776.20         0.0   \n",
      "12228          12.79       1.0        18.0   2116581.68         0.0   \n",
      "12229           9.62       2.0        19.0   2112651.36         0.0   \n",
      "12230          17.59       2.0        20.0   2108703.42         0.0   \n",
      "12231          10.38       2.0        21.0   2104461.09         0.0   \n",
      "12232          14.25       2.0        22.0   2100480.64         0.0   \n",
      "12233          13.79       2.0        23.0   2096205.75         0.0   \n",
      "12234           6.00       2.0        24.0   2092190.96         0.0   \n",
      "12235          11.93       2.0        25.0   2088159.82         0.0   \n",
      "12236          13.11       2.0        26.0   2088159.82         0.0   \n",
      "12237           4.33       2.0        27.0   2079219.88         0.0   \n",
      "12238          14.12       2.0        28.0   2074861.87         0.0   \n",
      "12239          11.20       2.0        29.0   2070759.76         0.0   \n",
      "12240          13.48       2.0        30.0   2066367.17         0.0   \n",
      "12241          14.38       3.0        31.0   2062230.89         0.0   \n",
      "12242          10.30       3.0        32.0   2058076.63         0.0   \n",
      "12243           1.78       3.0        33.0   2053633.82         0.0   \n",
      "12244          12.73       3.0        34.0   2049444.86         0.0   \n",
      "12245          12.27       3.0        35.0   2044967.87         0.0   \n",
      "12246           9.03       3.0        36.0   2040744.63         0.0   \n",
      "12247           9.64       3.0        37.0   2036502.45         0.0   \n",
      "12248           4.15       3.0        38.0   2031705.51         0.0   \n",
      "12249           7.54       3.0        39.0   2027428.03         0.0   \n",
      "12250          15.86       3.0        40.0   2022863.85         0.0   \n",
      "12251           7.95       0.0         1.0    140000.00    140000.0   \n",
      "12252           8.42       0.0         1.0     25000.00         0.0   \n",
      "12253           5.30       0.0         2.0     25000.00     25000.0   \n",
      "12254          10.03       0.0         2.0     55000.00     55000.0   \n",
      "12255           2.56       0.0         1.0     85000.00     85000.0   \n",
      "\n",
      "       prepay_ratio  sch_payment  tenor  tenor_m  time_to_maturity month  \\\n",
      "0               0.0      4520.83    1.0     12.0               1.0     1   \n",
      "1               0.0      4083.33    1.0     12.0               1.0     2   \n",
      "2               0.0      4520.84    1.0     12.0               1.0     3   \n",
      "3               0.0      4375.00    1.0     12.0               1.0     4   \n",
      "4               0.0      4520.83    1.0     12.0               1.0     5   \n",
      "5               0.0      4375.00    1.0     12.0               0.0     6   \n",
      "6               1.0      4520.83    1.0     12.0               0.0     7   \n",
      "7               1.0         0.00    0.0      0.0               0.0     1   \n",
      "8               0.0        37.89    0.0      0.0               0.0     1   \n",
      "9               1.0        34.22    0.0      0.0               0.0     2   \n",
      "10              1.0         0.00    0.0      0.0               0.0     9   \n",
      "11              1.0       818.06    0.0      0.0               0.0     1   \n",
      "12              1.0         0.00    0.0      0.0               0.0     6   \n",
      "13              1.0         0.00    0.0      0.0               0.0    11   \n",
      "14              1.0         0.00    0.0      0.0               0.0     4   \n",
      "15              1.0       542.82    0.0      0.0               0.0    12   \n",
      "16              0.0     21143.69    7.0     84.0               7.0     5   \n",
      "17              0.0     21143.69    7.0     84.0               7.0     6   \n",
      "18              0.0     21143.69    7.0     84.0               7.0     7   \n",
      "19              0.0     21143.69    7.0     84.0               7.0     8   \n",
      "20              0.0     21143.69    7.0     84.0               7.0     9   \n",
      "21              0.0     21143.69    7.0     84.0               6.0    10   \n",
      "22              0.0     21143.69    7.0     84.0               6.0    11   \n",
      "23              0.0     21143.69    7.0     84.0               6.0    12   \n",
      "24              0.0     21703.36    7.0     84.0               6.0     1   \n",
      "25              0.0     21703.36    7.0     84.0               6.0     2   \n",
      "26              0.0     21703.36    7.0     84.0               6.0     3   \n",
      "27              0.0     21703.36    7.0     84.0               6.0     4   \n",
      "28              0.0       215.62    0.0      0.0               0.0     9   \n",
      "29              1.0       222.82    0.0      0.0               0.0    10   \n",
      "...             ...          ...    ...      ...               ...   ...   \n",
      "12226           0.0     12587.72    7.0     84.0               6.0     4   \n",
      "12227           0.0     12587.72    7.0     84.0               6.0     5   \n",
      "12228           0.0     12587.72    7.0     84.0               6.0     6   \n",
      "12229           0.0     12587.72    7.0     84.0               5.0     7   \n",
      "12230           0.0     12587.72    7.0     84.0               5.0     8   \n",
      "12231           0.0     12587.72    7.0     84.0               5.0     9   \n",
      "12232           0.0     12587.72    7.0     84.0               5.0    10   \n",
      "12233           0.0     12587.72    7.0     84.0               5.0    11   \n",
      "12234           0.0     12587.72    7.0     84.0               5.0    12   \n",
      "12235           0.0     12587.72    7.0     84.0               5.0     1   \n",
      "12236           0.0     12587.72    7.0     84.0               5.0     2   \n",
      "12237           0.0     12587.72    7.0     84.0               5.0     3   \n",
      "12238           0.0     12587.72    7.0     84.0               5.0     4   \n",
      "12239           0.0     12587.72    7.0     84.0               5.0     5   \n",
      "12240           0.0     12587.72    7.0     84.0               5.0     6   \n",
      "12241           0.0     12587.72    7.0     84.0               4.0     7   \n",
      "12242           0.0     12587.72    7.0     84.0               4.0     8   \n",
      "12243           0.0     12587.72    7.0     84.0               4.0     9   \n",
      "12244           0.0     12587.72    7.0     84.0               4.0    10   \n",
      "12245           0.0     12587.72    7.0     84.0               4.0    11   \n",
      "12246           0.0     12587.72    7.0     84.0               4.0    12   \n",
      "12247           0.0     12587.72    7.0     84.0               4.0     1   \n",
      "12248           0.0     12587.72    7.0     84.0               4.0     2   \n",
      "12249           0.0     12587.72    7.0     84.0               4.0     3   \n",
      "12250           0.0     12587.72    7.0     84.0               4.0     4   \n",
      "12251           1.0         0.00    0.0      0.0               0.0     9   \n",
      "12252           0.0     25000.00    0.0      0.0               0.0    10   \n",
      "12253           1.0     25129.86    0.0      0.0               0.0    11   \n",
      "12254           1.0       275.00    0.0      0.0               0.0     6   \n",
      "12255           1.0         0.00    0.0      0.0               0.0    12   \n",
      "\n",
      "       partial_pred  partial_prob  \n",
      "0               0.0      0.419663  \n",
      "1               0.0      0.395735  \n",
      "2               0.0      0.417357  \n",
      "3               0.0      0.390728  \n",
      "4               0.0      0.415511  \n",
      "5               0.0      0.458477  \n",
      "6               0.0      0.287275  \n",
      "7               0.0      0.202124  \n",
      "8               0.0      0.440001  \n",
      "9               0.0      0.197998  \n",
      "10              0.0      0.206035  \n",
      "11              0.0      0.217179  \n",
      "12              0.0      0.205450  \n",
      "13              0.0      0.217739  \n",
      "14              0.0      0.192998  \n",
      "15              0.0      0.235770  \n",
      "16              0.0      0.254650  \n",
      "17              0.0      0.236981  \n",
      "18              0.0      0.215325  \n",
      "19              0.0      0.226896  \n",
      "20              0.0      0.267327  \n",
      "21              0.0      0.275827  \n",
      "22              0.0      0.282728  \n",
      "23              0.0      0.293551  \n",
      "24              0.0      0.263386  \n",
      "25              0.0      0.286124  \n",
      "26              0.0      0.302206  \n",
      "27              0.0      0.285700  \n",
      "28              0.0      0.467942  \n",
      "29              0.0      0.187781  \n",
      "...             ...           ...  \n",
      "12226           0.0      0.369827  \n",
      "12227           0.0      0.390301  \n",
      "12228           0.0      0.384755  \n",
      "12229           0.0      0.366101  \n",
      "12230           0.0      0.380890  \n",
      "12231           0.0      0.416092  \n",
      "12232           0.0      0.405288  \n",
      "12233           0.0      0.416448  \n",
      "12234           0.0      0.455903  \n",
      "12235           0.0      0.427108  \n",
      "12236           0.0      0.430057  \n",
      "12237           0.0      0.460900  \n",
      "12238           0.0      0.411017  \n",
      "12239           0.0      0.426797  \n",
      "12240           0.0      0.436983  \n",
      "12241           0.0      0.407915  \n",
      "12242           0.0      0.452145  \n",
      "12243           0.0      0.491928  \n",
      "12244           0.0      0.463601  \n",
      "12245           0.0      0.475090  \n",
      "12246           1.0      0.504128  \n",
      "12247           0.0      0.487887  \n",
      "12248           1.0      0.507283  \n",
      "12249           1.0      0.508732  \n",
      "12250           0.0      0.461633  \n",
      "12251           0.0      0.213623  \n",
      "12252           0.0      0.352036  \n",
      "12253           0.0      0.147694  \n",
      "12254           0.0      0.204452  \n",
      "12255           0.0      0.231517  \n",
      "\n",
      "[12256 rows x 13 columns]\n"
     ]
    }
   ],
   "source": [
    "X_p = logist_df.drop(['record_date', 'account_number', 'IS_PREPAY', 'IS_FULL', 'IS_PARTIAL'], axis=1)\n",
    "X_p = X_p.drop(dummies_cols, axis=1)\n",
    "y_p = logist_df['IS_PARTIAL'].values \n",
    "\n",
    "select_features = get_rfe(X_p, y_p)\n",
    "selectcols = X_p.columns[np.where(select_features)]\n",
    "print(f'Select index:{selectcols}')\n",
    "\n",
    "Xp_train = X_p[selectcols]\n",
    "Xp_train = pd.concat([Xp_train, dummies_df], axis=1) \n",
    "\n",
    "yp_pred, yp_prob = model_predict(Xp_train, y_p)\n",
    "yp_result_df = pd.DataFrame({'partial_pred': yp_pred, 'partial_prob': yp_prob}, columns=['partial_pred', 'partial_prob'])\n",
    "\n",
    "partial_result = pd.concat([Xp_train, month, yp_result_df], axis=1, join_axes=[Xp_train.index])\n",
    "partial_result = partial_result.drop(dummies_cols, axis=1)\n",
    "\n",
    "print(f'IS_PARTIAL result: \\n {partial_result}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rfe(X,y,n_features_to_select=10):\n",
    "    \"\"\"\n",
    "    Perform Recursive Feature Elimination (RFE), which is based on the idea to repeatedly construct a model and \n",
    "    choose either the best or worst performing feature.\n",
    "    \"\"\"\n",
    "        \n",
    "    # Initialise the Scaler \n",
    "    scaled_X = StandardScaler().fit_transform(X.values)\n",
    "    \n",
    "    #SMOTE\n",
    "    sm = SMOTE(random_state=0)\n",
    "    X_oversampled, y_oversampled = sm.fit_sample(scaled_X,y)\n",
    "    \n",
    "    #logistic regression\n",
    "    logreg = LogisticRegression()\n",
    "\n",
    "    #rfe\n",
    "    rfe = RFE(estimator = logreg, n_features_to_select=10,step=1)\n",
    "    rfe = rfe.fit(X_oversampled, y_oversampled)\n",
    "    select_features = rfe.support_\n",
    "    \n",
    "    print(f'Num Features:{rfe.n_features_}')\n",
    "    print(f'Selected Features:{rfe.support_}')\n",
    "    print(f'Feature Ranking:{rfe.ranking_}')\n",
    "\n",
    "    return select_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# append index and list together\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stepwise_selection(X,y,threshold = 0.05): \n",
    "        '''\n",
    "        Perform backward elimination, get rid of pvalue smaller than 0.05\n",
    "        '''\n",
    "        logit_model = st.Logit(y,X)\n",
    "        result = logit_model.fit()\n",
    "        \n",
    "        excluded = []\n",
    "        included = list(set(X.columns))\n",
    "        for i, column in enumerate (included):\n",
    "            if result.pvalues[i]<= threshold:\n",
    "                excluded.append(column)\n",
    "        \n",
    "        included = list(set(X.columns)-set(excluded))\n",
    "        print(f'resulting features:{included}')\n",
    "        \n",
    "        return included   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.004331\n",
      "         Iterations 20\n",
      "resulting features:['interest_rate', 'prepay_amt', 'prepay_ratio', 'sch_payment', 'time_to_maturity']\n"
     ]
    }
   ],
   "source": [
    "included = stepwise_selection(X_V,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_predict(X_train,y_train):\n",
    "    \"\"\"\n",
    "    Output model prediction.\n",
    "\n",
    "    Arguments:\n",
    "    X_train: dataframe after filtering unnecessary features \n",
    "    \"\"\"   \n",
    "        \n",
    "    #summarize class distribution\n",
    "    counter = Counter(y_train)\n",
    "    print(f'original distribution: {counter}')\n",
    "\n",
    "    # Initialise the Scaler \n",
    "    scaled_X = StandardScaler().fit_transform(X_train.values)\n",
    "\n",
    "    #Oversample with SMOTE\n",
    "    sm = SMOTE(random_state=0)\n",
    "    X_train_oversampled, y_train_oversampled = sm.fit_sample(scaled_X,y_train)\n",
    "\n",
    "    #logistic regression\n",
    "    logreg = LogisticRegression()\n",
    "    logreg.fit(X_train_oversampled, y_train_oversampled)\n",
    "    y_pred = logreg.predict(scaled_X)\n",
    "    y_prob = logreg.predict_proba(scaled_X)\n",
    "   \n",
    "    #summarize the new class distribution\n",
    "    counter = Counter(y_pred)\n",
    "    print(f'new distribution: {counter}')\n",
    "\n",
    "    return y_pred, y_prob[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self,X,y):\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        \n",
    "    def remove_cor_features(self,threshold = 0.8):\n",
    "        '''\n",
    "        Removing Correlated Features: check if the correlation between two columns is greater than threshold correlation\n",
    "        '''\n",
    "        \n",
    "        correlated_features = set()\n",
    "        correlation_matrix = self.X.drop(['IS_FULL','IS_PARTIAL'], axis=1).corr()\n",
    "\n",
    "        for i in range(len(correlation_matrix.columns)):\n",
    "            for j in range(i):\n",
    "                if abs(correlation_matrix.iloc[i, j]) > threshold:\n",
    "                    colname = correlation_matrix.columns[i]\n",
    "                    correlated_features.add(colname)\n",
    "        return correlated_features\n",
    "\n",
    "    def get_rfe(self,n_features_to_select=10):\n",
    "        '''\n",
    "        Perform Recursive Feature Elimination (RFE), which is based on the idea to repeatedly construct a model and \n",
    "        choose either the best or worst performing feature.\n",
    "        '''\n",
    "        \n",
    "        # Initialise the Scaler \n",
    "        scaled_X = StandardScaler().fit_transform(self.X.values)\n",
    "        \n",
    "        #SMOTE\n",
    "        sm = SMOTE(random_state=0)\n",
    "        X_oversampled, y_oversampled = sm.fit_sample(scaled_X,self.y)\n",
    "        \n",
    "        #logistic regression\n",
    "        logreg = LogisticRegression()\n",
    "    \n",
    "        #rfe\n",
    "        rfe = RFE(estimator=logreg, n_features_to_select=10, step=1)\n",
    "        rfe = rfe.fit(X_oversampled, y_oversampled)\n",
    "        select_features = rfe.support_\n",
    "        \n",
    "        print(f'Num Features:{rfe.n_features_}')\n",
    "        print(f'Selected Features:{rfe.support_}')\n",
    "        print(f'Feature Ranking:{rfe.ranking_}')\n",
    "        return select_features\n",
    "          \n",
    "    def stepwise_selection(self,threshold = 0.05): \n",
    "        '''\n",
    "        Perform backward elimination: get rid of pvalue smaller than threshold(0.05)\n",
    "        '''\n",
    "        logit_model = st.Logit(self.y,self.X)\n",
    "        result = logit_modelt_model.fit()\n",
    "        \n",
    "        excluded = []\n",
    "        included = list(set(self.X.colums))\n",
    "        for i, column in enumerate (included):\n",
    "            if result.pvalues[i]<= threshold:\n",
    "                excluded.append(column)\n",
    "        included = list(set(self.X.columns)-set(excluded))\n",
    "        print(f'resulting features:{included}')\n",
    "        \n",
    "        return included       \n",
    "    \n",
    "    def model_predict(self):\n",
    "        \"\"\"\n",
    "        Output model prediction.\n",
    "\n",
    "        Arguments:\n",
    "        X_train: dataframe after filtering unnecessary features \n",
    "        \"\"\"    \n",
    "        # need to edit \n",
    "        select_features = self.get_rfe()\n",
    "        select_cols = self.X.columns[np.where(select_features)]\n",
    "        X_train = self.X[select_cols]\n",
    "        y_train = self.y\n",
    "        \n",
    "        #summarize class distribution\n",
    "        counter = Counter(y)\n",
    "        print(f'original distribution: {counter}')\n",
    "        \n",
    "        # Initialise the Scaler \n",
    "        scaled_X = StandardScaler().fit_transform(X_train.values)\n",
    "        \n",
    "        # oversample with SMOTE\n",
    "        sm = SMOTE(random_state=0)\n",
    "        X_train_oversampled, y_train_oversampled = sm.fit_sample(scaled_X,y_train)\n",
    "        \n",
    "        #logistic regression\n",
    "        logreg = LogisticRegression()\n",
    "        \n",
    "        #logistic regression\n",
    "        logreg = LogisticRegression()\n",
    "        logreg.fit(X_train_oversampled, y_train_oversampled)   \n",
    "        y_pred = logreg.predict(scaled_X)\n",
    "        y_prob = logreg.predict_proba(scaled_X)\n",
    "        \n",
    "        #summarize the new class distribution\n",
    "        counter = Counter(y_pred)\n",
    "        print(f'new distribution: {counter}')\n",
    "        \n",
    "        return y_pred     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(X,y)\n",
    "\n",
    "result = model.model_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Model.predict of <__main__.Model object at 0x0000013920047B00>>"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-Fold cross validation + Over-sampling\n",
    "\n",
    "### Imbalanced Dataset\n",
    "<p> The prepayment observations in the data set account for just approximately xx% of the total number of observations.<br>Prepayments are rare events.This imbalance usually leads to an <b>underestimation</b> of the probability of belonging to the <b>rare class</b> and an overestimation of the probability of belonging to the majority class.</p>\n",
    "<p>We talk of an <i>imbalanced data</i> set when the amount of observations of one class is signiﬁcantly smaller than the amount of the other class.High imbalance is really common in real world problems where the model goal is to ﬁnd rare but important cases. This kind of framework is identiﬁed as anomaly detection problem, and the prediction of prepayment events in a pool of mortgages is one of these cases, where most of the mortgages will not prepay and just a small part will do.</p>\n",
    "\n",
    "### Over-sampling\n",
    "<p>Two approaches to make a balanced dataset out of an imbalanced one are under-sampling and over-sampling. Here we use oversmpling to deal with the dataset.<b>Oversampling</b> is used when the quantity of data is insufficient. It tries to balance dataset by increasing the size of rare samples. Rather than getting rid of abundant samples, new rare samples are generated by using e.g. repetition, bootstrapping or SMOTE (Synthetic Minority Over-Sampling Technique)</p>\n",
    "\n",
    "\n",
    "###  Use K-fold Cross-Validation in the right way\n",
    "<p>It is noteworthy that cross-validation should be applied properly while using over-sampling method to address imbalance problems.</p>\n",
    "\n",
    "<p>Keep in mind that over-sampling takes observed rare samples and applies bootstrapping to generate new random data based on a distribution function. If cross-validation is applied after over-sampling, basically what we are doing is overfitting our model to a specific artificial bootstrapping result. That is why cross-validation should always be done before over-sampling the data, just as how feature selection should be implemented. Only by resampling the data repeatedly, randomness can be introduced into the dataset to make sure that there won’t be an overfitting problem.</p>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![title](https://i.stack.imgur.com/L0p6c.png)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_acc = np.array(list(set(logist_df.account_number)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Random shuffle the account of this data\n",
    "random.seed(101)\n",
    "random.shuffle(total_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = KFold(n_splits= 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Over-sampling after split the data into 10 folds\n",
    "# from imblearn.over_sampling import RandomOverSampler \n",
    "# ros = RandomOverSampler(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "## test \n",
    "class oversampled_Kfold():\n",
    "    def __init__(self, n_splits, n_repeats=1):\n",
    "        self.n_splits = n_splits\n",
    "        self.n_repeats = n_repeats\n",
    "\n",
    "    def get_n_splits(self, X, y, groups=None):\n",
    "        return self.n_splits*self.n_repeats\n",
    "\n",
    "    def split(self, X, y, groups=None):\n",
    "        splits = np.split(np.random.choice(len(X), len(X),replace=False), 10)\n",
    "        train, test = [], []\n",
    "        for repeat in range(self.n_repeats):\n",
    "            for idx in range(len(splits)):\n",
    "                trainingIdx = np.delete(splits, idx)\n",
    "                Xidx_r, y_r = ros.fit_resample(trainingIdx.reshape((-1,1)), y[trainingIdx])\n",
    "                train.append(Xidx_r.flatten())\n",
    "                test.append(splits[idx])\n",
    "        return list(zip(train, test))\n",
    "\n",
    "\n",
    "rkf = oversampled_Kfold(n_splits=10, n_repeats=2)\n",
    "\n",
    "output = cross_validate(clf,x,y, scoring='r2',cv=rkf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<B>\"IS_FULL\" MODEL</B>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_validation(total_acc, model_type, df):\n",
    "    '''\n",
    "    Perform a 10-folds cross-validation \n",
    "    Arguments:\n",
    "        model_type - \"IS_FULL\" or \"IS_PARTIAL\"\n",
    "        total_acc - an unique list of total account\n",
    "    Returns: \n",
    "        result of 10-fold cross validation\n",
    "\n",
    "    '''\n",
    "    ##### KFold cross validation ####\n",
    "    kfold = KFold(n_splits = 10) # Define the split - into 10 folds\n",
    "    for train_acc_index, test_acc_index in kfold.split(total_acc):\n",
    "                \n",
    "        train = df[df['account_number'].isin(total_acc[train_acc_index])]\n",
    "        test = df[df['account_number'].isin(total_acc[test_acc_index])]\n",
    "        \n",
    "        X_train = train.drop(['account_number', 'IS_FULL', 'IS_PARTIAL'], axis=1)\n",
    "        X_test = test.drop(['account_number', 'IS_FULL', 'IS_PARTIAL'], axis=1)\n",
    "\n",
    "        y_train = train[model_type].values\n",
    "        y_test = test[model_type].values\n",
    "        \n",
    "        # Initialise the Scaler \n",
    "        scaled_Xtrain = StandardScaler().fit_transform(X_train.values)\n",
    "        scaled_Xtest = StandardScaler().fit_transform(X_test.values)\n",
    "        print(f'scaled_Xtrain:\\n {scaled_Xtrain}')\n",
    "        print(f'scaled_Xtest:\\n {scaled_Xtest}')\n",
    "        \n",
    "        # SMOTE\n",
    "        smote = SMOTE(random_state=0)\n",
    "        X_train_oversampled, y_train_oversampled = smote.fit_sample(scaled_Xtrain, y_train)\n",
    "        \n",
    "        #logistic regression\n",
    "        logreg = LogisticRegression()\n",
    "        logreg.fit(X_train_oversampled, y_train_oversampled)\n",
    "        #logreg.fit(scaled_Xtrain, y_train)\n",
    "        y_pred = logreg.predict(scaled_Xtest)\n",
    "        print(f'Accuracy: {logreg.score(scaled_Xtest, y_test)}')\n",
    "        print(f'f-score: {f1_score(y_test, y_pred)}')\n",
    "        \n",
    "        # AUC \n",
    "        #roc_auc_score(y_train,logreg.predict_proba(scaled_Xtrain)[:,1])\n",
    "        print(f'AUC score:{roc_auc_score(y_train,logreg.predict_proba(scaled_Xtrain)[:,1])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xf_kf_df = pd.concat([account_number, Xf_train,IS_FULL, IS_PARTIAL], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xp_kf_df = pd.concat([account_number, Xp_train,IS_FULL, IS_PARTIAL], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scaled_Xtrain:\n",
      " [[-0.90041734 -1.03516024 -1.03099168 ... -0.30581769 -0.30420192\n",
      "  -0.30185795]\n",
      " [ 1.41914476 -1.03516024 -0.99648304 ... -0.30581769 -0.30420192\n",
      "  -0.30185795]\n",
      " [ 0.37048046 -1.03516024 -0.96197441 ... -0.30581769 -0.30420192\n",
      "  -0.30185795]\n",
      " ...\n",
      " [-0.26323224 -1.03516024 -1.03099168 ...  3.26992204 -0.30420192\n",
      "  -0.30185795]\n",
      " [-0.80492638 -1.03516024 -0.99648304 ... -0.30581769  3.28729023\n",
      "  -0.30185795]\n",
      " [-1.28064496 -1.03516024 -1.03099168 ... -0.30581769 -0.30420192\n",
      "   3.31281648]]\n",
      "scaled_Xtest:\n",
      " [[ 1.58666302 -1.11441913 -1.0737917  ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]\n",
      " [ 0.80852482 -1.11441913 -1.03467345 ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]\n",
      " [-0.90126184 -1.11441913 -1.11290996 ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]\n",
      " ...\n",
      " [-0.84303381 -0.19910049 -0.25230838 ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]\n",
      " [-0.29427875 -0.19910049 -0.21319013 ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]\n",
      " [ 0.01627073 -1.11441913 -1.0737917  ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]]\n",
      "Accuracy: 1.0\n",
      "f-score: 1.0\n",
      "AUC score:0.9998742048385435\n",
      "scaled_Xtrain:\n",
      " [[-0.89960277 -1.06488267 -1.06188003 ... -0.30595747 -0.30228706\n",
      "  -0.3017603 ]\n",
      " [ 1.41837936 -1.06488267 -1.02657711 ... -0.30595747 -0.30228706\n",
      "  -0.3017603 ]\n",
      " [ 0.37042936 -1.06488267 -0.99127418 ... -0.30595747 -0.30228706\n",
      "  -0.3017603 ]\n",
      " ...\n",
      " [-0.26285169 -1.06488267 -1.06188003 ...  3.2684281  -0.30228706\n",
      "  -0.3017603 ]\n",
      " [-0.80417685 -1.06488267 -1.02657711 ... -0.30595747  3.30811382\n",
      "  -0.3017603 ]\n",
      " [ 0.01648598 -1.06488267 -1.02657711 ... -0.30595747 -0.30228706\n",
      "  -0.3017603 ]]\n",
      "scaled_Xtest:\n",
      " [[-1.1709479  -0.8313837  -0.78843498 ... -0.2981424  -0.31707217\n",
      "   3.33254893]\n",
      " [-0.3844713  -0.8313837  -0.8202307  ... -0.2981424  -0.31707217\n",
      "  -0.30007061]\n",
      " [-0.82418322 -0.8313837  -0.78843498 ... -0.2981424  -0.31707217\n",
      "  -0.30007061]\n",
      " ...\n",
      " [-1.47660131 -0.8313837  -0.8202307  ...  3.35410197 -0.31707217\n",
      "  -0.30007061]\n",
      " [-1.46230173 -0.8313837  -0.78843498 ... -0.2981424   3.15385613\n",
      "  -0.30007061]\n",
      " [-1.32109343 -0.8313837  -0.8202307  ... -0.2981424  -0.31707217\n",
      "   3.33254893]]\n",
      "Accuracy: 0.9970845481049563\n",
      "f-score: 0.9904153354632589\n",
      "AUC score:0.9998954307454033\n",
      "scaled_Xtrain:\n",
      " [[-0.90377642 -1.02611777 -1.02167391 ... -0.30615048 -0.30436024\n",
      "  -0.30075866]\n",
      " [ 1.42479682 -1.02611777 -0.98702253 ... -0.30615048 -0.30436024\n",
      "  -0.30075866]\n",
      " [ 0.37205862 -1.02611777 -0.95237115 ... -0.30615048 -0.30436024\n",
      "  -0.30075866]\n",
      " ...\n",
      " [-0.8079145  -1.02611777 -0.98702253 ... -0.30615048  3.28558028\n",
      "  -0.30075866]\n",
      " [ 0.01649803 -1.02611777 -0.98702253 ... -0.30615048 -0.30436024\n",
      "  -0.30075866]\n",
      " [-1.28548117 -1.02611777 -1.02167391 ... -0.30615048 -0.30436024\n",
      "   3.32492508]]\n",
      "scaled_Xtest:\n",
      " [[ 0.48303456 -1.19173043 -1.19283892 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]\n",
      " [ 0.77706853 -1.19173043 -1.19283892 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]\n",
      " [-1.55297971 -1.19173043 -1.19283892 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]\n",
      " ...\n",
      " [-0.09648588 -1.19173043 -1.19283892 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]\n",
      " [-1.54956071 -1.19173043 -1.15516822 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]\n",
      " [-1.07773877 -1.19173043 -1.15516822 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]]\n",
      "Accuracy: 0.9969742813918305\n",
      "f-score: 0.9865771812080537\n",
      "AUC score:0.9998769006124233\n",
      "scaled_Xtrain:\n",
      " [[-0.90737541 -1.0310731  -1.02618165 ... -0.30421619 -0.30211218\n",
      "  -0.30105652]\n",
      " [ 1.41777689 -1.0310731  -0.99140545 ... -0.30421619 -0.30211218\n",
      "  -0.30105652]\n",
      " [ 0.36658528 -1.0310731  -0.95662925 ... -0.30421619 -0.30211218\n",
      "  -0.30105652]\n",
      " ...\n",
      " [-0.81165432 -1.0310731  -0.99140545 ... -0.30421619  3.3100287\n",
      "  -0.30105652]\n",
      " [ 0.01154705 -1.0310731  -0.99140545 ... -0.30421619 -0.30211218\n",
      "  -0.30105652]\n",
      " [-1.28851939 -1.0310731  -1.02618165 ... -0.30421619 -0.30211218\n",
      "   3.32163541]]\n",
      "scaled_Xtest:\n",
      " [[ 1.74822593 -1.1730966  -1.18095714 ... -0.31689562 -0.31872763\n",
      "  -0.30763164]\n",
      " [ 0.39746552 -1.1730966  -1.14404891 ... -0.31689562 -0.31872763\n",
      "  -0.30763164]\n",
      " [-0.77169458 -1.1730966  -1.18095714 ... -0.31689562 -0.31872763\n",
      "   3.25064096]\n",
      " ...\n",
      " [-0.14560588 -1.1730966  -1.10714067 ... -0.31689562 -0.31872763\n",
      "  -0.30763164]\n",
      " [-0.63160291 -1.1730966  -1.14404891 ... -0.31689562 -0.31872763\n",
      "   3.25064096]\n",
      " [ 0.7710433  -1.1730966  -1.14404891 ... -0.31689562  3.1374751\n",
      "  -0.30763164]]\n",
      "Accuracy: 1.0\n",
      "f-score: 1.0\n",
      "AUC score:0.9998698579788052\n",
      "scaled_Xtrain:\n",
      " [[-0.90097298 -1.03690932 -1.03367721 ... -0.30462451 -0.3038939\n",
      "  -0.30206227]\n",
      " [ 1.4210549  -1.03690932 -0.99882409 ... -0.30462451 -0.3038939\n",
      "  -0.30206227]\n",
      " [ 0.37127583 -1.03690932 -0.96397098 ... -0.30462451 -0.3038939\n",
      "  -0.30206227]\n",
      " ...\n",
      " [-0.80538052 -1.03690932 -0.99882409 ... -0.30462451  3.2906221\n",
      "  -0.30206227]\n",
      " [ 0.01671468 -1.03690932 -0.99882409 ... -0.30462451 -0.3038939\n",
      "  -0.30206227]\n",
      " [-1.2816048  -1.03690932 -1.03367721 ... -0.30462451 -0.3038939\n",
      "   3.31057562]]\n",
      "scaled_Xtest:\n",
      " [[ 0.45722989 -1.08087939 -1.03554258 ... -0.31010926 -0.3010772\n",
      "  -0.2984635 ]\n",
      " [ 1.21228004  0.17540465  0.03001267 ... -0.31010926 -0.3010772\n",
      "  -0.2984635 ]\n",
      " [ 1.48493704  0.17540465  0.06553118 ... -0.31010926 -0.3010772\n",
      "  -0.2984635 ]\n",
      " ...\n",
      " [-0.6246591  -1.08087939 -1.03554258 ... -0.31010926 -0.3010772\n",
      "   3.35049346]\n",
      " [ 0.25798054 -1.08087939 -1.07106109 ... -0.31010926 -0.3010772\n",
      "  -0.2984635 ]\n",
      " [-1.23988515 -1.08087939 -1.03554258 ... -0.31010926 -0.3010772\n",
      "  -0.2984635 ]]\n",
      "Accuracy: 0.996042216358839\n",
      "f-score: 0.9801324503311257\n",
      "AUC score:0.9999275240767342\n",
      "scaled_Xtrain:\n",
      " [[-0.89999722 -1.03749239 -1.03286736 ... -0.30563722 -0.30335942\n",
      "  -0.30089364]\n",
      " [ 1.42026603 -1.03749239 -0.99801991 ... -0.30563722 -0.30335942\n",
      "  -0.30089364]\n",
      " [ 0.37128474 -1.03749239 -0.96317245 ... -0.30563722 -0.30335942\n",
      "  -0.30089364]\n",
      " ...\n",
      " [-0.34424555 -1.03749239 -1.03286736 ... -0.30563722 -0.30335942\n",
      "  -0.30089364]\n",
      " [ 0.01699304 -1.03749239 -0.99801991 ... -0.30563722 -0.30335942\n",
      "  -0.30089364]\n",
      " [-1.28033978 -1.03749239 -1.03286736 ... -0.30563722 -0.30335942\n",
      "   3.32343345]]\n",
      "scaled_Xtest:\n",
      " [[-0.76309891 -1.09370121 -1.09662222 ... -0.3018198   3.27326835\n",
      "  -0.30916125]\n",
      " [-0.83728795 -1.09370121 -1.06073583 ... -0.3018198  -0.30550505\n",
      "  -0.30916125]\n",
      " [ 1.53852786 -1.09370121 -1.02484943 ...  3.31323529 -0.30550505\n",
      "  -0.30916125]\n",
      " ...\n",
      " [ 1.03863501  0.17173167  0.30294715 ... -0.3018198  -0.30550505\n",
      "  -0.30916125]\n",
      " [-0.27557091 -1.09370121 -1.09662222 ...  3.31323529 -0.30550505\n",
      "  -0.30916125]\n",
      " [-0.82668952 -1.09370121 -1.06073583 ... -0.3018198   3.27326835\n",
      "  -0.30916125]]\n",
      "Accuracy: 1.0\n",
      "f-score: 1.0\n",
      "AUC score:0.9998806268330255\n",
      "scaled_Xtrain:\n",
      " [[ 0.49175164 -1.03091185 -1.027245   ... -0.30400682 -0.30292612\n",
      "  -0.30202358]\n",
      " [ 1.56342561 -1.03091185 -0.99270815 ... -0.30400682 -0.30292612\n",
      "  -0.30202358]\n",
      " [ 0.79620447 -1.03091185 -0.9581713  ... -0.30400682 -0.30292612\n",
      "  -0.30202358]\n",
      " ...\n",
      " [-0.80782702 -1.03091185 -0.99270815 ... -0.30400682  3.30113493\n",
      "  -0.30202358]\n",
      " [ 0.01506549 -1.03091185 -0.99270815 ... -0.30400682 -0.30292612\n",
      "  -0.30202358]\n",
      " [-1.28451317 -1.03091185 -1.027245   ... -0.30400682 -0.30292612\n",
      "   3.31099974]]\n",
      "scaled_Xtest:\n",
      " [[-0.89006874 -1.1504887  -1.14390038 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]\n",
      " [ 1.42841392 -1.1504887  -1.10515825 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]\n",
      " [ 0.38023762 -1.1504887  -1.06641611 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]\n",
      " ...\n",
      " [ 0.62666318 -0.24171065 -0.33031556 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]\n",
      " [-1.2145869  -0.24171065 -0.29157342 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]\n",
      " [-0.25491556 -0.24171065 -0.25283129 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]]\n",
      "Accuracy: 0.9962852897473997\n",
      "f-score: 0.9841269841269841\n",
      "AUC score:0.9998702397902917\n",
      "scaled_Xtrain:\n",
      " [[-0.9044801  -1.03983183 -1.03531616 ... -0.30608711 -0.3045234\n",
      "  -0.30260497]\n",
      " [ 1.42557224 -1.03983183 -1.00025747 ... -0.30608711 -0.3045234\n",
      "  -0.30260497]\n",
      " [ 0.37216534 -1.03983183 -0.96519877 ... -0.30608711 -0.3045234\n",
      "  -0.30260497]\n",
      " ...\n",
      " [-0.80855728 -1.03983183 -1.00025747 ... -0.30608711  3.28381985\n",
      "  -0.30260497]\n",
      " [ 0.01637891 -1.03983183 -1.00025747 ... -0.30608711 -0.3045234\n",
      "  -0.30260497]\n",
      " [-1.2864273  -1.03983183 -1.03531616 ... -0.30608711 -0.3045234\n",
      "   3.30463839]]\n",
      "scaled_Xtest:\n",
      " [[ 0.04235092 -1.07179351 -1.03968907 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]\n",
      " [-1.66006071 -1.07179351 -1.07337546 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]\n",
      " [ 1.12386316 -1.07179351 -1.03968907 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]\n",
      " ...\n",
      " [-1.32092817 -1.07179351 -1.07337546 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]\n",
      " [ 0.36967288 -1.07179351 -1.07337546 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]\n",
      " [-0.33558784 -1.07179351 -1.07337546 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 1.0\n",
      "f-score: 1.0\n",
      "AUC score:0.9998794643021802\n",
      "scaled_Xtrain:\n",
      " [[-0.89814586 -1.06844795 -1.06539691 ... -0.30452486 -0.30324883\n",
      "  -0.300686  ]\n",
      " [ 1.42814669 -1.06844795 -1.02907391 ... -0.30452486 -0.30324883\n",
      "  -0.300686  ]\n",
      " [ 0.37643958 -1.06844795 -0.99275092 ... -0.30452486 -0.30324883\n",
      "  -0.300686  ]\n",
      " ...\n",
      " [-0.80237783 -1.06844795 -1.02907391 ... -0.30452486  3.29762201\n",
      "  -0.300686  ]\n",
      " [ 0.02122725 -1.06844795 -1.02907391 ... -0.30452486 -0.30324883\n",
      "  -0.300686  ]\n",
      " [-1.27947675 -1.06844795 -1.06539691 ... -0.30452486 -0.30324883\n",
      "   3.32572854]]\n",
      "scaled_Xtest:\n",
      " [[ 0.84222843 -0.92511825 -0.88966179 ... -0.31093242 -0.30569767\n",
      "  -0.3083223 ]\n",
      " [-1.15844482 -0.92511825 -0.86144929 ... -0.31093242 -0.30569767\n",
      "  -0.3083223 ]\n",
      " [-0.94439522 -0.92511825 -0.8332368  ...  3.216133   -0.30569767\n",
      "  -0.3083223 ]\n",
      " ...\n",
      " [-0.6699284  -0.92511825 -0.88966179 ... -0.31093242 -0.30569767\n",
      "  -0.3083223 ]\n",
      " [-0.0243272  -0.92511825 -0.86144929 ... -0.31093242 -0.30569767\n",
      "  -0.3083223 ]\n",
      " [ 0.50389197 -0.92511825 -0.91787428 ... -0.31093242 -0.30569767\n",
      "  -0.3083223 ]]\n",
      "Accuracy: 0.9966352624495289\n",
      "f-score: 0.9832775919732442\n",
      "AUC score:0.9999253292751171\n",
      "scaled_Xtrain:\n",
      " [[-0.90187007 -1.05396669 -1.04977777 ... -0.30597581 -0.30457066\n",
      "  -0.30245486]\n",
      " [ 1.4203878  -1.05396669 -1.01516535 ... -0.30597581 -0.30457066\n",
      "  -0.30245486]\n",
      " [ 0.37050475 -1.05396669 -0.98055292 ... -0.30597581 -0.30457066\n",
      "  -0.30245486]\n",
      " ...\n",
      " [-0.80626813 -1.05396669 -1.01516535 ... -0.30597581  3.28331035\n",
      "  -0.30245486]\n",
      " [ 0.01590849 -1.05396669 -1.01516535 ... -0.30597581 -0.30457066\n",
      "  -0.30245486]\n",
      " [-1.28253958 -1.05396669 -1.04977777 ... -0.30597581 -0.30457066\n",
      "   3.3062785 ]]\n",
      "scaled_Xtest:\n",
      " [[-1.08909136  1.39881584  1.37502391 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]\n",
      " [ 0.23187569  1.39881584  1.41500369 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]\n",
      " [-1.43376754  1.39881584  1.45498347 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]\n",
      " ...\n",
      " [-1.132832   -0.00801251  0.05569125 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]\n",
      " [-0.0865561  -0.00801251  0.09567103 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]\n",
      " [-1.06634624 -0.00801251  0.13565081 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]]\n",
      "Accuracy: 0.9955035971223022\n",
      "f-score: 0.9838187702265372\n",
      "AUC score:0.9998341773453179\n"
     ]
    }
   ],
   "source": [
    "model_validation(total_acc, 'IS_FULL', Xf_kf_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scaled_Xtrain:\n",
      " [[-0.90041734 -1.03516024 -1.03099168 ... -0.30581769 -0.30420192\n",
      "  -0.30185795]\n",
      " [ 1.41914476 -1.03516024 -0.99648304 ... -0.30581769 -0.30420192\n",
      "  -0.30185795]\n",
      " [ 0.37048046 -1.03516024 -0.96197441 ... -0.30581769 -0.30420192\n",
      "  -0.30185795]\n",
      " ...\n",
      " [-0.26323224 -1.03516024 -1.03099168 ...  3.26992204 -0.30420192\n",
      "  -0.30185795]\n",
      " [-0.80492638 -1.03516024 -0.99648304 ... -0.30581769  3.28729023\n",
      "  -0.30185795]\n",
      " [-1.28064496 -1.03516024 -1.03099168 ... -0.30581769 -0.30420192\n",
      "   3.31281648]]\n",
      "scaled_Xtest:\n",
      " [[ 1.58666302 -1.11441913 -1.0737917  ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]\n",
      " [ 0.80852482 -1.11441913 -1.03467345 ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]\n",
      " [-0.90126184 -1.11441913 -1.11290996 ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]\n",
      " ...\n",
      " [-0.84303381 -0.19910049 -0.25230838 ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]\n",
      " [-0.29427875 -0.19910049 -0.21319013 ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]\n",
      " [ 0.01627073 -1.11441913 -1.0737917  ... -0.30114491 -0.29820265\n",
      "  -0.2996762 ]]\n",
      "Accuracy: 0.6198960653303638\n",
      "f-score: 0.10175438596491229\n",
      "AUC score:0.6870250342947114\n",
      "scaled_Xtrain:\n",
      " [[-0.89960277 -1.06488267 -1.06188003 ... -0.30595747 -0.30228706\n",
      "  -0.3017603 ]\n",
      " [ 1.41837936 -1.06488267 -1.02657711 ... -0.30595747 -0.30228706\n",
      "  -0.3017603 ]\n",
      " [ 0.37042936 -1.06488267 -0.99127418 ... -0.30595747 -0.30228706\n",
      "  -0.3017603 ]\n",
      " ...\n",
      " [-0.26285169 -1.06488267 -1.06188003 ...  3.2684281  -0.30228706\n",
      "  -0.3017603 ]\n",
      " [-0.80417685 -1.06488267 -1.02657711 ... -0.30595747  3.30811382\n",
      "  -0.3017603 ]\n",
      " [ 0.01648598 -1.06488267 -1.02657711 ... -0.30595747 -0.30228706\n",
      "  -0.3017603 ]]\n",
      "scaled_Xtest:\n",
      " [[-1.1709479  -0.8313837  -0.78843498 ... -0.2981424  -0.31707217\n",
      "   3.33254893]\n",
      " [-0.3844713  -0.8313837  -0.8202307  ... -0.2981424  -0.31707217\n",
      "  -0.30007061]\n",
      " [-0.82418322 -0.8313837  -0.78843498 ... -0.2981424  -0.31707217\n",
      "  -0.30007061]\n",
      " ...\n",
      " [-1.47660131 -0.8313837  -0.8202307  ...  3.35410197 -0.31707217\n",
      "  -0.30007061]\n",
      " [-1.46230173 -0.8313837  -0.78843498 ... -0.2981424   3.15385613\n",
      "  -0.30007061]\n",
      " [-1.32109343 -0.8313837  -0.8202307  ... -0.2981424  -0.31707217\n",
      "   3.33254893]]\n",
      "Accuracy: 0.8415937803692906\n",
      "f-score: 0.02395209580838323\n",
      "AUC score:0.7090051086693889\n",
      "scaled_Xtrain:\n",
      " [[-0.90377642 -1.02611777 -1.02167391 ... -0.30615048 -0.30436024\n",
      "  -0.30075866]\n",
      " [ 1.42479682 -1.02611777 -0.98702253 ... -0.30615048 -0.30436024\n",
      "  -0.30075866]\n",
      " [ 0.37205862 -1.02611777 -0.95237115 ... -0.30615048 -0.30436024\n",
      "  -0.30075866]\n",
      " ...\n",
      " [-0.8079145  -1.02611777 -0.98702253 ... -0.30615048  3.28558028\n",
      "  -0.30075866]\n",
      " [ 0.01649803 -1.02611777 -0.98702253 ... -0.30615048 -0.30436024\n",
      "  -0.30075866]\n",
      " [-1.28548117 -1.02611777 -1.02167391 ... -0.30615048 -0.30436024\n",
      "   3.32492508]]\n",
      "scaled_Xtest:\n",
      " [[ 0.48303456 -1.19173043 -1.19283892 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]\n",
      " [ 0.77706853 -1.19173043 -1.19283892 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]\n",
      " [-1.55297971 -1.19173043 -1.19283892 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]\n",
      " ...\n",
      " [-0.09648588 -1.19173043 -1.19283892 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]\n",
      " [-1.54956071 -1.19173043 -1.15516822 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]\n",
      " [-1.07773877 -1.19173043 -1.15516822 ... -0.29826517 -0.2967589\n",
      "  -0.30867061]]\n",
      "Accuracy: 0.5211800302571861\n",
      "f-score: 0.17470664928292048\n",
      "AUC score:0.7038455359131999\n",
      "scaled_Xtrain:\n",
      " [[-0.90737541 -1.0310731  -1.02618165 ... -0.30421619 -0.30211218\n",
      "  -0.30105652]\n",
      " [ 1.41777689 -1.0310731  -0.99140545 ... -0.30421619 -0.30211218\n",
      "  -0.30105652]\n",
      " [ 0.36658528 -1.0310731  -0.95662925 ... -0.30421619 -0.30211218\n",
      "  -0.30105652]\n",
      " ...\n",
      " [-0.81165432 -1.0310731  -0.99140545 ... -0.30421619  3.3100287\n",
      "  -0.30105652]\n",
      " [ 0.01154705 -1.0310731  -0.99140545 ... -0.30421619 -0.30211218\n",
      "  -0.30105652]\n",
      " [-1.28851939 -1.0310731  -1.02618165 ... -0.30421619 -0.30211218\n",
      "   3.32163541]]\n",
      "scaled_Xtest:\n",
      " [[ 1.74822593 -1.1730966  -1.18095714 ... -0.31689562 -0.31872763\n",
      "  -0.30763164]\n",
      " [ 0.39746552 -1.1730966  -1.14404891 ... -0.31689562 -0.31872763\n",
      "  -0.30763164]\n",
      " [-0.77169458 -1.1730966  -1.18095714 ... -0.31689562 -0.31872763\n",
      "   3.25064096]\n",
      " ...\n",
      " [-0.14560588 -1.1730966  -1.10714067 ... -0.31689562 -0.31872763\n",
      "  -0.30763164]\n",
      " [-0.63160291 -1.1730966  -1.14404891 ... -0.31689562 -0.31872763\n",
      "   3.25064096]\n",
      " [ 0.7710433  -1.1730966  -1.14404891 ... -0.31689562  3.1374751\n",
      "  -0.30763164]]\n",
      "Accuracy: 0.5408261287223823\n",
      "f-score: 0.10150375939849623\n",
      "AUC score:0.6977796193714383\n",
      "scaled_Xtrain:\n",
      " [[-0.90097298 -1.03690932 -1.03367721 ... -0.30462451 -0.3038939\n",
      "  -0.30206227]\n",
      " [ 1.4210549  -1.03690932 -0.99882409 ... -0.30462451 -0.3038939\n",
      "  -0.30206227]\n",
      " [ 0.37127583 -1.03690932 -0.96397098 ... -0.30462451 -0.3038939\n",
      "  -0.30206227]\n",
      " ...\n",
      " [-0.80538052 -1.03690932 -0.99882409 ... -0.30462451  3.2906221\n",
      "  -0.30206227]\n",
      " [ 0.01671468 -1.03690932 -0.99882409 ... -0.30462451 -0.3038939\n",
      "  -0.30206227]\n",
      " [-1.2816048  -1.03690932 -1.03367721 ... -0.30462451 -0.3038939\n",
      "   3.31057562]]\n",
      "scaled_Xtest:\n",
      " [[ 0.45722989 -1.08087939 -1.03554258 ... -0.31010926 -0.3010772\n",
      "  -0.2984635 ]\n",
      " [ 1.21228004  0.17540465  0.03001267 ... -0.31010926 -0.3010772\n",
      "  -0.2984635 ]\n",
      " [ 1.48493704  0.17540465  0.06553118 ... -0.31010926 -0.3010772\n",
      "  -0.2984635 ]\n",
      " ...\n",
      " [-0.6246591  -1.08087939 -1.03554258 ... -0.31010926 -0.3010772\n",
      "   3.35049346]\n",
      " [ 0.25798054 -1.08087939 -1.07106109 ... -0.31010926 -0.3010772\n",
      "  -0.2984635 ]\n",
      " [-1.23988515 -1.08087939 -1.03554258 ... -0.31010926 -0.3010772\n",
      "  -0.2984635 ]]\n",
      "Accuracy: 0.7275725593667546\n",
      "f-score: 0.14137214137214135\n",
      "AUC score:0.7129654802180845\n",
      "scaled_Xtrain:\n",
      " [[-0.89999722 -1.03749239 -1.03286736 ... -0.30563722 -0.30335942\n",
      "  -0.30089364]\n",
      " [ 1.42026603 -1.03749239 -0.99801991 ... -0.30563722 -0.30335942\n",
      "  -0.30089364]\n",
      " [ 0.37128474 -1.03749239 -0.96317245 ... -0.30563722 -0.30335942\n",
      "  -0.30089364]\n",
      " ...\n",
      " [-0.34424555 -1.03749239 -1.03286736 ... -0.30563722 -0.30335942\n",
      "  -0.30089364]\n",
      " [ 0.01699304 -1.03749239 -0.99801991 ... -0.30563722 -0.30335942\n",
      "  -0.30089364]\n",
      " [-1.28033978 -1.03749239 -1.03286736 ... -0.30563722 -0.30335942\n",
      "   3.32343345]]\n",
      "scaled_Xtest:\n",
      " [[-0.76309891 -1.09370121 -1.09662222 ... -0.3018198   3.27326835\n",
      "  -0.30916125]\n",
      " [-0.83728795 -1.09370121 -1.06073583 ... -0.3018198  -0.30550505\n",
      "  -0.30916125]\n",
      " [ 1.53852786 -1.09370121 -1.02484943 ...  3.31323529 -0.30550505\n",
      "  -0.30916125]\n",
      " ...\n",
      " [ 1.03863501  0.17173167  0.30294715 ... -0.3018198  -0.30550505\n",
      "  -0.30916125]\n",
      " [-0.27557091 -1.09370121 -1.09662222 ...  3.31323529 -0.30550505\n",
      "  -0.30916125]\n",
      " [-0.82668952 -1.09370121 -1.06073583 ... -0.3018198   3.27326835\n",
      "  -0.30916125]]\n",
      "Accuracy: 0.48874296435272047\n",
      "f-score: 0.2605156037991859\n",
      "AUC score:0.6904701685491671\n",
      "scaled_Xtrain:\n",
      " [[ 0.49175164 -1.03091185 -1.027245   ... -0.30400682 -0.30292612\n",
      "  -0.30202358]\n",
      " [ 1.56342561 -1.03091185 -0.99270815 ... -0.30400682 -0.30292612\n",
      "  -0.30202358]\n",
      " [ 0.79620447 -1.03091185 -0.9581713  ... -0.30400682 -0.30292612\n",
      "  -0.30202358]\n",
      " ...\n",
      " [-0.80782702 -1.03091185 -0.99270815 ... -0.30400682  3.30113493\n",
      "  -0.30202358]\n",
      " [ 0.01506549 -1.03091185 -0.99270815 ... -0.30400682 -0.30292612\n",
      "  -0.30202358]\n",
      " [-1.28451317 -1.03091185 -1.027245   ... -0.30400682 -0.30292612\n",
      "   3.31099974]]\n",
      "scaled_Xtest:\n",
      " [[-0.89006874 -1.1504887  -1.14390038 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]\n",
      " [ 1.42841392 -1.1504887  -1.10515825 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]\n",
      " [ 0.38023762 -1.1504887  -1.06641611 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]\n",
      " ...\n",
      " [ 0.62666318 -0.24171065 -0.33031556 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]\n",
      " [-1.2145869  -0.24171065 -0.29157342 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]\n",
      " [-0.25491556 -0.24171065 -0.25283129 ... -0.31571063 -0.30854392\n",
      "  -0.29832325]]\n",
      "Accuracy: 0.44205052005943535\n",
      "f-score: 0.22974358974358974\n",
      "AUC score:0.6982524891179267\n",
      "scaled_Xtrain:\n",
      " [[-0.9044801  -1.03983183 -1.03531616 ... -0.30608711 -0.3045234\n",
      "  -0.30260497]\n",
      " [ 1.42557224 -1.03983183 -1.00025747 ... -0.30608711 -0.3045234\n",
      "  -0.30260497]\n",
      " [ 0.37216534 -1.03983183 -0.96519877 ... -0.30608711 -0.3045234\n",
      "  -0.30260497]\n",
      " ...\n",
      " [-0.80855728 -1.03983183 -1.00025747 ... -0.30608711  3.28381985\n",
      "  -0.30260497]\n",
      " [ 0.01637891 -1.03983183 -1.00025747 ... -0.30608711 -0.3045234\n",
      "  -0.30260497]\n",
      " [-1.2864273  -1.03983183 -1.03531616 ... -0.30608711 -0.3045234\n",
      "   3.30463839]]\n",
      "scaled_Xtest:\n",
      " [[ 0.04235092 -1.07179351 -1.03968907 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]\n",
      " [-1.66006071 -1.07179351 -1.07337546 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]\n",
      " [ 1.12386316 -1.07179351 -1.03968907 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]\n",
      " ...\n",
      " [-1.32092817 -1.07179351 -1.07337546 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]\n",
      " [ 0.36967288 -1.07179351 -1.07337546 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]\n",
      " [-0.33558784 -1.07179351 -1.07337546 ... -0.29633695 -0.29228862\n",
      "  -0.29025002]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.36427850655903127\n",
      "f-score: 0.06250000000000001\n",
      "AUC score:0.6896103169704968\n",
      "scaled_Xtrain:\n",
      " [[-0.89814586 -1.06844795 -1.06539691 ... -0.30452486 -0.30324883\n",
      "  -0.300686  ]\n",
      " [ 1.42814669 -1.06844795 -1.02907391 ... -0.30452486 -0.30324883\n",
      "  -0.300686  ]\n",
      " [ 0.37643958 -1.06844795 -0.99275092 ... -0.30452486 -0.30324883\n",
      "  -0.300686  ]\n",
      " ...\n",
      " [-0.80237783 -1.06844795 -1.02907391 ... -0.30452486  3.29762201\n",
      "  -0.300686  ]\n",
      " [ 0.02122725 -1.06844795 -1.02907391 ... -0.30452486 -0.30324883\n",
      "  -0.300686  ]\n",
      " [-1.27947675 -1.06844795 -1.06539691 ... -0.30452486 -0.30324883\n",
      "   3.32572854]]\n",
      "scaled_Xtest:\n",
      " [[ 0.84222843 -0.92511825 -0.88966179 ... -0.31093242 -0.30569767\n",
      "  -0.3083223 ]\n",
      " [-1.15844482 -0.92511825 -0.86144929 ... -0.31093242 -0.30569767\n",
      "  -0.3083223 ]\n",
      " [-0.94439522 -0.92511825 -0.8332368  ...  3.216133   -0.30569767\n",
      "  -0.3083223 ]\n",
      " ...\n",
      " [-0.6699284  -0.92511825 -0.88966179 ... -0.31093242 -0.30569767\n",
      "  -0.3083223 ]\n",
      " [-0.0243272  -0.92511825 -0.86144929 ... -0.31093242 -0.30569767\n",
      "  -0.3083223 ]\n",
      " [ 0.50389197 -0.92511825 -0.91787428 ... -0.31093242 -0.30569767\n",
      "  -0.3083223 ]]\n",
      "Accuracy: 0.30955585464333785\n",
      "f-score: 0.18700475435816166\n",
      "AUC score:0.6862679398284184\n",
      "scaled_Xtrain:\n",
      " [[-0.90187007 -1.05396669 -1.04977777 ... -0.30597581 -0.30457066\n",
      "  -0.30245486]\n",
      " [ 1.4203878  -1.05396669 -1.01516535 ... -0.30597581 -0.30457066\n",
      "  -0.30245486]\n",
      " [ 0.37050475 -1.05396669 -0.98055292 ... -0.30597581 -0.30457066\n",
      "  -0.30245486]\n",
      " ...\n",
      " [-0.80626813 -1.05396669 -1.01516535 ... -0.30597581  3.28331035\n",
      "  -0.30245486]\n",
      " [ 0.01590849 -1.05396669 -1.01516535 ... -0.30597581 -0.30457066\n",
      "  -0.30245486]\n",
      " [-1.28253958 -1.05396669 -1.04977777 ... -0.30597581 -0.30457066\n",
      "   3.3062785 ]]\n",
      "scaled_Xtest:\n",
      " [[-1.08909136  1.39881584  1.37502391 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]\n",
      " [ 0.23187569  1.39881584  1.41500369 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]\n",
      " [-1.43376754  1.39881584  1.45498347 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]\n",
      " ...\n",
      " [-1.132832   -0.00801251  0.05569125 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]\n",
      " [-0.0865561  -0.00801251  0.09567103 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]\n",
      " [-1.06634624 -0.00801251  0.13565081 ... -0.29854364 -0.29315098\n",
      "  -0.29315098]]\n",
      "Accuracy: 0.7868705035971223\n",
      "f-score: 0.16842105263157897\n",
      "AUC score:0.6822338990844068\n"
     ]
    }
   ],
   "source": [
    "model_validation(total_acc, 'IS_PARTIAL', Xp_kf_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluating Logistic Models \n",
    "\n",
    "<p>The <b>Area Under Curve (AUC)</b> is the area under the ROC curve and serves as a single number performance summary of the classifier. AUC is often used as evaluation metric in imbalanced class problems, therefore it makes sense to directly optimize it in the training process. </p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p><b>F1 Score</b> might be a better measure to use if we need to seek a balance between Precision and Recall AND there is an uneven class distribution (large number of Actual Negatives). F1 score reaches its best value at 1 (perfect precision and recall) and worst at 0. </p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SMM Calculation "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>SMM is a measure of the monthly mortgage prepayment rate of the security’s mortgage pool.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [],
   "source": [
    "whole_result = pd.concat([record_date, Xp_train, month, IS_FULL, yf_result_df, IS_PARTIAL, yp_result_df], axis=1,\n",
    "                         join_axes=[Xp_train.index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [],
   "source": [
    "whole_result = whole_result.drop(dummies_df, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [],
   "source": [
    "prepay_perM = whole_result[whole_result['IS_PARTIAL']==1].groupby(['record_date']).agg({'prepay_amt':'sum'}).reset_index() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SMM calculation\n",
    "\n",
    "def SMM_Cal_perMonth(givenDate,prepay_perM,tenor_M,df):\n",
    "    \n",
    "    '''\n",
    "    givenMonth: specific month in a year\n",
    "    prepay_perM: a dataframe which shows aggregated prepay amount each month.  \n",
    "    tenor_M: A granularity variable \n",
    "    df: final result dataframe\n",
    "    partial_prob : result coming from \"IS_PARTIAL\" model\n",
    "    full_prob : result coming from \"IS_FULL\" model\n",
    "    '''\n",
    "    try:\n",
    "        selectDF = df[(df['record_date']== givenDate) & (df['tenor_m']== tenor_M)]\n",
    "        M_bar_t = sum(prepay_perM[prepay_perM['record_date'] == givenDate]['prepay_amt'])/sum(selectDF['partial_prob']*(selectDF['out_balance']-selectDF['sch_payment']))\n",
    "        partial_prepay = sum(selectDF['partial_prob'] *M_bar_t* (selectDF['out_balance']-selectDF['sch_payment']))\n",
    "        full_prepay = sum(selectDF['full_prob']*(selectDF['out_balance']-selectDF['sch_payment']))\n",
    "        total_prepay = sum(selectDF['out_balance']-selectDF['sch_payment'])\n",
    "        SMM_t = (partial_prepay + full_prepay) / total_prepay\n",
    "        #print(M_bar_t,partial_prepay,full_prepay,total_prepay)\n",
    "        return SMM_t\n",
    "\n",
    "    except ZeroDivisionError:\n",
    "        \n",
    "        return print('you have denominator 0')\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SMM summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*speed test*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2499985694885254\n"
     ]
    }
   ],
   "source": [
    "st=time.time()\n",
    "datelist = []\n",
    "SMMlist = []\n",
    "for index, row in prepay_perM.iterrows():\n",
    "    givenDate = row['record_date']\n",
    "    datelist.append(givenDate)\n",
    "    SMM = SMM_Cal_perMonth(givenDate, prepay_perM, 120, whole_result)\n",
    "    SMMlist.append(SMM)\n",
    "SMM_df = pd.DataFrame({'Date':datelist,'SMM_per': SMMlist}, columns=['Date','SMM_per'])\n",
    "print(time.time()-st)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2099921703338623\n"
     ]
    }
   ],
   "source": [
    "st=time.time()\n",
    "datelist = []\n",
    "SMMlist = []\n",
    "for  row in prepay_perM.itertuples():\n",
    "    givenDate = row.record_date\n",
    "    datelist.append(givenDate)\n",
    "    SMM = SMM_Cal_perMonth(givenDate, prepay_perM, 120, whole_result)\n",
    "    SMMlist.append(SMM)\n",
    "SMM_df = pd.DataFrame({'Date':datelist,'SMM_per': SMMlist}, columns=['Date','SMM_per'])\n",
    "print(time.time()-st)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SMM_summary(prepay_perM, tenor_M, df):\n",
    "    datelist = []\n",
    "    SMMlist = []\n",
    "    for row in prepay_perM.itertuples():\n",
    "        givenDate = row.record_date\n",
    "        datelist.append(givenDate)\n",
    "        SMM = SMM_Cal_perMonth(givenDate, prepay_perM, tenor_M, df)\n",
    "        SMMlist.append(SMM)\n",
    "    SMM_df = pd.DataFrame({'Date':datelist,'SMM_per': SMMlist}, columns=['Date','SMM_per'])\n",
    "    return SMM_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "partial_prepay_df = whole_result[whole_result['IS_PARTIAL']==1]\n",
    "\n",
    "partial_prepayM = partial_prepay_df.groupby(['record_date']).agg({'prepay_amt':'sum'}).reset_index()\n",
    "partial_prepayM_120 = partial_prepay_df[partial_prepay_df['tenor_m']== 120].groupby(['record_date']).agg({'prepay_amt':'sum'}).reset_index()\n",
    "partial_prepayM_180 = partial_prepay_df[partial_prepay_df['tenor_m']== 180].groupby(['record_date']).agg({'prepay_amt':'sum'}).reset_index()\n",
    "partial_prepayM_240 = partial_prepay_df[partial_prepay_df['tenor_m']== 240].groupby(['record_date']).agg({'prepay_amt':'sum'}).reset_index()\n",
    "partial_prepayM_360 = partial_prepay_df[partial_prepay_df['tenor_m']== 360].groupby(['record_date']).agg({'prepay_amt':'sum'}).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMM_total = SMM_summary(partial_prepayM, whole_result['tenor_m'], whole_result)\n",
    "SMM_120 = SMM_summary(partial_prepayM_120, 120, whole_result)\n",
    "SMM_180 = SMM_summary(partial_prepayM_180, 180, whole_result)\n",
    "SMM_240 = SMM_summary(partial_prepayM_240, 240, whole_result)\n",
    "SMM_360 = SMM_summary(partial_prepayM_360, 360, whole_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMM_total_mean = SMM_total.SMM_per.mean()\n",
    "SMM_120_mean = SMM_120.SMM_per.mean()\n",
    "SMM_180_mean = SMM_180.SMM_per.mean()\n",
    "SMM_240_mean = SMM_240.SMM_per.mean()\n",
    "SMM_360_mean = SMM_360.SMM_per.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMM_summary_list = [SMM_120_mean,SMM_180_mean,SMM_240_mean,SMM_360_mean]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMM total mean is : 0.024106090133460774\n",
      "SMM_120 total mean is : 0.006328286700979139\n",
      "SMM_180 total mean is : 0.0007514003968577605\n",
      "SMM_240 total mean is : nan\n",
      "SMM_360 total mean is : nan\n"
     ]
    }
   ],
   "source": [
    "print(f'SMM total mean is : {SMM_total.SMM_per.mean()}')\n",
    "print(f'SMM_120 total mean is : {SMM_120.SMM_per.mean()}')\n",
    "print(f'SMM_180 total mean is : {SMM_180.SMM_per.mean()}')\n",
    "print(f'SMM_240 total mean is : {SMM_240.SMM_per.mean()}')\n",
    "print(f'SMM_360 total mean is : {SMM_360.SMM_per.mean()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate the behavior tenor "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "def behavior_tenor(full_prob, contractor_tenor): \n",
    "    '''\n",
    "    Calculate the behavior tenor\n",
    "\n",
    "    Data description:\n",
    "        Behavior tenor is the expected life of a real estate loan before being paid off. \n",
    "        we make the following assumptions in deriving BT:\n",
    "         i). A real esate loan is paid off either at contractual maturity or after the borrower has made a full prepayment.\n",
    "         ii). BT varies with the contractual tenor of the loan.\n",
    "         iii). The full prepayment of a real estate loan is constant across time and account for a given tenor.\n",
    "\n",
    "    '''\n",
    "    sum_value = 0\n",
    "    for i in range(1,contractor_tenor-1):\n",
    "        calculate = i * full_prob * ((1-full_prob) ** (i-1))\n",
    "        sum_value += calculate\n",
    "    BT = sum_value + contractor_tenor * ((1-full_prob) ** (contractor_tenor-1))\n",
    "    return BT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_prob_120M = np.mean(whole_result[whole_result['tenor_m']==120]['full_prob'])\n",
    "full_prob_180M = np.mean(whole_result[whole_result['tenor_m']==180]['full_prob'])\n",
    "full_prob_240M = np.mean(whole_result[whole_result['tenor_m']==240]['full_prob'])\n",
    "full_prob_360M = np.mean(whole_result[whole_result['tenor_m']==360]['full_prob'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "BT_120 = behavior_tenor(full_prob_120M, 120)\n",
    "BT_180 = behavior_tenor(full_prob_180M, 180)\n",
    "BT_240 = behavior_tenor(full_prob_240M, 240)\n",
    "BT_360 = behavior_tenor(full_prob_360M, 360)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [],
   "source": [
    "BT_list = [BT_120, BT_180, BT_240, BT_360]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55.94527851443435 45.94570713696076 nan nan\n"
     ]
    }
   ],
   "source": [
    "print(BT_120,BT_180,BT_240,BT_360)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = pd.DataFrame(data = {'SMM':SMM_summary_list,'Behavior tenor': BT_list}, index=['120M','180M','240M','360M'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SMM</th>\n",
       "      <th>Behavior tenor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>120M</th>\n",
       "      <td>0.006328</td>\n",
       "      <td>55.945279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180M</th>\n",
       "      <td>0.000751</td>\n",
       "      <td>45.945707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240M</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360M</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           SMM  Behavior tenor\n",
       "120M  0.006328       55.945279\n",
       "180M  0.000751       45.945707\n",
       "240M       NaN             NaN\n",
       "360M       NaN             NaN"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}