Test inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import datatable as dt\n",
    "import numpy as np\n",
    "\n",
    "train = dt.fread(\"/input/riiid-test-answer-prediction/train.csv\").to_pandas()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Generating same feature files minus the validation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idx = pd.read_pickle(\"/home/dfs/cv1_valid.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train[~train.index.isin(val_idx.row_id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Convert users into sequences\n",
    "groups = train[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n",
    "            r['content_id'].values,\n",
    "            r['answered_correctly'].values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = groups.apply(lambda x: x[1][-140:].tolist())\n",
    "groups = groups.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "lec_data = pd.read_csv(\"/input/riiid-test-answer-prediction/lectures.csv\")\n",
    "lec_dict = lec_data[[\"lecture_id\", \"tag\"]].set_index(\"lecture_id\").tag.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "que_data = pd.read_pickle(\"/home/dfs/que_data.pickle\")\n",
    "questions = que_data.to_dict(\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_lec_data(train):\n",
    "    \n",
    "    train = train[train[\"content_type_id\"] == True]\n",
    "    \n",
    "    train = train.groupby(\"user_id\").apply(lambda x: x.content_id.tolist())\n",
    "    train = pd.DataFrame(train)\n",
    "    train.columns = [\"CUM_CONCAT\"]\n",
    "    \n",
    "    global lec_history\n",
    "    lec_history = {}\n",
    "\n",
    "    def f(row):\n",
    "\n",
    "        global lec_history\n",
    "        lec_history[str(row.name)] = {}\n",
    "\n",
    "        for i in row.CUM_CONCAT:\n",
    "            lec_tag  = lec_dict[i]\n",
    "            lec_history[str(row.name)][str(lec_tag)] = 1\n",
    "\n",
    "        return row\n",
    "\n",
    "    train.apply(f, axis=1)\n",
    "    \n",
    "    return lec_history\n",
    "    \n",
    "watched_tags = gen_lec_data(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#With lectures\n",
    "timetable = train.groupby(\"user_id\")[[\"user_id\",\"timestamp\"]].tail(5).groupby(\"user_id\")[\"timestamp\"].apply(lambda x: x.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "timetable_df = pd.DataFrame(columns=[\"first_timestamp\",\"second_timestamp\",\"third_timestamp\",\"fourth_timestamp\",\"fifth_timestamp\"], index=timetable.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "timetable_df[\"fifth_timestamp\"] = timetable.apply(lambda x: x[0])\n",
    "timetable_df[\"fourth_timestamp\"] = timetable.apply(lambda x: x[1] if len(x) > 1 else 0)\n",
    "timetable_df[\"third_timestamp\"] = timetable.apply(lambda x: x[2] if len(x) > 2 else 0)\n",
    "timetable_df[\"second_timestamp\"] = timetable.apply(lambda x: x[3] if len(x) > 3 else 0)\n",
    "timetable_df[\"first_timestamp\"] = timetable.apply(lambda x: x[4] if len(x) > 4 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "timetable_df = timetable_df/8.64e+7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"time_diff\"] = train.groupby(\"user_id\")[\"timestamp\"].diff().fillna(0).astype(\"float32\")\n",
    "train[\"ts_diff_shifted\"] = train.groupby(\"user_id\")[\"time_diff\"].shift(1).replace(0, np.nan)\n",
    "train[\"ts_diff_shifted\"] = train.groupby(\"user_id\")[\"ts_diff_shifted\"].fillna(method=\"ffill\").fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"time_diff1\"] = train.groupby(\"user_id\")[\"timestamp\"].diff(2).fillna(0).astype(\"float32\")\n",
    "train[\"ts_diff_shifted_2\"] = train.groupby(\"user_id\")[\"time_diff1\"].shift(1).replace(0, np.nan)\n",
    "train[\"ts_diff_shifted_2\"] = train.groupby(\"user_id\")[\"ts_diff_shifted_2\"].fillna(method=\"ffill\").fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_diff_shifted = train.groupby(\"user_id\")[\"ts_diff_shifted\"].last()\n",
    "ts_diff_shifted2 = train.groupby(\"user_id\")[\"ts_diff_shifted_2\"].last()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_diff_shifted = pd.DataFrame(ts_diff_shifted)\n",
    "ts_diff_shifted.columns = [\"ts_diff_shifted\"]\n",
    "\n",
    "ts_diff_shifted2 = pd.DataFrame(ts_diff_shifted2)\n",
    "ts_diff_shifted = ts_diff_shifted.join(ts_diff_shifted2, how=\"left\")\n",
    "ts_diff_shifted.columns = [\"ts_diff_shifted\",\"ts_diff_shifted_2\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "lecture_count = train.groupby(\"user_id\")[\"content_type_id\"].sum()\n",
    "interactions = train.groupby(\"user_id\")[\"content_type_id\"].count()\n",
    "\n",
    "\n",
    "base = pd.DataFrame(lecture_count)\n",
    "base.columns = [\"lecs_n\"]\n",
    "\n",
    "base = base.join(interactions, how=\"left\")\n",
    "base.columns = [\"lecs_n\", \"interaction_n\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = base.join(ts_diff_shifted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"timestamp\"] = train[\"timestamp\"]/8.64e+7\n",
    "train[\"time_diff\"] = train.groupby(\"user_id\")[\"timestamp\"].diff().fillna(0).astype(\"float32\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"sessions\"] = train[\"time_diff\"] > 0.083\n",
    "sessions = train.groupby(\"user_id\")[\"sessions\"].sum()\n",
    "\n",
    "base = base.join(sessions)\n",
    "base = base.rename(columns={\"user_id\":\"session\"})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"sessions\"] = train.groupby(\"user_id\")[\"sessions\"].cumsum()\n",
    "\n",
    "train[\"session_count\"] =1\n",
    "train[\"session_count\"] = train.groupby([\"user_id\",\"sessions\"])[\"session_count\"].cumsum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "session_count = train.groupby(\"user_id\")[[\"session_count\", \"user_id\"]].tail(1).set_index(\"user_id\", drop=True)\n",
    "base = base.join(session_count)\n",
    "\n",
    "del train[\"session_count\"]\n",
    "del train[\"sessions\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"mean_pause\"] = ((train.time_diff > 0.083)*train.time_diff).replace(0,np.nan)\n",
    "test = train[[\"mean_pause\",\"user_id\"]]\n",
    "test = test.dropna()\n",
    "\n",
    "base = base.join(test.groupby(\"user_id\")[\"mean_pause\"].sum())\n",
    "base = base.rename(columns={\"mean_pause\":\"sum_pauses\"})\n",
    "base.sum_pauses = base.sum_pauses.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train[train[\"content_type_id\"] == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.prior_question_had_explanation = train.prior_question_had_explanation.fillna(False).astype(\"bool\")\n",
    "train.prior_question_elapsed_time = train.prior_question_elapsed_time.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = base.join(train.groupby(\"user_id\")[\"prior_question_had_explanation\"].sum())\n",
    "base = base.rename(columns={\"prior_question_had_explanation\":\"had_exp\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.merge(que_data[[\"part\"]], how=\"left\", right_index=True, left_on=\"content_id\")\n",
    "train.part = train.part.astype(\"int8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"last_part\"] = train.groupby(\"user_id\")[\"part\"].shift().fillna(-1).astype(\"int8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = train.groupby(\"user_id\")[[\"user_id\",\"last_part\"]].tail(1).set_index(\"user_id\", drop=True)\n",
    "base = base.join(pd.DataFrame(a, columns=[\"last_part\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lecs_n</th>\n",
       "      <th>interaction_n</th>\n",
       "      <th>ts_diff_shifted</th>\n",
       "      <th>ts_diff_shifted_2</th>\n",
       "      <th>sessions</th>\n",
       "      <th>session_count</th>\n",
       "      <th>sum_pauses</th>\n",
       "      <th>had_exp</th>\n",
       "      <th>last_part</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>118231.0</td>\n",
       "      <td>118231.0</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7.712886</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>16819.0</td>\n",
       "      <td>16819.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2746</th>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>38370.0</td>\n",
       "      <td>65141.0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5382</th>\n",
       "      <td>3</td>\n",
       "      <td>128</td>\n",
       "      <td>269227.0</td>\n",
       "      <td>432686.0</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>23.964521</td>\n",
       "      <td>113</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8623</th>\n",
       "      <td>3</td>\n",
       "      <td>112</td>\n",
       "      <td>43466.0</td>\n",
       "      <td>101956.0</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>9.781732</td>\n",
       "      <td>96</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2147470770</th>\n",
       "      <td>2</td>\n",
       "      <td>228</td>\n",
       "      <td>40994.0</td>\n",
       "      <td>74437.0</td>\n",
       "      <td>14</td>\n",
       "      <td>20</td>\n",
       "      <td>32.620239</td>\n",
       "      <td>195</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2147470777</th>\n",
       "      <td>6</td>\n",
       "      <td>640</td>\n",
       "      <td>420113.0</td>\n",
       "      <td>420113.0</td>\n",
       "      <td>11</td>\n",
       "      <td>85</td>\n",
       "      <td>151.120880</td>\n",
       "      <td>603</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2147481750</th>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>27588.0</td>\n",
       "      <td>53729.0</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>0.597832</td>\n",
       "      <td>39</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2147482216</th>\n",
       "      <td>5</td>\n",
       "      <td>230</td>\n",
       "      <td>22649.0</td>\n",
       "      <td>43807.0</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>15.044339</td>\n",
       "      <td>216</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2147482888</th>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>35931.0</td>\n",
       "      <td>64406.0</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>4.946173</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>378537 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            lecs_n  interaction_n  ts_diff_shifted  ts_diff_shifted_2  \\\n",
       "user_id                                                                 \n",
       "115              0             46         118231.0           118231.0   \n",
       "124              0             30          16819.0            16819.0   \n",
       "2746             1             20          38370.0            65141.0   \n",
       "5382             3            128         269227.0           432686.0   \n",
       "8623             3            112          43466.0           101956.0   \n",
       "...            ...            ...              ...                ...   \n",
       "2147470770       2            228          40994.0            74437.0   \n",
       "2147470777       6            640         420113.0           420113.0   \n",
       "2147481750       0             50          27588.0            53729.0   \n",
       "2147482216       5            230          22649.0            43807.0   \n",
       "2147482888       0             27          35931.0            64406.0   \n",
       "\n",
       "            sessions  session_count  sum_pauses  had_exp  last_part  \n",
       "user_id                                                              \n",
       "115                1              7    7.712886        6          4  \n",
       "124                0             30    0.000000        0          7  \n",
       "2746               0             20    0.000000       11          2  \n",
       "5382              16              1   23.964521      113          5  \n",
       "8623              10             10    9.781732       96          2  \n",
       "...              ...            ...         ...      ...        ...  \n",
       "2147470770        14             20   32.620239      195          5  \n",
       "2147470777        11             85  151.120880      603          6  \n",
       "2147481750         1             20    0.597832       39          5  \n",
       "2147482216        13             10   15.044339      216          2  \n",
       "2147482888         2             12    4.946173       19          5  \n",
       "\n",
       "[378537 rows x 9 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x):\n",
    "    \n",
    "    parts = x.part.values.tolist()\n",
    "    nb = x.answered_correctly.tolist()\n",
    "    ret = [0]*7\n",
    "    \n",
    "    for i, k in enumerate(parts):\n",
    "        ret[k-1] = nb[i]\n",
    "        \n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = pd.DataFrame(train.groupby([\"user_id\",\"part\"])[\"answered_correctly\"].count())\n",
    "\n",
    "a[\"user_id\"] = a.index.get_level_values(0)\n",
    "a[\"part\"] = a.index.get_level_values(1)\n",
    "\n",
    "a.reset_index(drop=True, inplace=True)\n",
    "base = base.join(pd.DataFrame(a.groupby(\"user_id\").apply(f), columns=[\"part_count\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = pd.DataFrame(train.groupby([\"user_id\",\"part\"])[\"answered_correctly\"].sum())\n",
    "\n",
    "a[\"user_id\"] = a.index.get_level_values(0)\n",
    "a[\"part\"] = a.index.get_level_values(1)\n",
    "\n",
    "a.reset_index(drop=True, inplace=True)\n",
    "base = base.join(pd.DataFrame(a.groupby(\"user_id\").apply(f), columns=[\"part_corr\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_train = pd.read_pickle(\"/home/dfs/data_2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_train = tmp_train[[\"k\", \"content_id\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "que_2_k = tmp_train.groupby(\"content_id\")[\"k\"].first().to_dict()\n",
    "\n",
    "del tmp_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "que_2_k = pd.DataFrame(pd.Series(que_2_k), columns=[\"k\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.merge(que_2_k, how=\"left\", left_on=\"content_id\", right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x):\n",
    "    \n",
    "    k = x.k.values.tolist()\n",
    "    nb = x.answered_correctly.tolist()\n",
    "    ret = [0]*20\n",
    "    \n",
    "    for i, j in enumerate(k):\n",
    "        ret[j] = nb[i]\n",
    "        \n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = pd.DataFrame(train.groupby([\"user_id\", \"k\"])[\"answered_correctly\"].count())\n",
    "\n",
    "a[\"user_id\"] = a.index.get_level_values(0)\n",
    "a[\"k\"] = a.index.get_level_values(1)\n",
    "\n",
    "a.reset_index(drop=True, inplace=True)\n",
    "\n",
    "k_count = pd.DataFrame(a.groupby(\"user_id\").apply(f), columns=[\"k_count\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = pd.DataFrame(train.groupby([\"user_id\", \"k\"])[\"answered_correctly\"].sum())\n",
    "\n",
    "a[\"user_id\"] = a.index.get_level_values(0)\n",
    "a[\"k\"] = a.index.get_level_values(1)\n",
    "\n",
    "a.reset_index(drop=True, inplace=True)\n",
    "\n",
    "k_corr = pd.DataFrame(a.groupby(\"user_id\").apply(f), columns=[\"k_corr\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = base.join(k_corr).join(k_count)\n",
    "del k_corr\n",
    "del k_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "prior_part_mean_dict = {1: 22166.159642501425,\n",
    " 2: 18714.69673913695,\n",
    " 3: 23620.317746179924,\n",
    " 4: 23762.753651169547,\n",
    " 5: 25094.620302855932,\n",
    " 6: 32417.37918735745,\n",
    " 7: 47444.16407400242}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = pd.DataFrame(train.groupby([\"user_id\",\"part\"])[\"prior_question_elapsed_time\"].sum())\n",
    "\n",
    "a[\"user_id\"] = a.index.get_level_values(0)\n",
    "a[\"part\"] = a.index.get_level_values(1)\n",
    "\n",
    "a.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x):\n",
    "    \n",
    "    parts = x.part.values.tolist()\n",
    "    nb = x.prior_question_elapsed_time.tolist()\n",
    "    ret = [0]*7\n",
    "    \n",
    "    for i, k in enumerate(parts):\n",
    "        ret[k-1] = nb[i]\n",
    "        \n",
    "    return ret\n",
    "\n",
    "part_et = pd.DataFrame(a.groupby(\"user_id\").apply(f), columns=[\"part_et\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = pd.DataFrame(train.groupby([\"user_id\",\"part\"])[\"prior_question_had_explanation\"].sum())\n",
    "\n",
    "a[\"user_id\"] = a.index.get_level_values(0)\n",
    "a[\"part\"] = a.index.get_level_values(1)\n",
    "\n",
    "a.reset_index(drop=True, inplace=True)\n",
    "\n",
    "\n",
    "def f(x):\n",
    "    \n",
    "    parts = x.part.values.tolist()\n",
    "    nb = x.prior_question_had_explanation.tolist()\n",
    "    ret = [0]*7\n",
    "    \n",
    "    for i, k in enumerate(parts):\n",
    "        ret[k-1] = nb[i]\n",
    "        \n",
    "    return ret\n",
    "\n",
    "part_explan = pd.DataFrame(a.groupby(\"user_id\").apply(f), columns=[\"part_explan\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = train.groupby(\"user_id\")[[\"user_id\",\"prior_question_had_explanation\"]].tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "priors_5 = pd.DataFrame(a.groupby(\"user_id\")[\"prior_question_had_explanation\"].apply(lambda x: x.tolist()))\n",
    "priors_5.columns = [\"priors_5\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = base.join(part_et).join(part_explan).join(priors_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_ts = train[train.answered_correctly == True].groupby(\"user_id\")[\"timestamp\"].last()\n",
    "\n",
    "last_ts = pd.DataFrame(last_ts)\n",
    "last_ts.columns = [\"recent_corr\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "el_sum = train.groupby(\"user_id\")[\"prior_question_elapsed_time\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "el_sum = pd.DataFrame(el_sum)\n",
    "el_sum.columns = [\"el_sum\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = base.join(el_sum).join(last_ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n",
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    }
   ],
   "source": [
    "a = train.groupby(\"content_id\")[\"answered_correctly\"].agg([\"count\",\"mean\"])\n",
    "\n",
    "easy_questions = a[(a[\"mean\"] > 0.8)]\n",
    "easy_questions[\"easy_question\"] = True\n",
    "\n",
    "del easy_questions[\"mean\"]\n",
    "del easy_questions[\"count\"]\n",
    "\n",
    "\n",
    "hard_questions = a[(a[\"count\"] > 100) & (a[\"mean\"] < 0.4)]\n",
    "hard_questions[\"hard_question\"] = True\n",
    "\n",
    "del hard_questions[\"mean\"]\n",
    "del hard_questions[\"count\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "hardest = train[train.content_id.isin(hard_questions.index)].groupby(\"user_id\")[\"answered_correctly\"].agg([\"count\",\"sum\"])\n",
    "hardest.columns = [\"hard_ct\", \"hard_cr\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "hardest = train[train.content_id.isin(hard_questions.index)].groupby(\"user_id\")[\"answered_correctly\"].agg([\"count\",\"sum\"])\n",
    "hardest.columns = [\"hard_ct\", \"hard_cr\"]\n",
    "base = base.join(hardest)\n",
    "\n",
    "base.hard_ct = base.hard_ct.fillna(0).astype(\"int16\")\n",
    "base.hard_cr = base.hard_cr.fillna(0).astype(\"int16\")\n",
    "\n",
    "del hardest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "easiest = train[train.content_id.isin(easy_questions.index)].groupby(\"user_id\")[\"answered_correctly\"].agg([\"count\",\"sum\"])\n",
    "easiest.columns = [\"easy_ct\", \"easy_cr\"]\n",
    "base = base.join(easiest)\n",
    "\n",
    "base.easy_ct = base.easy_ct.fillna(0).astype(\"int16\")\n",
    "base.easy_cr = base.easy_cr.fillna(0).astype(\"int16\")\n",
    "\n",
    "del easiest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Dictionary containing accuracy, number of questions and number of correct questions and last lecture\"\"\"\n",
    "\n",
    "summ = train.groupby(\"user_id\")[[\"answered_correctly\"]].sum()\n",
    "mean = train.groupby(\"user_id\")[[\"answered_correctly\"]].mean()\n",
    "count = train.groupby(\"user_id\")[[\"answered_correctly\"]].count()\n",
    "\n",
    "last_lecs = train[[\"user_id\", \"content_id\"]].groupby(\"user_id\").tail(1)\n",
    "last_lecs.index = last_lecs.user_id\n",
    "del last_lecs[\"user_id\"]\n",
    "\n",
    "\n",
    "\n",
    "user_info = summ.join(mean, how=\"left\", rsuffix=\"_mean\").join(count, how=\"left\", rsuffix=\"_count\").join(last_lecs, how=\"left\", rsuffix=\"_lecs\").join(timetable_df, how=\"left\")\n",
    "user_info.columns = [\"correct_count\", \"mean_acc\", \"count\", \"last_lec\",\"first_timestamp\",\"second_timestamp\", \"third_timestamp\",\"fourth_timestamp\", \"fifth_timestamp\"]\n",
    "user_info[\"tmp\"] = 0\n",
    "\n",
    "\n",
    "user_info.last_lec = user_info.last_lec.fillna(0)\n",
    "\n",
    "user_info.mean_acc = user_info.mean_acc.astype(\"float16\")\n",
    "user_info.correct_count = user_info.correct_count.astype(\"int32\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_info = user_info.join(base)\n",
    "user_info = user_info.to_dict(\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'correct_count': 32,\n",
       " 'mean_acc': 0.69580078125,\n",
       " 'count': 46,\n",
       " 'last_lec': 3364,\n",
       " 'first_timestamp': 7.732523645833333,\n",
       " 'second_timestamp': 7.732523645833333,\n",
       " 'third_timestamp': 7.732523645833333,\n",
       " 'fourth_timestamp': 7.731155231481481,\n",
       " 'fifth_timestamp': 7.731155231481481,\n",
       " 'tmp': 0,\n",
       " 'lecs_n': 0,\n",
       " 'interaction_n': 46,\n",
       " 'ts_diff_shifted': 118231.0,\n",
       " 'ts_diff_shifted_2': 118231.0,\n",
       " 'sessions': 1,\n",
       " 'session_count': 7,\n",
       " 'sum_pauses': 7.712886333465576,\n",
       " 'had_exp': 6,\n",
       " 'last_part': 4,\n",
       " 'part_count': [37, 1, 3, 3, 2, 0, 0],\n",
       " 'part_corr': [26, 1, 2, 1, 2, 0, 0],\n",
       " 'k_corr': [2, 19, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 2, 0],\n",
       " 'k_count': [3, 25, 0, 0, 4, 2, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 3, 0],\n",
       " 'part_et': [745000.0, 21000.0, 51000.0, 42999.0, 37000.0, 0, 0],\n",
       " 'part_explan': [6, 0, 0, 0, 0, 0, 0],\n",
       " 'priors_5': [False, False, False, False, False],\n",
       " 'el_sum': 896999.0,\n",
       " 'recent_corr': 7.732523645833333,\n",
       " 'hard_ct': 6,\n",
       " 'hard_cr': 1,\n",
       " 'easy_ct': 20,\n",
       " 'easy_cr': 16}"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_info[115]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "containers_mean = train.groupby(\"task_container_id\")[\"answered_correctly\"].mean()\n",
    "containers_mean = containers_mean.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "hard_questions = hard_questions[\"hard_question\"].to_dict()\n",
    "easy_questions = easy_questions[\"easy_question\"].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "repeated_que_count = train.groupby([\"user_id\", \"content_id\"])[[\"answered_correctly\"]].count()\n",
    "\n",
    "def f(x):\n",
    "    return x.droplevel(0).answered_correctly.to_dict()\n",
    "\n",
    "repeated_que_count = repeated_que_count.groupby(level=0).apply(f).to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train\n",
    "train = dt.fread(\"/input/riiid-test-answer-prediction/train.csv\").to_pandas()\n",
    "train = train[train.content_type_id == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "heat = pd.read_pickle(\"/home/LGBM build and test/heat_pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlargest_que =train.groupby(\"content_id\")[\"user_id\"].count().nlargest(300).index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train = train[~train.index.isin(val_idx.row_id)]\n",
    "train_value = train[[\"user_id\",\"content_id\",\"answered_correctly\",\"task_container_id\"]].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_que_history = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 96817414/96817414 [07:33<00:00, 213572.52it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "for i in tqdm(range(train_value.shape[0])):\n",
    "\n",
    "    que_id = train_value[i, 1]\n",
    "    user_id = train_value[i, 0]\n",
    "    correct = train_value[i, 2]\n",
    "\n",
    "    if que_id in nlargest_que:\n",
    "\n",
    "        if correct == True: \n",
    "\n",
    "            if top_que_history.get(user_id, -1) == -1:\n",
    "                top_que_history[user_id] = {}\n",
    "            top_que_history[user_id][que_id] = True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = []\n",
    "for i in nlargest_que:\n",
    "    cols.append(\"q_\"+str(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "que_corr = {}\n",
    "\n",
    "for i in cols:\n",
    "    dct = heat[i].nlargest(6)[1:].to_dict()\n",
    "    que_corr[int(i[2:])] = {int(j[2:]):dct[j] for j in dct}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Import the model, test re-creating the features during inference**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = dt.fread(\"/input/riiid-test-answer-prediction/train.csv\").to_pandas()\n",
    "val_idx = pd.read_pickle(\"/home/dfs/cv1_valid.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.prior_question_elapsed_time = train.prior_question_elapsed_time.fillna(0)\n",
    "a = train.groupby(\"user_id\")[\"answered_correctly\"].count()\n",
    "\n",
    "prob_users = train[(train.timestamp == 0) & (train.prior_question_elapsed_time != 0)].user_id.unique().tolist()\n",
    "prob_users.extend([171953341,1958652827])  #These two users only answer to one bundle\n",
    "prob_users.extend(a[a == 1].index.tolist())\n",
    "prob_users = list(set(prob_users))\n",
    "\n",
    "del a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train[~train.user_id.isin(prob_users)]\n",
    "validation = train[train.index.isin(val_idx.row_id)]\n",
    "\n",
    "del train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tested_val = pd.read_pickle(\"validation_df\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for u in user_info:\n",
    "    if not u in top_que_history:\n",
    "        top_que_history[u] = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for u in user_info:\n",
    "    user_info[u][\"count_2\"] = user_info[u][\"count\"]\n",
    "    user_info[u][\"part_count_2\"] = user_info[u][\"part_count\"].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://www.kaggle.com/its7171/cv-strategy\n",
    "class Iter_Valid(object):\n",
    "    def __init__(self, df, max_user=1000):\n",
    "        df = df.reset_index(drop=True)\n",
    "        self.df = df\n",
    "        self.user_answer = df['user_answer'].astype(str).values\n",
    "        self.answered_correctly = df['answered_correctly'].astype(str).values\n",
    "        df['prior_group_responses'] = \"[]\"\n",
    "        df['prior_group_answers_correct'] = \"[]\"\n",
    "        self.sample_df = df[df['content_type_id'] == 0][['row_id']]\n",
    "        self.sample_df['answered_correctly'] = 0\n",
    "        self.len = len(df)\n",
    "        self.user_id = df.user_id.values\n",
    "        self.task_container_id = df.task_container_id.values\n",
    "        self.content_type_id = df.content_type_id.values\n",
    "        self.max_user = max_user\n",
    "        self.current = 0\n",
    "        self.pre_user_answer_list = []\n",
    "        self.pre_answered_correctly_list = []\n",
    "\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "    \n",
    "    def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n",
    "        df= self.df[pre_start:self.current].copy()\n",
    "        sample_df = self.sample_df[pre_start:self.current].copy()\n",
    "        df.loc[pre_start,'prior_group_responses'] = '[' + \",\".join(self.pre_user_answer_list) + ']'\n",
    "        df.loc[pre_start,'prior_group_answers_correct'] = '[' + \",\".join(self.pre_answered_correctly_list) + ']'\n",
    "        self.pre_user_answer_list = user_answer_list\n",
    "        self.pre_answered_correctly_list = answered_correctly_list\n",
    "        return df, sample_df\n",
    "\n",
    "    def __next__(self):\n",
    "        added_user = set()\n",
    "        pre_start = self.current\n",
    "        pre_added_user = -1\n",
    "        pre_task_container_id = -1\n",
    "\n",
    "        user_answer_list = []\n",
    "        answered_correctly_list = []\n",
    "        while self.current < self.len:\n",
    "            crr_user_id = self.user_id[self.current]\n",
    "            crr_task_container_id = self.task_container_id[self.current]\n",
    "            crr_content_type_id = self.content_type_id[self.current]\n",
    "            if crr_content_type_id == 1:\n",
    "                # no more than one task_container_id of \"questions\" from any single user\n",
    "                # so we only care for content_type_id == 0 to break loop\n",
    "                user_answer_list.append(self.user_answer[self.current])\n",
    "                answered_correctly_list.append(self.answered_correctly[self.current])\n",
    "                self.current += 1\n",
    "                continue\n",
    "            if crr_user_id in added_user and ((crr_user_id != pre_added_user) or (crr_task_container_id != pre_task_container_id)):\n",
    "                # known user(not prev user or differnt task container)\n",
    "                return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n",
    "            if len(added_user) == self.max_user:\n",
    "                if  crr_user_id == pre_added_user and crr_task_container_id == pre_task_container_id:\n",
    "                    user_answer_list.append(self.user_answer[self.current])\n",
    "                    answered_correctly_list.append(self.answered_correctly[self.current])\n",
    "                    self.current += 1\n",
    "                    continue\n",
    "                else:\n",
    "                    return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n",
    "            added_user.add(crr_user_id)\n",
    "            pre_added_user = crr_user_id\n",
    "            pre_task_container_id = crr_task_container_id\n",
    "            user_answer_list.append(self.user_answer[self.current])\n",
    "            answered_correctly_list.append(self.answered_correctly[self.current])\n",
    "            self.current += 1\n",
    "        if pre_start < self.current:\n",
    "            return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n",
    "        else:\n",
    "            raise StopIteration()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iter_test = Iter_Valid(validation,max_user=1000)\n",
    "predicted = []\n",
    "def set_predict(df):\n",
    "    predicted.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://stackoverflow.com/questions/42869495/numpy-version-of-exponential-weighted-moving-average-equivalent-to-pandas-ewm\n",
    "def numpy_ewma_vectorized_v2(data, window):\n",
    "\n",
    "    alpha = 2 /(window + 1.0)\n",
    "    alpha_rev = 1-alpha\n",
    "    n = data.shape[0]\n",
    "\n",
    "    pows = alpha_rev**(np.arange(n+1))\n",
    "\n",
    "    scale_arr = 1/pows[:-1]\n",
    "    offset = data[0]*pows[1:]\n",
    "    pw0 = alpha*alpha_rev**(n-1)\n",
    "\n",
    "    mult = data*pw0*scale_arr\n",
    "    cumsums = mult.cumsum()\n",
    "    out = offset + cumsums*scale_arr[::-1]\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_1_path = '/home/dfs/'\n",
    "que_data = pd.read_pickle(features_1_path + \"que_data.pickle\")\n",
    "\n",
    "questions = que_data.drop(columns=[\"options_number\",\"correctness_number\", \"correct_answer\",\"tags6\",\"tags5\", \"tags4\"]).to_dict(\"index\")\n",
    "questions1 = que_data[[\"tags1\", \"tags2\", \"tags3\",\"tags4\",\"tags5\", \"tags6\"]].to_dict(\"index\")\n",
    "\n",
    "parts = que_data.part.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lec_data = pd.read_csv(\"/input/riiid-test-answer-prediction/lectures.csv\")\n",
    "lec_dict = lec_data[[\"lecture_id\", \"tag\"]].set_index(\"lecture_id\").tag.to_dict()\n",
    "\n",
    "#Add dtype for every feature\n",
    "features = [\n",
    "    'task_container_id', \"ts_diff_shifted\", \"watched\",\"ts_diff_shifted_2\",\n",
    "    'content_id', \"k\", \"el_avg\", \"wut\", \"sum_corr\", \"gen_mean\", \"u_mean\", \"u_mean_opp\",\n",
    "    'prior_question_elapsed_time', \"time_diff2\", \"rolling_mean_5\", \"rolling_mean_10\", \"rolling_mean_15\", \"prior_question_had_explanation_u_part_avg\",\n",
    "    'prior_question_had_explanation', \"hard_ratio_opp\", \"easy_ratio_opp\", \"correct_recency\", \"prior_question_elapsed_time_u_part_avg\", \"ewm_mean_10\", \"rolling_mean_5_prior_question\",\n",
    "    'last_lecture', \"part_mean\", \"opp_mean\", \"mean_pause\", \"timestamp\", \"prior_part_mean\",\n",
    "    \"container_mean\", \"lecs_per\", \"hard_ratio\", \"easy_ratio\", 'corr_1', 'corr_2', 'corr_3', 'corr_4','corr_5', 'top_que',\n",
    "    'que_count_user', 'question_repeated', \"rolling_mean\",\"time_diff3\", \"time_diff4\",\n",
    "    'user_mean', \"time_diff1\", \"time_diff\", \"sessions\", \"session_count\", \"prior_question_had_explanation_ratio\"\n",
    "] + que_data.columns.tolist()[:-1]\n",
    "\n",
    "features.remove(\"options_number\")\n",
    "features.remove(\"correct_answer\")\n",
    "features.remove(\"tags6\")\n",
    "features.remove(\"tags5\")\n",
    "features.remove(\"tags4\")\n",
    "\n",
    "\n",
    "test_cols = ['row_id', 'timestamp', 'user_id', 'content_id', 'content_type_id',\n",
    "       'task_container_id', 'user_answer', 'answered_correctly',\n",
    "       'prior_question_elapsed_time', 'prior_question_had_explanation', \"st_preds\",\"lgb_preds\",\n",
    "       'prior_group_responses', 'prior_group_answers_correct']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "k_size = 20\n",
    "\n",
    "cols = {test_cols[k]:k for k in range(len(test_cols))}\n",
    "features_dict = {features[k]:k for k in range(len(features))}\n",
    "\n",
    "tmp_data = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import ast\n",
    "import copy\n",
    "from collections import Counter\n",
    "\n",
    "new_user = {'count': 0, 'mean_acc':0.5, 'correct_count': 0, 'last_lec':0, 'tmp':0,\"first_timestamp\":0, \"second_timestamp\":0,\n",
    "            \"third_timestamp\":0, \"fourth_timestamp\":0, \"fifth_timestamp\":0, \"lecs_n\":0,\"interaction_n\":0, \"ts_diff_shifted\":0.,\n",
    "            \"part_corr\":np.zeros((7), dtype=np.uint16), \"part_count\":np.zeros((7), dtype=np.uint16), \"hard_ct\":0, \"hard_cr\":0, \"easy_ct\":0, \"easy_cr\":0,\n",
    "           \"sessions\":0, \"session_count\":0, \"sum_pauses\":0., \"had_exp\":0, \"el_sum\":0, \"part_et\": np.zeros((7), dtype=np.float), \n",
    "            \"part_explan\": np.zeros((7), dtype=np.uint16), \"k_count\": np.zeros((k_size), dtype=np.uint16), \"k_corr\": np.zeros((k_size), dtype=np.uint16), \n",
    "            \"recent_corr\":0, \"priors_5\": [], \"ts_diff_shifted_2\":0., \"count_2\":0, \"part_count_2\":np.zeros((7), dtype=np.uint16), \"last_part\":-1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_meta_data(data_1):\n",
    "    \n",
    "    user_id = data_1[cols['user_id']]\n",
    "    content_type_id = data_1[cols['content_type_id']]\n",
    "    content_id = data_1[cols['content_id']]\n",
    "    prior_group_answers_correct = data_1[cols['prior_group_answers_correct']]\n",
    "    timestamp = data_1[cols['timestamp']]\n",
    "    task_container_id = data_1[cols['task_container_id']]\n",
    "    prior_question_had_explanation = data_1[cols['prior_question_had_explanation']]\n",
    "    elapsed = data_1[cols['prior_question_elapsed_time']]\n",
    "    \n",
    "    return user_id, content_type_id, content_id, prior_group_answers_correct, timestamp,task_container_id,prior_question_had_explanation, elapsed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_user(user_id): \n",
    "    user_info[user_id] = copy.deepcopy(new_user)\n",
    "    repeated_que_count[user_id] = {}\n",
    "    groups[user_id] = []\n",
    "    top_que_history[user_id] = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_user_part_acc(user_id, question, answered_correctly, elapsed, explan):\n",
    "    \n",
    "    part = parts.get(question, -1)\n",
    "    \n",
    "    user_info[user_id][\"part_count\"][part-1] += 1\n",
    "    user_info[user_id][\"part_corr\"][part-1] += answered_correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_user(user_id, had_exp, elapsed, content_id ,answered_correctly, timestamp):\n",
    "        \n",
    "    user_info[user_id]['count'] += 1\n",
    "\n",
    "    if repeated_que_count[user_id].get(content_id,  -1) == -1: #If first time question for the user\n",
    "        repeated_que_count[user_id][content_id] = 1\n",
    "    else:\n",
    "        repeated_que_count[user_id][content_id] += 1\n",
    "\n",
    "    if answered_correctly:\n",
    "        user_info[user_id]['correct_count'] += 1\n",
    "\n",
    "    user_info[user_id]['mean_acc'] = user_info[user_id]['correct_count']/user_info[user_id]['count']\n",
    "    \n",
    "    \n",
    "    update_user_part_acc(user_id, content_id, answered_correctly, elapsed, had_exp)\n",
    "    \n",
    "    if hard_questions.get(content_id, False):\n",
    "        user_info[user_id][\"hard_ct\"] += 1\n",
    "        user_info[user_id][\"hard_cr\"] += answered_correctly\n",
    "        \n",
    "    if easy_questions.get(content_id, False):\n",
    "        user_info[user_id][\"easy_ct\"] += 1\n",
    "        user_info[user_id][\"easy_cr\"] += answered_correctly     \n",
    "        \n",
    "    \n",
    "    k = que_2_k[content_id]\n",
    "    user_info[user_id][\"k_count\"][k] += 1\n",
    "    user_info[user_id][\"k_corr\"][k] += answered_correctly\n",
    "    \n",
    "    if answered_correctly:\n",
    "        user_info[user_id][\"recent_corr\"] = timestamp\n",
    "        \n",
    "    if answered_correctly and content_id in que_corr:\n",
    "        top_que_history[user_id][content_id] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_lec_data(user_id, content_id):\n",
    "    \n",
    "    if watched_tags.get(str(user_id), -1) == -1:  #If user's first lecture\n",
    "        watched_tags[str(user_id)] = {}\n",
    "        \n",
    "    if user_info.get(user_id, -1) == -1: #In case first action user does is watching a lecture\n",
    "        add_user(user_id)\n",
    "        \n",
    "        \n",
    "    user_info[user_id][\"lecs_n\"] += 1\n",
    "    \n",
    "    lec_tag  = lec_dict[content_id]\n",
    "\n",
    "    watched_tags[str(user_id)][str(lec_tag)] = 1\n",
    "    user_info[user_id]['last_lec'] = content_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def non_lag_update(user_id, timestamp, elapsed, explan, lec):\n",
    "    \n",
    "    \n",
    "    timestamp = timestamp/8.64e+7\n",
    "    \n",
    "    diff_timestamp_1 = timestamp - user_info[user_id][\"first_timestamp\"]\n",
    "    diff_timestamp_2 = timestamp - user_info[user_id][\"second_timestamp\"]\n",
    "    diff_timestamp_3 = timestamp - user_info[user_id][\"third_timestamp\"]\n",
    "    diff_timestamp_4 = timestamp - user_info[user_id][\"fourth_timestamp\"]\n",
    "    diff_timestamp_5 = timestamp - user_info[user_id][\"fifth_timestamp\"]\n",
    "\n",
    "    user_info[user_id][\"fifth_timestamp\"] = user_info[user_id][\"fourth_timestamp\"]\n",
    "    user_info[user_id][\"fourth_timestamp\"] = user_info[user_id][\"third_timestamp\"]\n",
    "    user_info[user_id][\"third_timestamp\"] = user_info[user_id][\"second_timestamp\"]\n",
    "    user_info[user_id][\"second_timestamp\"] = user_info[user_id][\"first_timestamp\"]\n",
    "    user_info[user_id][\"first_timestamp\"] = timestamp\n",
    "    \n",
    "    if (user_info[user_id][\"second_timestamp\"] - user_info[user_id][\"third_timestamp\"]) != 0:\n",
    "        user_info[user_id][\"ts_diff_shifted\"] = user_info[user_id][\"second_timestamp\"]*8.64e+7 - user_info[user_id][\"third_timestamp\"]*8.64e+7\n",
    "    \n",
    "    if (user_info[user_id][\"second_timestamp\"] - user_info[user_id][\"fourth_timestamp\"]) != 0:\n",
    "        user_info[user_id][\"ts_diff_shifted_2\"] = user_info[user_id][\"second_timestamp\"]*8.64e+7 - user_info[user_id][\"fourth_timestamp\"]*8.64e+7\n",
    "        \n",
    "        \n",
    "    if not isinstance(explan, pd._libs.missing.NAType) and explan == explan and not lec:\n",
    "        user_info[user_id][\"priors_5\"].append(explan) \n",
    "    \n",
    "    if diff_timestamp_1 > 0.083:\n",
    "        user_info[user_id][\"sessions\"] += 1\n",
    "        user_info[user_id][\"session_count\"] = 0\n",
    "        user_info[user_id][\"sum_pauses\"] += diff_timestamp_1\n",
    "        \n",
    "    user_info[user_id][\"session_count\"] += 1\n",
    "    \n",
    "    \n",
    "    return  diff_timestamp_1, diff_timestamp_2, diff_timestamp_3, diff_timestamp_4, diff_timestamp_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_data(prior_group_answers_correct):  #Update data retrospectively\n",
    "    \n",
    "    global tmp_data\n",
    "    \n",
    "    arr = np.array(ast.literal_eval(prior_group_answers_correct))\n",
    "\n",
    "    for i, line in enumerate(tmp_data): #Loop through users with correct answers\n",
    "        \n",
    "        user_id = line[cols['user_id']]\n",
    "        content_type_id = line[cols['content_type_id']]\n",
    "        content_id = line[cols['content_id']]\n",
    "        timestamp = line[cols['timestamp']]\n",
    "        task_container_id = line[cols['task_container_id']]\n",
    "        \n",
    "        explan = line[cols['prior_question_had_explanation']]\n",
    "        if isinstance(explan, pd._libs.missing.NAType) or explan == explan:\n",
    "            explan = 0\n",
    "            \n",
    "        elapsed = line[cols['prior_question_elapsed_time']]\n",
    "        if isinstance(elapsed, pd._libs.missing.NAType) or elapsed == elapsed:\n",
    "            elapsed = 0\n",
    "            \n",
    "        answered_correctly = arr[i]\n",
    "        \n",
    "        \n",
    "        user_arr = groups[user_id]\n",
    "        user_arr.insert(len(user_arr), answered_correctly)\n",
    "        groups[user_id] = user_arr\n",
    "\n",
    "        if content_type_id == False: #If question\n",
    "\n",
    "            if user_info.get(user_id, -1) == -1: #If user is new\n",
    "                add_user(user_id)\n",
    "                \n",
    "            update_user(user_id, explan, elapsed, content_id, answered_correctly, timestamp)\n",
    "\n",
    "     \n",
    "    tmp_data = []  #Flush tmp_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_data = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_1(chunk): #Optimized preprocess function for small batches, for batches > 2500 preprocess is better. Includes adding lecture\n",
    "    \n",
    "    \n",
    "    data_1 = chunk.values\n",
    "    out = np.zeros((data_1.shape[0], len(features)))\n",
    "    \n",
    "    batch_counts = data_1[:, [cols[\"user_id\"],cols[\"content_type_id\"]]]\n",
    "    batch_counts = Counter(batch_counts[batch_counts[:, 1] == False][:, 0])\n",
    "\n",
    "    global tmp_data\n",
    "\n",
    "\n",
    "    for i in range(data_1.shape[0]):\n",
    "\n",
    "        user_id, content_type_id, content_id ,prior_group_answers_correct, timestamp, task_container_id, explan, elapsed = get_meta_data(data_1[i])\n",
    "        part = parts.get(content_id, -1)\n",
    "        task_count = batch_counts[user_id]\n",
    "        \n",
    "        #When users answers are provided, update user states\n",
    "        if prior_group_answers_correct == prior_group_answers_correct and prior_group_answers_correct != '[]': #If user's previous responses are there\n",
    "            update_data(prior_group_answers_correct)\n",
    "\n",
    "        tmp_data.append(data_1[i].tolist()) #Append incoming data\n",
    "        \n",
    "        \n",
    "\n",
    "        if content_type_id: #If lecture, update the lecture state\n",
    "            update_lec_data(user_id, content_id)\n",
    "\n",
    "            \n",
    "        #Timestamp difference\n",
    "        if user_info.get(user_id, -1) == -1: #If user is new\n",
    "            add_user(user_id)        \n",
    "            \n",
    "        \n",
    "        user_info[user_id][\"interaction_n\"] += 1\n",
    "\n",
    "\n",
    "        _ = non_lag_update(user_id, timestamp, elapsed, explan, content_type_id)\n",
    "        diff_timestamp_1, diff_timestamp_2, diff_timestamp_3, diff_timestamp_4, diff_timestamp_5 = _\n",
    "\n",
    "        if not content_type_id:\n",
    "            if not isinstance(elapsed, pd._libs.missing.NAType) and elapsed == elapsed:\n",
    "                user_info[user_id][\"el_sum\"] += elapsed\n",
    "            else:\n",
    "                elapsed = 0\n",
    "\n",
    "            if isinstance(explan, pd._libs.missing.NAType) or explan != explan:\n",
    "                explan = 0 \n",
    "            else:\n",
    "                user_info[user_id][\"had_exp\"] += explan\n",
    "\n",
    "            if user_info[user_id][\"count_2\"] != 0 and user_info[user_id][\"last_part\"] != -1:\n",
    "                last_part = user_info[user_id][\"last_part\"]\n",
    "                user_info[user_id][\"part_et\"][last_part-1] += elapsed\n",
    "                user_info[user_id][\"part_explan\"][last_part-1] += explan\n",
    "            \n",
    "            user_info[user_id][\"last_part\"] = part\n",
    "            \n",
    "            user_info[user_id][\"part_count_2\"][part-1] += 1\n",
    "            user_info[user_id][\"count_2\"] += 1\n",
    "\n",
    "            #-----Filling the array -------------------------------------------->\n",
    "\n",
    "            out[i, features_dict['content_id']] = content_id\n",
    "            out[i, features_dict['task_container_id']] = data_1[i, cols['task_container_id']]\n",
    "\n",
    "\n",
    "            #prior_question_elapsed_time, and prior_question_elapsed_time\n",
    "            out[i, features_dict['prior_question_elapsed_time']] = elapsed\n",
    "            out[i, features_dict['el_avg']] = (user_info[user_id][\"el_sum\"]/(user_info[user_id][\"count_2\"]))/1000\n",
    "            \n",
    "            if user_info[user_id][\"part_count_2\"][part-1] != 1:\n",
    "                out[i, features_dict['prior_question_elapsed_time_u_part_avg']] = (user_info[user_id][\"part_et\"][part-1])/(user_info[user_id][\"part_count_2\"][part-1]-1)\n",
    "\n",
    "            out[i, features_dict['prior_question_had_explanation']] = explan\n",
    "            \n",
    "            if user_info[user_id][\"part_count_2\"][part-1] != 1:\n",
    "                out[i, features_dict['prior_question_had_explanation_u_part_avg']] = user_info[user_id][\"part_explan\"][part-1]/(user_info[user_id][\"part_count_2\"][part-1]-1)\n",
    "\n",
    "\n",
    "            #out[i, features_dict[\"content_type_id\"]] = content_type_id\n",
    "            #Fill all question features\n",
    "            out[i, features_dict['bundle_id']:] = np.array(list(questions.get(content_id).values())) #No unkown question expected\n",
    "\n",
    "            #que_count_user, question_repeated, user_mean, correct_count\n",
    "            out[i, features_dict['que_count_user']] = user_info.get(user_id, {}).get('count',0)\n",
    "            out[i, features_dict['question_repeated']] = repeated_que_count.get(user_id, {}).get(content_id, 0) + 1\n",
    "\n",
    "\n",
    "            m = user_info.get(user_id, {}).get('mean_acc', 0)\n",
    "            if m == 0: #Usually when user is new, mean drops to zero\n",
    "                out[i, features_dict['user_mean']] = 0.55\n",
    "            else:\n",
    "                out[i, features_dict['user_mean']] = m\n",
    "\n",
    "            out[i, features_dict['opp_mean']] = 1 - out[i, features_dict['user_mean']]\n",
    "\n",
    "            #out[i, features_dict['correct_count']] = user_info.get(user_id, {}).get('correct_count',0)\n",
    "            out[i, features_dict['last_lecture']] = user_info.get(user_id, {}).get('last_lec',0)\n",
    "\n",
    "\n",
    "            #Time gap\n",
    "            out[i, features_dict['time_diff']] = diff_timestamp_1\n",
    "            out[i, features_dict['time_diff1']] = diff_timestamp_2\n",
    "            out[i, features_dict['time_diff2']] = diff_timestamp_3\n",
    "            out[i, features_dict['time_diff3']] = diff_timestamp_4\n",
    "            out[i, features_dict['time_diff4']] = diff_timestamp_5\n",
    "\n",
    "            out[i, features_dict['timestamp']] = timestamp/8.64e+7\n",
    "            out[i, features_dict['correct_recency']] = (timestamp - user_info[user_id][\"recent_corr\"])/8.64e+7\n",
    "\n",
    "            kk = user_info[user_id][\"priors_5\"][-5:]\n",
    "            if len(kk) != 0: #If array not empty\n",
    "                #print(str(len(kk)) + \" \" + str(np.array(kk).mean()))\n",
    "                out[i, features_dict['rolling_mean_5_prior_question']] = np.array(kk).mean()\n",
    "\n",
    "            out[i, features_dict['ts_diff_shifted']] = user_info[user_id][\"ts_diff_shifted\"]\n",
    "            out[i, features_dict['ts_diff_shifted_2']] = user_info[user_id][\"ts_diff_shifted_2\"]\n",
    "\n",
    "            #Container mean\n",
    "            out[i, features_dict['container_mean']] = containers_mean[task_container_id]\n",
    "            out[i, features_dict['lecs_per']] = user_info[user_id][\"lecs_n\"]/user_info[user_id][\"interaction_n\"]*100\n",
    "            out[i, features_dict['sessions']] = user_info[user_id][\"sessions\"]\n",
    "            out[i, features_dict['session_count']] = user_info[user_id][\"session_count\"]\n",
    "\n",
    "            if user_info[user_id][\"count\"] != 0:\n",
    "                out[i, features_dict['prior_question_had_explanation_ratio']] = user_info[user_id][\"had_exp\"]/(user_info[user_id][\"count_2\"])\n",
    "\n",
    "            if user_info[user_id][\"sessions\"] != 0:\n",
    "                out[i, features_dict['mean_pause']] = user_info[user_id][\"sum_pauses\"]/user_info[user_id][\"sessions\"]\n",
    "\n",
    "\n",
    "            #Easy ratio\n",
    "            nn = user_info.get(user_id, {}).get('easy_ct',0)\n",
    "            if nn != 0:\n",
    "                out[i, features_dict['easy_ratio']] = user_info.get(user_id, {}).get('easy_cr',0)/nn\n",
    "\n",
    "            out[i, features_dict['easy_ratio_opp']] = 1 - out[i, features_dict['easy_ratio']]\n",
    "\n",
    "\n",
    "            #Hard ratio\n",
    "            nn = user_info.get(user_id, {}).get('hard_ct',0)\n",
    "            if nn != 0:\n",
    "                out[i, features_dict['hard_ratio']] = user_info.get(user_id, {}).get('hard_cr',0)/nn\n",
    "\n",
    "            out[i, features_dict['hard_ratio_opp']] = 1 - out[i, features_dict['hard_ratio']]\n",
    "\n",
    "\n",
    "\n",
    "            nn = user_info.get(user_id, {}).get('part_count',[])\n",
    "            if nn != []: #User exists\n",
    "                part = int(out[i, features_dict['part']] - 1)\n",
    "                ct = nn[part]\n",
    "                cr = user_info[user_id]['part_corr'][part]\n",
    "\n",
    "                if ct != 0:\n",
    "                    out[i, features_dict['part_mean']] = cr/ct\n",
    "\n",
    "\n",
    "            #Rolling mean\n",
    "            if groups.get(user_id, -1) != -1 and groups[user_id] != []:\n",
    "\n",
    "\n",
    "                ##NOTE: this feature was suppose to do ewm with span 5 and 10 to answers without lec, however due to a bug\n",
    "                ##in the generation of features, the lectures were in, and the model was trained by that, to be considered\n",
    "                #if it requires fixing for next training. PS: add lag to all features when training.\n",
    "                #k_acc needs to be shifted [\"user_id\",\"k\"] axis, add lag to it as well\n",
    "                #also mind the elapsed time and explan are wrong because they are prior, they need to be shifted\n",
    "\n",
    "                last_arr = np.array(groups[user_id][-40:])\n",
    "\n",
    "                out[i, features_dict['rolling_mean']] = numpy_ewma_vectorized_v2(np.array(last_arr[-30:]),5)[-1]  #Limit the length of groups array\n",
    "                \n",
    "                \n",
    "                last_arr = last_arr[last_arr != -1]  #Remove lecture data\n",
    "                out[i, features_dict['ewm_mean_10']] = numpy_ewma_vectorized_v2(np.array(last_arr[-30:]),5)[-1]\n",
    "\n",
    "                out[i, features_dict['rolling_mean_10']] = np.array(last_arr[-10:]).mean()\n",
    "                out[i, features_dict['rolling_mean_15']] = np.array(last_arr[-15:]).mean()\n",
    "                out[i, features_dict['rolling_mean_5']] = np.array(last_arr[-5:]).mean()\n",
    "\n",
    "\n",
    "            out[i, features_dict['prior_part_mean']] = prior_part_mean_dict[int(out[i, features_dict['part']])]\n",
    "\n",
    "            k = que_2_k[content_id]\n",
    "            out[i, features_dict['k']] = k     \n",
    "            out[i, features_dict['wut']] = user_info[user_id][\"ts_diff_shifted\"] - elapsed*task_count\n",
    "                \n",
    "              \n",
    "        \n",
    "            correlated = que_corr.get(content_id, -1)\n",
    "            nn = 0\n",
    "            if correlated != -1: #If this question among top questions\n",
    "                \n",
    "                out[i, features_dict['top_que']] = 1\n",
    "                \n",
    "                corr_keys = list(correlated.keys())\n",
    "                user_history = top_que_history.get(user_id, {})\n",
    "\n",
    "                if len(user_history) != 0:  #If user answered top questions, and if user is not new\n",
    "                    \n",
    "                    for j in range(len(corr_keys)):\n",
    "\n",
    "                        if top_que_history[user_id].get(corr_keys[j], False):   #If user answered this question\n",
    "                            out[i, features_dict['corr_'+str(j+1)]] = correlated[corr_keys[j]]   #Set the value to the degree of the correlation\n",
    "                            nn += correlated[corr_keys[j]]\n",
    "                            \n",
    "                            \n",
    "            tmp_m = out[i, features_dict['user_mean']]\n",
    "            out[i, features_dict[\"gen_mean\"]] = tmp_m - 0.657\n",
    "            out[i, features_dict[\"u_mean\"]] = tmp_m - out[i, features_dict['que_correct_per']]\n",
    "            out[i, features_dict[\"u_mean_opp\"]] = tmp_m - out[i, features_dict['que_false_per']]\n",
    "            out[i, features_dict[\"sum_corr\"]] = nn\n",
    "            \n",
    "             \n",
    "            #Watched_n\n",
    "            usr = watched_tags.get(str(user_id), -1)\n",
    "            if usr != -1:\n",
    "\n",
    "                nn = 0\n",
    "\n",
    "                for k in range(6):\n",
    "                    nn += usr.get(str(int(questions1[content_id][\"tags\"+str(k+1)])), 0) \n",
    "\n",
    "                out[i, features_dict['watched']] = nn\n",
    "\n",
    "\n",
    "\n",
    "    return out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "\n",
    "model = lgb.Booster(model_file='/home/final_model/lgb_classifier.txt')\n",
    "#stacked_model = lgb.Booster(model_file=\"/home/final_model/lgb_stack.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prior_part_mean_dict = {1: 22166.159642501425,\n",
    " 2: 18714.69673913695,\n",
    " 3: 23620.317746179924,\n",
    " 4: 23762.753651169547,\n",
    " 5: 25094.620302855932,\n",
    " 6: 32417.37918735745,\n",
    " 7: 47444.16407400242}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_ids = []\n",
    "preds = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "pbar = tqdm(total=890399, position=0, leave=True)\n",
    "for (current_test, current_prediction_df) in iter_test:\n",
    "\n",
    "    test_transform = preprocess_1(current_test)\n",
    "    preds.extend(model.predict(test_transform))\n",
    "    user_ids.extend(current_test.user_id.tolist())\n",
    "    \n",
    "    pbar.update(len(current_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "pd.set_option(\"display.max_columns\", 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "created = pd.DataFrame(created, columns=features)\n",
    "created[\"user_id\"] = user_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tested_val = tested_val.iloc[:len(user_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Do this with normal model\n",
    "roc_auc_score(tested_val[\"answered_correctly\"], model.predict(tested_val[features]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "roc_auc_score(tested_val[\"answered_correctly\"], model.predict(created[features]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Good strategy to figure which features you messed up\n",
    "gains = []\n",
    "\n",
    "for i in tqdm(features, position=0, leave=True):\n",
    "    \n",
    "    tmp = created.copy()\n",
    "    tmp[i] = tested_val[i].reset_index(drop=True)\n",
    "    \n",
    "    gain = roc_auc_score(tested_val[\"answered_correctly\"], model.predict(tmp[features]))\n",
    "    \n",
    "    gains.append((gain, i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gains = sorted(gains, key=lambda x: x[0], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, gain in gains:  \n",
    "    print(\"%s  == > ROC : %f\" %(gain, i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}