userCF_0.0914基线.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "sns.set(style=\"darkgrid\")\n",
    "plt.rcParams['font.sans-serif']=['SimHei'] #用来正常显示中文标签\n",
    "plt.rcParams['axes.unicode_minus']=False #用来正常显示\n",
    "from zipfile import ZipFile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "# 第一次运行时 usingSaved=False，生成临时文件后改为True\n",
    "usingSaved=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取item\n",
    "if not usingSaved:\n",
    "    output = open('item_attr.pkl', 'wb')\n",
    "    myzip=ZipFile('./data/Antai_AE_round1_item_attr_20190626.zip')\n",
    "    f=myzip.open('Antai_AE_round1_item_attr_20190626.csv')\n",
    "    item_attr=pd.read_csv(f)\n",
    "    f.close()\n",
    "    myzip.close()\n",
    "    pickle.dump(item_attr, output)\n",
    "    output.close()\n",
    "else:\n",
    "    item_attr = pickle.load(open('item_attr.pkl', 'rb'), encoding='iso-8859-1')\n",
    "#     item_attr = pickle.load(open('item_attr.pkl', 'rb'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取train  \n",
    "if not usingSaved:\n",
    "    output = open('train.pkl', 'wb')\n",
    "    myzip=ZipFile('./data/Antai_AE_round1_train_20190626.zip')\n",
    "    f=myzip.open('Antai_AE_round1_train_20190626.csv')\n",
    "    train=pd.read_csv(f)\n",
    "    f.close()\n",
    "    myzip.close()\n",
    "    train['create_order_time'] = train.create_order_time.apply(lambda x:pd.to_datetime(x))\n",
    "    train['hour']=train['create_order_time'].dt.hour\n",
    "    train['date']=train['create_order_time'].dt.day\n",
    "    train['month']=train['create_order_time'].dt.month\n",
    "    train['year']=train['create_order_time'].dt.year\n",
    "    train['month-date'] = train.month.astype(str)+'-'+train.date.astype(str)\n",
    "    train['count'] = 1\n",
    "    train['dayofweek']=train['create_order_time'].dt.dayofweek\n",
    "    train['isweekend']=train['dayofweek'].apply(lambda x:0 if x<5 else 1)\n",
    "    pickle.dump(train, output)\n",
    "    output.close()\n",
    "else:\n",
    "    train = pickle.load(open('train.pkl', 'rb'),encoding='iso-8859-1')\n",
    "#     train = pickle.load(open('train.pkl', 'rb'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取test\n",
    "if not usingSaved:\n",
    "    output = open('test.pkl', 'wb')\n",
    "    test=pd.read_csv('./data/Antai_AE_round1_test_20190626.csv')\n",
    "    test['create_order_time'] = test.create_order_time.apply(lambda x:pd.to_datetime(x))\n",
    "    test['hour']=test['create_order_time'].dt.hour\n",
    "    test['date']=test['create_order_time'].dt.day\n",
    "    test['month']=test['create_order_time'].dt.month\n",
    "    test['year']=test['create_order_time'].dt.year\n",
    "    test['count'] = 1\n",
    "    test['month-date'] = test.month.astype(str)+'-'+test.date.astype(str)\n",
    "    test['dayofweek']=test['create_order_time'].dt.dayofweek\n",
    "    test['isweekend']=test['dayofweek'].apply(lambda x:0 if x<5 else 1)\n",
    "    pickle.dump(test, output)\n",
    "    output.close()\n",
    "else:\n",
    "    test = pickle.load(open('test.pkl', 'rb'), encoding='iso-8859-1')\n",
    "submit = pd.read_csv('./data/Antai_AE_round1_submit_20190715.csv')\n",
    "#     test = pickle.load(open('test.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainyy = train[train.buyer_country_id =='yy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>152</th>\n",
       "      <th>343131</th>\n",
       "      <th>2140290</th>\n",
       "      <th>2600629</th>\n",
       "      <th>1554159</th>\n",
       "      <th>12211445</th>\n",
       "      <th>776612</th>\n",
       "      <th>1630164</th>\n",
       "      <th>3901301</th>\n",
       "      <th>8353580</th>\n",
       "      <th>...</th>\n",
       "      <th>2327622</th>\n",
       "      <th>2014295</th>\n",
       "      <th>5076158</th>\n",
       "      <th>5042708</th>\n",
       "      <th>818444</th>\n",
       "      <th>10451831</th>\n",
       "      <th>4333367</th>\n",
       "      <th>9437487</th>\n",
       "      <th>8469786</th>\n",
       "      <th>9465790</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>282</td>\n",
       "      <td>8988184</td>\n",
       "      <td>2210260</td>\n",
       "      <td>96479</td>\n",
       "      <td>4052816</td>\n",
       "      <td>7982989</td>\n",
       "      <td>11143899</td>\n",
       "      <td>10070236</td>\n",
       "      <td>2002400</td>\n",
       "      <td>8757854</td>\n",
       "      <td>...</td>\n",
       "      <td>5630838</td>\n",
       "      <td>8607737</td>\n",
       "      <td>7740462</td>\n",
       "      <td>7028968</td>\n",
       "      <td>9167824</td>\n",
       "      <td>12938507</td>\n",
       "      <td>6399956</td>\n",
       "      <td>3109441</td>\n",
       "      <td>10636176</td>\n",
       "      <td>6255625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>321</td>\n",
       "      <td>11140901</td>\n",
       "      <td>4618502</td>\n",
       "      <td>9666424</td>\n",
       "      <td>421308</td>\n",
       "      <td>8443514</td>\n",
       "      <td>12341414</td>\n",
       "      <td>4133929</td>\n",
       "      <td>4374880</td>\n",
       "      <td>9104983</td>\n",
       "      <td>...</td>\n",
       "      <td>12975999</td>\n",
       "      <td>10793761</td>\n",
       "      <td>7524523</td>\n",
       "      <td>666863</td>\n",
       "      <td>204808</td>\n",
       "      <td>8893340</td>\n",
       "      <td>7431033</td>\n",
       "      <td>1773069</td>\n",
       "      <td>12970909</td>\n",
       "      <td>9373627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>809</td>\n",
       "      <td>692592</td>\n",
       "      <td>9851186</td>\n",
       "      <td>5206030</td>\n",
       "      <td>1210732</td>\n",
       "      <td>6016462</td>\n",
       "      <td>6971778</td>\n",
       "      <td>11615323</td>\n",
       "      <td>7263674</td>\n",
       "      <td>10901782</td>\n",
       "      <td>...</td>\n",
       "      <td>3227122</td>\n",
       "      <td>10539551</td>\n",
       "      <td>2802000</td>\n",
       "      <td>8251066</td>\n",
       "      <td>9515496</td>\n",
       "      <td>7065316</td>\n",
       "      <td>11997137</td>\n",
       "      <td>4438427</td>\n",
       "      <td>6567724</td>\n",
       "      <td>7212577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>870</td>\n",
       "      <td>8361014</td>\n",
       "      <td>2327622</td>\n",
       "      <td>5481981</td>\n",
       "      <td>7530272</td>\n",
       "      <td>11680535</td>\n",
       "      <td>6555249</td>\n",
       "      <td>6137574</td>\n",
       "      <td>914424</td>\n",
       "      <td>10445923</td>\n",
       "      <td>...</td>\n",
       "      <td>8757854</td>\n",
       "      <td>10578287</td>\n",
       "      <td>10715090</td>\n",
       "      <td>7176446</td>\n",
       "      <td>3165153</td>\n",
       "      <td>10144005</td>\n",
       "      <td>8753413</td>\n",
       "      <td>2800387</td>\n",
       "      <td>12152573</td>\n",
       "      <td>1068526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2410</td>\n",
       "      <td>6320318</td>\n",
       "      <td>7525163</td>\n",
       "      <td>10734361</td>\n",
       "      <td>9050161</td>\n",
       "      <td>11309657</td>\n",
       "      <td>686434</td>\n",
       "      <td>10621679</td>\n",
       "      <td>11493273</td>\n",
       "      <td>12031376</td>\n",
       "      <td>...</td>\n",
       "      <td>7999857</td>\n",
       "      <td>6024506</td>\n",
       "      <td>7726279</td>\n",
       "      <td>245564</td>\n",
       "      <td>8636658</td>\n",
       "      <td>7684124</td>\n",
       "      <td>5386552</td>\n",
       "      <td>956425</td>\n",
       "      <td>5701392</td>\n",
       "      <td>7358804</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    152    343131  2140290   2600629  1554159  12211445    776612   1630164  \\\n",
       "0   282   8988184  2210260     96479  4052816   7982989  11143899  10070236   \n",
       "1   321  11140901  4618502   9666424   421308   8443514  12341414   4133929   \n",
       "2   809    692592  9851186   5206030  1210732   6016462   6971778  11615323   \n",
       "3   870   8361014  2327622   5481981  7530272  11680535   6555249   6137574   \n",
       "4  2410   6320318  7525163  10734361  9050161  11309657    686434  10621679   \n",
       "\n",
       "    3901301   8353580  ...   2327622   2014295   5076158  5042708   818444  \\\n",
       "0   2002400   8757854  ...   5630838   8607737   7740462  7028968  9167824   \n",
       "1   4374880   9104983  ...  12975999  10793761   7524523   666863   204808   \n",
       "2   7263674  10901782  ...   3227122  10539551   2802000  8251066  9515496   \n",
       "3    914424  10445923  ...   8757854  10578287  10715090  7176446  3165153   \n",
       "4  11493273  12031376  ...   7999857   6024506   7726279   245564  8636658   \n",
       "\n",
       "   10451831   4333367  9437487   8469786  9465790  \n",
       "0  12938507   6399956  3109441  10636176  6255625  \n",
       "1   8893340   7431033  1773069  12970909  9373627  \n",
       "2   7065316  11997137  4438427   6567724  7212577  \n",
       "3  10144005   8753413  2800387  12152573  1068526  \n",
       "4   7684124   5386552   956425   5701392  7358804  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submit.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>buyer_country_id</th>\n",
       "      <th>buyer_admin_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>create_order_time</th>\n",
       "      <th>irank</th>\n",
       "      <th>hour</th>\n",
       "      <th>date</th>\n",
       "      <th>month</th>\n",
       "      <th>year</th>\n",
       "      <th>count</th>\n",
       "      <th>month-date</th>\n",
       "      <th>dayofweek</th>\n",
       "      <th>isweekend</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>yy</td>\n",
       "      <td>1061132</td>\n",
       "      <td>189045</td>\n",
       "      <td>2018-07-17 07:43:40</td>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>7-17</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>yy</td>\n",
       "      <td>1081430</td>\n",
       "      <td>10713670</td>\n",
       "      <td>2018-07-17 09:43:15</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>7-17</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>yy</td>\n",
       "      <td>1918135</td>\n",
       "      <td>10546322</td>\n",
       "      <td>2018-07-17 11:11:34</td>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>7-17</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>yy</td>\n",
       "      <td>103277</td>\n",
       "      <td>5199603</td>\n",
       "      <td>2018-07-17 13:27:44</td>\n",
       "      <td>24</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>7-17</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>yy</td>\n",
       "      <td>103277</td>\n",
       "      <td>2477273</td>\n",
       "      <td>2018-07-17 13:27:44</td>\n",
       "      <td>22</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>7-17</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  buyer_country_id  buyer_admin_id   item_id   create_order_time  irank  hour  \\\n",
       "0               yy         1061132    189045 2018-07-17 07:43:40     18     7   \n",
       "1               yy         1081430  10713670 2018-07-17 09:43:15      8     9   \n",
       "2               yy         1918135  10546322 2018-07-17 11:11:34      9    11   \n",
       "3               yy          103277   5199603 2018-07-17 13:27:44     24    13   \n",
       "4               yy          103277   2477273 2018-07-17 13:27:44     22    13   \n",
       "\n",
       "   date  month  year  count month-date  dayofweek  isweekend  \n",
       "0    17      7  2018      1       7-17          1          0  \n",
       "1    17      7  2018      1       7-17          1          0  \n",
       "2    17      7  2018      1       7-17          1          0  \n",
       "3    17      7  2018      1       7-17          1          0  \n",
       "4    17      7  2018      1       7-17          1          0  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>buyer_country_id</th>\n",
       "      <th>buyer_admin_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>create_order_time</th>\n",
       "      <th>irank</th>\n",
       "      <th>hour</th>\n",
       "      <th>date</th>\n",
       "      <th>month</th>\n",
       "      <th>year</th>\n",
       "      <th>month-date</th>\n",
       "      <th>count</th>\n",
       "      <th>dayofweek</th>\n",
       "      <th>isweekend</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>yy</td>\n",
       "      <td>101887</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-08-27 08:31:26</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>27</td>\n",
       "      <td>8</td>\n",
       "      <td>2018</td>\n",
       "      <td>8-27</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>yy</td>\n",
       "      <td>288460</td>\n",
       "      <td>6</td>\n",
       "      <td>2018-08-31 13:41:25</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "      <td>2018</td>\n",
       "      <td>8-31</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>yy</td>\n",
       "      <td>341958</td>\n",
       "      <td>6</td>\n",
       "      <td>2018-08-31 15:42:10</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "      <td>2018</td>\n",
       "      <td>8-31</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>yy</td>\n",
       "      <td>12849617</td>\n",
       "      <td>6</td>\n",
       "      <td>2018-08-09 04:10:36</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>2018</td>\n",
       "      <td>8-9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>yy</td>\n",
       "      <td>770888</td>\n",
       "      <td>6</td>\n",
       "      <td>2018-08-31 23:33:54</td>\n",
       "      <td>12</td>\n",
       "      <td>23</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "      <td>2018</td>\n",
       "      <td>8-31</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   buyer_country_id  buyer_admin_id  item_id   create_order_time  irank  hour  \\\n",
       "2                yy          101887        3 2018-08-27 08:31:26      3     8   \n",
       "9                yy          288460        6 2018-08-31 13:41:25     11    13   \n",
       "10               yy          341958        6 2018-08-31 15:42:10      1    15   \n",
       "12               yy        12849617        6 2018-08-09 04:10:36     17     4   \n",
       "16               yy          770888        6 2018-08-31 23:33:54     12    23   \n",
       "\n",
       "    date  month  year month-date  count  dayofweek  isweekend  \n",
       "2     27      8  2018       8-27      1          0          0  \n",
       "9     31      8  2018       8-31      1          4          0  \n",
       "10    31      8  2018       8-31      1          4          0  \n",
       "12     9      8  2018        8-9      1          3          0  \n",
       "16    31      8  2018       8-31      1          4          0  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainyy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(138678,)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainyy.buyer_admin_id.unique().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(926771,)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainyy.item_id.unique().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "merge_yy = pd.concat([trainyy,test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((972388,), (150076,))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge_yy.item_id.unique().shape,merge_yy.buyer_admin_id.unique().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>buyer_admin_id</th>\n",
       "      <th>buyer_country_id</th>\n",
       "      <th>count</th>\n",
       "      <th>create_order_time</th>\n",
       "      <th>date</th>\n",
       "      <th>dayofweek</th>\n",
       "      <th>hour</th>\n",
       "      <th>irank</th>\n",
       "      <th>isweekend</th>\n",
       "      <th>item_id</th>\n",
       "      <th>month</th>\n",
       "      <th>month-date</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101887</td>\n",
       "      <td>yy</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-08-27 08:31:26</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>8-27</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>288460</td>\n",
       "      <td>yy</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-08-31 13:41:25</td>\n",
       "      <td>31</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>8-31</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>341958</td>\n",
       "      <td>yy</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-08-31 15:42:10</td>\n",
       "      <td>31</td>\n",
       "      <td>4</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>8-31</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12849617</td>\n",
       "      <td>yy</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-08-09 04:10:36</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>8-9</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>770888</td>\n",
       "      <td>yy</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-08-31 23:33:54</td>\n",
       "      <td>31</td>\n",
       "      <td>4</td>\n",
       "      <td>23</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>8-31</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    buyer_admin_id buyer_country_id  count   create_order_time  date  \\\n",
       "2           101887               yy      1 2018-08-27 08:31:26    27   \n",
       "9           288460               yy      1 2018-08-31 13:41:25    31   \n",
       "10          341958               yy      1 2018-08-31 15:42:10    31   \n",
       "12        12849617               yy      1 2018-08-09 04:10:36     9   \n",
       "16          770888               yy      1 2018-08-31 23:33:54    31   \n",
       "\n",
       "    dayofweek  hour  irank  isweekend  item_id  month month-date  year  \n",
       "2           0     8      3          0        3      8       8-27  2018  \n",
       "9           4    13     11          0        6      8       8-31  2018  \n",
       "10          4    15      1          0        6      8       8-31  2018  \n",
       "12          3     4     17          0        6      8        8-9  2018  \n",
       "16          4    23     12          0        6      8       8-31  2018  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge_yy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "from operator import itemgetter\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "random.seed(0)\n",
    "\n",
    "class UserBasedCF(object):\n",
    "    ''' TopN recommendation - User Based Collaborative Filtering '''\n",
    "\n",
    "    # 构造函数，用来初始化\n",
    "    def __init__(self):\n",
    "        # 定义 训练集 测试集 为字典类型\n",
    "        self.trainset = {}\n",
    "        self.testset = {}\n",
    "        # 训练集用的相似用户数\n",
    "        self.n_sim_user = 30\n",
    "        # 推荐Item数量\n",
    "        self.n_rec_item = 30\n",
    "\n",
    "        self.user_sim_mat = {}\n",
    "        self.item_popular = {}\n",
    "        self.item_count = 0\n",
    "        # sys.stderr 是用来重定向标准错误信息的\n",
    "        print ('相似用户数目为 = %d' % self.n_sim_user, file=sys.stderr)\n",
    "        print ('推荐Item数目为 = %d' %\n",
    "               self.n_rec_item, file=sys.stderr)\n",
    "\n",
    "    # 划分训练集和测试集 pivot用来定义训练集和测试集的比例\n",
    "    def generate_dataset(self, train,test=None, pivot=0.90):\n",
    "        ''' load rating data and split it to training set and test set '''\n",
    "        trainset_len = 0\n",
    "        testset_len = 0\n",
    "        if test is None:# 随机分配验证机实验 \n",
    "            print('算法尝试！')\n",
    "            for line in train.iterrows():\n",
    "                user, item, rating = line[1][0],line[1][1],1\n",
    "                # split the data by pivot\n",
    "                if random.random() < pivot:\n",
    "                    self.trainset.setdefault(user, {})\n",
    "                    self.trainset[user][item] = int(rating)\n",
    "                    trainset_len += 1\n",
    "                else:\n",
    "                    self.testset.setdefault(user, {})\n",
    "                    self.testset[user][item] = int(rating)\n",
    "                    testset_len += 1\n",
    "        else:#真正预测\n",
    "            print('预测尝试！')\n",
    "            for line in train.iterrows():\n",
    "                user, item, rating = line[1][0],line[1][1],1\n",
    "                # split the data by pivot\n",
    "                self.trainset.setdefault(user, {})\n",
    "                self.trainset[user][item] = int(rating)\n",
    "                trainset_len += 1\n",
    "            del user,item,rating\n",
    "            for line in test.iterrows():\n",
    "                user, item, rating = line[1][0],line[1][1],1\n",
    "                self.testset.setdefault(user, {})\n",
    "                self.testset[user][item] = int(rating)\n",
    "                testset_len += 1\n",
    "            \n",
    "        print ('split training set and test set succ')\n",
    "        print ('train set = %s' % trainset_len)\n",
    "        print ('test set = %s' % testset_len)\n",
    "    # 建立物品-用户 倒排表\n",
    "    def calc_user_sim(self):\n",
    "        ''' calculate user similarity matrix '''\n",
    "        # build inverse table for item-users\n",
    "        # key=itemID, value=list of userIDs who have seen this item\n",
    "        print ('构建物品-用户倒排表中，请等待......', file=sys.stderr)\n",
    "        item2users = dict()\n",
    "\n",
    "        # Python 字典(Dictionary) items() 函数以列表返回可遍历的(键, 值) 元组数组\n",
    "        for user, items in self.trainset.items():\n",
    "            for item in items:\n",
    "                # inverse table for item-users\n",
    "                if item not in item2users:\n",
    "                    # 根据商品id 构造set() 函数创建一个无序不重复元素集\n",
    "                    item2users[item] = set()\n",
    "                # 集合中值为用户id\n",
    "                # 数值形如\n",
    "                # {'914': {'1','6','10'}, '3408': {'1'} ......}\n",
    "                item2users[item].add(user)\n",
    "                # 记录电影的流行度\n",
    "                if item not in self.item_popular:\n",
    "                    self.item_popular[item] = 0\n",
    "                self.item_popular[item] += 1\n",
    "        print ('构建物品-用户倒排表成功', file=sys.stderr)\n",
    "\n",
    "        # save the total item number, which will be used in evaluation\n",
    "        self.item_count = len(item2users)\n",
    "        print ('总共被操作过的物品数目为 = %d' % self.item_count, file=sys.stderr)\n",
    "\n",
    "        # count co-rated items between users\n",
    "        usersim_mat = self.user_sim_mat\n",
    "\n",
    "        print ('building user co-rated items matrix...', file=sys.stderr)\n",
    "        # 令系数矩阵 C[u][v]表示N(u)∩N（v) ，假如用户u和用户v同时属于K个物品对应的用户列表，就有C[u][v]=K\n",
    "        for item, users in item2users.items():\n",
    "            for u in users:\n",
    "                usersim_mat.setdefault(u, defaultdict(int))\n",
    "                for v in users:\n",
    "                    if u == v:\n",
    "                        continue\n",
    "                    usersim_mat[u][v] += 1\n",
    "        print ('build user co-rated items matrix succ', file=sys.stderr)\n",
    "\n",
    "        # calculate similarity matrix\n",
    "        print ('calculating user similarity matrix...', file=sys.stderr)\n",
    "        simfactor_count = 0\n",
    "        PRINT_STEP = 2000000\n",
    "        # 循环遍历usersim_mat 根据余弦相似度公式计算出用户兴趣相似度\n",
    "        for u, related_users in usersim_mat.items():\n",
    "            for v, count in related_users.items():\n",
    "                # 以下是公式计算过程\n",
    "                usersim_mat[u][v] = count / math.sqrt(\n",
    "                    len(self.trainset[u]) * len(self.trainset[v]))\n",
    "                #计数 并没有什么卵用\n",
    "                simfactor_count += 1\n",
    "                if simfactor_count % PRINT_STEP == 0:\n",
    "                    print ('calculating user similarity factor(%d)' %\n",
    "                           simfactor_count, file=sys.stderr)\n",
    "\n",
    "        print ('calculate user similarity matrix(similarity factor) succ',\n",
    "               file=sys.stderr)\n",
    "        print ('Total similarity factor number = %d' %\n",
    "               simfactor_count, file=sys.stderr)\n",
    "    # 根据用户给予推荐结果\n",
    "    def recommend(self, user,predict=False):\n",
    "        '''定义给定K个相似用户和推荐N个商品'''\n",
    "        K = self.n_sim_user\n",
    "        N = self.n_rec_item\n",
    "        # 定义一个字典来存储为用户推荐的电影\n",
    "        rank = dict()\n",
    "        \n",
    "        watched_items = self.trainset[user]\n",
    "        # sorted() 函数对所有可迭代的对象进行排序操作。 key 指定比较的对象 ，reverse=True 降序\n",
    "        for similar_user, similarity_factor in sorted(self.user_sim_mat[user].items(),\n",
    "                                                      key=itemgetter(1), reverse=True)[0:K]:\n",
    "            for item in self.trainset[similar_user]:\n",
    "                # 判断 如果这个商品 该用户已经买过 则跳出循环 ##此处不太成立\n",
    "#                 if item in watched_items:\n",
    "#                     continue\n",
    "                # 记录用户对推荐的电影的兴趣度\n",
    "                rank.setdefault(item, 0)\n",
    "                rank[item] += similarity_factor\n",
    "        # return the N best items\n",
    "        return sorted(rank.items(), key=itemgetter(1), reverse=True)[0:N]\n",
    "    def predict(self):\n",
    "        print('Predict start...')\n",
    "\n",
    "        N = self.n_rec_item\n",
    "        #  varables for precision and recall\n",
    "        hit = 0\n",
    "        rec_count = 0\n",
    "        test_count = 0\n",
    "        # varables for coverage\n",
    "        all_rec_items = set()\n",
    "        # varables for popularity\n",
    "        popular_sum = 0\n",
    "        predict_result=[]\n",
    "        for i, user in enumerate(self.testset):\n",
    "            if i % 500 == 0:\n",
    "                print ('recommended for %d users' % i)\n",
    "            rec_items = self.recommend(user,predict=True)\n",
    "            predict_result.append((user,rec_items))\n",
    "        print('Predict end...')\n",
    "        return predict_result\n",
    "    # 计算 准确略，召回率，覆盖率，流行度\n",
    "    def evaluate(self):\n",
    "\n",
    "        ''' print evaluation result: precision, recall, coverage and popularity '''\n",
    "        print ('Evaluation start...', file=sys.stderr)\n",
    "\n",
    "        N = self.n_rec_item\n",
    "        #  varables for precision and recall\n",
    "        #记录推荐正确的电影数\n",
    "        hit = 0\n",
    "        #记录推荐电影的总数\n",
    "        rec_count = 0\n",
    "        #记录测试数据中总数\n",
    "        test_count = 0\n",
    "        # varables for coverage\n",
    "        all_rec_items = set()\n",
    "        # varables for popularity\n",
    "        popular_sum = 0\n",
    "\n",
    "        for i, user in enumerate(self.trainset):\n",
    "            if i % 500 == 0:\n",
    "                print ('recommended for %d users' % i, file=sys.stderr)\n",
    "            test_items = self.testset.get(user, {})\n",
    "            rec_items = self.recommend(user,predict=False)\n",
    "            for item, _ in rec_items:\n",
    "                if item in test_items:\n",
    "                    hit += 1\n",
    "                all_rec_items.add(item)\n",
    "                popular_sum += math.log(1 + self.item_popular[item])\n",
    "            rec_count += N\n",
    "            test_count += len(test_items)\n",
    "        # 计算准确度\n",
    "        precision = hit / (1.0 * rec_count)\n",
    "        # 计算召回率\n",
    "        recall = hit / (1.0 * test_count)\n",
    "        # 计算覆盖率\n",
    "        coverage = len(all_rec_items) / (1.0 * self.item_count)\n",
    "        #计算流行度\n",
    "        popularity = popular_sum / (1.0 * rec_count)\n",
    "\n",
    "        print ('precision=%.4f\\trecall=%.4f\\tcoverage=%.4f\\tpopularity=%.4f' %\n",
    "               (precision, recall, coverage, popularity), file=sys.stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "相似用户数目为 = 30\n",
      "推荐Item数目为 = 30\n"
     ]
    }
   ],
   "source": [
    "usercf = UserBasedCF()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "算法尝试！\n",
      "split training set and test set succ\n",
      "train set = 2159745\n",
      "test set = 239954\n"
     ]
    }
   ],
   "source": [
    "usercf.generate_dataset(merge_yy[['buyer_admin_id','item_id']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "构建物品-用户倒排表中，请等待......\n",
      "构建物品-用户倒排表成功\n",
      "总共被操作过的物品数目为 = 905532\n",
      "building user co-rated items matrix...\n",
      "build user co-rated items matrix succ\n",
      "calculating user similarity matrix...\n",
      "calculating user similarity factor(2000000)\n",
      "calculating user similarity factor(4000000)\n",
      "calculating user similarity factor(6000000)\n",
      "calculating user similarity factor(8000000)\n",
      "calculating user similarity factor(10000000)\n",
      "calculating user similarity factor(12000000)\n",
      "calculating user similarity factor(14000000)\n",
      "calculating user similarity factor(16000000)\n",
      "calculating user similarity factor(18000000)\n",
      "calculating user similarity factor(20000000)\n",
      "calculating user similarity factor(22000000)\n",
      "calculate user similarity matrix(similarity factor) succ\n",
      "Total similarity factor number = 23241670\n"
     ]
    }
   ],
   "source": [
    "usercf.calc_user_sim()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation start...\n",
      "recommended for 0 users\n",
      "recommended for 500 users\n",
      "recommended for 1000 users\n",
      "recommended for 1500 users\n",
      "recommended for 2000 users\n",
      "recommended for 2500 users\n",
      "recommended for 3000 users\n",
      "recommended for 3500 users\n",
      "recommended for 4000 users\n",
      "recommended for 4500 users\n",
      "recommended for 5000 users\n",
      "recommended for 5500 users\n",
      "recommended for 6000 users\n",
      "recommended for 6500 users\n",
      "recommended for 7000 users\n",
      "recommended for 7500 users\n",
      "recommended for 8000 users\n",
      "recommended for 8500 users\n",
      "recommended for 9000 users\n",
      "recommended for 9500 users\n",
      "recommended for 10000 users\n",
      "recommended for 10500 users\n",
      "recommended for 11000 users\n",
      "recommended for 11500 users\n",
      "recommended for 12000 users\n",
      "recommended for 12500 users\n",
      "recommended for 13000 users\n",
      "recommended for 13500 users\n",
      "recommended for 14000 users\n",
      "recommended for 14500 users\n",
      "recommended for 15000 users\n",
      "recommended for 15500 users\n",
      "recommended for 16000 users\n",
      "recommended for 16500 users\n",
      "recommended for 17000 users\n",
      "recommended for 17500 users\n",
      "recommended for 18000 users\n",
      "recommended for 18500 users\n",
      "recommended for 19000 users\n",
      "recommended for 19500 users\n",
      "recommended for 20000 users\n",
      "recommended for 20500 users\n",
      "recommended for 21000 users\n",
      "recommended for 21500 users\n",
      "recommended for 22000 users\n",
      "recommended for 22500 users\n",
      "recommended for 23000 users\n",
      "recommended for 23500 users\n",
      "recommended for 24000 users\n",
      "recommended for 24500 users\n",
      "recommended for 25000 users\n",
      "recommended for 25500 users\n",
      "recommended for 26000 users\n",
      "recommended for 26500 users\n",
      "recommended for 27000 users\n",
      "recommended for 27500 users\n",
      "recommended for 28000 users\n",
      "recommended for 28500 users\n",
      "recommended for 29000 users\n",
      "recommended for 29500 users\n",
      "recommended for 30000 users\n",
      "recommended for 30500 users\n",
      "recommended for 31000 users\n",
      "recommended for 31500 users\n",
      "recommended for 32000 users\n",
      "recommended for 32500 users\n",
      "recommended for 33000 users\n",
      "recommended for 33500 users\n",
      "recommended for 34000 users\n",
      "recommended for 34500 users\n",
      "recommended for 35000 users\n",
      "recommended for 35500 users\n",
      "recommended for 36000 users\n",
      "recommended for 36500 users\n",
      "recommended for 37000 users\n",
      "recommended for 37500 users\n",
      "recommended for 38000 users\n",
      "recommended for 38500 users\n",
      "recommended for 39000 users\n",
      "recommended for 39500 users\n",
      "recommended for 40000 users\n",
      "recommended for 40500 users\n",
      "recommended for 41000 users\n",
      "recommended for 41500 users\n",
      "recommended for 42000 users\n",
      "recommended for 42500 users\n",
      "recommended for 43000 users\n",
      "recommended for 43500 users\n",
      "recommended for 44000 users\n",
      "recommended for 44500 users\n",
      "recommended for 45000 users\n",
      "recommended for 45500 users\n",
      "recommended for 46000 users\n",
      "recommended for 46500 users\n",
      "recommended for 47000 users\n",
      "recommended for 47500 users\n",
      "recommended for 48000 users\n",
      "recommended for 48500 users\n",
      "recommended for 49000 users\n",
      "recommended for 49500 users\n",
      "recommended for 50000 users\n",
      "recommended for 50500 users\n",
      "recommended for 51000 users\n",
      "recommended for 51500 users\n",
      "recommended for 52000 users\n",
      "recommended for 52500 users\n",
      "recommended for 53000 users\n",
      "recommended for 53500 users\n",
      "recommended for 54000 users\n",
      "recommended for 54500 users\n",
      "recommended for 55000 users\n",
      "recommended for 55500 users\n",
      "recommended for 56000 users\n",
      "recommended for 56500 users\n",
      "recommended for 57000 users\n",
      "recommended for 57500 users\n",
      "recommended for 58000 users\n",
      "recommended for 58500 users\n",
      "recommended for 59000 users\n",
      "recommended for 59500 users\n",
      "recommended for 60000 users\n",
      "recommended for 60500 users\n",
      "recommended for 61000 users\n",
      "recommended for 61500 users\n",
      "recommended for 62000 users\n",
      "recommended for 62500 users\n",
      "recommended for 63000 users\n",
      "recommended for 63500 users\n",
      "recommended for 64000 users\n",
      "recommended for 64500 users\n",
      "recommended for 65000 users\n",
      "recommended for 65500 users\n",
      "recommended for 66000 users\n",
      "recommended for 66500 users\n",
      "recommended for 67000 users\n",
      "recommended for 67500 users\n",
      "recommended for 68000 users\n",
      "recommended for 68500 users\n",
      "recommended for 69000 users\n",
      "recommended for 69500 users\n",
      "recommended for 70000 users\n",
      "recommended for 70500 users\n",
      "recommended for 71000 users\n",
      "recommended for 71500 users\n",
      "recommended for 72000 users\n",
      "recommended for 72500 users\n",
      "recommended for 73000 users\n",
      "recommended for 73500 users\n",
      "recommended for 74000 users\n",
      "recommended for 74500 users\n",
      "recommended for 75000 users\n",
      "recommended for 75500 users\n",
      "recommended for 76000 users\n",
      "recommended for 76500 users\n",
      "recommended for 77000 users\n",
      "recommended for 77500 users\n",
      "recommended for 78000 users\n",
      "recommended for 78500 users\n",
      "recommended for 79000 users\n",
      "recommended for 79500 users\n",
      "recommended for 80000 users\n",
      "recommended for 80500 users\n",
      "recommended for 81000 users\n",
      "recommended for 81500 users\n",
      "recommended for 82000 users\n",
      "recommended for 82500 users\n",
      "recommended for 83000 users\n",
      "recommended for 83500 users\n",
      "recommended for 84000 users\n",
      "recommended for 84500 users\n",
      "recommended for 85000 users\n",
      "recommended for 85500 users\n",
      "recommended for 86000 users\n",
      "recommended for 86500 users\n",
      "recommended for 87000 users\n",
      "recommended for 87500 users\n",
      "recommended for 88000 users\n",
      "recommended for 88500 users\n",
      "recommended for 89000 users\n",
      "recommended for 89500 users\n",
      "recommended for 90000 users\n",
      "recommended for 90500 users\n",
      "recommended for 91000 users\n",
      "recommended for 91500 users\n",
      "recommended for 92000 users\n",
      "recommended for 92500 users\n",
      "recommended for 93000 users\n",
      "recommended for 93500 users\n",
      "recommended for 94000 users\n",
      "recommended for 94500 users\n",
      "recommended for 95000 users\n",
      "recommended for 95500 users\n",
      "recommended for 96000 users\n",
      "recommended for 96500 users\n",
      "recommended for 97000 users\n",
      "recommended for 97500 users\n",
      "recommended for 98000 users\n",
      "recommended for 98500 users\n",
      "recommended for 99000 users\n",
      "recommended for 99500 users\n",
      "recommended for 100000 users\n",
      "recommended for 100500 users\n",
      "recommended for 101000 users\n",
      "recommended for 101500 users\n",
      "recommended for 102000 users\n",
      "recommended for 102500 users\n",
      "recommended for 103000 users\n",
      "recommended for 103500 users\n",
      "recommended for 104000 users\n",
      "recommended for 104500 users\n",
      "recommended for 105000 users\n",
      "recommended for 105500 users\n",
      "recommended for 106000 users\n",
      "recommended for 106500 users\n",
      "recommended for 107000 users\n",
      "recommended for 107500 users\n",
      "recommended for 108000 users\n",
      "recommended for 108500 users\n",
      "recommended for 109000 users\n",
      "recommended for 109500 users\n",
      "recommended for 110000 users\n",
      "recommended for 110500 users\n",
      "recommended for 111000 users\n",
      "recommended for 111500 users\n",
      "recommended for 112000 users\n",
      "recommended for 112500 users\n",
      "recommended for 113000 users\n",
      "recommended for 113500 users\n",
      "recommended for 114000 users\n",
      "recommended for 114500 users\n",
      "recommended for 115000 users\n",
      "recommended for 115500 users\n",
      "recommended for 116000 users\n",
      "recommended for 116500 users\n",
      "recommended for 117000 users\n",
      "recommended for 117500 users\n",
      "recommended for 118000 users\n",
      "recommended for 118500 users\n",
      "recommended for 119000 users\n",
      "recommended for 119500 users\n",
      "recommended for 120000 users\n",
      "recommended for 120500 users\n",
      "recommended for 121000 users\n",
      "recommended for 121500 users\n",
      "recommended for 122000 users\n",
      "recommended for 122500 users\n",
      "recommended for 123000 users\n",
      "recommended for 123500 users\n",
      "recommended for 124000 users\n",
      "recommended for 124500 users\n",
      "recommended for 125000 users\n",
      "recommended for 125500 users\n",
      "recommended for 126000 users\n",
      "recommended for 126500 users\n",
      "recommended for 127000 users\n",
      "recommended for 127500 users\n",
      "recommended for 128000 users\n",
      "recommended for 128500 users\n",
      "recommended for 129000 users\n",
      "recommended for 129500 users\n",
      "recommended for 130000 users\n",
      "recommended for 130500 users\n",
      "recommended for 131000 users\n",
      "recommended for 131500 users\n",
      "recommended for 132000 users\n",
      "recommended for 132500 users\n",
      "recommended for 133000 users\n",
      "recommended for 133500 users\n",
      "recommended for 134000 users\n",
      "recommended for 134500 users\n",
      "recommended for 135000 users\n",
      "recommended for 135500 users\n",
      "recommended for 136000 users\n",
      "recommended for 136500 users\n",
      "recommended for 137000 users\n",
      "recommended for 137500 users\n",
      "recommended for 138000 users\n",
      "recommended for 138500 users\n",
      "recommended for 139000 users\n",
      "recommended for 139500 users\n",
      "recommended for 140000 users\n",
      "recommended for 140500 users\n",
      "recommended for 141000 users\n",
      "recommended for 141500 users\n",
      "recommended for 142000 users\n",
      "recommended for 142500 users\n",
      "recommended for 143000 users\n",
      "recommended for 143500 users\n",
      "recommended for 144000 users\n",
      "recommended for 144500 users\n",
      "recommended for 145000 users\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "recommended for 145500 users\n",
      "recommended for 146000 users\n",
      "recommended for 146500 users\n",
      "recommended for 147000 users\n",
      "recommended for 147500 users\n",
      "recommended for 148000 users\n",
      "recommended for 148500 users\n",
      "recommended for 149000 users\n",
      "recommended for 149500 users\n",
      "recommended for 150000 users\n",
      "precision=0.0070\trecall=0.1379\tcoverage=0.5394\tpopularity=2.2208\n"
     ]
    }
   ],
   "source": [
    "usercf.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "相似用户数目为 = 30\n",
      "推荐Item数目为 = 30\n"
     ]
    }
   ],
   "source": [
    "usercf_pred = UserBasedCF()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "预测尝试！\n",
      "split training set and test set succ\n",
      "train set = 2399699\n",
      "test set = 11398\n"
     ]
    }
   ],
   "source": [
    "usercf_pred.generate_dataset(merge_yy[['buyer_admin_id','item_id']],test[['buyer_admin_id','item_id']].drop_duplicates(subset=['buyer_admin_id'], keep='first'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "构建物品-用户倒排表中，请等待......\n",
      "构建物品-用户倒排表成功\n",
      "总共被操作过的物品数目为 = 972388\n",
      "building user co-rated items matrix...\n",
      "build user co-rated items matrix succ\n",
      "calculating user similarity matrix...\n",
      "calculating user similarity factor(2000000)\n",
      "calculating user similarity factor(4000000)\n",
      "calculating user similarity factor(6000000)\n",
      "calculating user similarity factor(8000000)\n",
      "calculating user similarity factor(10000000)\n",
      "calculating user similarity factor(12000000)\n",
      "calculating user similarity factor(14000000)\n",
      "calculating user similarity factor(16000000)\n",
      "calculating user similarity factor(18000000)\n",
      "calculating user similarity factor(20000000)\n",
      "calculating user similarity factor(22000000)\n",
      "calculating user similarity factor(24000000)\n",
      "calculating user similarity factor(26000000)\n",
      "calculate user similarity matrix(similarity factor) succ\n",
      "Total similarity factor number = 27251402\n"
     ]
    }
   ],
   "source": [
    "usercf_pred.calc_user_sim()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = open('usercf_pred.pkl', 'wb')\n",
    "pickle.dump(usercf_pred, output)\n",
    "output.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict start...\n",
      "recommended for 0 users\n",
      "recommended for 500 users\n",
      "recommended for 1000 users\n",
      "recommended for 1500 users\n",
      "recommended for 2000 users\n",
      "recommended for 2500 users\n",
      "recommended for 3000 users\n",
      "recommended for 3500 users\n",
      "recommended for 4000 users\n",
      "recommended for 4500 users\n",
      "recommended for 5000 users\n",
      "recommended for 5500 users\n",
      "recommended for 6000 users\n",
      "recommended for 6500 users\n",
      "recommended for 7000 users\n",
      "recommended for 7500 users\n",
      "recommended for 8000 users\n",
      "recommended for 8500 users\n",
      "recommended for 9000 users\n",
      "recommended for 9500 users\n",
      "recommended for 10000 users\n",
      "recommended for 10500 users\n",
      "recommended for 11000 users\n",
      "Predict end...\n"
     ]
    }
   ],
   "source": [
    "result = usercf_pred.predict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df = pd.DataFrame(result,columns=['buyer_admin_id','rec_items'])\n",
    "result_df['rec_items'] = result_df.rec_items.apply(lambda x:[i[0] for i in x])\n",
    "result_df['len'] = result_df.rec_items.apply(lambda x:len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>buyer_admin_id</th>\n",
       "      <th>rec_items</th>\n",
       "      <th>len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>726476</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>1896287</td>\n",
       "      <td>[956091, 2101539, 2104262, 2110417, 2116251, 2...</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>609047</td>\n",
       "      <td>[12928279, 12626807, 11717821, 12186473, 10945...</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>974416</td>\n",
       "      <td>[1578258, 2254804, 3589570, 6692791, 6721086, ...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>1095440</td>\n",
       "      <td>[436805, 3712123, 6975923, 7753914, 7982327, 1...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     buyer_admin_id                                          rec_items  len\n",
       "42           726476                                                 []    0\n",
       "88          1896287  [956091, 2101539, 2104262, 2110417, 2116251, 2...   16\n",
       "205          609047  [12928279, 12626807, 11717821, 12186473, 10945...   29\n",
       "225          974416  [1578258, 2254804, 3589570, 6692791, 6721086, ...    8\n",
       "338         1095440  [436805, 3712123, 6975923, 7753914, 7982327, 1...    7"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df[result_df['len'] < 30].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "popularity_check = usercf_pred.item_popular"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "popularity_check_sorted = sorted(popularity_check.items(), key=lambda obj: obj[1],reverse=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(30):\n",
    "    result_df['predict '+str(i+1)] = result_df.rec_items.apply(lambda x:x[i] if len(x) >i else random.choice( popularity_check_sorted [:100])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>buyer_admin_id</th>\n",
       "      <th>rec_items</th>\n",
       "      <th>len</th>\n",
       "      <th>predict 1</th>\n",
       "      <th>predict 2</th>\n",
       "      <th>predict 3</th>\n",
       "      <th>predict 4</th>\n",
       "      <th>predict 5</th>\n",
       "      <th>predict 6</th>\n",
       "      <th>predict 7</th>\n",
       "      <th>...</th>\n",
       "      <th>predict 21</th>\n",
       "      <th>predict 22</th>\n",
       "      <th>predict 23</th>\n",
       "      <th>predict 24</th>\n",
       "      <th>predict 25</th>\n",
       "      <th>predict 26</th>\n",
       "      <th>predict 27</th>\n",
       "      <th>predict 28</th>\n",
       "      <th>predict 29</th>\n",
       "      <th>predict 30</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>726476</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>13018359</td>\n",
       "      <td>11559746</td>\n",
       "      <td>2249889</td>\n",
       "      <td>2953605</td>\n",
       "      <td>10788165</td>\n",
       "      <td>11526051</td>\n",
       "      <td>3362266</td>\n",
       "      <td>...</td>\n",
       "      <td>7223628</td>\n",
       "      <td>10517552</td>\n",
       "      <td>9789145</td>\n",
       "      <td>10617258</td>\n",
       "      <td>11291525</td>\n",
       "      <td>3086129</td>\n",
       "      <td>10342698</td>\n",
       "      <td>9841579</td>\n",
       "      <td>10342698</td>\n",
       "      <td>6025215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>1896287</td>\n",
       "      <td>[956091, 2101539, 2104262, 2110417, 2116251, 2...</td>\n",
       "      <td>16</td>\n",
       "      <td>956091</td>\n",
       "      <td>2101539</td>\n",
       "      <td>2104262</td>\n",
       "      <td>2110417</td>\n",
       "      <td>2116251</td>\n",
       "      <td>2134952</td>\n",
       "      <td>3659075</td>\n",
       "      <td>...</td>\n",
       "      <td>10538002</td>\n",
       "      <td>8081951</td>\n",
       "      <td>3866632</td>\n",
       "      <td>13038480</td>\n",
       "      <td>13040827</td>\n",
       "      <td>9414379</td>\n",
       "      <td>10342698</td>\n",
       "      <td>908415</td>\n",
       "      <td>10761081</td>\n",
       "      <td>2953605</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    buyer_admin_id                                          rec_items  len  \\\n",
       "42          726476                                                 []    0   \n",
       "88         1896287  [956091, 2101539, 2104262, 2110417, 2116251, 2...   16   \n",
       "\n",
       "    predict 1  predict 2  predict 3  predict 4  predict 5  predict 6  \\\n",
       "42   13018359   11559746    2249889    2953605   10788165   11526051   \n",
       "88     956091    2101539    2104262    2110417    2116251    2134952   \n",
       "\n",
       "    predict 7  ...  predict 21  predict 22  predict 23  predict 24  \\\n",
       "42    3362266  ...     7223628    10517552     9789145    10617258   \n",
       "88    3659075  ...    10538002     8081951     3866632    13038480   \n",
       "\n",
       "    predict 25  predict 26  predict 27  predict 28  predict 29  predict 30  \n",
       "42    11291525     3086129    10342698     9841579    10342698     6025215  \n",
       "88    13040827     9414379    10342698      908415    10761081     2953605  \n",
       "\n",
       "[2 rows x 33 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df[result_df['len']<20].head(2) # 已经填上了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11398,)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.buyer_admin_id.unique().shape # 跟测试集个数匹配"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df.sort_values('buyer_admin_id').drop(['rec_items','len'],axis=1).to_csv('user_cf.csv',index=False,header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}