05_iMooc_Recommendation_System_Basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 慕课网-个性化入门实战课"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.404425Z",
     "start_time": "2019-07-18T08:41:41.401710Z"
    }
   },
   "outputs": [],
   "source": [
    "# Import packages\n",
    "from __future__ import division\n",
    "import os \n",
    "import math\n",
    "import operator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 公共信息抽取函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.414567Z",
     "start_time": "2019-07-18T08:41:41.406109Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "def get_user_click(rating_file):\n",
    "    \"\"\"\n",
    "    get user click list\n",
    "    Args:\n",
    "        rating_fiel: input file\n",
    "    Return:\n",
    "        dict, key: userid, value:[itemid1, itemid2]\n",
    "    \"\"\"\n",
    "    if not os.path.exists(rating_file):\n",
    "        return {},{}\n",
    "    fp = open(rating_file)\n",
    "    num = 0\n",
    "    user_click = {}\n",
    "    user_click_time = {}\n",
    "    for line in fp:\n",
    "        if num == 0:\n",
    "            num += 1\n",
    "            continue\n",
    "        item = line.strip().split(',')\n",
    "        if len(item) < 4:\n",
    "            continue\n",
    "        [userid, itemid, rating, timestamp] = item \n",
    "        if userid + '_' + itemid not in user_click_time:\n",
    "            user_click_time[userid + '_' + itemid] = int(timestamp)\n",
    "        if float(rating) < 3.0:\n",
    "            continue\n",
    "        if userid not in user_click:\n",
    "            user_click[userid] = []\n",
    "        user_click[userid].append(itemid)\n",
    "    fp.close()\n",
    "    return user_click, user_click_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.429509Z",
     "start_time": "2019-07-18T08:41:41.417373Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_item_info(item_file):\n",
    "    \"\"\"\n",
    "    get item info[title, genres]\n",
    "    Args:\n",
    "        item_file:input iteminfo file\n",
    "    return:\n",
    "        a dict, key itemid, value: [title, genres]\n",
    "    \"\"\"\n",
    "    if not os.path.exists(item_file):\n",
    "        return {}\n",
    "    num = 0\n",
    "    fp = open(item_file)\n",
    "    item_info ={}\n",
    "    for line in fp:\n",
    "        if num == 0:\n",
    "            num += 1\n",
    "            continue\n",
    "        item = line.strip().split(',')\n",
    "        if len(item) < 3:\n",
    "            continue\n",
    "        if len(item) == 3:\n",
    "            [itemid, title, genres] = item\n",
    "        elif len(item) > 3:\n",
    "            itemid = item[0]\n",
    "            genres = item[-1]\n",
    "            title = ','.join(item[1:-1])\n",
    "        if itemid not in item_info:\n",
    "            item_info[itemid] = [title, genres]\n",
    "    fp.close()\n",
    "    return item_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.573449Z",
     "start_time": "2019-07-18T08:41:41.431749Z"
    }
   },
   "outputs": [],
   "source": [
    "user_click,user_click_time = get_user_click(\"./tmp/dataset/ml-latest-small/ratings.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.586294Z",
     "start_time": "2019-07-18T08:41:41.574808Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "609\n",
      "['1', '3', '6', '47', '50', '70', '101', '110', '151', '157', '163', '216', '223', '231', '235', '260', '296', '316', '333', '349', '356', '362', '367', '423', '441', '457', '480', '500', '527', '543', '552', '553', '590', '592', '593', '596', '608', '648', '661', '673', '733', '736', '780', '804', '919', '923', '940', '943', '954', '1009', '1023', '1024', '1025', '1029', '1030', '1031', '1032', '1042', '1049', '1060', '1073', '1080', '1089', '1090', '1092', '1097', '1127', '1136', '1196', '1197', '1198', '1206', '1208', '1210', '1213', '1214', '1220', '1222', '1224', '1226', '1240', '1256', '1258', '1265', '1270', '1275', '1278', '1282', '1291', '1298', '1348', '1377', '1396', '1408', '1445', '1473', '1500', '1517', '1552', '1573', '1580', '1587', '1617', '1620', '1625', '1644', '1676', '1732', '1777', '1793', '1804', '1805', '1920', '1927', '1954', '1967', '2000', '2005', '2012', '2018', '2028', '2033', '2046', '2048', '2054', '2058', '2078', '2090', '2093', '2094', '2096', '2099', '2105', '2115', '2116', '2137', '2139', '2141', '2143', '2161', '2174', '2193', '2268', '2273', '2291', '2329', '2353', '2366', '2387', '2395', '2406', '2414', '2427', '2450', '2459', '2470', '2478', '2492', '2502', '2528', '2529', '2542', '2571', '2580', '2596', '2616', '2628', '2640', '2641', '2644', '2648', '2654', '2657', '2692', '2700', '2716', '2761', '2797', '2826', '2858', '2872', '2899', '2916', '2944', '2947', '2948', '2949', '2959', '2985', '2987', '2991', '2993', '2997', '3033', '3034', '3052', '3053', '3062', '3147', '3168', '3243', '3247', '3253', '3273', '3386', '3439', '3440', '3441', '3448', '3450', '3479', '3489', '3527', '3578', '3617', '3639', '3671', '3702', '3703', '3729', '3740', '3744', '3793', '3809', '4006', '5060']\n"
     ]
    }
   ],
   "source": [
    "print(len(user_click))\n",
    "print(user_click['1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.611513Z",
     "start_time": "2019-07-18T08:41:41.588737Z"
    }
   },
   "outputs": [],
   "source": [
    "item_info = get_item_info(\"./tmp/dataset/ml-latest-small/movies.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.615369Z",
     "start_time": "2019-07-18T08:41:41.612995Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9742\n",
      "['Toy Story (1995)', 'Adventure|Animation|Children|Comedy|Fantasy']\n"
     ]
    }
   ],
   "source": [
    "print(len(item_info))\n",
    "print(item_info['1'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 基于物品协同过滤实战\n",
    "\n",
    "步骤：\n",
    "\n",
    "1. 计算物品相似度矩阵\n",
    "2. 根据物品相似度矩阵推荐相似物品\n",
    "\n",
    "### 计算物品和用户相似度函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.620105Z",
     "start_time": "2019-07-18T08:41:41.616584Z"
    }
   },
   "outputs": [],
   "source": [
    "def base_contribution_score():\n",
    "    \"\"\"\n",
    "    item cf base sim contribution score by user，基础权重计算公式\n",
    "    \"\"\"\n",
    "    return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.625764Z",
     "start_time": "2019-07-18T08:41:41.621171Z"
    }
   },
   "outputs": [],
   "source": [
    "def update_one_contribution_score(user_total_click_num):\n",
    "    \"\"\"\n",
    "    升级版本权重计算公式1，惩罚点击数目过多的用户的权重\n",
    "    \"\"\"\n",
    "    return 1/math.log(1+user_total_click_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.631325Z",
     "start_time": "2019-07-18T08:41:41.626935Z"
    }
   },
   "outputs": [],
   "source": [
    "alpha = 1.0 # scale rate\n",
    "def update_two_contribution_score(click_time_one, click_time_two):\n",
    "    \"\"\"\n",
    "    升级版本权重计算公式2，惩罚时间差越大的用户的权重\n",
    "    \"\"\"\n",
    "    \n",
    "    delta_time = abs(click_time_one - click_time_two)\n",
    "    total_sec = 60*60*24\n",
    "    delta_time = delta_time/total_sec\n",
    "    return 1/(1+alpha * delta_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.636969Z",
     "start_time": "2019-07-18T08:41:41.632894Z"
    }
   },
   "outputs": [],
   "source": [
    "# choose the function to calculate the contribution score\n",
    "contribution_score_type = 2 # 0: base,  1: update_one, 2: update_two"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.651609Z",
     "start_time": "2019-07-18T08:41:41.638805Z"
    }
   },
   "outputs": [],
   "source": [
    "def cal_item_sim(user_click, user_click_time):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        user_click: key userid,  value: [itemid1, itemid2]\n",
    "    Return:\n",
    "        dict, key:itemid_i, value dict, value_key itemid_j, value_value simscore\n",
    "    \"\"\"\n",
    "    # 计算两个物品之间的贡献矩阵\n",
    "    co_appear = {}\n",
    "    item_user_click_time = {}\n",
    "    for user, itemlist in user_click.items():\n",
    "        for index_i in range(0, len(itemlist)):\n",
    "            itemid_i = itemlist[index_i]\n",
    "            item_user_click_time.setdefault(itemid_i, 0)\n",
    "            item_user_click_time[itemid_i] += 1\n",
    "            for index_j in range(index_i + 1, len(itemlist)):\n",
    "                itemid_j = itemlist[index_j]\n",
    "                co_appear.setdefault(itemid_i, {})\n",
    "                co_appear[itemid_i].setdefault(itemid_j, 0)\n",
    "                co_appear.setdefault(itemid_j, {})\n",
    "                co_appear[itemid_j].setdefault(itemid_i, 0)\n",
    "\n",
    "                if contribution_score_type == 0:\n",
    "                    # 基础贡献公式\n",
    "                    co_appear[itemid_i][itemid_j] += base_contribution_score()\n",
    "                    co_appear[itemid_j][itemid_i] += base_contribution_score()\n",
    "                elif contribution_score_type == 1:\n",
    "                    # 升级版本公式1\n",
    "                    co_appear[itemid_i][\n",
    "                        itemid_j] += update_one_contribution_score(len(itemlist))\n",
    "\n",
    "                    co_appear[itemid_j][itemid_i] += update_one_contribution_score(len(itemlist))\n",
    "                else:\n",
    "                    # 升级版本公式2\n",
    "                    if user + '_' + itemid_i not in user_click_time:\n",
    "                        click_time_one = 0\n",
    "                    else:\n",
    "                        click_time_one = user_click_time[user + '_' + itemid_i]\n",
    "                    if user + '_' + itemid_j not in user_click_time:\n",
    "                        click_time_two = 0\n",
    "                    else:\n",
    "                        click_time_two = user_click_time[user + '_' + itemid_j]\n",
    "                    co_appear[itemid_i][\n",
    "                        itemid_j] += update_two_contribution_score(\n",
    "                            click_time_one, click_time_two)\n",
    "                    co_appear[itemid_j][itemid_i] += update_two_contribution_score(click_time_one, click_time_two)\n",
    "\n",
    "    item_sim_score = {}\n",
    "    item_sim_score_sorted = {}\n",
    "    for itemid_i, relate_item in co_appear.items():\n",
    "        for itemid_j, co_time in relate_item.items():\n",
    "            sim_score = co_time / math.sqrt(item_user_click_time[itemid_i] *\n",
    "                                            item_user_click_time[itemid_j])\n",
    "            item_sim_score.setdefault(itemid_i, {})\n",
    "            item_sim_score[itemid_i].setdefault(itemid_j, 0)\n",
    "            item_sim_score[itemid_i][itemid_j] = sim_score\n",
    "\n",
    "    for itemid in item_sim_score:\n",
    "        item_sim_score_sorted[itemid] = sorted(item_sim_score[itemid].items(),\n",
    "                                               key=operator.itemgetter(1),\n",
    "                                               reverse=True)\n",
    "\n",
    "    return item_sim_score_sorted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:41:41.659788Z",
     "start_time": "2019-07-18T08:41:41.653300Z"
    }
   },
   "outputs": [],
   "source": [
    "def cal_recom_result(sim_info, user_click):\n",
    "    \"\"\"\n",
    "    recom by itemcf\n",
    "    Args:\n",
    "        sim_info: item sim dict\n",
    "        user_click: user click dict\n",
    "    Return:\n",
    "        dict, key:userid value dict, value_key itemid, value_value recom_score\n",
    "    \"\"\"\n",
    "    recent_click_num = 3\n",
    "    topk = 5\n",
    "    recom_info = {}\n",
    "    for user in user_click:\n",
    "        click_list = user_click[user]\n",
    "        recom_info.setdefault(user, {})\n",
    "        for itemid in click_list[:recent_click_num]:\n",
    "            if itemid not in sim_info:\n",
    "                continue\n",
    "            for itemidsim_tuple in sim_info[itemid][:topk]:\n",
    "                itemsimid = itemidsim_tuple[0]\n",
    "                itemsimscore = itemidsim_tuple[1]\n",
    "                recom_info[user][itemsimid] = itemsimscore\n",
    "    return recom_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.347382Z",
     "start_time": "2019-07-18T08:41:41.661092Z"
    }
   },
   "outputs": [],
   "source": [
    "sim_info = cal_item_sim(user_click, user_click_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.352681Z",
     "start_time": "2019-07-18T08:42:28.348760Z"
    }
   },
   "outputs": [],
   "source": [
    "recom_result = cal_recom_result(sim_info, user_click)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.374156Z",
     "start_time": "2019-07-18T08:42:28.353600Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'780': 0.46475374613998177, '3114': 0.43256226390108665, '356': 0.42982096383633256, '588': 0.4272105446504961, '260': 0.4217381348100523, '5': 0.3706893668636461, '7': 0.36722778250755456, '736': 0.355248820396473, '788': 0.35453382503684483, '784': 0.33956136189636077, '733': 0.4166114199492754, '16': 0.39177042306799037, '32': 0.3834927290133812, '112': 0.3691490738192548, '628': 0.34343752127181054}\n"
     ]
    }
   ],
   "source": [
    "print(recom_result[\"1\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.379892Z",
     "start_time": "2019-07-18T08:42:28.376042Z"
    }
   },
   "outputs": [],
   "source": [
    "recom_result_sorted = sorted(recom_result[\"1\"].items(), key=operator.itemgetter(1),reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.386821Z",
     "start_time": "2019-07-18T08:42:28.381672Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('780', 0.46475374613998177), ('3114', 0.43256226390108665), ('356', 0.42982096383633256), ('588', 0.4272105446504961), ('260', 0.4217381348100523), ('733', 0.4166114199492754), ('16', 0.39177042306799037), ('32', 0.3834927290133812), ('5', 0.3706893668636461), ('112', 0.3691490738192548), ('7', 0.36722778250755456), ('736', 0.355248820396473), ('788', 0.35453382503684483), ('628', 0.34343752127181054), ('784', 0.33956136189636077)]\n"
     ]
    }
   ],
   "source": [
    "print(recom_result_sorted)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 打印输出查看推荐的物品相似度情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.397976Z",
     "start_time": "2019-07-18T08:42:28.389260Z"
    }
   },
   "outputs": [],
   "source": [
    "def debug_itemsim(item_info, sim_info):\n",
    "    \"\"\"\n",
    "    Show itemsim info，强调的是输出计算出来的物品之间的相似度\n",
    "    Args:\n",
    "        item_info: dict, key itemid, value:[title, genres]\n",
    "        sim_info: dict, key itemid, value dict, key [(itemid1, simscore), (itemid2, simscore)]\n",
    "    \"\"\"\n",
    "    fixed_itemid = \"1\"\n",
    "    if fixed_itemid not in item_info:\n",
    "        print('Invalid itemid.')\n",
    "        return\n",
    "    [title_fix, genres_fix] = item_info[fixed_itemid]\n",
    "    print(title_fix + '\\t' + genres_fix)\n",
    "    for sim_info_tuple in sim_info[fixed_itemid][:10]:\n",
    "        itemid_sim = sim_info_tuple[0]\n",
    "        sim_score = sim_info_tuple[1]\n",
    "        if itemid_sim not in item_info:\n",
    "            continue\n",
    "        [title, genres] = item_info[itemid_sim]\n",
    "        print('sim:' + title + '\\t'+genres+'\\t'+str(sim_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.410087Z",
     "start_time": "2019-07-18T08:42:28.402043Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Toy Story (1995)\tAdventure|Animation|Children|Comedy|Fantasy\n",
      "sim:Independence Day (a.k.a. ID4) (1996)\tAction|Adventure|Sci-Fi|Thriller\t0.46475374613998177\n",
      "sim:Toy Story 2 (1999)\tAdventure|Animation|Children|Comedy|Fantasy\t0.43256226390108665\n",
      "sim:Forrest Gump (1994)\tComedy|Drama|Romance|War\t0.42982096383633256\n",
      "sim:Aladdin (1992)\tAdventure|Animation|Children|Comedy|Musical\t0.4272105446504961\n",
      "sim:Star Wars: Episode IV - A New Hope (1977)\tAction|Adventure|Sci-Fi\t0.4217381348100523\n",
      "sim:Mission: Impossible (1996)\tAction|Adventure|Mystery|Thriller\t0.42005115492687634\n",
      "sim:Jurassic Park (1993)\tAction|Adventure|Sci-Fi|Thriller\t0.41903077289856033\n",
      "sim:Star Wars: Episode VI - Return of the Jedi (1983)\tAction|Adventure|Sci-Fi\t0.41391971218503226\n",
      "sim:\"Lion King, The (1994)\"\tAdventure|Animation|Children|Drama|Musical|IMAX\t0.40592414467745963\n",
      "sim:Willy Wonka & the Chocolate Factory (1971)\tChildren|Comedy|Fantasy|Musical\t0.39680151931956986\n"
     ]
    }
   ],
   "source": [
    "debug_itemsim(item_info, sim_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.419515Z",
     "start_time": "2019-07-18T08:42:28.413410Z"
    }
   },
   "outputs": [],
   "source": [
    "def debug_recommendation_result(recom_result, item_info):\n",
    "    \"\"\"\n",
    "    Debug recommendation result，强调的是根据用户的偏好，输出来的推荐结果\n",
    "    Args:\n",
    "        recom_result: key userid value: dict, value_key:itemid, value_value:recom_score\n",
    "        item_info: dict, key itemid value: [title, genre]\n",
    "    \"\"\"\n",
    "    user_id = '1'\n",
    "    if user_id not in recom_result:\n",
    "        print(\"Invalid userid\")\n",
    "        return\n",
    "    for recom_result_info in sorted(recom_result[user_id].items(), key=operator.itemgetter(1), reverse=True):\n",
    "        itemid, score = recom_result_info\n",
    "        if itemid not in item_info:\n",
    "            continue\n",
    "        print(\",\".join(item_info[itemid])+'\\t' + str(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.428498Z",
     "start_time": "2019-07-18T08:42:28.421434Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Independence Day (a.k.a. ID4) (1996),Action|Adventure|Sci-Fi|Thriller\t0.46475374613998177\n",
      "Toy Story 2 (1999),Adventure|Animation|Children|Comedy|Fantasy\t0.43256226390108665\n",
      "Forrest Gump (1994),Comedy|Drama|Romance|War\t0.42982096383633256\n",
      "Aladdin (1992),Adventure|Animation|Children|Comedy|Musical\t0.4272105446504961\n",
      "Star Wars: Episode IV - A New Hope (1977),Action|Adventure|Sci-Fi\t0.4217381348100523\n",
      "\"Rock, The (1996)\",Action|Adventure|Thriller\t0.4166114199492754\n",
      "Casino (1995),Crime|Drama\t0.39177042306799037\n",
      "Twelve Monkeys (a.k.a. 12 Monkeys) (1995),Mystery|Sci-Fi|Thriller\t0.3834927290133812\n",
      "Father of the Bride Part II (1995),Comedy\t0.3706893668636461\n",
      "Rumble in the Bronx (Hont faan kui) (1995),Action|Adventure|Comedy|Crime\t0.3691490738192548\n",
      "Sabrina (1995),Comedy|Romance\t0.36722778250755456\n",
      "Twister (1996),Action|Adventure|Romance|Thriller\t0.355248820396473\n",
      "\"Nutty Professor, The (1996)\",Comedy|Fantasy|Romance|Sci-Fi\t0.35453382503684483\n",
      "Primal Fear (1996),Crime|Drama|Mystery|Thriller\t0.34343752127181054\n",
      "\"Cable Guy, The (1996)\",Comedy|Thriller\t0.33956136189636077\n"
     ]
    }
   ],
   "source": [
    "debug_recommendation_result(recom_result, item_info)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T06:53:19.187581Z",
     "start_time": "2019-07-18T06:53:19.184146Z"
    }
   },
   "source": [
    "## 基于用户的协同过滤实战\n",
    "\n",
    "步骤：\n",
    "\n",
    "1. 计算用户相似度矩阵\n",
    "2. 根据相似度矩阵推荐物品"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.558862Z",
     "start_time": "2019-07-18T08:42:28.431235Z"
    }
   },
   "outputs": [],
   "source": [
    "user_click,user_click_time = get_user_click(\"./tmp/dataset/ml-latest-small/ratings.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.567974Z",
     "start_time": "2019-07-18T08:42:28.562817Z"
    }
   },
   "outputs": [],
   "source": [
    "def transfer_user_click(user):\n",
    "    \"\"\"\n",
    "    将用户的点击转化成item被用户点击\n",
    "    Args:\n",
    "        user_click: key userid, value: [itemid1, itemid2]\n",
    "    Return:\n",
    "        dict, key itemid, value:[userid1,userid2]\n",
    "    \"\"\"\n",
    "    item_click_by_user = {}\n",
    "    for user in user_click:\n",
    "        item_list = user_click[user]\n",
    "        for itemid in item_list:\n",
    "            item_click_by_user.setdefault(itemid, [])\n",
    "            item_click_by_user[itemid].append(user)\n",
    "    return item_click_by_user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.576516Z",
     "start_time": "2019-07-18T08:42:28.569704Z"
    }
   },
   "outputs": [],
   "source": [
    "def usercf_base_contribution_score():\n",
    "    \"\"\"\n",
    "    基础的用户贡献权重\n",
    "    \"\"\"\n",
    "    return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:28.585183Z",
     "start_time": "2019-07-18T08:42:28.579746Z"
    }
   },
   "outputs": [],
   "source": [
    "def usercf_update_contribution_score(item_user_click_count):\n",
    "    \"\"\"\n",
    "    usercf contribution score update V1\n",
    "    Args:\n",
    "        item_user_click_count: how many user have clicked this item\n",
    "    Return：\n",
    "        contribution score\n",
    "    \"\"\"\n",
    "    return 1/math.log10(1 + item_user_click_count)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:42:54.663518Z",
     "start_time": "2019-07-18T08:42:54.657719Z"
    }
   },
   "outputs": [],
   "source": [
    "def usercf_update_two_contribution_score(click_time_one,click_time_two):\n",
    "    \"\"\"\n",
    "    usercf contribution score update V2\n",
    "    Args:\n",
    "        different user action time to the same item, click_time_one, click_time_two\n",
    "    return:\n",
    "        Contribution score\n",
    "    \"\"\"\n",
    "    delta_time = abs(click_time_two - click_time_one)\n",
    "    # 将时间转化为以天为单位\n",
    "    norm_num = 60*60*24\n",
    "    delta_time = delta_time/norm_num\n",
    "    return 1/(1+delta_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:43:19.431620Z",
     "start_time": "2019-07-18T08:43:19.429568Z"
    }
   },
   "outputs": [],
   "source": [
    "usercf_contribution_type = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:44:45.377640Z",
     "start_time": "2019-07-18T08:44:45.369811Z"
    }
   },
   "outputs": [],
   "source": [
    "def cal_user_sim(item_click_by_user,user_click_time):\n",
    "    \"\"\"\n",
    "    计算用户相似度\n",
    "    Args:\n",
    "        item_click_by_user: dict, key:itemid, value:[userid1, userid2]\n",
    "    Return:\n",
    "        dict, key itemid, value:dict, value_key:itemid_j, value_value:sim_score\n",
    "    \"\"\"\n",
    "    co_appear = {}\n",
    "    user_click_count = {}\n",
    "    for itemid, user_list in item_click_by_user.items():\n",
    "        for index_i in range(0, len(user_list)):\n",
    "            user_i = user_list[index_i]\n",
    "            user_click_count.setdefault(user_i, 0)\n",
    "            user_click_count[user_i] += 1\n",
    "            if user_i + \"_\" + itemid not in user_click_time:\n",
    "                click_time_one = 0\n",
    "            else:\n",
    "                click_time_one = user_click_time[user_i + \"_\" + itemid]\n",
    "            for index_j in range(index_i+1, len(user_list)):\n",
    "                user_j = user_list[index_j]\n",
    "                if user_j + \"_\" +itemid not in user_click_time:\n",
    "                    click_time_two = 0\n",
    "                else:\n",
    "                    click_time_two = user_click_time[user_j+'_'+itemid]\n",
    "                co_appear.setdefault(user_i, {})\n",
    "                co_appear[user_i].setdefault(user_j, 0)\n",
    "                co_appear.setdefault(user_j, {})\n",
    "                co_appear[user_j].setdefault(user_i, 0)\n",
    "                \n",
    "                # 基础用户贡献权重\n",
    "                if usercf_contribution_type == 0: \n",
    "                    co_appear[user_i][user_j] += usercf_base_contribution_score()\n",
    "                    co_appear[user_j][user_i] += usercf_base_contribution_score()\n",
    "                # 惩罚行为频繁用户的贡献权重\n",
    "                elif usercf_contribution_type == 1:\n",
    "                    co_appear[user_i][user_j] += usercf_update_contribution_score(len(user_list))\n",
    "                    co_appear[user_j][user_i] += usercf_update_contribution_score(len(user_list))\n",
    "                # 惩罚时间间隔太长的用户贡献权重\n",
    "                else usercf_contribution_type == 2:\n",
    "                    co_appear[user_i][user_j] += usercf_update_two_contribution_score(click_time_one,click_time_two)\n",
    "                    co_appear[user_j][user_i] += usercf_update_two_contribution_score(click_time_one,click_time_two)\n",
    "    user_sim_info = {}\n",
    "    user_sim_info_sorted = {}\n",
    "    for user_i, relate_user in co_appear.items():\n",
    "        user_sim_info.setdefault(user_i, {})\n",
    "        for user_j, cotime in relate_user.items():\n",
    "            user_sim_info[user_i].setdefault(user_j, 0)\n",
    "            user_sim_info[user_i][user_j] = cotime/math.sqrt(user_click_count[user_i]*user_click_count[user_j])\n",
    "    # 将相似度用户排序\n",
    "    for user in user_sim_info:\n",
    "        user_sim_info_sorted[user] = sorted(user_sim_info[user].items(), key=operator.itemgetter(1), reverse=True)\n",
    "    return user_sim_info_sorted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:46:11.602422Z",
     "start_time": "2019-07-18T08:46:11.598366Z"
    }
   },
   "outputs": [],
   "source": [
    "def cal_recom_result_using_usercf(user_click, user_sim):\n",
    "    \"\"\"\n",
    "    利用usercf产生推荐结果\n",
    "    Args:\n",
    "        user_clik: dict, key userid, value: [itemid1, itemid2]\n",
    "        user_sim: key: userid value: [(useridj, score1),(useridk, score2)]\n",
    "    Return:\n",
    "        dict, key userid, value: dict value_key:itemid, value_value:recom_score\n",
    "    \"\"\"\n",
    "    \n",
    "    recom_result = {}\n",
    "    topk_user = 3\n",
    "    item_num = 5\n",
    "    for user, item_list in user_click.items():\n",
    "        tmp_dict = {}\n",
    "        for itemid in item_list:\n",
    "            tmp_dict.setdefault(itemid, 1)\n",
    "        recom_result.setdefault(user, {})\n",
    "        for zuhe in user_sim[user][:topk_user]:\n",
    "            userid_j, sim_score = zuhe\n",
    "            if userid_j not in user_click:\n",
    "                continue\n",
    "            \n",
    "            for itemid_j in user_click[userid_j][:item_num]:\n",
    "                recom_result[user].setdefault(itemid_j, sim_score)\n",
    "    return recom_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:47:40.154979Z",
     "start_time": "2019-07-18T08:47:40.133882Z"
    }
   },
   "outputs": [],
   "source": [
    "item_click_by_user = transfer_user_click(user_click)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:49:12.501414Z",
     "start_time": "2019-07-18T08:49:08.230815Z"
    }
   },
   "outputs": [],
   "source": [
    "user_sim = cal_user_sim(item_click_by_user,user_click_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:50:41.623228Z",
     "start_time": "2019-07-18T08:50:41.609350Z"
    }
   },
   "outputs": [],
   "source": [
    "recom_result = cal_recom_result_using_usercf(user_click, user_sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:52:11.100913Z",
     "start_time": "2019-07-18T08:52:11.098634Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'6': 0.18803307299446, '10': 0.18803307299446, '29': 0.18803307299446, '32': 0.18803307299446, '39': 0.18803307299446, '1': 0.1879004753996153, '11': 0.1879004753996153, '21': 0.1879004753996153, '3': 0.18756012667350208, '17': 0.18756012667350208}\n"
     ]
    }
   ],
   "source": [
    "print(recom_result['1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:53:36.602353Z",
     "start_time": "2019-07-18T08:53:36.599154Z"
    }
   },
   "outputs": [],
   "source": [
    "def debug_user_sim(user_sim):\n",
    "    \"\"\"\n",
    "    测试用户间相似度\n",
    "    Args：\n",
    "        user_sim: key: userid vale:[(userid1, score1), (userid2,score2)]\n",
    "    \"\"\"\n",
    "    fixed_user = 120\n",
    "    topk = 5\n",
    "    if fixed_user not in user_sim:\n",
    "        print(\"Invalid user\")\n",
    "        return    \n",
    "    for zuhe in user_sim[fixed_user][:topk]:\n",
    "        userid, score = zuhe\n",
    "        print(\"fix_user\" + \"\\t sim_user\" + userid + \"\\t\" + str(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:55:02.904911Z",
     "start_time": "2019-07-18T08:55:02.901685Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Invalid user\n"
     ]
    }
   ],
   "source": [
    "debug_user_sim(user_sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:56:31.127607Z",
     "start_time": "2019-07-18T08:56:31.116989Z"
    }
   },
   "outputs": [],
   "source": [
    "item_info = get_item_info(\"./tmp/dataset/ml-latest-small/movies.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:58:01.527530Z",
     "start_time": "2019-07-18T08:58:01.522663Z"
    }
   },
   "outputs": [],
   "source": [
    "def debug_recom_result_using_usercf(item_info, recom_result):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        item_info: key itemid value:[title, genres]\n",
    "        recom_result: key userid, value dict, value key: itemid value_value: recom_score\n",
    "    \"\"\"\n",
    "    fix_user = '1'\n",
    "    if fix_user not in recom_result:\n",
    "        print(\"invalid user for recoming result.\")\n",
    "        return\n",
    "    for itemid in recom_result[\"1\"]:\n",
    "        if itemid not in item_info:\n",
    "            continue\n",
    "        recom_score = recom_result[\"1\"][itemid]\n",
    "        print(\"recom result: \" + \",\".join(item_info[itemid]) + \"\\t\" + str(recom_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T08:59:29.730003Z",
     "start_time": "2019-07-18T08:59:29.726713Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "recom result: Heat (1995),Action|Crime|Thriller\t0.18803307299446\n",
      "recom result: GoldenEye (1995),Action|Adventure|Thriller\t0.18803307299446\n",
      "recom result: \"City of Lost Children, The (Cité des enfants perdus, La) (1995)\",Adventure|Drama|Fantasy|Mystery|Sci-Fi\t0.18803307299446\n",
      "recom result: Twelve Monkeys (a.k.a. 12 Monkeys) (1995),Mystery|Sci-Fi|Thriller\t0.18803307299446\n",
      "recom result: Clueless (1995),Comedy|Romance\t0.18803307299446\n",
      "recom result: Toy Story (1995),Adventure|Animation|Children|Comedy|Fantasy\t0.1879004753996153\n",
      "recom result: \"American President, The (1995)\",Comedy|Drama|Romance\t0.1879004753996153\n",
      "recom result: Get Shorty (1995),Comedy|Crime|Thriller\t0.1879004753996153\n",
      "recom result: Grumpier Old Men (1995),Comedy|Romance\t0.18756012667350208\n",
      "recom result: Sense and Sensibility (1995),Drama|Romance\t0.18756012667350208\n"
     ]
    }
   ],
   "source": [
    "debug_recom_result_using_usercf(item_info, recom_result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}