faNFL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Statistics and Regression for NFL player performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You should start the IPython notebook in the dir where this faNFL.ipynb file is located:\n",
    "\n",
    "    ipython3 notebook .\n",
    "    \n",
    "For the server with certificate and with anaconda:\n",
    " \n",
    "    ~/anaconda/bin/ipython notebook --profile=faNFL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys; sys.path.insert(0, \"./nflgame\")\n",
    "from collections import defaultdict, OrderedDict\n",
    "import nflgame  # load NFL statistics\n",
    "import seaborn as sns  # nicer statistical plots\n",
    "import matplotlib.pyplot as plt  # the plots work-horse\n",
    "import sklearn  # machine learning\n",
    "import pandas as pd  # tabular data thingy\n",
    "from pandas import Series\n",
    "import numpy as np\n",
    "import itertools\n",
    "import qgrid\n",
    "import pyprind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "qgrid.nbinstall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='svg'\n",
    "%config InlineBackend.figure_format='retina'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pd.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defense team score\n",
    "\n",
    "Three questions we want to address here at first ...\n",
    "\n",
    "- You have to find all of the statistical plays that contribute to the teams score.\n",
    "- You have to find out how many points the other team scored on that defense.\n",
    "- You have to find out how many yards were given up by that defense."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "games = nflgame.games(2014, week=1, kind='REG')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "team_defense_score = defaultdict(int)\n",
    "# Also going to keep track of the scores that aren't by the offense\n",
    "not_against_team_defense = defaultdict(int)\n",
    "\n",
    "for p in nflgame.combine_plays(games).players():\n",
    "    team_defense_score[p.team] += p.kickret_tds*6\n",
    "    team_defense_score[p.team] += p.puntret_tds*6\n",
    "    not_against_team_defense[p.team] += p.kickret_tds*6\n",
    "    not_against_team_defense[p.team] += p.puntret_tds*6\n",
    "\n",
    "for p in nflgame.combine_plays(games).players().defense():\n",
    "    team_defense_score[p.team] += p.defense_int_tds*6\n",
    "    team_defense_score[p.team] += p.defense_frec_tds*6\n",
    "    team_defense_score[p.team] += p.defense_misc_tds*6\n",
    "    team_defense_score[p.team] += p.defense_int*2\n",
    "    team_defense_score[p.team] += p.defense_frec*2\n",
    "    team_defense_score[p.team] += p.kicking_fgb*2\n",
    "    team_defense_score[p.team] += p.kicking_xpb*2\n",
    "    team_defense_score[p.team] += p.punting_block*2\n",
    "    team_defense_score[p.team] += p.defense_safe*2\n",
    "    team_defense_score[p.team] += p.defense_sk*1\n",
    "    not_against_team_defense[p.team] += p.defense_int_tds*6\n",
    "    not_against_team_defense[p.team] += p.defense_frec_tds*6\n",
    "    not_against_team_defense[p.team] += p.defense_misc_tds*6\n",
    "    not_against_team_defense[p.team] += p.defense_safe*2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "team_defense_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 2)\n",
    "team_points_against = defaultdict(int)\n",
    "for g in games:\n",
    "    team_points_against[g.home] = g.score_away-not_against_team_defense[g.away]\n",
    "    team_points_against[g.away] = g.score_home-not_against_team_defense[g.home]\n",
    "for team in team_points_against:\n",
    "    pts = team_points_against[team]\n",
    "    if pts == 0: team_defense_score[team] += 5\n",
    "    elif pts <= 6: team_defense_score[team] += 4\n",
    "    elif pts <= 13: team_defense_score[team] += 3\n",
    "    elif pts <= 17: team_defense_score[team] += 1\n",
    "    elif pts <= 27: team_defense_score[team] += 0\n",
    "    elif pts <= 34: team_defense_score[team] -= 1\n",
    "    elif pts <= 45: team_defense_score[team] -= 3\n",
    "    elif pts > 45: team_defense_score[team] -= 5\n",
    "\n",
    "# 3)\n",
    "team_yards_against = defaultdict(int)\n",
    "plays = nflgame.combine_plays(games)\n",
    "for p in plays.players().defense():\n",
    "    team_yards_against[p.team] += p.rushing_yds\n",
    "    team_yards_against[p.team] += p.passing_yds\n",
    "    team_yards_against[p.team] -= p.defense_sk_yds  # ???\n",
    "for team in team_yards_against:\n",
    "    yds = team_yards_against[team]\n",
    "    if yds < 100: team_defense_score[team] += 5\n",
    "    elif yds < 200: team_defense_score[team] += 3\n",
    "    elif yds < 300: team_defense_score[team] += 2\n",
    "    elif yds < 350: team_defense_score[team] += 0\n",
    "    elif yds < 400: team_defense_score[team] -= 1\n",
    "    elif yds < 450: team_defense_score[team] -= 3\n",
    "    elif yds < 500: team_defense_score[team] -= 5\n",
    "    elif yds < 550: team_defense_score[team] -= 6\n",
    "    elif yds >= 550: team_defense_score[team] -= 7\n",
    "\n",
    "print(team_defense_score['NE'])\n",
    "print(team_defense_score['CHI'], team_points_against['CHI'])\n",
    "print(team_defense_score['MIN'], team_points_against['MIN'])\n",
    "print(team_defense_score['OAK'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computing a fantasy league score\n",
    "\n",
    "for a player in a game."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def yahoo_expected_points(yahoo_scoring_system=dict(points_passing_yds=25,\n",
    "                                              points_passing_ints=-1,\n",
    "                                              points_passing_twoptm=2,\n",
    "                                              points_passing_tds=4,\n",
    "                                              points_rushing_yds=10,\n",
    "                                              points_rushing_tds=6),\n",
    "                    player_passing_yds=None,\n",
    "                    player_passing_ints=None,\n",
    "                    player_passing_twoptm=None,\n",
    "                    player_passing_tds=None,\n",
    "                    player_rushing_yds=None,\n",
    "                    points_rushing_tds=None):\n",
    "    score = 0\n",
    "    \n",
    "    score += player_passing_yds / yahoo_scoring_system[\"points_passing_yds\"]\n",
    "    score += player_passing_ints * yahoo_scoring_system[\"points_passing_ints\"]\n",
    "    score += player_passing_twoptm * yahoo_scoring_system[\"points_passing_twoptm\"]\n",
    "    score += player_passing_tds * yahoo_scoring_system[\"points_passing_tds\"]\n",
    "    score += player_rushing_yds / yahoo_scoring_system[\"points_rushing_yds\"]\n",
    "    score += player_rushing_tds * yahoo_scoring_system[\"points_rushing_tds\"]\n",
    "    \n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def fantasy_score(game, player_id,\n",
    "                  points_passing_yards=25,\n",
    "                  points_passing_ints=-1,\n",
    "                  points_passing_twoptm=2,\n",
    "                  points_passing_tds=4,\n",
    "                  points_rushing_yds=10,\n",
    "                  points_rushing_tds=6):\n",
    "    score = 0\n",
    "    # passing\n",
    "    if player_id in game.data['away']['stats']['passing'].keys():\n",
    "        player_passing_data = game.data['away']['stats']['passing'][player_id]\n",
    "    elif player_id in game.data['home']['stats']['passing'].keys():\n",
    "        player_passing_data = game.data['home']['stats']['passing'][player_id]\n",
    "    else:\n",
    "        raise Exception(\"Given player_id {0} not found 'passing' in that game {1}\".format(\n",
    "            player_id,\n",
    "            game.nice_score()))\n",
    "        \n",
    "    score += player_passing_data['yds'] / points_passing_yards\n",
    "    score += player_passing_data['ints'] * points_passing_ints\n",
    "    score += player_passing_data['twoptm'] * points_passing_twoptm\n",
    "    score += player_passing_data['tds'] * points_passing_tds\n",
    "    \n",
    "    # rushing\n",
    "    if player_id in game.data['away']['stats']['rushing'].keys():\n",
    "        player_rushing_data = game.data['away']['stats']['rushing'][player_id]\n",
    "    elif player_id in game.data['home']['stats']['rushing'].keys():\n",
    "        player_rushing_data = game.data['home']['stats']['rushing'][player_id]\n",
    "    else:\n",
    "        raise Exception(\"Given player_id {0} not found 'rushing' in that game {1}\".format(\n",
    "            player_id,\n",
    "            game.nice_score()))\n",
    "\n",
    "    score += player_rushing_data['yds'] / points_rushing_yds\n",
    "    score += player_rushing_data['tds'] * points_rushing_tds\n",
    "    return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filling pandas table\n",
    "\n",
    "Let's have a big DataFrame (that is a table) with all games (we are interested in) and how all players performed in that games. When a player wasn't in a game, we denote that by `NaN` (that is `None`)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need two helper functions to extract all the player ids of a certain game (certain year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def players_in_game(game, side='away'):\n",
    "    \"\"\"Return unique set of `player_id`s for a certain `nflgame.Game` for the given `side`.\"\"\"\n",
    "    cats = ['receiving', 'defense', 'punting', 'kickret', 'kicking', 'rushing', 'puntret', 'passing', 'fumbles']\n",
    "    list_of_list_of_player_ids = (list(game.data[side]['stats'][cat].keys())\n",
    "                                  for cat in cats if cat in game.data[side]['stats'])\n",
    "    all_player_id = itertools.chain(*list_of_list_of_player_ids)\n",
    "    return set(all_player_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def active_players_in_year(year):\n",
    "    players = set()\n",
    "    for game in nflgame.games(year):\n",
    "        for side in ['away', 'home']:\n",
    "            players.update(players_in_game(game, side=side))\n",
    "    return players"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def lookup_player_name(player_id):\n",
    "    \"\"\"\n",
    "    Return the name (str) of a player, given his player_id (str).\n",
    "    \"\"\"\n",
    "    return nflgame.players[player_id].full_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def opposite_side(side):\n",
    "    if side == 'home':\n",
    "        return 'away'\n",
    "    elif side == 'away':\n",
    "        return 'home'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cats=('receiving', 'defense', 'punting', 'kickret', 'kicking', 'rushing', 'puntret', 'passing', 'fumbles')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def collect_stats_for_keys(keys, games):\n",
    "    cats = [ cat for cat, key in keys]\n",
    "    columns = ['site', 'date', 'week', 'team', 'op_team', 'season']\n",
    "    columns += [ cat+\"_\"+key for cat, key in keys]\n",
    "    tmp_list = []\n",
    "    tmp_index = []\n",
    "    for game in games:\n",
    "        playing = {}\n",
    "        for site in ['away', 'home']:\n",
    "            for cat in cats:\n",
    "                if cat not in game.data[site]['stats']:\n",
    "                    continue\n",
    "                stat = game.data[site]['stats'][cat]\n",
    "                for player_id in stat:\n",
    "                    playing[player_id] = defaultdict(lambda: None)\n",
    "                    for key in [key for c, key in keys if c == cat]:\n",
    "                        if key in stat[player_id]:                            \n",
    "                            playing[player_id][cat+\"_\"+key] = stat[player_id][key]\n",
    "                    playing[player_id]['site'] = site\n",
    "                    playing[player_id]['date'] = pd.datetime(game.schedule['year'],\n",
    "                                                             game.schedule['month'],\n",
    "                                                             game.schedule['day'],\n",
    "                                                             int(game.schedule['time'].split(':')[0]))\n",
    "                    playing[player_id]['week'] = game.schedule['week']\n",
    "                    playing[player_id]['team'] = game.data[site]['abbr']\n",
    "                    playing[player_id]['op_team'] = game.data[opposite_side(site)]['abbr']\n",
    "                    playing[player_id]['season'] = game.season()\n",
    "        for player_id in playing:\n",
    "            tmp_index.append([player_id, game.eid])\n",
    "            tmp_list.append([playing[player_id][col] for col in columns])\n",
    "    return pd.DataFrame(tmp_list,\n",
    "                        index=pd.MultiIndex.from_tuples(tmp_index, names=('player_id', 'eid')),\n",
    "                        columns=columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set = list(itertools.chain(*(nflgame.games(year) for year in [2009, 2010, 2011, 2012, 2013])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_set = list(itertools.chain(*(nflgame.games(year) for year in [2014])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "g1 = train_set[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "offense_keys= [('receiving', 'tds'),\n",
    "               ('receiving', 'yds'),\n",
    "               ('receiving', 'rec'),\n",
    "               ('receiving', 'lng'),\n",
    "               ('passing', 'yds'),\n",
    "               ('passing', 'tds'),\n",
    "               ('passing', 'att'),\n",
    "               ('passing', 'cmp'),\n",
    "               ('passing', 'ints'),\n",
    "               ('rushing', 'yds'),\n",
    "               ('rushing', 'tds'),\n",
    "               ('rushing', 'att'),\n",
    "               ('rushing', 'lng'),\n",
    "               ('fumbles', 'yds'),\n",
    "               ('fumbles', 'rcv'),\n",
    "               ('fumbles', 'trcv'),\n",
    "               ('fumbles', 'tot'),\n",
    "               ('fumbles', 'lost'),\n",
    "               ('kickret', 'tds'),\n",
    "               ('kickret', 'avg'),\n",
    "               ('kickret', 'ret'),\n",
    "               ('kicking', 'fga'),\n",
    "               ('kicking', 'fgyds'),\n",
    "               ('kicking', 'xpb'),\n",
    "               ('kicking', 'xptot'),\n",
    "               ('kicking', 'xpa'),\n",
    "               ('kicking', 'xpmissed'),\n",
    "               ('kicking', 'xpmade'),\n",
    "               ('kicking', 'fgm'),\n",
    "               ('kicking', 'totpfg'),\n",
    "               ('puntret', 'lng'),\n",
    "               ('puntret', 'ret'),\n",
    "               ('puntret', 'tds'),\n",
    "               ('puntret', 'avg')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "defense_keys=[('defense', 'ffum'),\n",
    "              ('defense', 'ast'),\n",
    "              ('defense', 'int'),\n",
    "              ('defense', 'sk'),\n",
    "              ('defense', 'tkl'),\n",
    "              ('punting', 'pts'),\n",
    "              ('punting', 'yds'),\n",
    "              ('punting', 'i20'),\n",
    "              ('punting', 'avg'),\n",
    "              ('punting', 'lng')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = collect_stats_for_keys(keys=offense_keys+defense_keys,\n",
    "                                 games=train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "offense = defense = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#offense['passing_success_rate'] = offense['passing_cmp'] / offense['passing_att']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "opposite_players = [ players_in_game(nflgame.game.Game(i[1]), side=opposite_side(offense.loc[i]['site'])) for i in offense.index.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "[ position(pid) for pid in opposite_players[4] ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "offense_means = offense.groupby(level='player_id').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense_means.loc['00-0010346']['passing_yds']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense_means.loc['00-0023459']['passing_yds']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lookup_player_name('00-0023459')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense_means.sort('passing_tds', ascending=False)['passing_tds'].head(200).plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense.sort('passing_yds', ascending=False)['passing_yds'].head(200).plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "g = sns.PairGrid(offense, vars=['passing_yds', 'passing_tds', 'passing_success_rate'],\n",
    "                 hue='op_team', palette='husl')\n",
    "g.map(plt.scatter, alpha=0.5)\n",
    "g.add_legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense[offense['season']>=2012].groupby('op_team').mean()[['receiving_tds', 'receiving_yds','passing_yds','passing_tds', 'passing_att', 'passing_cmp', 'passing_ints', 'rushing_yds', 'rushing_tds', 'passing_success_rate']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "qgrid.show_grid(_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "offense[(offense['season']==2013) & (offense['op_team'] == 'CAR')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.scatter_matrix(offense, alpha=0.2, figsize=(14, 14), diagonal='kde');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pandas.tools.plotting import lag_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "offense2014_by_player = offense2014.groupby(level='player_id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining active players of 2014\n",
    "\n",
    "...by basically saying that they need to participate in at least 10 games. Participate means that nfl has some stats for them in the database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n = 10\n",
    "# Doing a set comprehension here to get unique ids after we grouped and counted the 'site' column.\n",
    "# The 'site' column is probably always there, so it is a safe bet, but we could used other colums \n",
    "# that do not contain nans.\n",
    "active_players_2014 = {ind[0]\n",
    "    for ind in offense2014_by_player.filter(lambda x: x['site'].count() >= n).index.values}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So lets see how many players we removed by looking at the 0-th element for all the indices in `offense2014`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len({ v[0] for v in offense2014.index.values})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "... compared to the number of interesting players in 2014:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(active_players_2014)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for player in active_players_2014:\n",
    "    try:\n",
    "        lag_plot(offense2014.sort('date').loc[player]['receiving_yds'], s=1)\n",
    "    except KeyError:\n",
    "        print(\".\",end=\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "autocorrelation_plot?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pandas.tools.plotting import autocorrelation_plot\n",
    "for player in active_players_2014:\n",
    "    try:\n",
    "        autocorrelation_plot(offense2014.sort('date').loc[player]['receiving_yds'],lw=0.5)\n",
    "    except KeyError:\n",
    "        print(\".\",end=\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class SimpleOracle():\n",
    "\n",
    "    def __init__(self, offense, defense):\n",
    "        self.offense = offense\n",
    "        self.defense = defense\n",
    "    \n",
    "    def predict(self, player_id, what='passing_yds'):\n",
    "        try:\n",
    "            return offense.groupby(level='player_id').mean().loc[player_id][what]\n",
    "        except KeyError as e:\n",
    "            return np.nan\n",
    "    \n",
    "    def predict_passing_yds(self, player_id):\n",
    "        return self.predict(player_id, what='passing_yds')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SimpleOpTeamBasedOracle():\n",
    "    \n",
    "    def __init__(self, offense, defense):\n",
    "        self.offense = offense\n",
    "        self.defense = defense\n",
    "    \n",
    "    def predict(self, player_id, what='passing_yds', op_team=None):\n",
    "        try:\n",
    "            return offense[offense['op_team'] == op_team].groupby(level='player_id').mean().loc[player_id][what]\n",
    "        except KeyError:\n",
    "            return np.nan\n",
    "    \n",
    "    def predict_passing_yds(self, player_id, op_team):\n",
    "        return self.predict(player_id, what='passing_yds', op_team=op_team)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class OpTeamBasedOracle():\n",
    "    \n",
    "    def __init__(self, offense, defense):\n",
    "        self.offense = offense\n",
    "        self.defense = defense\n",
    "    \n",
    "    def predict(self, what='passing_yds', op_team=None):\n",
    "        try:\n",
    "            return offense[offense['op_team'] == op_team][what].mean()\n",
    "        except KeyError:\n",
    "            return np.nan\n",
    "    \n",
    "    def predict_passing_yds(self, op_team):\n",
    "        return self.predict(what='passing_yds', op_team=op_team)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def append_ops_columns(data, player_ids=None, keys=None):\n",
    "    mean_values = data.groupby(level='player_id').mean()\n",
    "    \n",
    "    if player_ids is None:\n",
    "        # If not provided, make a unique `Set` of player ids from the offense data\n",
    "        player_ids = { i[0] for i in data.index.values }\n",
    "    # Creating the new column names by going through the offense_keys\n",
    "    # and prepending \"ops_\".\n",
    "    new_columns = {}\n",
    "    for key in keys:\n",
    "        new_columns[\"ops_{0}_{1}\".format(*key)] = []\n",
    "    bar = pyprind.ProgBar(len(player_ids))\n",
    "    for player_id in player_ids:\n",
    "        for game_id, row in data.loc[player_id].iterrows():\n",
    "            opposing_players = players_in_game(nflgame.game.Game(game_id),\n",
    "                                               side=opposite_side(row['site']))\n",
    "            for what in keys:\n",
    "                what_key = \"{0}_{1}\".format(*what)\n",
    "                tmp = 0.0\n",
    "                for op_id in opposing_players:\n",
    "                    value = mean_values.loc[op_id][what_key]\n",
    "                    if not np.isnan(value):\n",
    "                        tmp += value\n",
    "                new_columns[\"ops_\"+what_key].append(tmp)\n",
    "        bar.update()\n",
    "    \n",
    "    # making a new DataFrame for the new columns (ops = opposing player's sum)\n",
    "    for column_name in new_columns:\n",
    "        data[column_name] = new_columns[column_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "append_ops_columns(data, keys=offense_keys)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some ideas for better oracles:\n",
    "\n",
    "- For the simple predictor don't use so many seasons but just the last season\n",
    "- check if there is a correlation if the game was on \"home\" or \"away\". That would be an easy plot based on the data we already have.\n",
    "- Incorporating the time (so treating performance as a time series) by looking at the previous n games.\n",
    "- we don't have these: (get value if injuries are reported)\n",
    "- weather\n",
    "- how about the position of the team in the league? Can we get the position from some kind of top list to the time of each game?\n",
    "- looking at the opposing players (their ids or something)\n",
    "- instead of individual opponents: Just use the mean value of each opponent that is going to play to make up new columns like \"op_intercepts\", \"op_fumble_successes\" ... and these columns could be the sum over all player of the opposing team. So we have certain fixed columns and don't have to care about individual player_ids and such.\n",
    "- same as for opponents, the mean value of the stats of his teammates might also matter\n",
    "- "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predict(playerid, team,"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Seeing how often the `op_team` was \"HOU\" for a certain \"player_id\", we realize that this is a *very* small sample size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense[offense['op_team'] == 'HOU'].groupby(level='player_id').groups[nflgame.find(\"Peyton Manning\")[0].playerid]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense2014 = collect_stats_for_keys(keys=[('receiving', 'tds'),\n",
    "                                       ('receiving', 'yds'),\n",
    "                                       ('passing', 'yds'),\n",
    "                                       ('passing', 'tds'),\n",
    "                                       ('passing', 'att'),\n",
    "                                       ('passing', 'cmp'),\n",
    "                                       ('passing', 'ints'),\n",
    "                                       ('rushing', 'yds'),\n",
    "                                       ('rushing', 'tds')],\n",
    "                                    games=test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "defense2014 = collect_stats_for_keys(keys=[('defense','ffum'),\n",
    "                                       ('defense','ast'),\n",
    "                                       ('defense','int'),\n",
    "                                       ('defense','sk')],\n",
    "                                    games=test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "simple_oracle_predictions = []\n",
    "simple_opteam_based_oracle_predictions = []\n",
    "opteam_based_oracle_predictions = []\n",
    "progbar = pyprind.ProgBar(len(offense2014))\n",
    "simple = SimpleOracle(offense, defense)\n",
    "simple_opteam = SimpleOpTeamBasedOracle(offense, defense)\n",
    "opteam = OpTeamBasedOracle(offense, defense)\n",
    "\n",
    "for index, row in offense2014.iterrows():\n",
    "    progbar.update()\n",
    "    pred1 = simple.predict_passing_yds(index[0])\n",
    "    pred2 = simple_opteam.predict_passing_yds(\n",
    "        index[0],\n",
    "        op_team=row['op_team'])\n",
    "    pred3 = opteam.predict_passing_yds(op_team=row['op_team'])\n",
    "    simple_oracle_predictions.append(pred1)\n",
    "    simple_opteam_based_oracle_predictions.append(pred2)\n",
    "    opteam_based_oracle_predictions.append(pred3)\n",
    "    \n",
    "\n",
    "offense2014['sp_pass_yds'] = simple_oracle_predictions\n",
    "offense2014['so_pass_yds'] = simple_opteam_based_oracle_predictions\n",
    "offense2014['op_pass_yds'] = opteam_based_oracle_predictions\n",
    "offense2014['sp_pass_yds_off'] = offense2014['passing_yds'] - offense2014['sp_pass_yds']\n",
    "offense2014['so_pass_yds_off'] = offense2014['passing_yds'] - offense2014['so_pass_yds']\n",
    "offense2014['op_pass_yds_off'] = offense2014['passing_yds'] - offense2014['op_pass_yds']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense2014[['op_team','passing_yds','sp_pass_yds', 'so_pass_yds', 'op_pass_yds', 'sp_pass_yds_off', 'so_pass_yds_off', 'op_pass_yds_off']].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense2014[['sp_pass_yds_off','op_pass_yds_off']].dropna().boxplot(return_type='axes')\n",
    "offense2014[['sp_pass_yds','op_pass_yds']].dropna().plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "seaborn.jointplot?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pd.scatter_matrix(offense2014[['passing_yds','sp_pass_yds','op_pass_yds']].dropna(), alpha=1, figsize=(10, 10), diagonal='kde')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense2014['so_pass_yds_off'].dropna().abs().plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "offense2014['so_pass_yds_off'].dropna().abs().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense2014[['sp_pass_yds_off', 'op_pass_yds_off', 'passing_yds']]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "offense2014[['sp_pass_yds_off', 'op_pass_yds_off', 'passing_yds']].plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "g1 = games[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "g1.data['home']['stats'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "g1.data['home']['stats']['team']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def color_players(x):\n",
    "    if x == '00-0010346':\n",
    "        return 'orange'\n",
    "    elif x == '00-0023459':\n",
    "        return 'green'\n",
    "    else:\n",
    "        return 'white'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lookup_player_name('00-0023459')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "offense_means.loc[['00-0023459','00-0010346']]['passing_yds']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense.groupby(level='player_id').std().loc[['00-0023459','00-0010346']]['passing_yds']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.unique(np.array(list(color_players(v[0]) for v in offense.index.values)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense.plot(kind='scatter', x='passing_success_rate', y='passing_yds',\n",
    "             c=list(color_players(v[0]) for v in offense.index.values),\n",
    "             s=offense['passing_att'], figsize=(16, 6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense_means.index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense_means.plot(kind='scatter', x='passing_success_rate', y='passing_yds',\n",
    "                   c=list(color_players(v) for v in offense_means.index.values),\n",
    "                   s=offense_means['passing_att']*2, figsize=(16, 6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I set the size of the circles to the passting_att. right.\n",
    "Now you see the \"outliers\" with success_rate 1.0 so tiny and it is clear that they don't matter."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I see. I dunno....\n",
    "good point ... "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense.plot(kind='scatter', x='passing_att', y='passing_yds', c='passing_tds', s=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense.plot(kind='scatter', x='passing_yds', y='passing_tds', s=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tmp = offense_means['passing_yds'].dropna().mean(level=0).sort(inplace=False).tail(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tmp.plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offense.loc[tmp.index[-1], '2009091312' ][['season','week','team','team']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "g1 = nflgame.games(*offense.loc[tmp.index[-1], '2009091312'][['season','week','team','team']])[0]\n",
    "g1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "g1.stats_home"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example for lookup_player_name of that `tmp` list that has the exceptionally good average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lookup_player_name(tmp.index[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What are the minimum number of players that have played in a game (by looking at all the statistics we have for each game)?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "players = active_players_in_year(2013)\n",
    "games = list(itertools.chain(*(nflgame.games(year) for year in [2009, 2010, 2011, 2012, 2013])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "games"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Just messing around"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def players_in_game(game, side='away'):\n",
    "    \"\"\"\n",
    "    Given a nflgame.Game object, return all the `player_id`s that are found in the stats for the given `side`.\n",
    "    \"\"\"\n",
    "    list_of_list_of_player_ids = [ list(game.data[side]['stats'][key].keys()) for key in\n",
    "                               ['receiving', 'defense', 'punting', 'kickret', 'kicking', 'rushing', 'puntret', 'passing', 'fumbles']\n",
    "                               if key in game.data[side]['stats']]\n",
    "    all_player_id = itertools.chain(*list_of_list_of_player_ids)\n",
    "    return set(all_player_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Demonstrate the ussage of nice_score of any game"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for game in nflgame.games(2014,1, home=\"IND\", away=\"IND\"):\n",
    "    print(game.nice_score())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How to use `nflgame.find` to lookup a player by (full) name and get his `player_id`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pl = nflgame.find(\"Andrew Luck\")[0]\n",
    "pl.player_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How to get the nicely (text) summary of the total stats for a certain period:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pl.stats(2013).formatted_stats()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The scheme after the ponints are assingned in Gregs fantasy league"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "greg_league_points = dict(points_passing_yards=20,\n",
    "                          points_passing_ints=-2,\n",
    "                          points_passing_twoptm=2,\n",
    "                          points_passing_tds=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fantasy_score(g1, player_id=pl.player_id, **greg_league_points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}