NFL_Blending.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/nfl-big-data-bowl-2020/train.csv\n",
      "/kaggle/input/nfl-big-data-bowl-2020/kaggle/competitions/nflrush/sample_submission.csv.encrypted\n",
      "/kaggle/input/nfl-big-data-bowl-2020/kaggle/competitions/nflrush/competition.cpython-36m-x86_64-linux-gnu.so\n",
      "/kaggle/input/nfl-big-data-bowl-2020/kaggle/competitions/nflrush/test.csv.encrypted\n",
      "/kaggle/input/nfl-big-data-bowl-2020/kaggle/competitions/nflrush/__init__.py\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from kaggle.competitions import nflrush\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "\n",
    "import datetime\n",
    "import time\n",
    "\n",
    "from string import punctuation\n",
    "import re\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "\n",
    "#Modeling\n",
    "#https://www.kaggle.com/bestpredict/location-eda-8eb410\n",
    "\n",
    "import os\n",
    "TRAIN_ABLE_FALSE=True\n",
    "if TRAIN_ABLE_FALSE:\n",
    "    os.environ['CUDA_VISIBLE_DEVICES'] = \"1\"\n",
    "\n",
    "import sklearn.metrics as mtr\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from keras.layers import Dense,Input,Flatten,concatenate,Dropout,Lambda,BatchNormalization\n",
    "from keras import optimizers\n",
    "from keras.models import Model\n",
    "import keras.backend as K\n",
    "from keras.callbacks import Callback\n",
    "from  keras.callbacks import EarlyStopping,ModelCheckpoint\n",
    "\n",
    "TRAIN_OFFLINE = False\n",
    "\n",
    "pd.set_option('display.max_columns', 50)\n",
    "pd.set_option('display.max_rows', 500)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.61 s, sys: 1.07 s, total: 6.68 s\n",
      "Wall time: 6.65 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train = pd.read_csv('../input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})\n",
    "train = pd.DataFrame(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tools for Preprocessing\n",
    "\n",
    "def clean_StadiumType(txt):\n",
    "    if pd.isna(txt):\n",
    "        return np.nan\n",
    "    txt = txt.lower()\n",
    "    txt = ''.join([c for c in txt if c not in punctuation])\n",
    "    txt = re.sub(' +', ' ', txt)\n",
    "    txt = txt.strip()\n",
    "    txt = txt.replace('outside', 'outdoor')\n",
    "    txt = txt.replace('outdor', 'outdoor')\n",
    "    txt = txt.replace('outddors', 'outdoor')\n",
    "    txt = txt.replace('outdoors', 'outdoor')\n",
    "    txt = txt.replace('oudoor', 'outdoor')\n",
    "    txt = txt.replace('indoors', 'indoor')\n",
    "    txt = txt.replace('ourdoor', 'outdoor')\n",
    "    txt = txt.replace('retractable', 'rtr.')\n",
    "    return txt\n",
    "\n",
    "def transform_StadiumType(txt):\n",
    "    if pd.isna(txt):\n",
    "        return np.nan\n",
    "    if 'outdoor' in txt or 'open' in txt:\n",
    "        return 2\n",
    "    if 'indoor' in txt or 'closed' in txt:\n",
    "        return 0\n",
    "    if 'dome' in txt:\n",
    "        return 1\n",
    "    \n",
    "    return np.nan\n",
    "\n",
    "Turf = {'Field Turf':1, 'A-Turf Titan':1, 'Grass':0, 'UBU Sports Speed S5-M':1, \n",
    "        'Artificial':1, 'DD GrassMaster':1, 'Natural Grass':0, \n",
    "        'UBU Speed Series-S5-M':1, 'FieldTurf':1, 'FieldTurf 360':1, 'Natural grass':0, 'grass':0, \n",
    "        'Natural':0, 'Artifical':1, 'FieldTurf360':1, 'Naturall Grass':0, 'Field turf':1, \n",
    "        'SISGrass':1, 'Twenty-Four/Seven Turf':1, 'natural grass':0} \n",
    "\n",
    "def str_to_float(txt):\n",
    "    try:\n",
    "        return float(txt)\n",
    "    except:\n",
    "        return -1\n",
    "    \n",
    "def clean_WindDirection(txt):\n",
    "    if pd.isna(txt):\n",
    "        return np.nan\n",
    "    txt = txt.lower()\n",
    "    txt = ''.join([c for c in txt if c not in punctuation])\n",
    "    txt = txt.replace('from', '')\n",
    "    txt = txt.replace(' ', '')\n",
    "    txt = txt.replace('north', 'N')\n",
    "    txt = txt.replace('south', 'S')\n",
    "    txt = txt.replace('west', 'W')\n",
    "    txt = txt.replace('east', 'E')\n",
    "    return txt\n",
    "\n",
    "def transform_WindDirection(txt):\n",
    "    if pd.isna(txt):\n",
    "        return np.nan\n",
    "    \n",
    "    if txt=='n':\n",
    "        return 0\n",
    "    if txt=='nne' or txt=='nen':\n",
    "        return 1/8\n",
    "    if txt=='ne':\n",
    "        return 2/8\n",
    "    if txt=='ene' or txt=='nee':\n",
    "        return 3/8\n",
    "    if txt=='e':\n",
    "        return 4/8\n",
    "    if txt=='ese' or txt=='see':\n",
    "        return 5/8\n",
    "    if txt=='se':\n",
    "        return 6/8\n",
    "    if txt=='ses' or txt=='sse':\n",
    "        return 7/8\n",
    "    if txt=='s':\n",
    "        return 8/8\n",
    "    if txt=='ssw' or txt=='sws':\n",
    "        return 9/8\n",
    "    if txt=='sw':\n",
    "        return 10/8\n",
    "    if txt=='sww' or txt=='wsw':\n",
    "        return 11/8\n",
    "    if txt=='w':\n",
    "        return 12/8\n",
    "    if txt=='wnw' or txt=='nww':\n",
    "        return 13/8\n",
    "    if txt=='nw':\n",
    "        return 14/8\n",
    "    if txt=='nwn' or txt=='nnw':\n",
    "        return 15/8\n",
    "    return np.nan\n",
    "\n",
    "def map_weather(txt):\n",
    "    ans = 1\n",
    "    if pd.isna(txt):\n",
    "        return 0\n",
    "    if 'partly' in txt:\n",
    "        ans*=0.5\n",
    "    if 'climate controlled' in txt or 'indoor' in txt:\n",
    "        return ans*3\n",
    "    if 'sunny' in txt or 'sun' in txt:\n",
    "        return ans*2\n",
    "    if 'clear' in txt:\n",
    "        return ans\n",
    "    if 'cloudy' in txt:\n",
    "        return -ans\n",
    "    if 'rain' in txt or 'rainy' in txt:\n",
    "        return -2*ans\n",
    "    if 'snow' in txt:\n",
    "        return -3*ans\n",
    "    return 0\n",
    "\n",
    "def OffensePersonnelSplit(x):\n",
    "    dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}\n",
    "    for xx in x.split(\",\"):\n",
    "        xxs = xx.split(\" \")\n",
    "        dic[xxs[-1]] = int(xxs[-2])\n",
    "    return dic\n",
    "\n",
    "def DefensePersonnelSplit(x):\n",
    "    dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}\n",
    "    for xx in x.split(\",\"):\n",
    "        xxs = xx.split(\" \")\n",
    "        dic[xxs[-1]] = int(xxs[-2])\n",
    "    return dic\n",
    "\n",
    "def YardGain(train):\n",
    "    YG = 0\n",
    "    if train['YardLine'] == 50:\n",
    "        YG = 50\n",
    "    #Home team offense at home field\n",
    "    elif (train['FieldPosition'] == train['HomeTeamAbbr']) & (train['isOffenseHome'] == True):\n",
    "        YG = train['YardLine']\n",
    "    #Home team offense at away field\n",
    "    elif (train['FieldPosition'] == train['VisitorTeamAbbr']) & (train['isOffenseHome'] == True):\n",
    "        YG = 100 - train['YardLine']\n",
    "    #Away team offense at away field\n",
    "    elif (train['FieldPosition'] == train['VisitorTeamAbbr']) & (train['isOffenseHome'] == False):    \n",
    "        YG = train['YardLine']\n",
    "    #Away team offense at home field\n",
    "    elif (train['FieldPosition'] == train['HomeTeamAbbr']) & (train['isOffenseHome'] == False):\n",
    "        YG = 100 - train['YardLine']\n",
    "    return YG\n",
    "\n",
    "def DistanceFromScrimmageLine(Df):\n",
    "    Distance = 0\n",
    "    if Df.PlayDirection == 'left':\n",
    "        Distance = - (10 + Df['OffenseYardLine'] - (120 - Df['X_Rusher']))\n",
    "    if Df.PlayDirection == 'right':\n",
    "        Distance = - (10 + Df['OffenseYardLine'] - (Df['X_Rusher']))\n",
    "    return Distance\n",
    "\n",
    "import matplotlib.patches as patches\n",
    "def PlayVisualization(linenumbers=True,\n",
    "                          endzones=True,\n",
    "                          highlight_line=False,\n",
    "                          highlight_line_number=50,\n",
    "                          highlighted_name='Line of Scrimmage',\n",
    "                          fifty_is_los=False,\n",
    "                          figsize=(12, 6.33)):\n",
    "    \"\"\"\n",
    "    Function that plots the football field for viewing plays.\n",
    "    Allows for showing or hiding endzones.\n",
    "    \"\"\"\n",
    "    rect = patches.Rectangle((0, 0), 120, 53.3, linewidth=0.1,\n",
    "                             edgecolor='r', facecolor='darkgreen', zorder=0)\n",
    "\n",
    "    fig, ax = plt.subplots(1, figsize=figsize)\n",
    "    ax.add_patch(rect)\n",
    "\n",
    "    plt.plot([10, 10, 10, 20, 20, 30, 30, 40, 40, 50, 50, 60, 60, 70, 70, 80,\n",
    "              80, 90, 90, 100, 100, 110, 110, 120, 0, 0, 120, 120],\n",
    "             [0, 0, 53.3, 53.3, 0, 0, 53.3, 53.3, 0, 0, 53.3, 53.3, 0, 0, 53.3,\n",
    "              53.3, 0, 0, 53.3, 53.3, 0, 0, 53.3, 53.3, 53.3, 0, 0, 53.3],\n",
    "             color='white')\n",
    "    if fifty_is_los:\n",
    "        plt.plot([60, 60], [0, 53.3], color='gold')\n",
    "        plt.text(62, 50, '<- Player Yardline at Snap', color='gold')\n",
    "    # Endzones\n",
    "    if endzones:\n",
    "        ez1 = patches.Rectangle((0, 0), 10, 53.3,\n",
    "                                linewidth=0.1,\n",
    "                                edgecolor='r',\n",
    "                                facecolor='blue',\n",
    "                                alpha=0.2,\n",
    "                                zorder=0)\n",
    "        ez2 = patches.Rectangle((110, 0), 120, 53.3,\n",
    "                                linewidth=0.1,\n",
    "                                edgecolor='r',\n",
    "                                facecolor='blue',\n",
    "                                alpha=0.2,\n",
    "                                zorder=0)\n",
    "        ax.add_patch(ez1)\n",
    "        ax.add_patch(ez2)\n",
    "    plt.xlim(0, 120)\n",
    "    plt.ylim(-5, 58.3)\n",
    "    plt.axis('off')\n",
    "    if linenumbers:\n",
    "        for x in range(20, 110, 10):\n",
    "            numb = x\n",
    "            if x > 50:\n",
    "                numb = 120 - x\n",
    "            plt.text(x, 5, str(numb - 10),\n",
    "                     horizontalalignment='center',\n",
    "                     fontsize=20,  # fontname='Arial',\n",
    "                     color='white')\n",
    "            plt.text(x - 0.95, 53.3 - 5, str(numb - 10),\n",
    "                     horizontalalignment='center',\n",
    "                     fontsize=20,  # fontname='Arial',\n",
    "                     color='white', rotation=180)\n",
    "    if endzones:\n",
    "        hash_range = range(11, 110)\n",
    "    else:\n",
    "        hash_range = range(1, 120)\n",
    "\n",
    "    for x in hash_range:\n",
    "        ax.plot([x, x], [0.4, 0.7], color='white')\n",
    "        ax.plot([x, x], [53.0, 52.5], color='white')\n",
    "        ax.plot([x, x], [22.91, 23.57], color='white')\n",
    "        ax.plot([x, x], [29.73, 30.39], color='white')\n",
    "\n",
    "    if highlight_line:\n",
    "        hl = highlight_line_number + 10\n",
    "        plt.plot([hl, hl], [0, 53.3], color='yellow')\n",
    "        plt.text(hl + 2, 50, '<- {}'.format(highlighted_name),\n",
    "                 color='yellow')\n",
    "    return fig, ax\n",
    "\n",
    "def TeamAbbrMapper(Abbr):\n",
    "    if Abbr == 'ARI':\n",
    "        return 'ARZ'\n",
    "    if Abbr == 'BAL':\n",
    "        return 'BLT'\n",
    "    if Abbr == 'CLE':\n",
    "        return 'CLV'\n",
    "    if Abbr == 'HOU':\n",
    "        return 'HST'\n",
    "    else:\n",
    "        return Abbr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "def OffenseFormationMapper(Formation):\n",
    "    Split = Formation.split(\",\")\n",
    "    Positions = []\n",
    "    Nums = []\n",
    "    \n",
    "    for s in Split:\n",
    "        Position = s.split(\" \")[-1]\n",
    "        Positions.append(Position)\n",
    "        \n",
    "        Num = s.split(\" \")[-2]\n",
    "        Nums.append(Num)\n",
    "\n",
    "    PositionLists = []\n",
    "    for i in range(len(Positions)):\n",
    "        PositionList = list(itertools.repeat(Positions[i], int(Nums[i])))\n",
    "        PositionLists = PositionLists + PositionList\n",
    "\n",
    "    OffenseDic = {'OL' : 0,'QB' : 0,'RB' : 0,'TE' : 0,'WR' : 0}\n",
    "\n",
    "    OffenseDic['OL'] = PositionLists.count('OL')\n",
    "    OffenseDic['QB'] = PositionLists.count('QB')\n",
    "    OffenseDic['RB'] = PositionLists.count('RB')\n",
    "    OffenseDic['TE'] = PositionLists.count('TE')\n",
    "    OffenseDic['WR'] = PositionLists.count('WR')\n",
    "    \n",
    "    return OffenseDic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def DefenseFormationMapper(Formation):\n",
    "    Split = Formation.split(\",\")\n",
    "    Positions = []\n",
    "    Nums = []\n",
    "    \n",
    "    for s in Split:\n",
    "        Position = s.split(\" \")[-1]\n",
    "        Positions.append(Position)\n",
    "        \n",
    "        Num = s.split(\" \")[-2]\n",
    "        Nums.append(Num)\n",
    "\n",
    "    PositionLists = []\n",
    "    for i in range(len(Positions)):\n",
    "        PositionList = list(itertools.repeat(Positions[i], int(Nums[i])))\n",
    "        PositionLists = PositionLists + PositionList\n",
    "\n",
    "    DefenseDic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}\n",
    "\n",
    "    DefenseDic['DB'] = PositionLists.count('DB')\n",
    "    DefenseDic['DL'] = PositionLists.count('DL')\n",
    "    DefenseDic['LB'] = PositionLists.count('LB')\n",
    "    DefenseDic['OL'] = PositionLists.count('OL')\n",
    "    \n",
    "    return DefenseDic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def PreprocessingNNTrainSet(train):\n",
    "    #GroupFeatures\n",
    "    GroupFeatures = ['Yards_OffenseFormation_mean', 'Yards_DefendersInTheBox_mean', 'Yards_Down_mean', 'Yards_PossessionTeam_mean']\n",
    "    OffenseFormationTemp = train.groupby('OffenseFormation')['Yards'].agg(['mean']).rename({'mean':'Yards_OffenseFormation_mean'},axis=1).astype('float64')\n",
    "    DefendersInTheBoxTemp = train.groupby('DefendersInTheBox')['Yards'].agg(['mean']).rename({'mean':'Yards_DefendersInTheBox_mean'},axis=1).astype('float64')\n",
    "    DownTemp = train.groupby('Down')['Yards'].agg(['mean']).rename({'mean':'Yards_Down_mean'},axis=1).astype('float64')\n",
    "    PossessionTeamTemp = train.groupby('PossessionTeam')['Yards'].agg(['mean']).rename({'mean':'Yards_PossessionTeam_mean'},axis=1).astype('float64')\n",
    "\n",
    "    OffenseFormationDic = OffenseFormationTemp.to_dict()\n",
    "    DefendersInTheBoxDic = DefendersInTheBoxTemp.to_dict()\n",
    "    DownDic = DownTemp.to_dict()\n",
    "    PossessionTeamDic = PossessionTeamTemp.to_dict()\n",
    "\n",
    "    GroupFeatureDic = {**OffenseFormationDic, **DefendersInTheBoxDic, **DownDic, **PossessionTeamDic}\n",
    "\n",
    "    train['Yards_OffenseFormation_mean'] = train['OffenseFormation'].replace(GroupFeatureDic['Yards_OffenseFormation_mean']).astype('float64')\n",
    "    train['Yards_DefendersInTheBox_mean'] = train['DefendersInTheBox'].replace(GroupFeatureDic['Yards_DefendersInTheBox_mean'])\n",
    "    train['Yards_Down_mean'] = train['Down'].replace(GroupFeatureDic['Yards_Down_mean'])\n",
    "    train['Yards_PossessionTeam_mean'] = train['PossessionTeam'].replace(GroupFeatureDic['Yards_PossessionTeam_mean']).astype('float64')\n",
    "    \n",
    "    #Ground & Weather Conditions \n",
    "    ConditionFeatures = ['StadiumType','Turf','GameWeather','Temperature','Humidity','WindSpeed',\n",
    "    'WindDirection', 'Stadium', 'Location']\n",
    "    Drop = ['Stadium','Location']\n",
    "    Drops = Drop\n",
    "\n",
    "    train['StadiumType'] = train['StadiumType'].apply(clean_StadiumType)\n",
    "    train['StadiumType'] = train['StadiumType'].apply(transform_StadiumType)\n",
    "    train['Turf'] = train['Turf'].map(Turf)\n",
    "\n",
    "    #WindSpeed\n",
    "    train['WindSpeed'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '').strip() if not pd.isna(x) else x)\n",
    "    train['WindSpeed'] = train['WindSpeed'].apply(lambda x: (int(x.split('-')[0])+int(x.split('-')[1]))/2 if not pd.isna(x) and '-' in x else x)\n",
    "    train['WindSpeed'] = train['WindSpeed'].apply(lambda x: (int(x.split()[0])+int(x.split()[-1]))/2 if not pd.isna(x) and type(x)!=float and 'gusts up to' in x else x)\n",
    "    train['WindSpeed'] = train['WindSpeed'].apply(str_to_float)\n",
    "\n",
    "    #WindDirection\n",
    "    train['WindDirection'] = train['WindDirection'].apply(clean_WindDirection)\n",
    "    train['WindDirection'] = train['WindDirection'].apply(transform_WindDirection)\n",
    "\n",
    "    #GameWeather\n",
    "    train['GameWeather'] = train['GameWeather'].str.lower()\n",
    "    indoor = \"indoor\"\n",
    "    train['GameWeather'] = train['GameWeather'].apply(lambda x: indoor if not pd.isna(x) and indoor in x else x)\n",
    "    train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy').replace('clouidy', 'cloudy').replace('party', 'partly') if not pd.isna(x) else x)\n",
    "    train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear') if not pd.isna(x) else x)\n",
    "    train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('skies', '').replace(\"mostly\", \"\").strip() if not pd.isna(x) else x)\n",
    "    train['GameWeather'] = train['GameWeather'].apply(map_weather)\n",
    "\n",
    "    #Drop Location\n",
    "    train = train.drop(columns = Drop)\n",
    "\n",
    "    #Time Features\n",
    "    TimeFeatures = ['TimeHandoff','TimeSnap','Season','Quarter','Week']\n",
    "    Drop = ['TimeHandoff','TimeSnap','GameClock']\n",
    "    Drops = Drops + Drop\n",
    "\n",
    "    #Handle Time Data\n",
    "    train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, \"%Y-%m-%dT%H:%M:%S.%fZ\"))\n",
    "    train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, \"%Y-%m-%dT%H:%M:%S.%fZ\"))\n",
    "    train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, \"%m/%d/%Y\"))\n",
    "\n",
    "    #New Features\n",
    "    train['SnaptoHandoff'] = train.apply(lambda row: (row['TimeHandoff'] - row['TimeSnap']).total_seconds(), axis=1)\n",
    "    train['QuarterSecondsLeft'] = train['GameClock'].apply(lambda x: int(x[0:2])*60 + int(x[3:5]))\n",
    "    train['GameSecondsLeft'] = train['Quarter'].map({1:2700, 2:1800, 3:900, 4:0}) + train['QuarterSecondsLeft']\n",
    "    train['GameSecondsPassed'] = 3600 - train['GameSecondsLeft']\n",
    "    train['PlayerAge'] = 2019 - train.PlayerBirthDate.apply(lambda x : x.year)\n",
    "\n",
    "    #For 5th Quarter\n",
    "    train.loc[(train['Quarter'] == 5),'GameSecondsLeft'] = -1\n",
    "    train.loc[(train['Quarter'] == 5),'QuarterSecondsLeft'] = -1\n",
    "    train.loc[(train['Quarter'] == 5),'GameSecondsPassed'] = 3600\n",
    "\n",
    "    #Drop useless features\n",
    "    train = train.drop(columns = Drop)\n",
    "\n",
    "    #Team Features\n",
    "    TeamFeatures = ['Team','HomeTeamAbbr','VisitorTeamAbbr','HomeScoreBeforePlay',\n",
    "    'VisitorScoreBeforePlay','PossessionTeam','FieldPosition','Dis','YardLine','Down','Distance',\n",
    "    'OffenseFormation','OffensePersonnel','DefendersInTheBox','DefensePersonnel']\n",
    "    Drop = ['HomeTeamAbbr','VisitorTeamAbbr','HomeScoreBeforePlay','VisitorScoreBeforePlay',\n",
    "    'PossessionTeam', 'FieldPosition']\n",
    "    Drops = Drops + Drop\n",
    "\n",
    "    #Possession\n",
    "    #To Handle some typos\n",
    "    map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}\n",
    "\n",
    "    for abb in train['PossessionTeam'].unique():\n",
    "        map_abbr[abb] = abb\n",
    "\n",
    "    train['PossessionTeam'] = train['PossessionTeam'].map(map_abbr)\n",
    "    train['HomeTeamAbbr'] = train['HomeTeamAbbr'].map(map_abbr)\n",
    "    train['VisitorTeamAbbr'] = train['VisitorTeamAbbr'].map(map_abbr)\n",
    "\n",
    "    #New Features\n",
    "    train['OffenseTeamAbbr'] = train['PossessionTeam']\n",
    "    train['DefenseTeamAbbr'] = train.apply(lambda row : row['VisitorTeamAbbr'] if row['OffenseTeamAbbr'] == row['HomeTeamAbbr'] else row['HomeTeamAbbr'], axis = 1)\n",
    "    train['isOffenseHome'] = train['HomeTeamAbbr'] == train['PossessionTeam']\n",
    "    train['isOffenseField'] = train['FieldPosition'] == train['PossessionTeam']\n",
    "    train['OffenseLeadScore'] = train.apply(lambda row : row['HomeScoreBeforePlay'] - row['VisitorScoreBeforePlay'] if row['isOffenseHome'] == 'True' else row['VisitorScoreBeforePlay'] - row['HomeScoreBeforePlay'], axis = 1)\n",
    "    train['OffenseYardLine'] =  train.apply(YardGain, axis = 1)\n",
    "\n",
    "    #Strategies & Formations\n",
    "    FormationFeatures = ['OffensePersonnel','DefendersInTheBox',\n",
    "                         'DefensePersonnel']\n",
    "    Drop = ['OffensePersonnel','DefensePersonnel']\n",
    "    Drops.append(Drop)\n",
    "\n",
    "\n",
    "\n",
    "    #OffensivePersonnel\n",
    "    temp = train[\"OffensePersonnel\"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(OffenseFormationMapper(x)))\n",
    "    temp.columns = [\"Offense\" + c for c in temp.columns]\n",
    "    temp[\"PlayId\"] = train[\"PlayId\"].iloc[np.arange(0, len(train), 22)]\n",
    "    train = train.merge(temp, on = 'PlayId')\n",
    "\n",
    "    #DefensivePersonnel\n",
    "    temp = train[\"DefensePersonnel\"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(DefenseFormationMapper(x)))\n",
    "    temp.columns = [\"Defense\" + c for c in temp.columns]\n",
    "    temp[\"PlayId\"] = train[\"PlayId\"].iloc[np.arange(0, len(train), 22)]\n",
    "    train = train.merge(temp, on = 'PlayId')\n",
    "\n",
    "    #Drop\n",
    "    train = train.drop(columns = Drop)\n",
    "\n",
    "    #Player Features\n",
    "    PlayerFeatures = ['Team','DisplayName','JerseyNumber','NflIdRusher','PlayerCollegeName',\n",
    "    'NflId','X','Y','S','A','Orientation','Dir','PlayerHeight','PlayerWeight','Position',\n",
    "    'PlayerBirthDate']\n",
    "\n",
    "    Drop = ['Team','DisplayName','JerseyNumber','NflIdRusher','PlayerCollegeName','NflId',\n",
    "    'PlayerBirthDate']\n",
    "    Drops = Drops + Drop\n",
    "\n",
    "    #Height , Weight , BMI\n",
    "    train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))\n",
    "    train['PlayerBMI'] = 703*(train['PlayerWeight']/(train['PlayerHeight'])**2)\n",
    "\n",
    "    #Position Cleaning\n",
    "\n",
    "    train['Position'] = train.Position.replace('DE','DL')\n",
    "    train['Position'] = train.Position.replace('DT','DL')\n",
    "    train['Position'] = train.Position.replace('NT','DL')\n",
    "\n",
    "    train['Position'] = train.Position.replace('ILB','LB')\n",
    "    train['Position'] = train.Position.replace('MLB','LB')\n",
    "    train['Position'] = train.Position.replace('OLB','LB')\n",
    "\n",
    "    train['Position'] = train.Position.replace('CB','DB')\n",
    "\n",
    "    train['Position'] = train.Position.replace('FS','S')\n",
    "    train['Position'] = train.Position.replace('SS','S')\n",
    "    train['Position'] = train.Position.replace('SAF','S')\n",
    "\n",
    "    train['Position'] = train.Position.replace('HB','RB')\n",
    "\n",
    "    train['Position'] = train.Position.replace('C','OL')\n",
    "    train['Position'] = train.Position.replace('G','OL')\n",
    "    train['Position'] = train.Position.replace('T','OL')\n",
    "    train['Position'] = train.Position.replace('OT','OL')\n",
    "    train['Position'] = train.Position.replace('OG','OL')\n",
    "    train['Position'] = train.Position.replace('OG','OL')\n",
    "\n",
    "    #Position Encoding\n",
    "    train['Position'] = train.Position.replace('DL',-4)\n",
    "    train['Position'] = train.Position.replace('LB',-3)\n",
    "    train['Position'] = train.Position.replace('DB',-2)\n",
    "    train['Position'] = train.Position.replace('S',-1)\n",
    "\n",
    "    train['Position'] = train.Position.replace('RB',6)\n",
    "    train['Position'] = train.Position.replace('WR',5)\n",
    "    train['Position'] = train.Position.replace('FB',4)\n",
    "    train['Position'] = train.Position.replace('QB',3)\n",
    "    train['Position'] = train.Position.replace('OL',2)\n",
    "    train['Position'] = train.Position.replace('TE',1)\n",
    "\n",
    "    #Transform Orientation to Radian\n",
    "    train['Orientation_rad'] = np.mod(train.Orientation, 360) * math.pi/180.0\n",
    "\n",
    "    #Fix shifted orientation \n",
    "    train.loc[train.Season >= 2018, 'Orientation_rad'] = np.mod(train.loc[train.Season >= 2018, 'Orientation'] - 90, 360) * math.pi/180.0\n",
    "\n",
    "    #Overwrite Orientation\n",
    "    train['Orientation'] = train['Orientation_rad']\n",
    "    train.drop(columns = 'Orientation_rad',inplace = True)\n",
    "\n",
    "    #PlayerInfo Dataframe\n",
    "    PlayerInfo = train[['PlayId','DisplayName','NflId','NflIdRusher','X','Y','Position','Team','isOffenseHome','A','S','PlayerAge','PlayerWeight','PlayerHeight','PlayerBMI','Dir','Orientation']]\n",
    "    PlayerInfo.set_index('PlayId',inplace = True)\n",
    "\n",
    "    #Create RusherInfo Dataframe\n",
    "    RusherInfo = train[(train['NflId'] == train['NflIdRusher'])][['PlayId','DisplayName','Position','X','Y','A','S','Dir','Orientation','PlayerAge','PlayerHeight','PlayerWeight','PlayerBMI']].set_index('PlayId')[['DisplayName','Position','X','Y','A','S','Dir','Orientation','PlayerAge','PlayerHeight','PlayerWeight','PlayerBMI']]\n",
    "    RusherInfo.columns = ['RusherName','RusherPosition','X_Rusher','Y_Rusher','A_Rusher','S_Rusher','Dir_Rusher','Orientation_Rusher','PlayerAge_Rusher','PlayerHeight_Rusher','PlayerWeight_Rusher','PlayerBMI_Rusher']\n",
    "\n",
    "    #Add Player's performance\n",
    "    Top10Rusher = ['Christian McCaffrey','Kenyan Drake','Nick Chubb','Dalvin Cook','Ezekiel Elliott',\n",
    "    'Leonard Fournette','Josh Jacobs','Chris Carson','Marlon Mack','Lamar Jackson']\n",
    "    RusherInfo['isTop10Rusher'] = RusherInfo['RusherName'].isin(Top10Rusher)\n",
    "\n",
    "    #Rusher's vertical / horizontal physics\n",
    "    RadianAngle = (90 - RusherInfo['Dir_Rusher']) * np.pi / 180.0\n",
    "    RusherInfo['A_Horizontal_Rusher'] = np.abs(RusherInfo['A_Rusher'] * np.cos(RadianAngle))\n",
    "    RusherInfo['A_Vertical_Rusher'] = np.abs(RusherInfo['A_Rusher'] * np.sin(RadianAngle))\n",
    "\n",
    "    RusherInfo['S_Horizontal_Rusher'] = np.abs(RusherInfo['S_Rusher'] * np.cos(RadianAngle))\n",
    "    RusherInfo['S_Vertical_Rusher'] = np.abs(RusherInfo['S_Rusher'] * np.sin(RadianAngle))\n",
    "\n",
    "    RusherInfo['F_Rusher'] = np.sqrt(RusherInfo['A_Horizontal_Rusher']**2 + RusherInfo['A_Vertical_Rusher']**2) * RusherInfo['PlayerWeight_Rusher']\n",
    "    RusherInfo['KE_Rusher'] = (1/2) *  RusherInfo['PlayerWeight_Rusher'] * (RusherInfo['S_Horizontal_Rusher']**2 + RusherInfo['S_Vertical_Rusher']**2)\n",
    "\n",
    "    #Concat PlayerInfo & RusherInfo\n",
    "    InfoDf = pd.concat([PlayerInfo,RusherInfo],1)\n",
    "\n",
    "    #Create DistanceFromRusher / isOffense\n",
    "    InfoDf['DistanceFromRusher'] = np.sqrt((InfoDf['X_Rusher'] - InfoDf['X'])**2 + (InfoDf['Y_Rusher'] - InfoDf['Y'])**2)\n",
    "    InfoDf['isOffense'] = InfoDf.apply(lambda row : True if ((row['Team'] == 'home') & (row['isOffenseHome'] == True)) | (row['Team'] == 'away') & (row['isOffenseHome'] == False) else False, axis = 1)\n",
    "\n",
    "\n",
    "    #Nearest distance between defensive team & rusher\n",
    "    MinDistance = InfoDf[InfoDf['isOffense']==False][['DistanceFromRusher']].groupby('PlayId').agg(min)\n",
    "    MinDistance.columns = ['NearestDistanceFromRusher']\n",
    "    DistanceDf = pd.concat([InfoDf,MinDistance],1)\n",
    "\n",
    "    #Create DefenderInfo Dataframe\n",
    "    DefensivePosition = [4,-3,-2,-1]\n",
    "    DefenderInfo = DistanceDf[DistanceDf['isOffense'] == False]\n",
    "    NearestDefenderInfo = DefenderInfo[DefenderInfo['DistanceFromRusher'] == DefenderInfo['NearestDistanceFromRusher']]\n",
    "\n",
    "    DistanceDf['A_NearestDefender'] = NearestDefenderInfo['A']\n",
    "    DistanceDf['S_NearestDefender'] = NearestDefenderInfo['S']\n",
    "    DistanceDf['Orientation_NearestDefender'] = NearestDefenderInfo['Orientation']\n",
    "    DistanceDf['Dir_NearestDefender'] = NearestDefenderInfo['Dir']\n",
    "\n",
    "    RadianAngle = (90 - NearestDefenderInfo['Dir']) * np.pi / 180.0\n",
    "    DistanceDf['A_Horizontal_NearestDefender'] = np.abs(DistanceDf['A'] * np.cos(RadianAngle))\n",
    "    DistanceDf['A_Vertical_NearestDefender'] = np.abs(DistanceDf['A'] * np.sin(RadianAngle))\n",
    "\n",
    "    DistanceDf['S_Horizontal_NearestDefender'] = np.abs(DistanceDf['S'] * np.cos(RadianAngle))\n",
    "    DistanceDf['S_Vertical_NearestDefender'] = np.abs(DistanceDf['S'] * np.sin(RadianAngle))\n",
    "\n",
    "    DistanceDf['F_NearestDefender'] = np.sqrt(DistanceDf['A_Horizontal_NearestDefender']**2 + DistanceDf['A_Vertical_NearestDefender']**2) * DistanceDf['PlayerWeight']\n",
    "    DistanceDf['KE_Rusher'] = (1/2) *  DistanceDf['PlayerWeight'] * (DistanceDf['S_Horizontal_NearestDefender']**2 + DistanceDf['S_Vertical_NearestDefender']**2)\n",
    "\n",
    "    PlayerFeatures = ['PlayId','Position','X','Y','S','A','Dis','Orientation','Dir','PlayerHeight',\n",
    "    'PlayerWeight','PlayerAge']\n",
    "\n",
    "    ToBeDropped = ['GameId','Team','NflId','DisplayName','JerseyNumber','YardLine',\n",
    "    'PossessionTeam','FieldPosition','HomeScoreBeforePlay','VisitorScoreBeforePlay',\n",
    "    'RusherName','NflIdRusher','PlayDirection','PlayerBirthDate','PlayerCollegeName',\n",
    "    'HomeTeamAbbr','VisitorTeamAbbr','isRusher','isOffense']\n",
    "\n",
    "    NflDf = pd.concat([train[[f for f in list(train.columns) if f not in list(DistanceDf.columns)]],DistanceDf.reset_index().drop(columns = 'PlayId')], 1)\n",
    "\n",
    "    PlayerDf = NflDf[PlayerFeatures]\n",
    "\n",
    "    count = PlayerDf.groupby(['PlayId']).cumcount() + 1\n",
    "    PlayerDf = PlayerDf.set_index(['PlayId', count]).unstack().sort_index(1, level=1)\n",
    "    PlayerDf = PlayerDf.reset_index()\n",
    "\n",
    "    feats = []\n",
    "    for i in range(len(PlayerDf.columns)):\n",
    "        feat = ''.join(list(PlayerDf.columns[i][0] + str(PlayerDf.columns[i][1])))\n",
    "        feats.append(feat)\n",
    "    PlayerDf.columns = feats\n",
    "\n",
    "    OneHotFeatures = ['OffenseTeamAbbr','DefenseTeamAbbr','RusherPosition','OffenseFormation']\n",
    "\n",
    "    Booleans = ['isOffenseHome','isTop10Rusher','isOffenseField']\n",
    "    BoolMap = {True : 1, False : 0}\n",
    "\n",
    "    #Create TeamDf\n",
    "    TeamFeatures = [f for f in NflDf.columns if f not in ToBeDropped + PlayerFeatures]\n",
    "\n",
    "    TeamDf = NflDf[TeamFeatures]\n",
    "    TeamDf['PlayId'] = train['PlayId']\n",
    "\n",
    "    #PlayerCode Decoding\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(-4,'DL')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(-3,'LB')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(-2,'DB')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(-1,'S')\n",
    "\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(6,'RB')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(5,'WR')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(4,'FB')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(3,'QB')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(2,'OL')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(1,'TE')\n",
    "\n",
    "\n",
    "\n",
    "    NumDf = TeamDf[[f for f in TeamDf.columns if f not in Booleans + OneHotFeatures]]\n",
    "    BoolDf = TeamDf[Booleans].stack().map(BoolMap).unstack()\n",
    "\n",
    "    Dummies = TeamDf[OneHotFeatures]\n",
    "    #Fill Nulls\n",
    "    for col in Dummies.columns:\n",
    "        Dummies[col].fillna(Dummies[col].mode()[0],inplace = True)\n",
    "\n",
    "    enc = OneHotEncoder(handle_unknown='ignore')\n",
    "    DummyDf = enc.fit_transform(Dummies)\n",
    "    DummyDf = pd.DataFrame(DummyDf.toarray())\n",
    "    DummyDf.columns = enc.get_feature_names()\n",
    "\n",
    "\n",
    "    TeamDf = pd.concat([NumDf,BoolDf,DummyDf], 1)\n",
    "\n",
    "    TeamDf = TeamDf.groupby(['PlayId']).agg('mean')\n",
    "    Df = pd.concat([PlayerDf,TeamDf.reset_index().drop(columns = 'PlayId')],1)\n",
    "\n",
    "    #Distance From Scrimmage Line\n",
    "    Df['PlayDirection'] = train.groupby('PlayId')[['PlayDirection']].agg(pd.Series.mode).reset_index().drop(columns = 'PlayId')['PlayDirection']\n",
    "    Df['DistanceFromScrimmageLine'] = Df.apply(DistanceFromScrimmageLine, axis = 1)\n",
    "\n",
    "    Df = Df.drop(columns = 'PlayDirection')\n",
    "    Df = Df.drop(columns = 'PlayId')\n",
    "\n",
    "    #Fill Nulls : Mode\n",
    "    for col in Df.columns:\n",
    "        Df[col].fillna(Df[col].mode()[0],inplace = True)\n",
    "    \n",
    "    return Df, enc, GroupFeatureDic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def PreprocessingNNTestSet(TestDf, enc, GroupFeatureDic):\n",
    "    \n",
    "    #GroupFeatures\n",
    "    TestDf['Yards_OffenseFormation_mean'] = TestDf['OffenseFormation'].replace(GroupFeatureDic['Yards_OffenseFormation_mean']).astype('float64')\n",
    "    TestDf['Yards_DefendersInTheBox_mean'] = TestDf['DefendersInTheBox'].replace(GroupFeatureDic['Yards_DefendersInTheBox_mean']).astype('float64')\n",
    "    TestDf['Yards_Down_mean'] = TestDf['Down'].replace(GroupFeatureDic['Yards_Down_mean']).astype('float64')\n",
    "    TestDf['Yards_PossessionTeam_mean'] = TestDf['PossessionTeam'].replace(GroupFeatureDic['Yards_PossessionTeam_mean']).astype('float64')\n",
    "    \n",
    "    #Ground & Weather Conditions \n",
    "    ConditionFeatures = ['StadiumType','Turf','GameWeather','Temperature','Humidity','WindSpeed',\n",
    "                         'WindDirection', 'Stadium', 'Location']\n",
    "    Drop = ['Stadium','Location']\n",
    "    Drops = Drop\n",
    "\n",
    "    TestDf['StadiumType'] = TestDf['StadiumType'].apply(clean_StadiumType)\n",
    "    TestDf['StadiumType'] = TestDf['StadiumType'].apply(transform_StadiumType)\n",
    "    TestDf['Turf'] = TestDf['Turf'].map(Turf)\n",
    "\n",
    "    #WindSpeed\n",
    "    TestDf['WindSpeed'] = TestDf['WindSpeed'].apply(lambda x: x.lower().replace('mph', '').strip() if not pd.isna(x) else x)\n",
    "    TestDf['WindSpeed'] = TestDf['WindSpeed'].apply(lambda x: (int(x.split('-')[0])+int(x.split('-')[1]))/2 if not pd.isna(x) and '-' in x else x)\n",
    "    TestDf['WindSpeed'] = TestDf['WindSpeed'].apply(lambda x: (int(x.split()[0])+int(x.split()[-1]))/2 if not pd.isna(x) and type(x)!=float and 'gusts up to' in x else x)\n",
    "    TestDf['WindSpeed'] = TestDf['WindSpeed'].apply(str_to_float)\n",
    "\n",
    "    #WindDirection\n",
    "    TestDf['WindDirection'] = TestDf['WindDirection'].apply(clean_WindDirection)\n",
    "    TestDf['WindDirection'] = TestDf['WindDirection'].apply(transform_WindDirection)\n",
    "\n",
    "    #GameWeather\n",
    "    TestDf['GameWeather'] = TestDf['GameWeather'].str.lower()\n",
    "    indoor = \"indoor\"\n",
    "    TestDf['GameWeather'] = TestDf['GameWeather'].apply(lambda x: indoor if not pd.isna(x) and indoor in x else x)\n",
    "    TestDf['GameWeather'] = TestDf['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy').replace('clouidy', 'cloudy').replace('party', 'partly') if not pd.isna(x) else x)\n",
    "    TestDf['GameWeather'] = TestDf['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear') if not pd.isna(x) else x)\n",
    "    TestDf['GameWeather'] = TestDf['GameWeather'].apply(lambda x: x.replace('skies', '').replace(\"mostly\", \"\").strip() if not pd.isna(x) else x)\n",
    "    TestDf['GameWeather'] = TestDf['GameWeather'].apply(map_weather)\n",
    "\n",
    "    #Drop Location\n",
    "    TestDf = TestDf.drop(columns = Drop)\n",
    "\n",
    "    #Time Features\n",
    "    TimeFeatures = ['TimeHandoff','TimeSnap','Season','Quarter','Week']\n",
    "    Drop = ['TimeHandoff','TimeSnap','GameClock']\n",
    "    Drops = Drops + Drop\n",
    "\n",
    "    #Handle Time Data\n",
    "    TestDf['TimeHandoff'] = TestDf['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, \"%Y-%m-%dT%H:%M:%S.%fZ\"))\n",
    "    TestDf['TimeSnap'] = TestDf['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, \"%Y-%m-%dT%H:%M:%S.%fZ\"))\n",
    "    TestDf['PlayerBirthDate'] = TestDf['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, \"%m/%d/%Y\"))\n",
    "\n",
    "    #New Features\n",
    "    TestDf['SnaptoHandoff'] = TestDf.apply(lambda row: (row['TimeHandoff'] - row['TimeSnap']).total_seconds(), axis=1)\n",
    "    TestDf['QuarterSecondsLeft'] = TestDf['GameClock'].apply(lambda x: int(x[0:2])*60 + int(x[3:5]))\n",
    "    TestDf['GameSecondsLeft'] = TestDf['Quarter'].map({1:2700, 2:1800, 3:900, 4:0}) + TestDf['QuarterSecondsLeft']\n",
    "    TestDf['GameSecondsPassed'] = 3600 - TestDf['GameSecondsLeft']\n",
    "    TestDf['PlayerAge'] = 2019 - TestDf.PlayerBirthDate.apply(lambda x : x.year)\n",
    "\n",
    "    #For 5th Quarter\n",
    "    TestDf.loc[(TestDf['Quarter'] == 5),'GameSecondsLeft'] = -1\n",
    "    TestDf.loc[(TestDf['Quarter'] == 5),'QuarterSecondsLeft'] = -1\n",
    "    TestDf.loc[(TestDf['Quarter'] == 5),'GameSecondsPassed'] = 3600\n",
    "\n",
    "    #Drop useless features\n",
    "    TestDf = TestDf.drop(columns = Drop)\n",
    "\n",
    "    #Team Features\n",
    "    TeamFeatures = ['Team','HomeTeamAbbr','VisitorTeamAbbr','HomeScoreBeforePlay',\n",
    "    'VisitorScoreBeforePlay','PossessionTeam','FieldPosition','Dis','YardLine','Down','Distance',\n",
    "    'OffenseFormation','OffensePersonnel','DefendersInTheBox','DefensePersonnel']\n",
    "    Drop = ['HomeTeamAbbr','VisitorTeamAbbr','HomeScoreBeforePlay','VisitorScoreBeforePlay',\n",
    "    'PossessionTeam', 'FieldPosition']\n",
    "    Drops = Drops + Drop\n",
    "\n",
    "    TestDf['PossessionTeam'] = TestDf['PossessionTeam'].apply(TeamAbbrMapper)\n",
    "    TestDf['HomeTeamAbbr'] = TestDf['HomeTeamAbbr'].apply(TeamAbbrMapper)\n",
    "    TestDf['VisitorTeamAbbr'] = TestDf['VisitorTeamAbbr'].apply(TeamAbbrMapper)\n",
    "\n",
    "    #New Features\n",
    "    TestDf['OffenseTeamAbbr'] = TestDf['PossessionTeam']\n",
    "    TestDf['DefenseTeamAbbr'] = TestDf.apply(lambda row : row['VisitorTeamAbbr'] if row['OffenseTeamAbbr'] == row['HomeTeamAbbr'] else row['HomeTeamAbbr'], axis = 1)\n",
    "    TestDf['isOffenseHome'] = TestDf['HomeTeamAbbr'] == TestDf['PossessionTeam']\n",
    "    TestDf['isOffenseField'] = TestDf['FieldPosition'] == TestDf['PossessionTeam']\n",
    "    TestDf['OffenseLeadScore'] = TestDf.apply(lambda row : row['HomeScoreBeforePlay'] - row['VisitorScoreBeforePlay'] if row['isOffenseHome'] == 'True' else row['VisitorScoreBeforePlay'] - row['HomeScoreBeforePlay'], axis = 1)\n",
    "    TestDf['OffenseYardLine'] =  TestDf.apply(YardGain, axis = 1)\n",
    "\n",
    "    #Strategies & Formations\n",
    "    FormationFeatures = ['OffensePersonnel','DefendersInTheBox',\n",
    "                         'DefensePersonnel']\n",
    "    Drop = ['OffensePersonnel','DefensePersonnel']\n",
    "    Drops.append(Drop)\n",
    "\n",
    "\n",
    "    #OffensivePersonnel\n",
    "    temp = TestDf[\"OffensePersonnel\"].iloc[np.arange(0, len(TestDf), 22)].apply(lambda x : pd.Series(OffenseFormationMapper(x)))\n",
    "    temp.columns = [\"Offense\" + c for c in temp.columns]\n",
    "    temp[\"PlayId\"] = TestDf[\"PlayId\"].iloc[np.arange(0, len(TestDf), 22)]\n",
    "    TestDf = TestDf.merge(temp, on = \"PlayId\")\n",
    "\n",
    "    #DefensivePersonnel\n",
    "    temp = TestDf[\"DefensePersonnel\"].iloc[np.arange(0, len(TestDf), 22)].apply(lambda x : pd.Series(DefenseFormationMapper(x)))\n",
    "    temp.columns = [\"Defense\" + c for c in temp.columns]\n",
    "    temp[\"PlayId\"] = TestDf[\"PlayId\"].iloc[np.arange(0, len(TestDf), 22)]\n",
    "    TestDf = TestDf.merge(temp, on = \"PlayId\")\n",
    "\n",
    "    #Drop\n",
    "    TestDf = TestDf.drop(columns = Drop)\n",
    "\n",
    "    #Player Features\n",
    "    PlayerFeatures = ['Team','DisplayName','JerseyNumber','NflIdRusher','PlayerCollegeName',\n",
    "    'NflId','X','Y','S','A','Orientation','Dir','PlayerHeight','PlayerWeight','Position',\n",
    "    'PlayerBirthDate']\n",
    "\n",
    "    Drop = ['Team','DisplayName','JerseyNumber','NflIdRusher','PlayerCollegeName','NflId',\n",
    "    'PlayerBirthDate']\n",
    "    Drops = Drops + Drop\n",
    "\n",
    "    #Height , Weight , BMI\n",
    "    TestDf['PlayerHeight'] = TestDf['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))\n",
    "    TestDf['PlayerBMI'] = 703*(TestDf['PlayerWeight']/(TestDf['PlayerHeight'])**2)\n",
    "\n",
    "    #Position Cleaning\n",
    "\n",
    "    TestDf['Position'] = TestDf.Position.replace('DE','DL')\n",
    "    TestDf['Position'] = TestDf.Position.replace('DT','DL')\n",
    "    TestDf['Position'] = TestDf.Position.replace('NT','DL')\n",
    "\n",
    "    TestDf['Position'] = TestDf.Position.replace('ILB','LB')\n",
    "    TestDf['Position'] = TestDf.Position.replace('MLB','LB')\n",
    "    TestDf['Position'] = TestDf.Position.replace('OLB','LB')\n",
    "\n",
    "    TestDf['Position'] = TestDf.Position.replace('CB','DB')\n",
    "\n",
    "    TestDf['Position'] = TestDf.Position.replace('FS','S')\n",
    "    TestDf['Position'] = TestDf.Position.replace('SS','S')\n",
    "    TestDf['Position'] = TestDf.Position.replace('SAF','S')\n",
    "\n",
    "    TestDf['Position'] = TestDf.Position.replace('HB','RB')\n",
    "\n",
    "    TestDf['Position'] = TestDf.Position.replace('C','OL')\n",
    "    TestDf['Position'] = TestDf.Position.replace('G','OL')\n",
    "    TestDf['Position'] = TestDf.Position.replace('T','OL')\n",
    "    TestDf['Position'] = TestDf.Position.replace('OT','OL')\n",
    "    TestDf['Position'] = TestDf.Position.replace('OG','OL')\n",
    "    TestDf['Position'] = TestDf.Position.replace('OG','OL')\n",
    "\n",
    "    #Position Encoding\n",
    "    TestDf['Position'] = TestDf.Position.replace('DL',-4)\n",
    "    TestDf['Position'] = TestDf.Position.replace('LB',-3)\n",
    "    TestDf['Position'] = TestDf.Position.replace('DB',-2)\n",
    "    TestDf['Position'] = TestDf.Position.replace('S',-1)\n",
    "\n",
    "    TestDf['Position'] = TestDf.Position.replace('RB',6)\n",
    "    TestDf['Position'] = TestDf.Position.replace('WR',5)\n",
    "    TestDf['Position'] = TestDf.Position.replace('FB',4)\n",
    "    TestDf['Position'] = TestDf.Position.replace('QB',3)\n",
    "    TestDf['Position'] = TestDf.Position.replace('OL',2)\n",
    "    TestDf['Position'] = TestDf.Position.replace('TE',1)\n",
    "    \n",
    "    #Transform Orientation to Radian\n",
    "    TestDf['Orientation_rad'] = np.mod(TestDf.Orientation, 360) * math.pi/180.0\n",
    "    \n",
    "    #Fix shifted orientation \n",
    "    TestDf.loc[TestDf.Season >= 2018, 'Orientation_rad'] = np.mod(TestDf.loc[TestDf.Season >= 2018, 'Orientation'] - 90, 360) * math.pi/180.0\n",
    "    \n",
    "    #Overwrite Orientation\n",
    "    TestDf['Orientation'] = TestDf['Orientation_rad']\n",
    "    TestDf.drop(columns = 'Orientation_rad',inplace = True)\n",
    "    \n",
    "\n",
    "    #PlayerInfo Dataframe\n",
    "    PlayerInfo = TestDf[['PlayId','DisplayName','NflId','NflIdRusher','X','Y','Position','Team','isOffenseHome','A','S','PlayerAge','PlayerWeight','PlayerHeight','PlayerBMI','Dir','Orientation']]\n",
    "    PlayerInfo.set_index('PlayId',inplace = True)\n",
    "\n",
    "    #Create RusherInfo Dataframe\n",
    "    RusherInfo = TestDf[(TestDf['NflId'] == TestDf['NflIdRusher'])][['PlayId','DisplayName','Position','X','Y','A','S','Dir','Orientation','PlayerAge','PlayerHeight','PlayerWeight','PlayerBMI']].set_index('PlayId')[['DisplayName','Position','X','Y','A','S','Dir','Orientation','PlayerAge','PlayerHeight','PlayerWeight','PlayerBMI']]\n",
    "    RusherInfo.columns = ['RusherName','RusherPosition','X_Rusher','Y_Rusher','A_Rusher','S_Rusher','Dir_Rusher','Orientation_Rusher','PlayerAge_Rusher','PlayerHeight_Rusher','PlayerWeight_Rusher','PlayerBMI_Rusher']\n",
    "\n",
    "    #Add Player's performance\n",
    "    Top10Rusher = ['Christian McCaffrey','Kenyan Drake','Nick Chubb','Dalvin Cook','Ezekiel Elliott',\n",
    "    'Leonard Fournette','Josh Jacobs','Chris Carson','Marlon Mack','Lamar Jackson']\n",
    "\n",
    "    RusherInfo['isTop10Rusher'] = RusherInfo['RusherName'].isin(Top10Rusher)\n",
    "    \n",
    "    #Rusher's vertical / horizontal physics\n",
    "    RadianAngle = (90 - RusherInfo['Dir_Rusher']) * np.pi / 180.0\n",
    "    RusherInfo['A_Horizontal_Rusher'] = np.abs(RusherInfo['A_Rusher'] * np.cos(RadianAngle))\n",
    "    RusherInfo['A_Vertical_Rusher'] = np.abs(RusherInfo['A_Rusher'] * np.sin(RadianAngle))\n",
    "\n",
    "    RusherInfo['S_Horizontal_Rusher'] = np.abs(RusherInfo['S_Rusher'] * np.cos(RadianAngle))\n",
    "    RusherInfo['S_Vertical_Rusher'] = np.abs(RusherInfo['S_Rusher'] * np.sin(RadianAngle))\n",
    "\n",
    "    RusherInfo['F_Rusher'] = np.sqrt(RusherInfo['A_Horizontal_Rusher']**2 + RusherInfo['A_Vertical_Rusher']**2) * RusherInfo['PlayerWeight_Rusher']\n",
    "    RusherInfo['KE_Rusher'] = (1/2) *  RusherInfo['PlayerWeight_Rusher'] * (RusherInfo['S_Horizontal_Rusher']**2 + RusherInfo['S_Vertical_Rusher']**2)\n",
    "\n",
    "    #Concat PlayerInfo & RusherInfo\n",
    "    InfoDf = pd.concat([PlayerInfo,RusherInfo],1)\n",
    "\n",
    "    #Create DistanceFromRusher / isOffense\n",
    "    InfoDf['DistanceFromRusher'] = np.sqrt((InfoDf['X_Rusher'] - InfoDf['X'])**2 + (InfoDf['Y_Rusher'] - InfoDf['Y'])**2)\n",
    "    InfoDf['isOffense'] = InfoDf.apply(lambda row : True if ((row['Team'] == 'home') & (row['isOffenseHome'] == True)) | (row['Team'] == 'away') & (row['isOffenseHome'] == False) else False, axis = 1)\n",
    "\n",
    "\n",
    "    #Nearest distance between defensive team & rusher\n",
    "    MinDistance = InfoDf[InfoDf['isOffense']==False][['DistanceFromRusher']].groupby('PlayId').agg(min)\n",
    "    MinDistance.columns = ['NearestDistanceFromRusher']\n",
    "    DistanceDf = pd.concat([InfoDf,MinDistance],1)\n",
    "\n",
    "    #Create DefenderInfo Dataframe\n",
    "    DefensivePosition = [4,-3,-2,-1]\n",
    "    DefenderInfo = DistanceDf[DistanceDf['isOffense'] == False]\n",
    "    NearestDefenderInfo = DefenderInfo[DefenderInfo['DistanceFromRusher'] == DefenderInfo['NearestDistanceFromRusher']]\n",
    "\n",
    "    DistanceDf['A_NearestDefender'] = NearestDefenderInfo['A']\n",
    "    DistanceDf['S_NearestDefender'] = NearestDefenderInfo['S']\n",
    "    DistanceDf['Orientation_NearestDefender'] = NearestDefenderInfo['Orientation']\n",
    "    DistanceDf['Dir_NearestDefender'] = NearestDefenderInfo['Dir']\n",
    "    \n",
    "    RadianAngle = (90 - NearestDefenderInfo['Dir']) * np.pi / 180.0\n",
    "    DistanceDf['A_Horizontal_NearestDefender'] = np.abs(DistanceDf['A'] * np.cos(RadianAngle))\n",
    "    DistanceDf['A_Vertical_NearestDefender'] = np.abs(DistanceDf['A'] * np.sin(RadianAngle))\n",
    "\n",
    "    DistanceDf['S_Horizontal_NearestDefender'] = np.abs(DistanceDf['S'] * np.cos(RadianAngle))\n",
    "    DistanceDf['S_Vertical_NearestDefender'] = np.abs(DistanceDf['S'] * np.sin(RadianAngle))\n",
    "\n",
    "    DistanceDf['F_NearestDefender'] = np.sqrt(DistanceDf['A_Horizontal_NearestDefender']**2 + DistanceDf['A_Vertical_NearestDefender']**2) * DistanceDf['PlayerWeight']\n",
    "    DistanceDf['KE_Rusher'] = (1/2) *  DistanceDf['PlayerWeight'] * (DistanceDf['S_Horizontal_NearestDefender']**2 + DistanceDf['S_Vertical_NearestDefender']**2)\n",
    "\n",
    "    PlayerFeatures = ['PlayId','Position','X','Y','S','A','Dis','Orientation','Dir','PlayerHeight',\n",
    "    'PlayerWeight','PlayerAge']\n",
    "\n",
    "    ToBeDropped = ['GameId','Team','NflId','DisplayName','JerseyNumber','YardLine',\n",
    "    'PossessionTeam','FieldPosition','HomeScoreBeforePlay','VisitorScoreBeforePlay',\n",
    "    'RusherName','NflIdRusher','PlayDirection','PlayerBirthDate','PlayerCollegeName',\n",
    "    'HomeTeamAbbr','VisitorTeamAbbr','isRusher','isOffense']\n",
    "\n",
    "    NflDf = pd.concat([TestDf[[f for f in list(TestDf.columns) if f not in list(DistanceDf.columns)]],DistanceDf.reset_index().drop(columns = 'PlayId')], 1)\n",
    "\n",
    "    PlayerDf = NflDf[PlayerFeatures]\n",
    "\n",
    "    count = PlayerDf.groupby(['PlayId']).cumcount() + 1\n",
    "    PlayerDf = PlayerDf.set_index(['PlayId', count]).unstack().sort_index(1, level=1)\n",
    "    PlayerDf = PlayerDf.reset_index()\n",
    "\n",
    "    feats = []\n",
    "    for i in range(len(PlayerDf.columns)):\n",
    "        feat = ''.join(list(PlayerDf.columns[i][0] + str(PlayerDf.columns[i][1])))\n",
    "        feats.append(feat)\n",
    "    PlayerDf.columns = feats\n",
    "\n",
    "    OneHotFeatures = ['OffenseTeamAbbr','DefenseTeamAbbr','RusherPosition','OffenseFormation']\n",
    "\n",
    "    Booleans = ['isOffenseHome','isTop10Rusher','isOffenseField']\n",
    "    BoolMap = {True : 1, False : 0}\n",
    "\n",
    "    #Create TeamDf\n",
    "    TeamFeatures = [f for f in NflDf.columns if f not in ToBeDropped + PlayerFeatures]\n",
    "\n",
    "    TeamDf = NflDf[TeamFeatures]\n",
    "    TeamDf['PlayId'] = TestDf['PlayId']\n",
    "\n",
    "    #PlayerCode Decoding\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(-4,'DL')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(-3,'LB')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(-2,'DB')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(-1,'S')\n",
    "\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(6,'RB')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(5,'WR')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(4,'FB')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(3,'QB')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(2,'OL')\n",
    "    TeamDf['RusherPosition'] = TeamDf.RusherPosition.replace(1,'TE')\n",
    "\n",
    "\n",
    "\n",
    "    NumDf = TeamDf[[f for f in TeamDf.columns if f not in Booleans + OneHotFeatures]]\n",
    "    BoolDf = TeamDf[Booleans].stack().map(BoolMap).unstack()\n",
    "\n",
    "    Dummies = TeamDf[OneHotFeatures]\n",
    "    #Fill Nulls\n",
    "    for col in Dummies.columns:\n",
    "        Dummies[col].fillna(Dummies[col].mode()[0],inplace = True)\n",
    "\n",
    "    DummyDf = enc.transform(Dummies)\n",
    "    DummyDf = pd.DataFrame(DummyDf.toarray())\n",
    "    DummyDf.columns = enc.get_feature_names()\n",
    "\n",
    "\n",
    "    TeamDf = pd.concat([NumDf,BoolDf,DummyDf], 1)\n",
    "\n",
    "    TeamDf = TeamDf.groupby(['PlayId']).agg('mean')\n",
    "    Df = pd.concat([PlayerDf,TeamDf.reset_index().drop(columns = 'PlayId')],1)\n",
    "\n",
    "    #Distance From Scrimmage Line\n",
    "    Df['PlayDirection'] = TestDf.groupby('PlayId')[['PlayDirection']].agg(pd.Series.mode).reset_index().drop(columns = 'PlayId')['PlayDirection']\n",
    "    Df['DistanceFromScrimmageLine'] = Df.apply(DistanceFromScrimmageLine, axis = 1)\n",
    "\n",
    "    Df = Df.drop(columns = 'PlayDirection')\n",
    "    Df = Df.drop(columns = 'PlayId')\n",
    "\n",
    "    #Fill Nulls : Mode\n",
    "    for col in Df.columns:\n",
    "        idx = 0\n",
    "        Mode = pd.DataFrame(Df[col].value_counts(dropna = False)).index[idx]\n",
    "        while True:\n",
    "            if np.isnan(Mode) == False:\n",
    "                Df[col].fillna(Mode, inplace = True)\n",
    "                break\n",
    "            else:\n",
    "                Df[col].fillna(-1, inplace = True)\n",
    "                break\n",
    "    \n",
    "    return Df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IMPORTS \n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "import lightgbm as lgb\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# evaluation metric\n",
    "def crps(y_true, y_pred):\n",
    "    y_true = np.clip(np.cumsum(y_true, axis=1), 0, 1)\n",
    "    y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)\n",
    "    return ((y_true - y_pred) ** 2).sum(axis=1).sum(axis=0) / (199 * y_true.shape[0]) \n",
    "\n",
    "\n",
    "# author : ryancaldwell\n",
    "# Link : https://www.kaggle.com/ryancaldwell/location-eda\n",
    "def PreprocessingTreeDataSet(df, deploy=False):\n",
    "    def new_X(x_coordinate, play_direction):\n",
    "        if play_direction == 'left':\n",
    "            return 120.0 - x_coordinate\n",
    "        else:\n",
    "            return x_coordinate\n",
    "\n",
    "    def new_line(rush_team, field_position, yardline):\n",
    "        if rush_team == field_position:\n",
    "            # offense starting at X = 0 plus the 10 yard endzone plus the line of scrimmage\n",
    "            return 10.0 + yardline\n",
    "        else:\n",
    "            # half the field plus the yards between midfield and the line of scrimmage\n",
    "            return 60.0 + (50 - yardline)\n",
    "\n",
    "    def new_orientation(angle, play_direction):\n",
    "        if play_direction == 'left':\n",
    "            new_angle = 360.0 - angle\n",
    "            if new_angle == 360.0:\n",
    "                new_angle = 0.0\n",
    "            return new_angle\n",
    "        else:\n",
    "            return angle\n",
    "\n",
    "    def euclidean_distance(x1,y1,x2,y2):\n",
    "        x_diff = (x1-x2)**2\n",
    "        y_diff = (y1-y2)**2\n",
    "\n",
    "        return np.sqrt(x_diff + y_diff)\n",
    "\n",
    "    def back_direction(orientation):\n",
    "        if orientation > 180.0:\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "\n",
    "    def update_yardline(df):\n",
    "        new_yardline = df[df['NflId'] == df['NflIdRusher']]\n",
    "        new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)\n",
    "        new_yardline = new_yardline[['GameId','PlayId','YardLine']]\n",
    "\n",
    "        return new_yardline\n",
    "\n",
    "    def update_orientation(df, yardline):\n",
    "        df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)\n",
    "        df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)\n",
    "        df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)\n",
    "\n",
    "        df = df.drop('YardLine', axis=1)\n",
    "        df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')\n",
    "\n",
    "        return df\n",
    "\n",
    "    def back_features(df):\n",
    "        carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]\n",
    "        carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']\n",
    "        carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))\n",
    "        carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))\n",
    "        carriers = carriers.rename(columns={'X':'back_X',\n",
    "                                            'Y':'back_Y'})\n",
    "        carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]\n",
    "\n",
    "        return carriers\n",
    "\n",
    "    def features_relative_to_back(df, carriers):\n",
    "        player_distance = df[['GameId','PlayId','NflId','X','Y']]\n",
    "        player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')\n",
    "        player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]\n",
    "        player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)\n",
    "\n",
    "        player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\\\n",
    "                                         .agg({'dist_to_back':['min','max','mean','std']})\\\n",
    "                                         .reset_index()\n",
    "        player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',\n",
    "                                   'min_dist','max_dist','mean_dist','std_dist']\n",
    "\n",
    "        return player_distance\n",
    "\n",
    "    def defense_features(df):\n",
    "        rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]\n",
    "        rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']\n",
    "\n",
    "        defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')\n",
    "        defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]\n",
    "        defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)\n",
    "\n",
    "        defense = defense.groupby(['GameId','PlayId'])\\\n",
    "                         .agg({'def_dist_to_back':['min','max','mean','std']})\\\n",
    "                         .reset_index()\n",
    "        defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']\n",
    "\n",
    "        return defense\n",
    "\n",
    "    def static_features(df):\n",
    "        static_features = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','X','Y','S','A','Dis','Orientation','Dir',\n",
    "                                                            'YardLine','Quarter','Down','Distance','DefendersInTheBox']].drop_duplicates()\n",
    "        static_features['DefendersInTheBox'] = static_features['DefendersInTheBox'].fillna(np.mean(static_features['DefendersInTheBox']))\n",
    "\n",
    "        return static_features\n",
    "    \n",
    "    def split_personnel(s):\n",
    "        splits = s.split(',')\n",
    "        for i in range(len(splits)):\n",
    "            splits[i] = splits[i].strip()\n",
    "\n",
    "        return splits\n",
    "\n",
    "    def defense_formation(l):\n",
    "        dl = 0\n",
    "        lb = 0\n",
    "        db = 0\n",
    "        other = 0\n",
    "\n",
    "        for position in l:\n",
    "            sub_string = position.split(' ')\n",
    "            if sub_string[1] == 'DL':\n",
    "                dl += int(sub_string[0])\n",
    "            elif sub_string[1] in ['LB','OL']:\n",
    "                lb += int(sub_string[0])\n",
    "            else:\n",
    "                db += int(sub_string[0])\n",
    "\n",
    "        counts = (dl,lb,db,other)\n",
    "\n",
    "        return counts\n",
    "\n",
    "    def offense_formation(l):\n",
    "        qb = 0\n",
    "        rb = 0\n",
    "        wr = 0\n",
    "        te = 0\n",
    "        ol = 0\n",
    "\n",
    "        sub_total = 0\n",
    "        qb_listed = False\n",
    "        for position in l:\n",
    "            sub_string = position.split(' ')\n",
    "            pos = sub_string[1]\n",
    "            cnt = int(sub_string[0])\n",
    "\n",
    "            if pos == 'QB':\n",
    "                qb += cnt\n",
    "                sub_total += cnt\n",
    "                qb_listed = True\n",
    "            # Assuming LB is a line backer lined up as full back\n",
    "            elif pos in ['RB','LB']:\n",
    "                rb += cnt\n",
    "                sub_total += cnt\n",
    "            # Assuming DB is a defensive back and lined up as WR\n",
    "            elif pos in ['WR','DB']:\n",
    "                wr += cnt\n",
    "                sub_total += cnt\n",
    "            elif pos == 'TE':\n",
    "                te += cnt\n",
    "                sub_total += cnt\n",
    "            # Assuming DL is a defensive lineman lined up as an additional line man\n",
    "            else:\n",
    "                ol += cnt\n",
    "                sub_total += cnt\n",
    "\n",
    "        # If not all 11 players were noted at given positions we need to make some assumptions\n",
    "        # I will assume if a QB is not listed then there was 1 QB on the play\n",
    "        # If a QB is listed then I'm going to assume the rest of the positions are at OL\n",
    "        # This might be flawed but it looks like RB, TE and WR are always listed in the personnel\n",
    "        if sub_total < 11:\n",
    "            diff = 11 - sub_total\n",
    "            if not qb_listed:\n",
    "                qb += 1\n",
    "                diff -= 1\n",
    "            ol += diff\n",
    "\n",
    "        counts = (qb,rb,wr,te,ol)\n",
    "\n",
    "        return counts\n",
    "    \n",
    "    def personnel_features(df):\n",
    "        personnel = df[['GameId','PlayId','OffensePersonnel','DefensePersonnel']].drop_duplicates()\n",
    "        personnel['DefensePersonnel'] = personnel['DefensePersonnel'].apply(lambda x: split_personnel(x))\n",
    "        personnel['DefensePersonnel'] = personnel['DefensePersonnel'].apply(lambda x: defense_formation(x))\n",
    "        personnel['num_DL'] = personnel['DefensePersonnel'].apply(lambda x: x[0])\n",
    "        personnel['num_LB'] = personnel['DefensePersonnel'].apply(lambda x: x[1])\n",
    "        personnel['num_DB'] = personnel['DefensePersonnel'].apply(lambda x: x[2])\n",
    "\n",
    "        personnel['OffensePersonnel'] = personnel['OffensePersonnel'].apply(lambda x: split_personnel(x))\n",
    "        personnel['OffensePersonnel'] = personnel['OffensePersonnel'].apply(lambda x: offense_formation(x))\n",
    "        personnel['num_QB'] = personnel['OffensePersonnel'].apply(lambda x: x[0])\n",
    "        personnel['num_RB'] = personnel['OffensePersonnel'].apply(lambda x: x[1])\n",
    "        personnel['num_WR'] = personnel['OffensePersonnel'].apply(lambda x: x[2])\n",
    "        personnel['num_TE'] = personnel['OffensePersonnel'].apply(lambda x: x[3])\n",
    "        personnel['num_OL'] = personnel['OffensePersonnel'].apply(lambda x: x[4])\n",
    "\n",
    "        # Let's create some features to specify if the OL is covered\n",
    "        personnel['OL_diff'] = personnel['num_OL'] - personnel['num_DL']\n",
    "        personnel['OL_TE_diff'] = (personnel['num_OL'] + personnel['num_TE']) - personnel['num_DL']\n",
    "        # Let's create a feature to specify if the defense is preventing the run\n",
    "        # Let's just assume 7 or more DL and LB is run prevention\n",
    "        personnel['run_def'] = (personnel['num_DL'] + personnel['num_LB'] > 6).astype(int)\n",
    "\n",
    "        personnel.drop(['OffensePersonnel','DefensePersonnel'], axis=1, inplace=True)\n",
    "        \n",
    "        return personnel\n",
    "\n",
    "    def combine_features(relative_to_back, defense, static, personnel, deploy=deploy):\n",
    "        df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')\n",
    "        df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')\n",
    "        df = pd.merge(df,personnel,on=['GameId','PlayId'],how='inner')\n",
    "\n",
    "        if not deploy:\n",
    "            df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')\n",
    "\n",
    "        return df\n",
    "    \n",
    "    yardline = update_yardline(df)\n",
    "    df = update_orientation(df, yardline)\n",
    "    back_feats = back_features(df)\n",
    "    rel_back = features_relative_to_back(df, back_feats)\n",
    "    def_feats = defense_features(df)\n",
    "    static_feats = static_features(df)\n",
    "    personnel = personnel_features(df)\n",
    "    basetable = combine_features(rel_back, def_feats, static_feats, personnel, deploy=deploy)\n",
    "    \n",
    "    return basetable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Radam\n",
    "import keras.optimizers\n",
    "__all__ = ['RAdam']\n",
    "class RAdam(keras.optimizers.Optimizer):\n",
    "    \"\"\"RAdam optimizer.\n",
    "    # Arguments\n",
    "        learning_rate: float >= 0. Learning rate.\n",
    "        beta_1: float, 0 < beta < 1. Generally close to 1.\n",
    "        beta_2: float, 0 < beta < 1. Generally close to 1.\n",
    "        epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n",
    "        decay: float >= 0. Learning rate decay over each update.\n",
    "        weight_decay: float >= 0. Weight decay for each param.\n",
    "        amsgrad: boolean. Whether to apply the AMSGrad variant of this\n",
    "            algorithm from the paper \"On the Convergence of Adam and\n",
    "            Beyond\".\n",
    "        total_steps: int >= 0. Total number of training steps. Enable warmup by setting a positive value.\n",
    "        warmup_proportion: 0 < warmup_proportion < 1. The proportion of increasing steps.\n",
    "        min_lr: float >= 0. Minimum learning rate after warmup.\n",
    "    # References\n",
    "        - [Adam - A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980v8)\n",
    "        - [On the Convergence of Adam and Beyond](https://openreview.net/forum?id=ryQu7f-RZ)\n",
    "        - [On The Variance Of The Adaptive Learning Rate And Beyond](https://arxiv.org/pdf/1908.03265v1.pdf)\n",
    "    \"\"\"\n",
    "    def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999,\n",
    "                 epsilon=None, decay=0., weight_decay=0., amsgrad=False,\n",
    "                 total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):\n",
    "        learning_rate = kwargs.pop('lr', learning_rate)\n",
    "        super(RAdam, self).__init__(**kwargs)\n",
    "        with K.name_scope(self.__class__.__name__):\n",
    "            self.iterations = K.variable(0, dtype='int64', name='iterations')\n",
    "            self.learning_rate = K.variable(learning_rate, name='learning_rate')\n",
    "            self.beta_1 = K.variable(beta_1, name='beta_1')\n",
    "            self.beta_2 = K.variable(beta_2, name='beta_2')\n",
    "            self.decay = K.variable(decay, name='decay')\n",
    "            self.weight_decay = K.variable(weight_decay, name='weight_decay')\n",
    "            self.total_steps = K.variable(total_steps, name='total_steps')\n",
    "            self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')\n",
    "            self.min_lr = K.variable(min_lr, name='min_lr')\n",
    "        if epsilon is None:\n",
    "            epsilon = K.epsilon()\n",
    "        self.epsilon = epsilon\n",
    "        self.initial_decay = decay\n",
    "        self.initial_weight_decay = weight_decay\n",
    "        self.initial_total_steps = total_steps\n",
    "        self.amsgrad = amsgrad\n",
    "\n",
    "    def get_updates(self, loss, params):\n",
    "        grads = self.get_gradients(loss, params)\n",
    "        self.updates = [K.update_add(self.iterations, 1)]\n",
    "\n",
    "        lr = self.lr\n",
    "\n",
    "        if self.initial_decay > 0:\n",
    "            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))\n",
    "\n",
    "        t = K.cast(self.iterations, K.floatx()) + 1\n",
    "\n",
    "        if self.initial_total_steps > 0:\n",
    "            warmup_steps = self.total_steps * self.warmup_proportion\n",
    "            decay_steps = K.maximum(self.total_steps - warmup_steps, 1)\n",
    "            decay_rate = (self.min_lr - lr) / decay_steps\n",
    "            lr = K.switch(\n",
    "                t <= warmup_steps,\n",
    "                lr * (t / warmup_steps),\n",
    "                lr + decay_rate * K.minimum(t - warmup_steps, decay_steps),\n",
    "            )\n",
    "\n",
    "        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for (i, p) in enumerate(params)]\n",
    "        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for (i, p) in enumerate(params)]\n",
    "\n",
    "        if self.amsgrad:\n",
    "            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for (i, p) in enumerate(params)]\n",
    "        else:\n",
    "            vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))]\n",
    "\n",
    "        self.weights = [self.iterations] + ms + vs + vhats\n",
    "\n",
    "        beta_1_t = K.pow(self.beta_1, t)\n",
    "        beta_2_t = K.pow(self.beta_2, t)\n",
    "\n",
    "        sma_inf = 2.0 / (1.0 - self.beta_2) - 1.0\n",
    "        sma_t = sma_inf - 2.0 * t * beta_2_t / (1.0 - beta_2_t)\n",
    "\n",
    "        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):\n",
    "            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n",
    "            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n",
    "\n",
    "            m_corr_t = m_t / (1.0 - beta_1_t)\n",
    "            if self.amsgrad:\n",
    "                vhat_t = K.maximum(vhat, v_t)\n",
    "                v_corr_t = K.sqrt(vhat_t / (1.0 - beta_2_t))\n",
    "                self.updates.append(K.update(vhat, vhat_t))\n",
    "            else:\n",
    "                v_corr_t = K.sqrt(v_t / (1.0 - beta_2_t))\n",
    "\n",
    "            r_t = K.sqrt((sma_t - 4.0) / (sma_inf - 4.0) *\n",
    "                         (sma_t - 2.0) / (sma_inf - 2.0) *\n",
    "                         sma_inf / sma_t)\n",
    "\n",
    "            p_t = K.switch(sma_t >= 5, r_t * m_corr_t / (v_corr_t + self.epsilon), m_corr_t)\n",
    "\n",
    "            if self.initial_weight_decay > 0:\n",
    "                p_t += self.weight_decay * p\n",
    "\n",
    "            p_t = p - lr * p_t\n",
    "\n",
    "            self.updates.append(K.update(m, m_t))\n",
    "            self.updates.append(K.update(v, v_t))\n",
    "            new_p = p_t\n",
    "\n",
    "            # Apply constraints.\n",
    "            if getattr(p, 'constraint', None) is not None:\n",
    "                new_p = p.constraint(new_p)\n",
    "\n",
    "            self.updates.append(K.update(p, new_p))\n",
    "        return self.updates\n",
    "\n",
    "    @property\n",
    "    def lr(self):\n",
    "        return self.learning_rate\n",
    "\n",
    "    @lr.setter\n",
    "    def lr(self, learning_rate):\n",
    "        self.learning_rate = learning_rate\n",
    "\n",
    "    def get_config(self):\n",
    "        config = {\n",
    "            'learning_rate': float(K.get_value(self.learning_rate)),\n",
    "            'beta_1': float(K.get_value(self.beta_1)),\n",
    "            'beta_2': float(K.get_value(self.beta_2)),\n",
    "            'decay': float(K.get_value(self.decay)),\n",
    "            'weight_decay': float(K.get_value(self.weight_decay)),\n",
    "            'epsilon': self.epsilon,\n",
    "            'amsgrad': self.amsgrad,\n",
    "            'total_steps': float(K.get_value(self.total_steps)),\n",
    "            'warmup_proportion': float(K.get_value(self.warmup_proportion)),\n",
    "            'min_lr': float(K.get_value(self.min_lr)),\n",
    "        }\n",
    "        base_config = super(RAdam, self).get_config()\n",
    "        return dict(list(base_config.items()) + list(config.items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CRPS \n",
    "class CRPSCallBack(Callback):\n",
    "    \n",
    "    def __init__(self,validation, predict_batch_size=20, include_on_batch=False):\n",
    "        super(CRPSCallBack, self).__init__()\n",
    "        self.validation = validation\n",
    "        self.predict_batch_size = predict_batch_size\n",
    "        self.include_on_batch = include_on_batch\n",
    "        \n",
    "        print('validation shape',len(self.validation))\n",
    "\n",
    "    def on_batch_begin(self, batch, logs={}):\n",
    "        pass\n",
    "\n",
    "    def on_train_begin(self, logs={}):\n",
    "        if not ('CRPS_Score_Val' in self.params['metrics']):\n",
    "            self.params['metrics'].append('CRPS_Score_Val')\n",
    "\n",
    "    def on_batch_end(self, batch, logs={}):\n",
    "        if (self.include_on_batch):\n",
    "            logs['CRPS_Score_Val'] = float('-inf')\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        logs['CRPS_Score_Val'] = float('-inf')\n",
    "            \n",
    "        if (self.validation):\n",
    "            X_valid, y_valid = self.validation[0], self.validation[1]\n",
    "            y_pred = self.model.predict(X_valid)\n",
    "            y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)\n",
    "            y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)\n",
    "            Val_Score = ((y_true - y_pred) ** 2).sum(axis=1).sum(axis=0) / (199 * X_valid.shape[0])\n",
    "            Val_Score = np.round(Val_Score, 6)\n",
    "            logs['CRPS_Score_Val'] = Val_Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def BuildModel(x_tr,y_tr,x_val,y_val):\n",
    "    inp = Input(shape = (x_tr.shape[1],))\n",
    "    x = Dense(1024, input_dim=X.shape[1], activation='relu')(inp)\n",
    "    x = Dropout(0.7)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    x = Dense(512, activation='relu')(x)\n",
    "    x = Dropout(0.7)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    x = Dense(256, activation='relu')(x)\n",
    "    x = Dropout(0.7)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    out = Dense(199, activation='softmax')(x)\n",
    "    model = Model(inp,out)\n",
    "    Radam = RAdam()\n",
    "    Adam = optimizers.Adam()\n",
    "    model.compile(optimizer=Adam, loss='categorical_crossentropy', metrics=[])\n",
    "    \n",
    "    #add lookahead\n",
    "#     lookahead = Lookahead(k=5, alpha=0.5) # Initialize Lookahead\n",
    "#     lookahead.inject(model) # add into model\n",
    "\n",
    "    \n",
    "    es = EarlyStopping(monitor='CRPS_Score_Val', mode='min',restore_best_weights=True,\n",
    "                       verbose=1,patience=10)\n",
    "\n",
    "    #mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',\n",
    "                        #save_best_only=True, verbose=1, save_weights_only=True)\n",
    "    \n",
    "    bsz = 1024\n",
    "    steps = x_tr.shape[0]/bsz\n",
    "    lr = 1e-05\n",
    "\n",
    "    model.fit(x_tr, y_tr,callbacks=[CRPSCallBack(validation = (x_val,y_val)),es], epochs=100, batch_size=bsz,verbose=1)\n",
    "    #model.load_weights(\"best_model.h5\")\n",
    "    \n",
    "    y_pred = model.predict(x_val)\n",
    "    y_valid = y_val\n",
    "    y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)\n",
    "    y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)\n",
    "    val_s = ((y_true - y_pred) ** 2).sum(axis=1).sum(axis=0) / (199 * x_val.shape[0])\n",
    "    crps = np.round(val_s, 6)\n",
    "\n",
    "    return model,crps\n",
    "\n",
    "# def predict(x_te):\n",
    "#     model_num = len(models)\n",
    "#     for k,m in enumerate(models):\n",
    "#         if k==0:\n",
    "#             y_pred = m.predict(x_te,batch_size=1024)\n",
    "#         else:\n",
    "#             y_pred += m.predict(x_te,batch_size=1024)\n",
    "            \n",
    "#     y_pred = y_pred / model_num\n",
    "    \n",
    "#     return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------\n",
      "-----------\n",
      "validation shape 2\n",
      "Epoch 1/100\n",
      "18536/18536 [==============================] - 3s 188us/step - loss: 5.7160\n",
      "Epoch 2/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 5.4168\n",
      "Epoch 3/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 5.1003\n",
      "Epoch 4/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 4.6794\n",
      "Epoch 5/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 4.1656\n",
      "Epoch 6/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 3.6973\n",
      "Epoch 7/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 3.3927\n",
      "Epoch 8/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 3.2183\n",
      "Epoch 9/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 3.1000\n",
      "Epoch 10/100\n",
      "18536/18536 [==============================] - 2s 127us/step - loss: 3.0453\n",
      "Epoch 11/100\n",
      "18536/18536 [==============================] - 2s 127us/step - loss: 2.9955\n",
      "Epoch 12/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.9520\n",
      "Epoch 13/100\n",
      "18536/18536 [==============================] - 2s 127us/step - loss: 2.9164\n",
      "Epoch 14/100\n",
      "18536/18536 [==============================] - 2s 131us/step - loss: 2.9053\n",
      "Epoch 15/100\n",
      "18536/18536 [==============================] - 2s 131us/step - loss: 2.8782\n",
      "Epoch 16/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.8625\n",
      "Epoch 17/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.8445\n",
      "Epoch 18/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.8318\n",
      "Epoch 19/100\n",
      "18536/18536 [==============================] - 2s 123us/step - loss: 2.8273\n",
      "Epoch 20/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 2.8161\n",
      "Epoch 21/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.7972\n",
      "Epoch 22/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.7949\n",
      "Epoch 23/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.7888\n",
      "Epoch 24/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.7814\n",
      "Epoch 25/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.7735\n",
      "Epoch 26/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.7677\n",
      "Epoch 27/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.7635\n",
      "Epoch 28/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.7611\n",
      "Epoch 29/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.7546\n",
      "Epoch 30/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.7467\n",
      "Epoch 31/100\n",
      "18536/18536 [==============================] - 4s 231us/step - loss: 2.7398\n",
      "Epoch 32/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.7367\n",
      "Epoch 33/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.7358\n",
      "Epoch 34/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 2.7316\n",
      "Epoch 35/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 2.7252\n",
      "Epoch 36/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 2.7256\n",
      "Epoch 37/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.7209\n",
      "Epoch 38/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.7122\n",
      "Epoch 39/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.7093\n",
      "Epoch 40/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 2.7076\n",
      "Epoch 41/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.7014\n",
      "Epoch 42/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 2.6960\n",
      "Epoch 43/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.6932\n",
      "Epoch 44/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.6892\n",
      "Epoch 45/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.6833\n",
      "Epoch 46/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 2.6783\n",
      "Epoch 47/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 2.6758\n",
      "Epoch 48/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.6726\n",
      "Epoch 49/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.6644\n",
      "Epoch 50/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.6641\n",
      "Epoch 51/100\n",
      "18536/18536 [==============================] - 3s 145us/step - loss: 2.6602\n",
      "Epoch 52/100\n",
      "18536/18536 [==============================] - 2s 122us/step - loss: 2.6550\n",
      "Epoch 53/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.6466\n",
      "Epoch 54/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.6483\n",
      "Epoch 55/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.6414\n",
      "Epoch 56/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.6390\n",
      "Epoch 57/100\n",
      "18536/18536 [==============================] - 2s 121us/step - loss: 2.6353\n",
      "Epoch 58/100\n",
      "18536/18536 [==============================] - 2s 119us/step - loss: 2.6322\n",
      "Epoch 59/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.6249\n",
      "Epoch 60/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.6202\n",
      "Epoch 61/100\n",
      "18536/18536 [==============================] - 2s 119us/step - loss: 2.6117\n",
      "Epoch 62/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.6072\n",
      "Epoch 63/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.6078\n",
      "Epoch 64/100\n",
      "18536/18536 [==============================] - 2s 119us/step - loss: 2.5999\n",
      "Epoch 65/100\n",
      "18536/18536 [==============================] - 2s 119us/step - loss: 2.6020\n",
      "Epoch 66/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.5962\n",
      "Epoch 67/100\n",
      "18536/18536 [==============================] - 2s 120us/step - loss: 2.5888\n",
      "Epoch 68/100\n",
      "18536/18536 [==============================] - 2s 119us/step - loss: 2.5851\n",
      "Epoch 69/100\n",
      "18536/18536 [==============================] - 2s 119us/step - loss: 2.5818\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00069: early stopping\n",
      "the 1 fold CRPS : 0.013079\n",
      "-----------\n",
      "-----------\n",
      "validation shape 2\n",
      "Epoch 1/100\n",
      "18537/18537 [==============================] - 3s 182us/step - loss: 5.7061\n",
      "Epoch 2/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 5.4245\n",
      "Epoch 3/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 5.1000\n",
      "Epoch 4/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 4.6846\n",
      "Epoch 5/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 4.1548\n",
      "Epoch 6/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 3.6855\n",
      "Epoch 7/100\n",
      "18537/18537 [==============================] - 2s 127us/step - loss: 3.3872\n",
      "Epoch 8/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 3.2006\n",
      "Epoch 9/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 3.1019\n",
      "Epoch 10/100\n",
      "18537/18537 [==============================] - 2s 127us/step - loss: 3.0346\n",
      "Epoch 11/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.9735\n",
      "Epoch 12/100\n",
      "18537/18537 [==============================] - 2s 128us/step - loss: 2.9348\n",
      "Epoch 13/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.9068\n",
      "Epoch 14/100\n",
      "18537/18537 [==============================] - 2s 128us/step - loss: 2.8906\n",
      "Epoch 15/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.8705\n",
      "Epoch 16/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.8522\n",
      "Epoch 17/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.8426\n",
      "Epoch 18/100\n",
      "18537/18537 [==============================] - 3s 149us/step - loss: 2.8311\n",
      "Epoch 19/100\n",
      "18537/18537 [==============================] - 3s 151us/step - loss: 2.8214\n",
      "Epoch 20/100\n",
      "18537/18537 [==============================] - 2s 130us/step - loss: 2.8096\n",
      "Epoch 21/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.7994\n",
      "Epoch 22/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.7921\n",
      "Epoch 23/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.7902\n",
      "Epoch 24/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7784\n",
      "Epoch 25/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7698\n",
      "Epoch 26/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7701\n",
      "Epoch 27/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7629\n",
      "Epoch 28/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7580\n",
      "Epoch 29/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7469\n",
      "Epoch 30/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.7470\n",
      "Epoch 31/100\n",
      "18537/18537 [==============================] - 2s 127us/step - loss: 2.7447\n",
      "Epoch 32/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7415\n",
      "Epoch 33/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.7369\n",
      "Epoch 34/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7268\n",
      "Epoch 35/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7252\n",
      "Epoch 36/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7223\n",
      "Epoch 37/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7213\n",
      "Epoch 38/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7108\n",
      "Epoch 39/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7089\n",
      "Epoch 40/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7097\n",
      "Epoch 41/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7007\n",
      "Epoch 42/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7033\n",
      "Epoch 43/100\n",
      "18537/18537 [==============================] - 3s 135us/step - loss: 2.6930\n",
      "Epoch 44/100\n",
      "18537/18537 [==============================] - 2s 134us/step - loss: 2.6879\n",
      "Epoch 45/100\n",
      "18537/18537 [==============================] - 2s 132us/step - loss: 2.6886\n",
      "Epoch 46/100\n",
      "18537/18537 [==============================] - 2s 134us/step - loss: 2.6810\n",
      "Epoch 47/100\n",
      "18537/18537 [==============================] - 2s 130us/step - loss: 2.6804\n",
      "Epoch 48/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.6718\n",
      "Epoch 49/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.6659\n",
      "Epoch 50/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.6669\n",
      "Epoch 51/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.6589\n",
      "Epoch 52/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.6580\n",
      "Epoch 53/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 2.6504\n",
      "Epoch 54/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6529\n",
      "Epoch 55/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.6454\n",
      "Epoch 56/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.6466\n",
      "Epoch 57/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6357\n",
      "Epoch 58/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6338\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00058: early stopping\n",
      "the 2 fold CRPS : 0.012977\n",
      "-----------\n",
      "-----------\n",
      "validation shape 2\n",
      "Epoch 1/100\n",
      "18537/18537 [==============================] - 3s 186us/step - loss: 5.7148\n",
      "Epoch 2/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 5.4206\n",
      "Epoch 3/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 5.1057\n",
      "Epoch 4/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 4.6858\n",
      "Epoch 5/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 4.1541\n",
      "Epoch 6/100\n",
      "18537/18537 [==============================] - 2s 127us/step - loss: 3.6706\n",
      "Epoch 7/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 3.3645\n",
      "Epoch 8/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 3.1908\n",
      "Epoch 9/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 3.0992\n",
      "Epoch 10/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 3.0303\n",
      "Epoch 11/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.9826\n",
      "Epoch 12/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.9416\n",
      "Epoch 13/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.9171\n",
      "Epoch 14/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.8906\n",
      "Epoch 15/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.8695\n",
      "Epoch 16/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.8605\n",
      "Epoch 17/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8459\n",
      "Epoch 18/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8328\n",
      "Epoch 19/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8259\n",
      "Epoch 20/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8170\n",
      "Epoch 21/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8049\n",
      "Epoch 22/100\n",
      "18537/18537 [==============================] - 3s 143us/step - loss: 2.7956\n",
      "Epoch 23/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7947\n",
      "Epoch 24/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7805\n",
      "Epoch 25/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7750\n",
      "Epoch 26/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7708\n",
      "Epoch 27/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7659\n",
      "Epoch 28/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7600\n",
      "Epoch 29/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7533\n",
      "Epoch 30/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7460\n",
      "Epoch 31/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7495\n",
      "Epoch 32/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7385\n",
      "Epoch 33/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7375\n",
      "Epoch 34/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7341\n",
      "Epoch 35/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7284\n",
      "Epoch 36/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7223\n",
      "Epoch 37/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7208\n",
      "Epoch 38/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7142\n",
      "Epoch 39/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7110\n",
      "Epoch 40/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7102\n",
      "Epoch 41/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7042\n",
      "Epoch 42/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7044\n",
      "Epoch 43/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6981\n",
      "Epoch 44/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6915\n",
      "Epoch 45/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6871\n",
      "Epoch 46/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.6834\n",
      "Epoch 47/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6752\n",
      "Epoch 48/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.6830\n",
      "Epoch 49/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.6691\n",
      "Epoch 50/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6682\n",
      "Epoch 51/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6622\n",
      "Epoch 52/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6559\n",
      "Epoch 53/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6555\n",
      "Epoch 54/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6486\n",
      "Epoch 55/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6431\n",
      "Epoch 56/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6415\n",
      "Epoch 57/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6348\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00057: early stopping\n",
      "the 3 fold CRPS : 0.012855\n",
      "-----------\n",
      "-----------\n",
      "validation shape 2\n",
      "Epoch 1/100\n",
      "18537/18537 [==============================] - 3s 172us/step - loss: 5.7010\n",
      "Epoch 2/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 5.4267\n",
      "Epoch 3/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 5.1147\n",
      "Epoch 4/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 4.6784\n",
      "Epoch 5/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 4.1478\n",
      "Epoch 6/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 3.6887\n",
      "Epoch 7/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 3.3771\n",
      "Epoch 8/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 3.2074\n",
      "Epoch 9/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 3.1038\n",
      "Epoch 10/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 3.0275\n",
      "Epoch 11/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.9777\n",
      "Epoch 12/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.9471\n",
      "Epoch 13/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.9190\n",
      "Epoch 14/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8951\n",
      "Epoch 15/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8711\n",
      "Epoch 16/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8581\n",
      "Epoch 17/100\n",
      "18537/18537 [==============================] - 2s 130us/step - loss: 2.8437\n",
      "Epoch 18/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8264\n",
      "Epoch 19/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8230\n",
      "Epoch 20/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8072\n",
      "Epoch 21/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.8021\n",
      "Epoch 22/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7908\n",
      "Epoch 23/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7887\n",
      "Epoch 24/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7795\n",
      "Epoch 25/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7768\n",
      "Epoch 26/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7700\n",
      "Epoch 27/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7612\n",
      "Epoch 28/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7543\n",
      "Epoch 29/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7567\n",
      "Epoch 30/100\n",
      "18537/18537 [==============================] - 2s 127us/step - loss: 2.7458\n",
      "Epoch 31/100\n",
      "18537/18537 [==============================] - 2s 133us/step - loss: 2.7457\n",
      "Epoch 32/100\n",
      "18537/18537 [==============================] - 2s 133us/step - loss: 2.7356\n",
      "Epoch 33/100\n",
      "18537/18537 [==============================] - 2s 130us/step - loss: 2.7344\n",
      "Epoch 34/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.7305\n",
      "Epoch 35/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.7265\n",
      "Epoch 36/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.7212\n",
      "Epoch 37/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7167\n",
      "Epoch 38/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7170\n",
      "Epoch 39/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.7082\n",
      "Epoch 40/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7098\n",
      "Epoch 41/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7045\n",
      "Epoch 42/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.6945\n",
      "Epoch 43/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6961\n",
      "Epoch 44/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.6917\n",
      "Epoch 45/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6873\n",
      "Epoch 46/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6834\n",
      "Epoch 47/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6772\n",
      "Epoch 48/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6686\n",
      "Epoch 49/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6701\n",
      "Epoch 50/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6643\n",
      "Epoch 51/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.6557\n",
      "Epoch 52/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6596\n",
      "Epoch 53/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6569\n",
      "Epoch 54/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6517\n",
      "Epoch 55/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6451\n",
      "Epoch 56/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6343\n",
      "Epoch 57/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6392\n",
      "Epoch 58/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6313\n",
      "Epoch 59/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6296\n",
      "Epoch 60/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6245\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00060: early stopping\n",
      "the 4 fold CRPS : 0.012785\n",
      "-----------\n",
      "-----------\n",
      "validation shape 2\n",
      "Epoch 1/100\n",
      "18537/18537 [==============================] - 3s 174us/step - loss: 5.7137\n",
      "Epoch 2/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 5.4185\n",
      "Epoch 3/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 5.1115\n",
      "Epoch 4/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 4.6798\n",
      "Epoch 5/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 4.1552\n",
      "Epoch 6/100\n",
      "18537/18537 [==============================] - 3s 142us/step - loss: 3.6947\n",
      "Epoch 7/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 3.3816\n",
      "Epoch 8/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 3.2044\n",
      "Epoch 9/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 3.1041\n",
      "Epoch 10/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 3.0340\n",
      "Epoch 11/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.9796\n",
      "Epoch 12/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.9541\n",
      "Epoch 13/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.9156\n",
      "Epoch 14/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.8944\n",
      "Epoch 15/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8793\n",
      "Epoch 16/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8588\n",
      "Epoch 17/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8492\n",
      "Epoch 18/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.8311\n",
      "Epoch 19/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8182\n",
      "Epoch 20/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8104\n",
      "Epoch 21/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8041\n",
      "Epoch 22/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7856\n",
      "Epoch 23/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7824\n",
      "Epoch 24/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7796\n",
      "Epoch 25/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7751\n",
      "Epoch 26/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7651\n",
      "Epoch 27/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7598\n",
      "Epoch 28/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7567\n",
      "Epoch 29/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7541\n",
      "Epoch 30/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7395\n",
      "Epoch 31/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7429\n",
      "Epoch 32/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7423\n",
      "Epoch 33/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7319\n",
      "Epoch 34/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7249\n",
      "Epoch 35/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7188\n",
      "Epoch 36/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7209\n",
      "Epoch 37/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7100\n",
      "Epoch 38/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7081\n",
      "Epoch 39/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7085\n",
      "Epoch 40/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7008\n",
      "Epoch 41/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.7004\n",
      "Epoch 42/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6886\n",
      "Epoch 43/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6892\n",
      "Epoch 44/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6855\n",
      "Epoch 45/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6876\n",
      "Epoch 46/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6736\n",
      "Epoch 47/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6741\n",
      "Epoch 48/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6664\n",
      "Epoch 49/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.6643\n",
      "Epoch 50/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6641\n",
      "Epoch 51/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6579\n",
      "Epoch 52/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.6564\n",
      "Epoch 53/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6475\n",
      "Epoch 54/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6463\n",
      "Epoch 55/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6396\n",
      "Epoch 56/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6353\n",
      "Epoch 57/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6298\n",
      "Epoch 58/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6237\n",
      "Epoch 59/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6239\n",
      "Epoch 60/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6148\n",
      "Epoch 61/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6155\n",
      "Epoch 62/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6116\n",
      "Epoch 63/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.5989\n",
      "Epoch 64/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6008\n",
      "Epoch 65/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.5939\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00065: early stopping\n",
      "the 5 fold CRPS : 0.013312\n",
      "-----------\n",
      "-----------\n",
      "validation shape 2\n",
      "Epoch 1/100\n",
      "18536/18536 [==============================] - 3s 182us/step - loss: 5.7125\n",
      "Epoch 2/100\n",
      "18536/18536 [==============================] - 2s 126us/step - loss: 5.4182\n",
      "Epoch 3/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 5.1017\n",
      "Epoch 4/100\n",
      "18536/18536 [==============================] - 2s 126us/step - loss: 4.6762\n",
      "Epoch 5/100\n",
      "18536/18536 [==============================] - 2s 127us/step - loss: 4.1620\n",
      "Epoch 6/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 3.6858\n",
      "Epoch 7/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 3.3950\n",
      "Epoch 8/100\n",
      "18536/18536 [==============================] - 2s 126us/step - loss: 3.2117\n",
      "Epoch 9/100\n",
      "18536/18536 [==============================] - 2s 131us/step - loss: 3.1065\n",
      "Epoch 10/100\n",
      "18536/18536 [==============================] - 3s 139us/step - loss: 3.0379\n",
      "Epoch 11/100\n",
      "18536/18536 [==============================] - 2s 134us/step - loss: 2.9827\n",
      "Epoch 12/100\n",
      "18536/18536 [==============================] - 2s 133us/step - loss: 2.9509\n",
      "Epoch 13/100\n",
      "18536/18536 [==============================] - 2s 127us/step - loss: 2.9216\n",
      "Epoch 14/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.8928\n",
      "Epoch 15/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.8758\n",
      "Epoch 16/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.8614\n",
      "Epoch 17/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.8400\n",
      "Epoch 18/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.8318\n",
      "Epoch 19/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.8275\n",
      "Epoch 20/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.8189\n",
      "Epoch 21/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.8052\n",
      "Epoch 22/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.7887\n",
      "Epoch 23/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.7895\n",
      "Epoch 24/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.7873\n",
      "Epoch 25/100\n",
      "18536/18536 [==============================] - 3s 168us/step - loss: 2.7773\n",
      "Epoch 26/100\n",
      "18536/18536 [==============================] - 3s 140us/step - loss: 2.7727\n",
      "Epoch 27/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.7671\n",
      "Epoch 28/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.7614\n",
      "Epoch 29/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.7620\n",
      "Epoch 30/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.7508\n",
      "Epoch 31/100\n",
      "18536/18536 [==============================] - 2s 126us/step - loss: 2.7445\n",
      "Epoch 32/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.7429\n",
      "Epoch 33/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.7336\n",
      "Epoch 34/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.7356\n",
      "Epoch 35/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.7265\n",
      "Epoch 36/100\n",
      "18536/18536 [==============================] - 2s 123us/step - loss: 2.7236\n",
      "Epoch 37/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.7210\n",
      "Epoch 38/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.7122\n",
      "Epoch 39/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.7120\n",
      "Epoch 40/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.7053\n",
      "Epoch 41/100\n",
      "18536/18536 [==============================] - 2s 127us/step - loss: 2.7017\n",
      "Epoch 42/100\n",
      "18536/18536 [==============================] - 3s 148us/step - loss: 2.6949\n",
      "Epoch 43/100\n",
      "18536/18536 [==============================] - 2s 127us/step - loss: 2.6915\n",
      "Epoch 44/100\n",
      "18536/18536 [==============================] - 2s 126us/step - loss: 2.6895\n",
      "Epoch 45/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6806\n",
      "Epoch 46/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6814\n",
      "Epoch 47/100\n",
      "18536/18536 [==============================] - 2s 127us/step - loss: 2.6858\n",
      "Epoch 48/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6739\n",
      "Epoch 49/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6713\n",
      "Epoch 50/100\n",
      "18536/18536 [==============================] - 2s 126us/step - loss: 2.6739\n",
      "Epoch 51/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6599\n",
      "Epoch 52/100\n",
      "18536/18536 [==============================] - 2s 130us/step - loss: 2.6562\n",
      "Epoch 53/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.6517\n",
      "Epoch 54/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6514\n",
      "Epoch 55/100\n",
      "18536/18536 [==============================] - 2s 123us/step - loss: 2.6462\n",
      "Epoch 56/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6375\n",
      "Epoch 57/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.6416\n",
      "Epoch 58/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6339\n",
      "Epoch 59/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.6281\n",
      "Epoch 60/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.6212\n",
      "Epoch 61/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6168\n",
      "Epoch 62/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6142\n",
      "Epoch 63/100\n",
      "18536/18536 [==============================] - 2s 124us/step - loss: 2.6125\n",
      "Epoch 64/100\n",
      "18536/18536 [==============================] - 2s 125us/step - loss: 2.6020\n",
      "Epoch 65/100\n",
      "18536/18536 [==============================] - 2s 126us/step - loss: 2.6038\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00065: early stopping\n",
      "the 1 fold CRPS : 0.012720\n",
      "-----------\n",
      "-----------\n",
      "validation shape 2\n",
      "Epoch 1/100\n",
      "18537/18537 [==============================] - 3s 189us/step - loss: 5.7019\n",
      "Epoch 2/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 5.4135\n",
      "Epoch 3/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 5.1161\n",
      "Epoch 4/100\n",
      "18537/18537 [==============================] - 2s 127us/step - loss: 4.6804\n",
      "Epoch 5/100\n",
      "18537/18537 [==============================] - 2s 126us/step - loss: 4.1678\n",
      "Epoch 6/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 3.6958\n",
      "Epoch 7/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 3.3834\n",
      "Epoch 8/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 3.1990\n",
      "Epoch 9/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 3.1048\n",
      "Epoch 10/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 3.0354\n",
      "Epoch 11/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.9900\n",
      "Epoch 12/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.9513\n",
      "Epoch 13/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.9313\n",
      "Epoch 14/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.8973\n",
      "Epoch 15/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.8799\n",
      "Epoch 16/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.8651\n",
      "Epoch 17/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.8491\n",
      "Epoch 18/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.8389\n",
      "Epoch 19/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.8311\n",
      "Epoch 20/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8171\n",
      "Epoch 21/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8064\n",
      "Epoch 22/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8017\n",
      "Epoch 23/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.7932\n",
      "Epoch 24/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7848\n",
      "Epoch 25/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7776\n",
      "Epoch 26/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7731\n",
      "Epoch 27/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7635\n",
      "Epoch 28/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7600\n",
      "Epoch 29/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7558\n",
      "Epoch 30/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7505\n",
      "Epoch 31/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7452\n",
      "Epoch 32/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 2.7411\n",
      "Epoch 33/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7336\n",
      "Epoch 34/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7343\n",
      "Epoch 35/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7291\n",
      "Epoch 36/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7229\n",
      "Epoch 37/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7176\n",
      "Epoch 38/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7176\n",
      "Epoch 39/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7095\n",
      "Epoch 40/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7092\n",
      "Epoch 41/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7048\n",
      "Epoch 42/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7022\n",
      "Epoch 43/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6926\n",
      "Epoch 44/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.6940\n",
      "Epoch 45/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.6867\n",
      "Epoch 46/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.6768\n",
      "Epoch 47/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.6806\n",
      "Epoch 48/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.6738\n",
      "Epoch 49/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.6687\n",
      "Epoch 50/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6640\n",
      "Epoch 51/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6624\n",
      "Epoch 52/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6564\n",
      "Epoch 53/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6518\n",
      "Epoch 54/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6538\n",
      "Epoch 55/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6424\n",
      "Epoch 56/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6420\n",
      "Epoch 57/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6420\n",
      "Epoch 58/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6341\n",
      "Epoch 59/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.6230\n",
      "Epoch 60/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6263\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00060: early stopping\n",
      "the 2 fold CRPS : 0.013285\n",
      "-----------\n",
      "-----------\n",
      "validation shape 2\n",
      "Epoch 1/100\n",
      "18537/18537 [==============================] - 3s 173us/step - loss: 5.7093\n",
      "Epoch 2/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 5.4238\n",
      "Epoch 3/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 5.0957\n",
      "Epoch 4/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 4.6888\n",
      "Epoch 5/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 4.1396\n",
      "Epoch 6/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 3.6731\n",
      "Epoch 7/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 3.3649\n",
      "Epoch 8/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 3.2058\n",
      "Epoch 9/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 3.1024\n",
      "Epoch 10/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 3.0406\n",
      "Epoch 11/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 2.9796\n",
      "Epoch 12/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.9452\n",
      "Epoch 13/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 2.9082\n",
      "Epoch 14/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.8949\n",
      "Epoch 15/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.8734\n",
      "Epoch 16/100\n",
      "18537/18537 [==============================] - 3s 143us/step - loss: 2.8595\n",
      "Epoch 17/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.8406\n",
      "Epoch 18/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.8352\n",
      "Epoch 19/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.8228\n",
      "Epoch 20/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8119\n",
      "Epoch 21/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.8063\n",
      "Epoch 22/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.7937\n",
      "Epoch 23/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7885\n",
      "Epoch 24/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7824\n",
      "Epoch 25/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7786\n",
      "Epoch 26/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7675\n",
      "Epoch 27/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7665\n",
      "Epoch 28/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7591\n",
      "Epoch 29/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7516\n",
      "Epoch 30/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7477\n",
      "Epoch 31/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7465\n",
      "Epoch 32/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7350\n",
      "Epoch 33/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7359\n",
      "Epoch 34/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7283\n",
      "Epoch 35/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7310\n",
      "Epoch 36/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7173\n",
      "Epoch 37/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7184\n",
      "Epoch 38/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7117\n",
      "Epoch 39/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7056\n",
      "Epoch 40/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7093\n",
      "Epoch 41/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6927\n",
      "Epoch 42/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6931\n",
      "Epoch 43/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.6879\n",
      "Epoch 44/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6863\n",
      "Epoch 45/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6819\n",
      "Epoch 46/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6750\n",
      "Epoch 47/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6739\n",
      "Epoch 48/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6771\n",
      "Epoch 49/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6667\n",
      "Epoch 50/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 2.6666\n",
      "Epoch 51/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6552\n",
      "Epoch 52/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6556\n",
      "Epoch 53/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6534\n",
      "Epoch 54/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 2.6468\n",
      "Epoch 55/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6408\n",
      "Epoch 56/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6362\n",
      "Epoch 57/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6300\n",
      "Epoch 58/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6337\n",
      "Epoch 59/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6252\n",
      "Epoch 60/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6111\n",
      "Epoch 61/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6141\n",
      "Epoch 62/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6107\n",
      "Epoch 63/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6118\n",
      "Epoch 64/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6023\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00064: early stopping\n",
      "the 3 fold CRPS : 0.012684\n",
      "-----------\n",
      "-----------\n",
      "validation shape 2\n",
      "Epoch 1/100\n",
      "18537/18537 [==============================] - 3s 172us/step - loss: 5.7112\n",
      "Epoch 2/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 5.4111\n",
      "Epoch 3/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 5.1009\n",
      "Epoch 4/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 4.6653\n",
      "Epoch 5/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 4.1500\n",
      "Epoch 6/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 3.6849\n",
      "Epoch 7/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 3.3789\n",
      "Epoch 8/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 3.2063\n",
      "Epoch 9/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 3.1009\n",
      "Epoch 10/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 3.0378\n",
      "Epoch 11/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.9840\n",
      "Epoch 12/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.9466\n",
      "Epoch 13/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.9157\n",
      "Epoch 14/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8855\n",
      "Epoch 15/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 2.8732\n",
      "Epoch 16/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8533\n",
      "Epoch 17/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.8405\n",
      "Epoch 18/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8332\n",
      "Epoch 19/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8209\n",
      "Epoch 20/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.8087\n",
      "Epoch 21/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 2.8015\n",
      "Epoch 22/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7913\n",
      "Epoch 23/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.7828\n",
      "Epoch 24/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 2.7777\n",
      "Epoch 25/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7730\n",
      "Epoch 26/100\n",
      "18537/18537 [==============================] - 2s 121us/step - loss: 2.7683\n",
      "Epoch 27/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7623\n",
      "Epoch 28/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7571\n",
      "Epoch 29/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7489\n",
      "Epoch 30/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7479\n",
      "Epoch 31/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7481\n",
      "Epoch 32/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7389\n",
      "Epoch 33/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7341\n",
      "Epoch 34/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7268\n",
      "Epoch 35/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7192\n",
      "Epoch 36/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7159\n",
      "Epoch 37/100\n",
      "18537/18537 [==============================] - 2s 118us/step - loss: 2.7154\n",
      "Epoch 38/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7104\n",
      "Epoch 39/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7090\n",
      "Epoch 40/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.7058\n",
      "Epoch 41/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.7023\n",
      "Epoch 42/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6889\n",
      "Epoch 43/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6921\n",
      "Epoch 44/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6810\n",
      "Epoch 45/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6816\n",
      "Epoch 46/100\n",
      "18537/18537 [==============================] - 2s 120us/step - loss: 2.6818\n",
      "Epoch 47/100\n",
      "18537/18537 [==============================] - 2s 119us/step - loss: 2.6706\n",
      "Epoch 48/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.6655\n",
      "Epoch 49/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6648\n",
      "Epoch 50/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.6664\n",
      "Epoch 51/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.6535\n",
      "Epoch 52/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6522\n",
      "Epoch 53/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6558\n",
      "Epoch 54/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6444\n",
      "Epoch 55/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6460\n",
      "Epoch 56/100\n",
      "18537/18537 [==============================] - 3s 150us/step - loss: 2.6330\n",
      "Epoch 57/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6368\n",
      "Epoch 58/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.6359\n",
      "Epoch 59/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.6271\n",
      "Epoch 60/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.6227\n",
      "Epoch 61/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.6167\n",
      "Epoch 62/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.6108\n",
      "Epoch 63/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.6151\n",
      "Epoch 64/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.6045\n",
      "Epoch 65/100\n",
      "18537/18537 [==============================] - 2s 122us/step - loss: 2.5952\n",
      "Epoch 66/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.5921\n",
      "Epoch 67/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.5920\n",
      "Epoch 68/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.5880\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00068: early stopping\n",
      "the 4 fold CRPS : 0.013510\n",
      "-----------\n",
      "-----------\n",
      "validation shape 2\n",
      "Epoch 1/100\n",
      "18537/18537 [==============================] - 3s 179us/step - loss: 5.7134\n",
      "Epoch 2/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 5.4175\n",
      "Epoch 3/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 5.1026\n",
      "Epoch 4/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 4.6755\n",
      "Epoch 5/100\n",
      "18537/18537 [==============================] - 2s 127us/step - loss: 4.1496\n",
      "Epoch 6/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 3.6748\n",
      "Epoch 7/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 3.3689\n",
      "Epoch 8/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 3.1936\n",
      "Epoch 9/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 3.1006\n",
      "Epoch 10/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 3.0186\n",
      "Epoch 11/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.9791\n",
      "Epoch 12/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.9404\n",
      "Epoch 13/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.9190\n",
      "Epoch 14/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.8898\n",
      "Epoch 15/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.8707\n",
      "Epoch 16/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.8555\n",
      "Epoch 17/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.8430\n",
      "Epoch 18/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.8268\n",
      "Epoch 19/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.8183\n",
      "Epoch 20/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.8111\n",
      "Epoch 21/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.8010\n",
      "Epoch 22/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7892\n",
      "Epoch 23/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7848\n",
      "Epoch 24/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7824\n",
      "Epoch 25/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7755\n",
      "Epoch 26/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7657\n",
      "Epoch 27/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7670\n",
      "Epoch 28/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7544\n",
      "Epoch 29/100\n",
      "18537/18537 [==============================] - 2s 125us/step - loss: 2.7511\n",
      "Epoch 30/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7484\n",
      "Epoch 31/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7439\n",
      "Epoch 32/100\n",
      "18537/18537 [==============================] - 2s 124us/step - loss: 2.7342\n",
      "Epoch 33/100\n",
      "18537/18537 [==============================] - 2s 123us/step - loss: 2.7360\n",
      "Epoch 34/100\n",
      "18537/18537 [==============================] - 3s 141us/step - loss: 2.7252\n",
      "Epoch 35/100\n",
      "18432/18537 [============================>.] - ETA: 0s - loss: 2.7254"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Training Session\n",
    "from sklearn.model_selection import train_test_split, KFold\n",
    "import time\n",
    "\n",
    "NNTrainSet, enc, GroupFeatureDic = PreprocessingNNTrainSet(train)\n",
    "\n",
    "#Split train / val set\n",
    "X = NNTrainSet.copy()\n",
    "X.drop(['Yards'], axis=1, inplace=True)\n",
    "\n",
    "yards = NNTrainSet.Yards\n",
    "y = np.zeros((yards.shape[0], 199))\n",
    "for idx, target in enumerate(list(yards.astype(int))):\n",
    "    y[idx][99 + target] = 1\n",
    "\n",
    "#Scaling\n",
    "NNScaler = StandardScaler()\n",
    "X = NNScaler.fit_transform(X)\n",
    "\n",
    "#Data Split\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=12345)\n",
    "\n",
    "losses = []\n",
    "Models = []\n",
    "crps_csv = []\n",
    "\n",
    "s_time = time.time()\n",
    "yards = NNTrainSet.Yards\n",
    "\n",
    "for k in range(2):\n",
    "    kfold = KFold(5, random_state = 42 + k, shuffle = True)\n",
    "    for k_fold, (tr_inds, val_inds) in enumerate(kfold.split(yards)):\n",
    "        print(\"-----------\")\n",
    "        print(\"-----------\")\n",
    "        tr_x,tr_y = X[tr_inds],y[tr_inds]\n",
    "        val_x,val_y = X[val_inds],y[val_inds]\n",
    "        Model,crps = BuildModel(tr_x,tr_y,val_x,val_y)\n",
    "        Models.append(Model)\n",
    "        print(\"the %d fold CRPS : %f\"%((k_fold+1),crps))\n",
    "        crps_csv.append(crps)\n",
    " \n",
    "print(\"mean CRPS : %f\"%np.mean(crps_csv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\SYLee\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:60: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold : 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\SYLee\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1205: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n",
      "C:\\Users\\SYLee\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:762: UserWarning: categorical_feature in param dict is overridden.\n",
      "  warnings.warn('categorical_feature in param dict is overridden.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttraining's multi_logloss: 2.25124\tvalid_1's multi_logloss: 2.79396\n",
      "[200]\ttraining's multi_logloss: 1.94467\tvalid_1's multi_logloss: 2.76195\n",
      "[300]\ttraining's multi_logloss: 1.72803\tvalid_1's multi_logloss: 2.75293\n",
      "[400]\ttraining's multi_logloss: 1.5595\tvalid_1's multi_logloss: 2.75541\n",
      "[500]\ttraining's multi_logloss: 1.4226\tvalid_1's multi_logloss: 2.76411\n",
      "Early stopping, best iteration is:\n",
      "[317]\ttraining's multi_logloss: 1.69658\tvalid_1's multi_logloss: 2.75266\n",
      "Fold : 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\SYLee\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1205: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n",
      "C:\\Users\\SYLee\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:762: UserWarning: categorical_feature in param dict is overridden.\n",
      "  warnings.warn('categorical_feature in param dict is overridden.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttraining's multi_logloss: 2.23736\tvalid_1's multi_logloss: 2.8107\n",
      "[200]\ttraining's multi_logloss: 1.92839\tvalid_1's multi_logloss: 2.78397\n",
      "[300]\ttraining's multi_logloss: 1.71031\tvalid_1's multi_logloss: 2.77707\n",
      "[400]\ttraining's multi_logloss: 1.54069\tvalid_1's multi_logloss: 2.78183\n",
      "[500]\ttraining's multi_logloss: 1.40221\tvalid_1's multi_logloss: 2.79265\n",
      "Early stopping, best iteration is:\n",
      "[307]\ttraining's multi_logloss: 1.69716\tvalid_1's multi_logloss: 2.777\n",
      "Fold : 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\SYLee\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1205: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n",
      "C:\\Users\\SYLee\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:762: UserWarning: categorical_feature in param dict is overridden.\n",
      "  warnings.warn('categorical_feature in param dict is overridden.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttraining's multi_logloss: 2.20137\tvalid_1's multi_logloss: 2.89527\n",
      "[200]\ttraining's multi_logloss: 1.89387\tvalid_1's multi_logloss: 2.8897\n",
      "[300]\ttraining's multi_logloss: 1.67808\tvalid_1's multi_logloss: 2.89864\n",
      "Early stopping, best iteration is:\n",
      "[178]\ttraining's multi_logloss: 1.95088\tvalid_1's multi_logloss: 2.88926\n",
      "Wall time: 13min 23s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Train Trees\n",
    "\n",
    "train = pd.read_csv('./data/train.csv', dtype={'WindSpeed': 'object'})\n",
    "outcomes = train[['GameId','PlayId','Yards']].drop_duplicates()\n",
    "\n",
    "\n",
    "TreeTrainSet = PreprocessingTreeDataSet(train, False)\n",
    "\n",
    "X = TreeTrainSet.copy()#.head(5000)\n",
    "yards = X.Yards\n",
    "\n",
    "\n",
    "y = np.zeros((yards.shape[0], 199))\n",
    "for idx, target in enumerate(list(yards)):\n",
    "    y[idx][99 + target] = 1\n",
    "X.drop(['GameId','PlayId','Yards'], axis=1, inplace=True)\n",
    "X_columns = X.columns.values.tolist()\n",
    "\n",
    "TreeScaler = StandardScaler()\n",
    "X = TreeScaler.fit_transform(X)\n",
    "\n",
    "X = pd.DataFrame(X, columns=X_columns)\n",
    "# Categorical variables if <= 50 unique values\n",
    "# Warning: These have to be lists for lgbm training to work\n",
    "cat_feats = X.columns[(X.nunique() <= 50)].values.tolist()\n",
    "num_feats = X.columns[(X.nunique() > 50)].values.tolist()\n",
    "\n",
    "metric = \"multi_logloss\"\n",
    "param = {'num_leaves': 50, #Original 50\n",
    "         'min_data_in_leaf': 30, #Original 30\n",
    "         'objective':'multiclass',\n",
    "         'num_class': 199, # 199 possible places\n",
    "         'max_depth': -1,\n",
    "         'learning_rate': 0.01,\n",
    "         \"min_child_samples\": 20,\n",
    "         \"boosting\": \"gbdt\",\n",
    "         \"feature_fraction\": 0.7, #0.9\n",
    "         \"bagging_freq\": 1,\n",
    "         \"bagging_fraction\": 0.9,\n",
    "         \"bagging_seed\": 11,\n",
    "         \"metric\": metric,\n",
    "         \"lambda_l1\": 0.1,\n",
    "         \"verbosity\": -1,\n",
    "         \"seed\":1234}\n",
    "\n",
    "Trees = []\n",
    "kf = KFold(n_splits=3, random_state=42)\n",
    "score = []\n",
    "feature_importance_df = pd.DataFrame()\n",
    "best_validation_scores = []\n",
    "y = np.argmax(y, axis=1)\n",
    "\n",
    "for i, (tdx, vdx) in enumerate(kf.split(X, y)):\n",
    "    print(f'Fold : {i}')\n",
    "    X_train, X_val, y_train, y_val = X.iloc[tdx], X.iloc[vdx], y[tdx], y[vdx]\n",
    "    trn_data = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_feats)\n",
    "    val_data = lgb.Dataset(X_val, label=y_val, categorical_feature=cat_feats)\n",
    "    num_round = 10000\n",
    "    Tree = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 200)\n",
    "    score_ = crps(np.expand_dims(y_val, axis=1), Tree.predict(X_val, num_iteration=Tree.best_iteration))\n",
    "    \n",
    "    #fold_importance_df = pd.DataFrame()\n",
    "    #fold_importance_df[\"feature\"] = X_columns\n",
    "    #fold_importance_df[\"importance\"] = Tree.feature_importance()\n",
    "    #fold_importance_df[\"fold\"] = i + 1\n",
    "    #feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n",
    "    best_validation_scores.append(Tree.best_score['valid_1'][metric])\n",
    "    score.append(score_)\n",
    "    Trees.append(Tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your submission file has been saved!  Once you `Commit` your Notebook and it finishes running, you can submit the file to the competition from the Notebook Viewer `Output` tab.\n",
      "CPU times: user 1h 5min 34s, sys: 24.5 s, total: 1h 5min 59s\n",
      "Wall time: 1h 4min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Submission\n",
    "if  TRAIN_OFFLINE==False:\n",
    "    from kaggle.competitions import nflrush\n",
    "    env = nflrush.make_env()\n",
    "    iter_test = env.iter_test()\n",
    "\n",
    "    for (test_df, sample_prediction_df) in iter_test:\n",
    "        NNTestSet = PreprocessingNNTestSet(test_df,enc,GroupFeatureDic)\n",
    "        NNTestSet = NNScaler.transform(NNTestSet)\n",
    "        \n",
    "        TreeTestSet = PreprocessingTreeDataSet(test_df, deploy=True)\n",
    "        TreeTestSet.drop(['GameId','PlayId'], axis=1, inplace=True)\n",
    "        TreeTestSet = TreeScaler.transform(TreeTestSet)\n",
    "        \n",
    "        y_pred_NN = np.mean([Model.predict(NNTestSet, batch_size = 1024) for Model in Models], axis = 0)\n",
    "        y_pred_Trees = np.mean([Tree.predict(scaled_basetable, num_iteration=Tree.best_iteration) for Tree in Trees], axis = 0)\n",
    "        y_pred = (y_pred_NN + y_pred_Trees) / len(Models + Trees)\n",
    "        y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1).tolist()[0]\n",
    "\n",
    "        preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)\n",
    "        env.predict(preds_df)\n",
    "\n",
    "    env.write_submission_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['submission.csv']\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "print([filename for filename in os.listdir('/kaggle/working') if '.csv' in filename])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}