MovieLens 1M Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "Using gpu device 0: GRID K520 (CNMeM is disabled, cuDNN 5006)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import math\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint\n",
    "from CFModel import CFModel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define constants\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "RATINGS_CSV_FILE = 'ml1m_ratings.csv'\n",
    "MODEL_WEIGHTS_FILE = 'ml1m_weights.h5'\n",
    "K_FACTORS = 120\n",
    "RNG_SEED = 1446557"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load MovieLens 1M data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000209 ratings loaded.\n"
     ]
    }
   ],
   "source": [
    "ratings = pd.read_csv(RATINGS_CSV_FILE, \n",
    "                      sep='\\t', \n",
    "                      encoding='latin-1', \n",
    "                      usecols=['userid', 'movieid', 'user_emb_id', 'movie_emb_id', 'rating'])\n",
    "max_userid = ratings['userid'].drop_duplicates().max()\n",
    "max_movieid = ratings['movieid'].drop_duplicates().max()\n",
    "print len(ratings), 'ratings loaded.'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Users: [4403 1819 2571 ...,  354 3510  831] , shape = (1000209,)\n",
      "Movies: [2717 3692  149 ..., 2700 2920 3711] , shape = (1000209,)\n",
      "Ratings: [5 4 4 ..., 2 4 4] , shape = (1000209,)\n"
     ]
    }
   ],
   "source": [
    "shuffled_ratings = ratings.sample(frac=1., random_state=RNG_SEED)\n",
    "Users = shuffled_ratings['user_emb_id'].values\n",
    "print 'Users:', Users, ', shape =', Users.shape\n",
    "Movies = shuffled_ratings['movie_emb_id'].values\n",
    "print 'Movies:', Movies, ', shape =', Movies.shape\n",
    "Ratings = shuffled_ratings['rating'].values\n",
    "print 'Ratings:', Ratings, ', shape =', Ratings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = CFModel(max_userid, max_movieid, K_FACTORS)\n",
    "model.compile(loss='mse', optimizer='adamax')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 900188 samples, validate on 100021 samples\n",
      "Epoch 1/30\n",
      "68s - loss: 7.9214 - val_loss: 2.0847\n",
      "Epoch 2/30\n",
      "68s - loss: 1.3914 - val_loss: 1.0758\n",
      "Epoch 3/30\n",
      "68s - loss: 0.9606 - val_loss: 0.9078\n",
      "Epoch 4/30\n",
      "68s - loss: 0.8582 - val_loss: 0.8514\n",
      "Epoch 5/30\n",
      "68s - loss: 0.8129 - val_loss: 0.8213\n",
      "Epoch 6/30\n",
      "68s - loss: 0.7831 - val_loss: 0.8033\n",
      "Epoch 7/30\n",
      "68s - loss: 0.7597 - val_loss: 0.7889\n",
      "Epoch 8/30\n",
      "68s - loss: 0.7379 - val_loss: 0.7772\n",
      "Epoch 9/30\n",
      "68s - loss: 0.7162 - val_loss: 0.7680\n",
      "Epoch 10/30\n",
      "68s - loss: 0.6948 - val_loss: 0.7601\n",
      "Epoch 11/30\n",
      "68s - loss: 0.6721 - val_loss: 0.7529\n",
      "Epoch 12/30\n",
      "68s - loss: 0.6490 - val_loss: 0.7477\n",
      "Epoch 13/30\n",
      "68s - loss: 0.6254 - val_loss: 0.7451\n",
      "Epoch 14/30\n",
      "68s - loss: 0.6019 - val_loss: 0.7436\n",
      "Epoch 15/30\n",
      "68s - loss: 0.5778 - val_loss: 0.7435\n",
      "Epoch 16/30\n",
      "68s - loss: 0.5537 - val_loss: 0.7459\n",
      "Epoch 17/30\n",
      "68s - loss: 0.5302 - val_loss: 0.7488\n",
      "Epoch 18/30\n",
      "68s - loss: 0.5070 - val_loss: 0.7543\n"
     ]
    }
   ],
   "source": [
    "callbacks = [EarlyStopping('val_loss', patience=2), \n",
    "             ModelCheckpoint(MODEL_WEIGHTS_FILE, save_best_only=True)]\n",
    "history = model.fit([Users, Movies], Ratings, nb_epoch=30, validation_split=.1, verbose=2, callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot training and validation RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0, 3.0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAG2CAYAAAA+zfE7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl8VPW5x/Hvk4R9C4uABQMoAgoqat1rjdprXWqtVrhu\nVWxv3evW1qqtpa3X1vbWautuqxZ7q17B1rrbKgZra92xKO4CQUFQSSAQlpA894/fnBjChEwmc2bL\n5/16ndfMOXPmzI8vMT6c3zPnmLsLAAAAuVWS6wEAAACAogwAACAvUJQBAADkAYoyAACAPEBRBgAA\nkAcoygAAAPJArEWZmfUws2fN7GUzm2dm05Ps093M7jazt83sGTOriHNMAAAA+SjWoszd10s60N13\nlTRZ0mFmtmer3b4haYW7by/pGkm/iHNMAAAA+Sj26Ut3r0887SGpTFLrq9UeJWlG4vksSQfHPSYA\nAIB8E3tRZmYlZvaypA8l/c3dn2+1ywhJiyXJ3Rsl1ZrZoLjHBQAAkE/K4v4Ad2+StKuZ9Zd0n5nt\n6O7zW+xird5i2vxsmsyM+0EBAICC4e6ta5wtytq3L919laQqSYe2emmxpG0kycxKJfV395o2jpHT\nZfr06TkfQz4s5EAG5EAOZEAO5LDlJR1xf/tyiJkNSDzvJekLkt5otdsDkk5JPJ8iaXacY+qMhQsX\n5noIeYEcyCBCDgE5kEGEHAJySE/c05dbS5phZiUKBeD/ufvDZvZjSc+7+4OSbpX0BzN7W9Inko6L\neUwAAAB5J9aizN3nSdotyfbpLZ6vlzQ1znFkyrRp03I9hLxADmQQIYeAHMggQg4BOaTH0p33zDYz\n80IZKwAA6NrMTJ6vjf7FoKqqKtdDyAvkQAYRcgjIgQwiceYwevRomRlLni2jR4/O2N9x7JfEAAAA\nnbdo0aK0v9WH+Jh16GTYlo9VKH/BTF8CALqyxHRYroeBVtr6e2H6EgAAoEBRlHUAPRMBOZBBhBwC\nciCDCDmgMyjKAAAA8gA9ZQAAFIBi7ik788wzNXLkSH3/+9/P6L7ZkMmeMooyAAAKQD4XZWPGjNGt\nt96qgw46KNdDyToa/XOEXoGAHMggQg4BOZBBhBw219jYmOshFAyKMgAAkLaTTz5Z1dXV+tKXvqT+\n/fvrf/7nf1RSUqLbbrtNo0aN0sEHHyxJmjp1qrbeemsNHDhQlZWVmj9/fvMxTj31VP3whz+UJM2Z\nM0fbbLONfvWrX2nYsGEaMWKEfv/736e174oVK3TkkUdqwIAB2muvvXTZZZdp//33jz+UNFGUdUBl\nZWWuh5AXyIEMIuQQkAMZRLpiDnfccYcqKir00EMPadWqVZo6NdzO+qmnntIbb7yhxx57TJJ0+OGH\n691339Xy5cu122676cQTT2zzmB9++KHq6uq0ZMkS/e53v9PZZ5+tlStXdnjfs846S/369dPy5cv1\n+9//XjNmzMjoxV4zjaIMAIAiYJaZJV0t+6rMTD/+8Y/Vq1cv9ejRQ1K4SXnv3r3VrVs3/fCHP9Qr\nr7yiurq6pMfq3r27LrvsMpWWluqwww5T37599eabb3Zo36amJv3pT3/ST37yE/Xo0UM77LCDTjnl\nlPT/gFlAUdYB9AoE5EAGEXIIyIEMIrnMwT0zS6aMHDmy+XlTU5MuvvhijR07VuXl5RozZozMTB9/\n/HHS9w4ePFglJZ+WKL1799bq1as7tO9HH32kxsbGTcaxzTbbdPaPFSuKMgAA0CnJpgRbbrvzzjv1\nwAMPaPbs2aqtrdXChQvl7rF+m3SrrbZSWVmZ3n///eZtixcvju3zMoGirAO6Yq9AMuRABhFyCMiB\nDCJdNYfhw4frvffek6SkxVZdXZ169OihgQMHas2aNbrkkkti7+0qKSnRMcccox/96Edau3at3njj\nDd1xxx2xfmZnUZQBAIBOufjii3X55Zdr0KBBuvfeezcruE4++WRVVFRoxIgRmjRpkvbdd98OHb8j\nBVzLfa+99lrV1tZq66231imnnKITTjihucctH3Hx2A6oqqrqsv8KaokcyCBCDgE5kEEkzhzy+eKx\nheLiiy/WsmXLdPvtt2fsmFw8FgAAoB1vvvmm5s2bJ0l67rnndOutt+qYY47J8ajaxpkyAAAKAGfK\nOu6FF17Q8ccfr6VLl2ro0KE644wzdNFFF2X0M7j3JQAAXQxFWX7qstOXa9bk9vO5Dk9ADmQQIYeA\nHMggQg7ojIIqyt56K9cjAAAAiEdBTV/edZfruONyPRIAALKP6cv81GWnL994I9cjAAAAiEdBFWVt\n3Is0a+gVCMiBDCLkEJADGUTIAZ1BUQYAALJuzpw5m9wgfNKkSXrqqadS2rejzjzzTF1xxRVpvz9b\nCqqnrE8f16pVUklBlZIAAHResfWUzZkzR1/72tdUXV2d0X1nzJih3/3ud/r73/+eiWG2q8v2lPXv\nL33wQa5HAQAA8pW7x36z87gUVFE2YUJum/3pFQjIgQwi5BCQAxlEumIOP//5zzVlypRNtp1//vk6\n//zz9fvf/1477rij+vfvr7Fjx+qWW25p8zhjxozR7NmzJUnr1q3TtGnTNGjQIE2aNEnPP//8Zp85\nduxY9e/fX5MmTdJ9990nSXrjjTd05pln6plnnlG/fv00aNAgSdKpp56qH/7wh83v/+1vf6vtt99e\nQ4YM0Ve+8hUtXbq0+bWSkhLdfPPNGjdunAYPHqxzzjmncwF1QEEVZePH01cGAEA+Of744/XII49o\n9erVkqSmpibdc889OuGEEzRs2DA99NBDWrVqlW6//XZdcMEFmjt3brvH/NGPfqQFCxZowYIFeuyx\nxzRjxoxNXh87dqz+8Y9/aNWqVZo+fbpOOukkLVu2TBMmTNBNN92kffbZR3V1dVqxYsVmx549e7Yu\nvfRSzZo1S0uXLlVFRYWOa3W9rYceekgvvvii5s6dq3vuuUd//etfO5FQ6sqy8ikZMn58bs+UVVZW\n5u7D8wg5kEGEHAJyIINILnOwH2dmys6nd6xvraKiQrvttpvuu+8+nXTSSXriiSfUp08f7bnnnpvs\nt//+++uQQw7R3//+d02ePHmLx5w5c6ZuuukmDRgwQAMGDNC5556ryy+/vPn1r371q83Pp0yZop/+\n9Kd67rnndOSRR7Y73jvvvFPf+MY3tMsuu0iSfvazn2ngwIGqrq5WRUWFJOmSSy5Rv3791K9fPx14\n4IGaO3euDjnkkJQzSVdBFWUTJkgPPZTrUQAAkH86Wkxl0vHHH6+77rpLJ510ku666y6dcMIJkqRH\nHnlEP/nJT/TWW2+pqalJa9eu1c4779zu8ZYsWaKRI0c2r48aNWqT1++44w5dffXVWrhwoSRpzZo1\n+vjjj1Ma65IlS7T77rs3r/fp00eDBw/WBx980FyUDRs2rPn13r17N58FjFvBTV/SU5Z75EAGEXII\nyIEMIl01hylTpqiqqkoffPCB/vznP+vEE0/Uhg0bdOyxx+qiiy7SRx99pJqaGh122GEpfYN06623\n1uLFi5vXFy1a1Py8urpap512mm644QbV1NSopqZGEydObD5ue03+n/nMZzY53po1a/TJJ59sUgTm\nSkEVZRUV0iefSFkqWAEAQAqGDBmiAw44QKeeeqq23XZbjRs3Ths2bNCGDRs0ZMgQlZSU6JFHHkm5\nN2vq1Kn62c9+ptraWr3//vu67rrrml9bs2aNSkpKNGTIEDU1Nen222/Xq6++2vz6sGHD9P7776uh\noSHpsU844QTdfvvt+ve//63169fr0ksv1d57792p66BlSkEVZaWl0tixubsxOT0TATmQQYQcAnIg\ng0hXzuGEE07QE088oRNPPFGS1LdvX/3mN7/RlClTNGjQIN1999066qij2nx/yzNc06dPV0VFhcaM\nGaNDDz1UJ598cvNrO+ywg7797W9r77331vDhw/Xaa6/pc5/7XPPrBx10kCZOnKjhw4dr6NChm33O\nQQcdpMsvv1zHHHOMRowYoQULFujuu+9OOo5k63EqqIvHurumTpWOPlo6/vhcjwgAgOwptovHFosu\ne/FYKbd9ZV21V6A1ciCDCDkE5EAGEXJAZxRcUTZhAtcqAwAAxafgpi9feEH6r/+SUrj2HAAARYPp\ny/yUyenLgivK6uqk4cOlujpuTA4A6DooyvJTl+4p69dPKi+XWly+JGvoFQjIgQwi5BCQAxlEyAGd\nUXBFmURfGQAAKD4FN30pSWedFQqzc8/N8aAAAMiS0aNHb3IleuSHUaNGNd/uqaV0pi8L6t6XEc6U\nAQC6mmT/40dxKcjpy1xdq4xegYAcyCBCDgE5kEGEHAJySE9BFmWcKQMAAMWmIHvKmprCtzA//DA8\nAgAA5JMucUkMKVyfbPvtc3djcgAAgEwryKJMCn1l2Z7CZI48IAcyiJBDQA5kECGHgBzSU7BF2YQJ\nubsxOQAAQKYVZE+ZJN15p/SXv0j/9385HBQAAEASXaanTMrdZTEAAADiUNBF2dtvh29iZgtz5AE5\nkEGEHAJyIIMIOQTkkJ6CLcr69pUGDZKqq3M9EgAAgM4r2J4ySfrCF6Tvflf64hdzNCgAAIAk8q6n\nzMxGmtlsM5tvZvPMbLNbiJvZAWZWa2YvJZYfpHp8+soAAECxiHv6cqOkC919R0n7SDrbzCYk2e8p\nd98tsfx3qgfP9u2WmCMPyIEMIuQQkAMZRMghIIf0xFqUufuH7j438Xy1pNcljUiya4dO70U4UwYA\nAIpF1nrKzGy0pCpJkxIFWrT9AEmzJL0vaYmk77r7/CTv36ynrLpa2mcf6YMP4hs3AABAR6XTU1YW\n12BaMrO+CoXXeS0LsoQXJY1y93ozO0zSfZLGJTvOtGnTNHr0aElSeXm5dt55smprK7VqlfTSS1WS\npMrKSkmfnjplnXXWWWedddZZj3s9er5w4UKlK/YzZWZWJulBSY+4+69T2H+BpN3dfUWr7ZudKZOk\nXXeVfvtb6bOfzdSI21ZVVdX8l9CVkQMZRMghIAcyiJBDQA55+O3LhNskzW+rIDOzYS2e76lQKK5I\ntm8y9JUBAIBiEOuZMjPbT9JTkuZJ8sRyqaRRktzdbzGzsyWdKalB0lpJF7j7s0mOlfRM2Y9+JDU2\nSpdfHtsfAwAAoEPyrqfM3f8hqbSdfa6XdH26nzF+vPSnP6X7bgAAgPyQjenLWGXzWmUtm/m6MnIg\ngwg5BORABhFyCMghPQVflI0bF25M3tiY65EAAACkr6DvfRmpqJDmzJHGjMnyoAAAAJLI129fxm78\n+OzebgkAACDTiqIomzAhO5fFYI48IAcyiJBDQA5kECGHgBzSUxRFGWfKAABAoSuKnrLHH5euuEJ6\n8sksDwoAACAJesoAAAAKVFEUZSNGSKtWhSVOzJEH5EAGEXIIyIEMIuQQkEN6iqIoKykJ1yvjbBkA\nAChURdFTJknHHy8dfrj0ta9lcVAAAABJdNmeMim7t1sCAADItKIpysaPj/9aZcyRB+RABhFyCMiB\nDCLkEJBDeoqmKONMGQAAKGRF01O2Zo00ZIi0erVUWprFgQEAALTSpXvK+vSRhg6VFi3K9UgAAAA6\nrmiKMin+vjLmyANyIIMIOQTkQAYRcgjIIT1FVZTRVwYAAApV0fSUSdL110v//rd0881ZGhQAAEAS\nXbqnTOJMGQAAKFxFVZTRU5Yd5EAGEXIIyIEMIuQQkEN6iqooGzEiXBpj5cpcjwQAAKBjiqqnTJJ2\n31268UZpzz2zMCgAAIAkunxPmRT6yuK+3RIAAECmFV1RNn58fM3+zJEH5EAGEXIIyIEMIuQQkEN6\niq4o40wZAAAoREXXU/bKK9KJJ0qvvpqFQQEAACSRTk9Z0RVl9fXS4MHcmBwAAOQOjf6SeveWhg2T\nFi7M/LGZIw/IgQwi5BCQAxlEyCEgh/QUXVEmxX8RWQAAgEwruulLSTrvPGnUKOnCC2MeFAAAQBJM\nXyZwpgwAABSaoizK4roxOXPkATmQQYQcAnIggwg5BOSQnqIsyjhTBgAACk1R9pS5S/37S9XV0sCB\nMQ8MAACgFXrKEszivd0SAABAphVlUSbF01fGHHlADmQQIYeAHMggQg4BOaSnaIsy+soAAEAhKcqe\nMkmaOVO66y7pT3+KcVAAAABJ0FPWAmfKAABAISnaomz77aUFC6SNGzN3TObIA3Iggwg5BORABhFy\nCMghPUVblPXqJQ0fHgozAACAfFe0PWWSdNhh0tlnS1/6UkyDAgAASIKesla4VhkAACgURV2UTZiQ\n2WZ/5sgDciCDCDkE5EAGEXIIyCE9RV2UcaYMAAAUiqLuKVu6VNplF2n58pgGBQAAkAQ9Za0MHy6t\nWyetWJHrkQAAAGxZURdlZpm9ByZz5AE5kEGEHAJyIIMIOQTkkJ6iLsok+soAAEBhKOqeMkm64gqp\nrk668soYBgUAAJAEPWVJcKYMAAAUgi5RlGXqWmXMkQfkQAYRcgjIgQwi5BCQQ3qKviiLbkze0JDr\nkQAAALSt6HvKJGnbbaVHH5XGjcvwoAAAAJKgp6wNmbwsBgAAQBxiLcrMbKSZzTaz+WY2z8zObWO/\n35jZ22Y218wmZ3ocmeorY448IAcyiJBDQA5kECGHgBzSE/eZso2SLnT3HSXtI+lsM5vQcgczO0zS\ndu6+vaTTJd2U6UFwpgwAAOS7rPaUmdl9kq519ydabLtJ0pPu/n+J9dclVbr7slbvTbunrKpK+sEP\npKefTnvoAAAAKcvrnjIzGy1psqRnW700QtLiFusfJLZlDGfKAABAvivLxoeYWV9JsySd5+6rW7+c\n5C1JT4lNmzZNo0ePliSVl5dr8uTJqqyslPTp/HWy9WHDpPr6Kv3lL9JRR7W/f1vrc+fO1fnnn5/2\n+4tlvWWvQD6MJxfr11xzTco/f8W8Hm3Ll/Hw85C7dX4/8vuxq/88RM8XLlyodMU+fWlmZZIelPSI\nu/86yeutpy/fkHRAJqcvJWmvvaSrr5b23TftQ6iqqqr5L6ErIwcyiJBDQA5kECGHgBzSm77MRlF2\nh6SP3f3CNl4/XNLZ7n6Eme0t6Rp33zvJfp0qyk4+WaqslL7+9bQPAQAAkJJ0irJYpy/NbD9JJ0qa\nZ2YvK0xLXipplCR391vc/WEzO9zM3pG0RtKpcYyFvjIAAJDPSuI8uLv/w91L3X2yu+/q7ru5+6Pu\nfrO739Jiv3Pcfay77+LuL8UxlkzcmLzlvHFXRg5kECGHgBzIIEIOATmkJ9aiLJ9MmJC5G5MDAABk\n2hZ7ysysVNLj7n5g9obU5lg61VO2fr00YIBUVyd165bBgQEAALSS8euUuXujpCYzG9CpkeWBHj2k\nkSOl997L9UgAAAA2l8r05WqFRv1bE/eo/I2Z/SbugcWhs31lzJEH5EAGEXIIyIEMIuQQkEN6Uvn2\n5Z8SS8GL+sq+/OVcjwQAAGBTKV2nzMy6SxqXWH3T3RtiHVXyMXSqp0ySbrlFevZZ6dZbMzQoAACA\nJGK596WZVUp6W9L1km6Q9JaZfT6tEeYY38AEAAD5KpWesqskHeLuB7j75yV9UdLV8Q4rHvSUZQY5\nkEGEHAJyIIMIOQTkkJ5UirJu7t5cyrj7W5IK8qISQ4dKGzdKH3+c65EAAABsqt2eMjO7TeH2SH9I\nbDpRUpm7x3I7pC2Mo9M9ZZK0997SVVdJ++2XgUEBAAAkEUtPmaQzJb0m6VxJ50maL+mMjg8vP4wf\nT18ZAADIP1ssyhJX9L/N3X/l7se4+9HufrW7r8/S+DKuMzcmZ448IAcyiJBDQA5kECGHgBzSk8oV\n/UclLolRFDhTBgAA8lEqPWV3SNpB0v2S1kTb3f1X8Q5ts3FkpKds/nzp6KM79y1MAACALUmnpyyV\nK/q/m1hKJPVLZ2D5ZLvtpEWLpA0bpO5Fc/4PAAAUulR6yvq5+49bL1kaX8b16CFts016NyZnjjwg\nBzKIkENADmQQIYeAHNKTSk9Z0V08gr4yAACQb1LpKbtR0ghJM7VpT1lWb1KeqZ4ySfrOd6SttpK+\n972MHA4AAGATcfWU9ZT0iaSDWmxzSVktyjJp/Hjpn//M9SgAAAA+1e7FY9391CTL17MxuLike60y\n5sgDciCDCDkE5EAGEXIIyCE97RZlZjbOzJ4ws1cT6zub2Q/iH1p8op6yDM2GAgAAdFoqPWVzJH1X\n0s3uvmti26vuPikL42s5joz1lLlLgweHs2VbbZWRQwIAADSL696Xvd39uVbbNnbkQ/KNWThbxgVk\nAQBAvkilKPvYzLZTaO6XmR0raWmso8qCCRM6flkM5sgDciCDCDkE5EAGEXIIyCE9qXz78mxJt0ia\nYGYfSFog6cRYR5UFnCkDAAD5pN2esuYdzfpIKnH3uniH1ObnZ6ynTJLuu0+69VbpgQcydkgAAABJ\n8V2nTJLk7mva36twcKYMAADkk1R6yorSdttJ1dXhxuSpYo48IAcyiJBDQA5kECGHgBzS02WLsu7d\npYoK6d13cz0SAACALfSUmdkxW3pjId/7MnLkkdLXvy4dfXRGDwsAALq4TPeUHZl4HCppX0mzE+sH\nSvqnCvjel5F0b7cEAACQaW1OX0b3uZTUTdKO7v5Vd/+qpImJbQUvut1SqpgjD8iBDCLkEJADGUTI\nISCH9KTSU7aNu7e8WOwySRUxjSerOFMGAADyRSr3vrxO0vaS7lK4qv9xkt5x92/FP7xNxpHxnrKP\nPpLGjZNWrAi3XgIAAMiEdHrKUrp4rJkdLenzidWn3P3PaYyvU+IoytylIUOk11+Xhg7N6KEBAEAX\nFtcNySXpJUkPufsFkh4zs34dHl0eim5MnmpfGXPkATmQQYQcAnIggwg5BOSQnnaLMjP7pqRZkm5O\nbBoh6b44B5VN9JUBAIB8kEpP2VxJe0p61t13TWyb5+47ZWF8LceR8elLSfr5z6Xly6Wrrsr4oQEA\nQBcV1/TlendvvhmRmZUpNPwXBc6UAQCAfJBKUTbHzC6V1MvM/kPSTEkPxDus7KGnrOPIgQwi5BCQ\nAxlEyCEgh/SkUpRdLOkjSfMknS7pYUk/iHNQ2bTddtL770vr1+d6JAAAoCvbYk+ZmZVKusPdT8ze\nkNocSyw9ZVI4W/bnP0s77hjL4QEAQBeT8Z4yd2+UNMrMundqZHluwoSO3W4JAAAg01KZvnxP0j/M\n7DIzuzBa4h5YNo0fn1qzP3PkATmQQYQcAnIggwg5BOSQnrIU9nk3sZRIKoqLxrY2YYI0Z06uRwEA\nALqylG6zlA/i7Cn7xz+kb39b+te/Yjk8AADoYmK596WZbSXpIkkTJfWMtrv7QekMMl1xFmWffBK+\nhVlTw43JAQBA58V18dg/SnpD0hhJP5a0UNLzHR5dHhs8WCorC1f23xLmyANyIIMIOQTkQAYRcgjI\nIT2pFGWD3f1WSQ3uPsfdvy4pq2fJsoFvYAIAgFxKZfryX+6+t5k9Juk3kpZImuXu22VjgC3GEdv0\npSR94xvSXntJp50W20cAAIAuIp3py1S+ffnfZjZA0rclXSupv6QL0hhfXuvI7ZYAAAAyrd3pS3d/\n0N1Xuvur7n6gu+/u7vdnY3DZlMqNyZkjD8iBDCLkEJADGUTIISCH9LR7pszMbpe02bxhoresaHCm\nDAAA5FIqPWVfbbHaU9LRkpa4+7lxDizJOGLtKWtokPr1k2prpZ49298fAACgLbH0lLn7va0+5C5J\nT3dwbHmvWzdp9GjpnXekSZNyPRoAANDVpHJJjNa2lzQ00wPJB+31lTFHHpADGUTIISAHMoiQQ0AO\n6Wm3KDOzOjNbFT1KekDS91I5uJndambLzOzfbbx+gJnVmtlLieUHHRt+ZtFXBgAAciXWe1+a2eck\nrZZ0h7vvnOT1AyR9292/nMKxYu0pk6Tbb5eefFK6445YPwYAABS5WHrKzGy3Lb3u7i9t4bWnzWxU\nex/R3hiyZfx46cYbcz0KAADQFaXSU3aDpH9JukXSbyU9K+l6SVdJ+mUGxrC3mb1sZg+Z2Y4ZOF7a\nxo8PPWVtnZBjjjwgBzKIkENADmQQIYeAHNKTSlG2RNLu7v5Zd99d0q6SPkhcSLaz98B8UdIod99V\n0nWS7uvk8Tpl8GCpe3fpww9zOQoAANAVpXKbpfHuPi9acfdXzWyHTHy4u69u8fwRM7vBzAa5+4pk\n+0+bNk2jR4+WJJWXl2vy5MmqrKyU9GlV3tn1CRMq9eab0ptvJn89kqnPK8T1ysrKvBpPLtajbfky\nHtZzux5ty5fx5Go9ki/jycV6Jb8fu+zPQ/R84cKFSlcqF4+9S9IaSf+rcGX/kyT1dffjU/oAs9GS\nHnD3nZK8NszdlyWe7ynpHncf3cZxYm/0l6T/+i9pjz2k00+P/aMAAECRSqfRvySFfU6V9Jqk8ySd\nL2l+YlsqA7pT0j8ljTOzajM71cxON7PTErsca2avmtnLkq6R9J8dGXwcJkxo+7IYrav/roocyCBC\nDgE5kEGEHAJySE8qV/RfJ+lqSVeb2SBJIxPb2uXuJ7Tz+vUKXxrIG+PHS7Nn53oUAACgq0ll+rJK\n0pcVCrgXJS2X9E93vyD20W06jqxMX779tvTFL0rvvRf7RwEAgCIV1/TlAHdfJekYhYvA7iXp4HQG\nWAjGjJGWLpXWpXQuEAAAIDNSKcrKzGxrSVMlPRjzeHKurCwUZm+/vflrzJEH5EAGEXIIyIEMIuQQ\nkEN6UinKfiLpMUnvuPvzZratpCQlS/yeWfxMVj4nuogsAABAtsR678tMMjPf6Yad9OJpL6pbabdY\nP+uSS6Q+faQf5PT26AAAoFDF1VOWN4b3Ha5fP/vr2D+HM2UAACDbCqoou+GIG3Tl01dqUe2iWD9n\n/Pjk1ypjjjwgBzKIkENADmQQIYeAHNJTUEXZ2EFjdf7e5+tbj3xLcU67tndjcgAAgExL5TplPSR9\nVdJotbjYrLv/JNaRbT4Od3et37hek2+erJ8e9FMdvcPRsX3e0KHS3LnSZz4T20cAAIAiFVdP2V8k\nHSVpo8I9MKMlJ3qU9dCNR9yocx89V3Xr62L7nAkT6CsDAADZk0pRNtLd/9Pdf+HuV0VL7CPbgsrR\nlTp4zMEDgeyaAAAgAElEQVSaXjU9ts9I1lfGHHlADmQQIYeAHMggQg4BOaQnlaLsn2a2U+wj6aBf\nHvJL/XHeH/Xy0pdjOT5nygAAQDal0lM2X9JYSQskrZdkktzdd45/eJuMY7N7X9728m266YWb9Mw3\nnlFpSWlGP+/BB6XrrpMefTSjhwUAAF1AOj1lqRRlo5Jtd/d4r0ux+Tg2K8qavEkH/P4AHT/peJ21\nx1kZ/bx33pH+4z+kBQsyelgAANAFxNLo7+6LEgXYWkneYsm5EivRTUfcpOlV07W0bmlGjz16dLgx\n+dq1n25jjjwgBzKIkENADmQQIYeAHNLTblFmZl82s7cVpi/nSFoo6ZGYx5WyiUMn6pu7fVMXPHZB\nRo9bViZtt13yG5MDAABkWirTl69IOkjS4+6+q5kdKOkkd/9GNgbYYhybTV9G6hvqNemGSbrxiBv1\nxbFfzNhnHnOMdNxx0tSpGTskAADoAuK6TlmDu38iqcTMStz9SUmfTWuEMendrbeuP/x6nf3w2Vrb\nsLb9N6SIb2ACAIBsSaUoqzWzvpL+LumPZvZr5fDisW05bPvDtNvWu+mKv1+RsWO2vjE5c+QBOZBB\nhBwCciCDCDkE5JCeVIqyoyTVSzpf0qOS3pV0ZJyDStc1h16jm1+8Wa9/9HpGjjdhQvIbkwMAAGRa\nuz1lUvNlMbZ398fNrLekUneP7x5HycfQZk9ZS9c+e63uff1ePXnKkzLr0FTuZmprpW22kVatkjp5\nKAAA0IXE0lNmZt+UNEvSzYlNIyTd1/HhZcdZe5ylNQ1rNOOVGZ0+Vnm51KePtGRJBgYGAACwBalM\nX54taT9JqyTJ3d+WNDTOQXVGaUmpbjriJn3v8e/p4/qPO328ln1lzJEH5EAGEXIIyIEMIuQQkEN6\nUinK1rv7hmjFzMqUJxePbcvun9ldx008Tt/72/c6fSz6ygAAQDakcp2yX0iqlXSypG9JOkvSfHf/\nfvzD22QcKfWURVatX6WJN0zUncfcqf1H7Z/25/7qV9KiRdKvf532IQAAQBcT13XKLpb0kaR5kk6X\n9LCkH3R8eNnVv0d/XfPFa3T6g6drQ+OG9t/QBs6UAQCAbEjl3pdN7v5bd5/i7scmnuf19GXkmB2O\n0bYDt9VV/7wq7WPQU7Y5ciCDCDkE5EAGEXIIyCE9qXz78ktm9rKZrTCzVWZWZ2arsjG4zjIzXXf4\ndbrqmav0Xs17aR1j9Ghp2TKpvj6zYwMAAGgplZ6ydyQdI2leLs+QdbSnrKWfP/1zVS2q0sMnPJzW\ntcsmTZL++Edpl13S+ngAANDFxNVTtljSq4UyZZnMhftcqMUrF2vW/FlpvX/8ePrKAABAvFIpyi6S\n9LCZXWJmF0ZL3APLpG6l3XTzl27WBY9doJXrVnb4/dGNyZkjD8iBDCLkEJADGUTIISCH9KRSlF2h\ncO/LnpL6tVgKyn4V++mwsYfpB7M7/sVRzpQBAIC4pdJT9qq7T8rSeLY0jk7PoK5Yu0ITb5io+4+7\nX3uM2CPl9z33nHTmmdKLL3bq4wEAQBcRV0/Zw2Z2SJpjyiuDeg3SL77wC53+4Ona2LQx5fdFl8Uo\n3K46AACQ71Ipys6U9KiZrS20S2Ikc9LOJ6m8Z7muf+76lN8zYIDUr580a1ZVfAMrIPQKkEGEHAJy\nIIMIOQTkkJ5ULh7bz91L3L2Xu/dPrPfPxuDiYGa68YgbdflTl+v9Ve+n/L7x46Xq6hgHBgAAurR2\ne8ryRSZ6ylqa/uR0vfbRa5o1NbXLZJxxRrhe2TnnZGwIAACgSMXVU1aULtn/Er2y7BU99NZDKe1/\nxBHSf/+39PjjMQ8MAAB0SV22KOtZ1lM3HnGjznnkHK3ZsKbd/Y88Uvre96r0ta9Jv/hF1276p1eA\nDCLkEJADGUTIISCH9KRy78s/pLKtEH1h2y9o32321U/m/CSl/XfdNVweY9YsaepUqa4u5gECAIAu\nI5XrlL3k7ru1WC9VuA/mjnEPrtU4YrnT07LVyzTpxkmaffJs7TRsp5Tes25d6C175hnpz3+Wxo3L\n+LAAAEABy2hPWeK2SnWSdm5xKYw6Scsl/aWTY80bw/oO0+UHXq4zHjpDTd6U0nt69pR++1vpvPOk\nz31Ouv/+mAcJAACKXptFmbv/zN37SfqfFpfC6Ofug939kiyOMXan7X6amrxJt7506xb3azlHbiad\ndlooyM4+W5o+XWpKraYrePQKkEGEHAJyIIMIOQTkkJ5UrlN2iZl92cx+mVi+lI2BZVOJleimI27S\n92d/X8vXLO/Qe/feW3r+eenJJ8OXAWpqYhokAAAoaqn0lP1M0p6S/pjYdLykF7J9tiyunrKWvvPX\n72j5muW64+g7OvzehgbpO9+RHnoo9JntlFp7GgAAKELp9JSlUpT9W9Jk99BwlWj0f9ndd057pGnI\nRlG2esNqTbxhom4/6nYdNOagtI7xv/8rXXCBdO210nHHZXiAAACgIMR58djyFs8HdOQDCknf7n11\n7WHX6syHztT6jes3ez2VOfKTTpL+9jfp0kvDmbONqd/3vGDQK0AGEXIIyIEMIuQQkEN6UinKfibp\nZTP7vZnNkPSipJ/GO6zc+fL4L2vHrXbUz//x87SPMXmy9MIL0rx50iGHSB99lMEBAgCAopTSvS/N\nbGtJe0gySc+6+4dxDyzJGGKfvowsXrlYu968q575xjPafvD2aR+nsVH64Q/DlOasWdIee2RwkAAA\nIG/FOX25h6TPS9o/8byobTNgG126/6U686Ez1ZlCsLRUuuIK6ZprpMMPl267LYODBAAARSWV2yxd\nKek8SfMTy7mJb2QWtXP3Olcf13+su169q3lbunPkRx8tPfVUuGfmGWdI6zdvVyso9AqQQYQcAnIg\ngwg5BOSQnlTOlB0u6T/c/TZ3v03SoZKOiHdYuVdWUqabv3SzvvPX76hmbecvPrbDDuG+mcuWSZWV\n0gcfdH6MAACgeKR6SYxKd1+RWB8kqaoYL4mRzFkPnaUmb9JNX7opI8drapKuvFK6/nrp7rul/ffP\nyGEBAEAeies6ZcdLulLSkwqN/p+XdIm7353uQNORq6Ksdl2tdrx+R9079V7ts80+GTvuo49Kp5wi\nff/70re+FW7bBAAAikPGG/3NzCQ9LWlvSX9KLPtkuyDLpfKe5frVF3+lMx46Q48/8XjGjnvoodIz\nz4Tm/1NOkerrM3bo2NErQAYRcgjIgQwi5BCQQ3q2WJQlTk097O5L3f1+d/9LLi6HkWv/OfE/NazP\nMP14zo/16DuPqqGxISPH3XZb6Z//DJfO2G8/acGCjBwWAAAUoFSmL2dIus7dn8/OkNocR06mLyM1\na2s045UZuue1e/TWJ2/pKxO+oik7TtFBYw5St9JunTq2u/Sb30g//an0hz+EC84CAIDCFVdP2RuS\nxkpaJGmNQl+Zp9Lob2a3SvqSpGVt7W9mv5F0WOLY09x9bhv75bQoa6l6ZbVmzZ+lmfNn6u1P3tbR\nE47W1IlTdeCYA1VWUpb2cefMkY4/PvSYXXwxfWYAABSquC4e+0VJ20k6SNKRCkXWkSke//bE+5My\ns8Mkbefu20s6XVJmvuIYk2iOvGJAhS7c50I9841n9OJpL2rCkAn6wZM/0NZXba3THjhNj7/3uDY2\ndfymlwccEC6b8Ze/SMceK9XVZfgPkCH0CpBBhBwCciCDCDkE5JCedosyd1+UbEnl4O7+tKQtXeTr\nKEl3JPZ9VtIAMxuWyrHzxajyUfr2vt/Ws//1rJ7/5vMaN3icLn3iUn3mqs/o9AdO1xPvPdGhAm3k\nyHDGbMgQac89pTffjHHwAAAgb6R078tOfYDZKEkPJJu+NLMHJP3M3f+ZWH9c0kXu/lKSffNm+jIV\nC2oWaOb8mZo5f6aqV1brmAnHaMrEKTpg1AEqLSlN6Ri/+5106aXSLbdIX/lKzAMGAAAZk870ZfoN\nUJmRbLBtVl7Tpk3T6NGjJUnl5eWaPHmyKisrJX16qjRf1he9skh7ak9d9M2L9F7Ne7ryf6/UGU+e\noZVbr9RXd/iqxtWN005Dd9LBBx3c5vHGjpUefLBSxx4r3XtvlaZNkw4+OD/+fKyzzjrrrLPO+qfr\n0fOFCxcqXbk+U3aTpCfd/f8S629IOsDdlyXZN+dnyqqqqpr/EtL17op3NXP+TN3z2j1aUrdEX93h\nq5o6cao+V/G5Ns+gLV8uTZ0q9eoV7gYwcaJUlsNyOhM5FDoyCMghIAcyiJBDQA7xNfp3lin5GTFJ\nul/SyZJkZntLqk1WkBWT7QZtp4s/d7FeOv0lPf31pzWy/0id/9j5Gnn1SJ3z8Dl6atFTamxq3OQ9\nQ4dKf/ub9NnPSscdJw0cKH3+89J3vyvNnClVV4fLagAAgMIV65kyM7tTUqWkwZKWSZouqbvCJTVu\nSexzncJNztdIOjVZP1liv5yfKYvTW5+8pZmvhR605WuW69gdj9WUHadov4r9VGKb1s61tdILL0jP\nPvvpYibttVdY9txT2mMPacCAHP1hAADo4mK5Tlm+KPairKU3P36zeYrzk7Wf6NgdjtXUiVO1zzb7\nbFagSeEsWXV1uJzGs8+Gx5dekrbZ5tMiba+9pJ12krp3z8EfCACALoaiLGa5mCN/4+M3NPO1mbpn\n/j2qWVujPUfsqYoBFRo1YJRGlY9qfj6k9xBZi6vNbtwovfbap2fSnntOeu89aZddNi3Uxozp+EVq\n6RUggwg5BORABhFyCMihML99iXZMGDJBlx1wmS474DK9+fGbmrd8nqpXVmth7UI9Vf2UFtUuUvXK\natU31KtiQEVzkVYxoEKjykdp/CGj9B9TKjSy/0itq++mF18MRdqsWaEnbf36UKBFRdoee0iDB+f6\nTw0AQNfDmbIisXrDalWvrFb1yurmQm3RykVatDI8X1q3VEP7DN3k7FrFgAr13ThKKxaO0qJXKvTK\nc/31wgvSsGGbnk2bPFnq0SPXf0IAAAoH05do08amjVpSt2TTgq12kapXhSJu0cpF6l7aXRX9KzSo\ndJRK6iq0ZskoffhmhT58Y5QmbD1K++0yTHvtWaIxY6Sttw5Lnz65/pMBAJB/KMpiVsxz5O6uFWtX\nbFqwJZ4vrKnWe58sUl3DSvXYMEKNb3WXDRyl9SsHqrShXP3KBqq850AN7lOuYf0H6jODBqpiq3KN\nHj5QY0cO1Pbb9NfA8tKiusF6Mf8sdAQ5BORABhFyCMiBnjJ0gplpcO/BGtx7sHbdetek+6xtWKv3\nV72vv83+m8ZMHqMVa2u0tKZWiz+u0dKaZVq+6k0trK/RK6trtLq2VvVv1aihtEZNZXXShn4qaxio\nHl6u3iUD1b/7QA3sWa4hfQdq2ICBGjGoXBVDB2qbIQM1sFe5BvYaqIE9B6q8Z7l6lDF3CgAofpwp\nQ+wamxq1ZMUqvbW4RguW1GrhhzV6/5MafVhbq+V1NVpRX6NVG2q1urFGG8tq1a1fjUp618h71Kqh\nrEalVqY+JQM1oEco2Ib0LVd57z7q17OPenfrnXTp020Lr3UPr3Ur6bbJN1YBAMgUpi9R8NaulT78\nUFq6NCxLlriqP6zXomW1+uCTGi1bWaOPV9eqbn29SnvWq3f/evXsv0bd+9are+96lfUK261HvdRt\njby0Xo2l9dpo9WpQvdY3rdG6xnrVN9SryZvaLNg2Wco2396rrJd6deulnmU91bOsp3qVffq8eVu3\nTbclu8YcAKA4UZTFjDnyIB9ycJfWrAl3N6ipSW1pua8klQ9u0IDBa9V/cL36DlqjPuX16j2gXj37\n1qtH33p171Ov0l5rVNqrXiXd62Xd6+Vl9dqger374jsatOMgrd24Vus2rttkWduw+bZ1G9eprKRs\ns0Jti0VdO4Vej9Ie6lHWo/mxe2n3pM97lCbWy3pk/OxgPvws5ANyIIMIOQTkQE8ZuhAzqW/fsIwc\n2fH3r10r1dR0Syz9Ny/elrRd3G3cKPXuXaUhQyrVr5/Uv39YhiWe92v5ODA89uvn6tVvg7r3Xqfu\nvdepW891KumxVhvVfkEXFX6frP1E6+o23b6hcYPWb1wfHhvXt/u8oakhpeIt6fMk+73/7/f1fLfn\n1b20u7qVdlO3km7Nz7uXdle3km4pPU/2/lIrZXoZQJfCmTKgg9avD4VbXZ20alXbj+29tnp1uO1V\nVNT1S1bUtXhsva31UpbCP7GavEkbGjc0F3PrG9d3+nlDY0NzwdfQ2KANTUm2JXkevbfl85b7NXpj\nSgVcWUlZ83r0vKykbJP15udJ9km2fzqvtXy95dLy9ZYLBSdQ3DhTBmRBjx7hArvDhnXuOO5SfX1q\nxd3y5W3vs3p1eCwr27xQ69u39XqJ+vXrmVg2f31gi/Vu3TKTV7qavKndgm5D4wZtbNqohqaG8NjY\n0Lze8nnr16L16PnahrWqa6pL+lp7x44KyGhbyyV67ybbEvuXWummxVuKBV2y/cpKylRqpSotKd30\nMdm2FB7bPF47j2amEiuRKfGYWE+2LZ19ovW29jFZm4+SOvyapDaPWYjcXU3eJFfiMcl6pl9rua15\nXa3WO/B6R967sWmjGpsa1eiNmzxubNqY3rYOvj8dnCnrAObIA3LIvwzcpXXrQnHWslBruXR0W1Tk\nbV7Yffp8xYoqTZpUmXS/dM7kFaqO/jy4uxq9cZNCLdWCrq39kv3PJ53HLR5rC+/76LWPNHCHgW3+\nj7+9YiBT+7h8s8co8468JqnN/VtqXaz5AlfJmNS+1NOR4i4qHlMRjbdlNi2Ps6WiNlOvrXtnnfqO\n67tJ0d26sG5dlLf3ekfea2Yqs7LN/uFQVpLatrb+cZLqtlIr1edGfY4zZUBXZCb16hWWoUM7f7yW\nRd6WCrc1a0KfXXX15vu03rd79y0Xbe2d5Wv9WkkBf5k1+h9GWUnx/ArOt3+oxK2tIm9O1RwdUHlA\nSu9P+bOSFINbOu6WziJmS1f7ecgUzpQBiJ17+HJFW4Vbe2fvWi/19aEATbeoa7307q2iuuMEgNzj\nkhgAuoSmpnCWriOF3JaW9evDfVzTOZPX+ksY/fvnvh8PQO5RlMWM07EBOZBBpFhy2Ljx08IunQJv\n6dIqNTVVNn8xo1u3Twu0ziw9exbOGbxi+VnoLHIIyIFvXwJAWsrKpPLysKSjqkqK/v8TTdVGBVpb\ny8qV0uLFm25r+Y3bVaukhoaOFXEDBmz+fMAApmeBQsGZMgDIUxs2bF6otVXg1dWFx2i95eOGDcmL\nt7aKuLa29eyZ60SAwsH0JQBgMw0NmxZwrZ9vaVvLR7P2i7j2lkL/5iyQKoqymDFHHpADGUTIIegq\nOaxb13YR98ILVRo6tFIrV2qLy9q14UsRqRRwbS39+uVvYddVfhbaQw70lAEAYtSzZ1iSXQuvouLT\nvrot2bjx02KurWXFCmnBgrZfr68PZ9zaKtrKy5M/b7nety99dsg/nCkDABSUxsb2C7uVK8M9apM9\nX7kynPVrPeXaVjHXVmFXSN+ORfYxfQkAQAqiPru2irYtFXTRelNT8oItlSWahqWoK14UZTFjjjwg\nBzKIkENADl0zg/XrNy/a/vGPKo0cWanaWrW7rF0bzta1V7y19Rq9dfmNnjIAALKkR4/QX9eyx66s\nLLXeOunT/rotFW7vvvtp4dd6WbMmFGbJireBA9tfp68u/3CmDACAAhT11kVF2sqVUk3N5sVby20t\nn69d+2kx17poS6Ww69WLom5LmL4EAAApaWjY/CxcWwVcsvWop66toq29wq5791wnEC+KspgxRx6Q\nAxlEyCEgBzKIdKUc1q1ru5h76aUqDRpU2WZBV1sb7hGbyjRrsvUBA6TS0lwnsGX0lAEAgKzo2VMa\nPjwsrbW8H2wy7qEnbktn4pYskebPT/76qlVSnz7tF3LJnudzPx1nygAAQEFpakr+JYmocGtr6jV6\nXLcunG1rq2hrb1uPHu2PkTNlAACg6JWUfFokpaOhof0+uurq5K/X1IRv2bZXyKWDoqwDulKvwJaQ\nAxlEyCEgBzKIkEOQ7zl06yZttVVYOso9fHM12Vm56PmSJemNi6IMAAAgRWZS795hGTGi7f2uvjqN\nYxdKnxY9ZQAAoFCk01OWpzdoAAAA6Fooyjqgqqoq10PIC+RABhFyCMiBDCLkEJBDeijKAAAA8gA9\nZQAAABlGTxkAAECBoijrAObIA3Iggwg5BORABhFyCMghPRRlAAAAeYCeMgAAgAyjpwwAAKBAUZR1\nAHPkATmQQYQcAnIggwg5BOSQHooyAACAPEBPGQAAQIbRUwYAAFCgKMo6gDnygBzIIEIOATmQQYQc\nAnJID0UZAABAHqCnDAAAIMPoKQMAAChQFGUdwBx5QA5kECGHgBzIIEIOATmkh6IMAAAgD9BTBgAA\nkGH0lAEAABSo2IsyMzvUzN4ws7fM7HtJXj/FzJab2UuJ5etxjyldzJEH5EAGEXIIyIEMIuQQkEN6\nyuI8uJmVSLpO0sGSlkh63sz+4u5vtNr1bnc/N86xAAAA5LNYe8rMbG9J0939sMT6xZLc3X/eYp9T\nJH3W3b/VzrHoKQMAAAUhH3vKRkha3GL9/cS21o4xs7lmdo+ZjYx5TAAAAHkn1ulLSckqxNanu+6X\ndKe7N5jZ6ZJmKEx3bmbatGkaPXq0JKm8vFyTJ09WZWWlpE/nr+Ncnzt3rs4///ysfV6+rrfsFciH\n8eRi/Zprrsn6z18+rkfb8mU8/Dzkbp3fj/x+7Oo/D9HzhQsXKl3ZmL78kbsfmljfbPqy1f4lkla4\ne3mS13I+fVlVVdX8l9CVkQMZRMghIAcyiJBDQA7pTV/GXZSVSnpT4czXUknPSTre3V9vsc9wd/8w\n8fxoSd91932THCvnRRkAAEAq0inKYp2+dPdGMztH0l8V+tdudffXzezHkp539wclnWtmX5bUIGmF\npGlxjgkAACAflcT9Ae7+qLuPd/ft3f3KxLbpiYJM7n6pu09y913d/WB3fyvuMaWr5bxxV0YOZBAh\nh4AcyCBCDgE5pCf2ogwAAADt496XAAAAGZaP1ykDAABACijKOoA58oAcyCBCDgE5kEGEHAJySA9F\nGQAAQB6gpwwAACDD6CkDAAAoUBRlHcAceUAOZBAhh4AcyCBCDgE5pIeiDAAAIA/QUwYAAJBh9JQB\nAAAUKIqyDmCOPCAHMoiQQ0AOZBAhh4Ac0kNRBgAAkAfoKQMAAMgwesoAAAAKFEVZBzBHHpADGUTI\nISAHMoiQQ0AO6aEoAwAAyAP0lAEAAGQYPWUAAAAFiqKsA5gjD8iBDCLkEJADGUTIISCH9FCUAQAA\n5AF6ygAAADKMnjIAAIACRVHWAcyRB+RABhFyCMiBDCLkEJBDeijKAAAA8gA9ZQAAABlGTxkAAECB\noijrAObIA3Iggwg5BORABhFyCMghPRRlAAAAeYCeMgAAgAyjpwwAAKBAUZR1AHPkATmQQYQcAnIg\ngwg5BOSQHooyAACAPEBPGQAAQIbRUwYAAFCgKMo6gDnygBzIIEIOATmQQYQcAnJID0UZAABAHqCn\nDAAAIMPoKQMAAChQFGUdwBx5QA5kECGHgBzIIEIOATmkh6IMAAAgD9BTBgAAkGH0lAEAABQoirIO\nYI48IAcyiJBDQA5kECGHgBzSQ1EGAACQB+gpAwAAyDB6ygAAAAoURVkHMEcekAMZRMghIAcyiJBD\nQA7poSgDAADIA/SUAQAAZBg9ZQAAAAWKoqwDmCMPyIEMIuQQkAMZRMghIIf0UJQBAADkAXrKAAAA\nMoyeMgAAgAJFUdYBzJEH5EAGEXIIyIEMIuQQkEN6Yi/KzOxQM3vDzN4ys+8leb27md1tZm+b2TNm\nVhH3mNI1d+7cXA8hL5ADGUTIISAHMoiQQ0AO6Ym1KDOzEknXSfqipImSjjezCa12+4akFe6+vaRr\nJP0izjF1Rm1tba6HkBfIgQwi5BCQAxlEyCEgh/TEfaZsT0lvu/sid2+QdLeko1rtc5SkGYnnsyQd\nHPOYAAAA8k7cRdkISYtbrL+f2JZ0H3dvlFRrZoNiHldaFi5cmOsh5AVyIIMIOQTkQAYRcgjIIT2x\nXhLDzI6VdIi7n5ZYP0nSHu5+Xot9Xk3ssySx/k5in5pWx+J6GAAAoGB09JIYZXENJOF9SS0b90dK\nWtJqn8WStpG0xMxKJfVvXZBJHf+DAQAAFJK4py+flzTWzEaZWXdJx0m6v9U+D0g6JfF8iqTZMY8J\nAAAg78R6pszdG83sHEl/VSgAb3X3183sx5Ked/cHJd0q6Q9m9rakTxQKNwAAgC6lYG6zBAAAUMy4\non8KzGykmc02s/lmNs/Mzs31mHLFzErM7CUzaz0N3WWY2QAzm2lmr5vZa2a2V67HlG1mdoGZvWpm\n/zazPybaE4qemd1qZsvM7N8ttg00s7+a2Ztm9piZDcjlGLOhjRx+kfhvYq6Z3Wtm/XM5xmxIlkOL\n175jZk35ejWBTGkrAzP7VuLC8fPM7MpcjS9b2vhvYpfERfFfNrPnzOyz7R2Hoiw1GyVd6O47StpH\n0tlJLoLbVZwnaX6uB5Fjv5b0sLvvIGkXSa/neDxZZWafkfQtSbu5+84KbRBdpe3gdoWLYbd0saTH\n3X28Qk/sJVkfVfYly+Gvkia6+2RJb6vr5iAzGynpC5IWZX1E2bdZBmZWKelISZPcfSdJv8zBuLIt\n2c/CLyRNd/ddJU2X9D/tHYSiLAXu/qG7z008X63wP+HW11sreolfNIdL+l2ux5IrZtZP0v7ufrsk\nuftGd1+V42HlQqmkPmZWJqm3Nv9WdVFy96cltf52eMsLYM+Q9JWsDioHkuXg7o+7e1Ni9V8K37Yv\nam38PEjS1ZK+m+Xh5EQbGZwp6Up335jY5+OsDyzL2sihSVJ05rxc0gftHYeirIPMbLSkyZKeze1I\nciL6RdOVGxG3lfSxmd2emMa9xcx65XpQ2ZS4puBVkqoVfsnUuvvjuR1VTg1192VS+AecpK1yPJ58\n8JoJZcwAAASESURBVHVJj+R6ELlgZkdKWuzu83I9lhwaJ+nzZvYvM3sylWm7InWBpF+aWbXCWbN2\nzx5TlHWAmfVVuBXUeYkzZl2GmR0haVnijKEllq6oTNJukq53990k1StMX3UZZlaucHZolKTPSOpr\nZifkdlTIF2b2fUkN7n5nrseSbYl/oH1fYaqqeXOOhpNLZZLK3X1vSRdJuifH48mVMxXqhQqFAu22\n9t5AUZaixDTNLEl/cPe/5Ho8ObCfpC+b2XuS7pJ0oJndkeMx5cL7Cv8KfiGxPkuhSOtKviDpPXdf\nkbg12p8k7ZvjMeXSMjMbJklmNlzS8hyPJ2fM7BSFFoeuWqRvJ2m0pFfMbIHCFO6LZjY0p6PKvsUK\nvxfk7s9LajKzwbkdUk6c4u73SZK7z1K4H/gWUZSl7jZJ893917keSC64+6XuXuHu2yo0dc9295Nz\nPa5sS0xTLTazcYlNB6vrffGhWtLeZtbTzEwhg670ZYfWZ4rvlzQt8fwUSV3lH22b5GBmhyqcFfmy\nu6/P2aiyrzkHd3/V3Ye7+7buPkbhH3G7unuxF+qt/5u4T+H3ghK/K7u5+ye5GFiWtc7hAzM7QJLM\n7GBJb7V3gLhvs1QUzGw/SSdKmmdmLyv0VF3q7o/mdmTIkXMl/dHMukl6T9KpOR5PVrn7c2Y2S9LL\nkhoSj7fkdlTZYWZ3SqqUNDjRJzJd0pWSZprZ1xUK1im5G2F2tJHDpZK6S/pbqNX1L3c/K2eDzIJk\nOURfAkpwFfn0ZRs/C7dJut3M5klaL6no/wHfRg7flPSbxC0k10k6rd3jcPFYAACA3GP6EgAAIA9Q\nlAEAAOQBijIAAIA8QFEGAACQByjKAAAA8gBFGQAAQB6gKAOAdpjZAWb2QK7HAaC4UZQBQGq4qCOA\nWFGUASgaZnaimT1rZi+Z2Y1mVmJmdWb2KzN71cz+Ft2Dz8wmm9kzZjbXzO41swGJ7dsl9ptrZi+Y\n2ZjE4fuZ2Uwze93M/pCzPySAokVRBqAomNkESf8paV93301Sk8Lt0XpLes7dJ0l6SuH2J5I0Q9J3\n3X2ypFdbbP+jpGsT2/eVtDSxfbLCLbZ2lLSdmXXlm7ADiAH3vgRQLA6WtJuk5xM3Su8paZlCcXZP\nYp//lXSvmfWXNMDdn05snyHpHjPrK2mEu98vSe6+QZIS93J8zt2XJtbnShot6Z9Z+HMB6CIoygAU\nC5M0w92/v8lGs8ta7ect9k92jLasb/G8Ufz+BJBhTF8CKBZPSDrWzLaSJDMbaGYVkkolHZvY50RJ\nT7v7KkkrzGy/xPavSZrj7nWSFpvZUYljdDez/2/XXpEaCoIogN7GIbIddoJMpfBZAopdkKVkI3Hx\n0RQmpiMYSRUKMrx3jhwxHzN1u2ce//QUwGqp9IBF6O5TVb0mOVbVQ5Jrkn2SzyRPo2N2yde/syTZ\nJXkfoeuc5GWMb5McquptzPH83XK/dxJgrarb3QIsV1V9dPfm3vsA+InnS2DpVJ7Av6BTBgAwAZ0y\nAIAJCGUAABMQygAAJiCUAQBMQCgDAJjADWr/QNdo5VaqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd33b961e90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "loss = pd.DataFrame({'epoch': [ i + 1 for i in history.epoch ],\n",
    "                     'training': [ math.sqrt(loss) for loss in history.history['loss'] ],\n",
    "                     'validation': [ math.sqrt(loss) for loss in history.history['val_loss'] ]})\n",
    "ax = loss.ix[:,:].plot(x='epoch', figsize={7,10}, grid=True)\n",
    "ax.set_ylabel(\"root mean squared error\")\n",
    "ax.set_ylim([0.0,3.0]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Print best validation RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum RMSE at epoch 15 = 0.8623\n"
     ]
    }
   ],
   "source": [
    "min_val_loss, idx = min((val, idx) for (idx, val) in enumerate(history.history['val_loss']))\n",
    "print 'Minimum RMSE at epoch', '{:d}'.format(idx+1), '=', '{:.4f}'.format(math.sqrt(min_val_loss))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}