T-RexRL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# dependencies\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from DQN import DQN\n",
    "from Experience import ExperienceBuffer\n",
    "from Game import GameEnv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# opts\n",
    "load_model = False\n",
    "checkpoints_path = 'checkpoints/'\n",
    "# choose one from checkpoints path\n",
    "model_path = checkpoints_path + '-500'\n",
    "save_period = 500\n",
    "info_period = 10\n",
    "update_period = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "batch_size = 50\n",
    "gamma = 0.9\n",
    "s_epsilon = 0.9\n",
    "e_epsilon = 0.1\n",
    "epsilon = s_epsilon\n",
    "drop_step = (s_epsilon - e_epsilon) / 5000\n",
    "num_episodes = 10000\n",
    "pre_train = 5000\n",
    "max_ep_len = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# create game env, model and exp buff\n",
    "env = GameEnv()\n",
    "dqn = DQN()\n",
    "experience_buffer = ExperienceBuffer(30000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 seconds before start, focus tab with game!\n",
      "model saved\n",
      "|------------------------|\n",
      "total episodes: 1\n",
      "total steps: 59\n",
      "last 10 episodes average reward: 58.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 11\n",
      "total steps: 685\n",
      "last 10 episodes average reward: 61.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 21\n",
      "total steps: 1280\n",
      "last 10 episodes average reward: 58.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 31\n",
      "total steps: 1845\n",
      "last 10 episodes average reward: 55.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 41\n",
      "total steps: 2411\n",
      "last 10 episodes average reward: 55.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 51\n",
      "total steps: 2972\n",
      "last 10 episodes average reward: 55.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 61\n",
      "total steps: 3517\n",
      "last 10 episodes average reward: 53.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 71\n",
      "total steps: 4057\n",
      "last 10 episodes average reward: 53.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 81\n",
      "total steps: 4598\n",
      "last 10 episodes average reward: 53.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 91\n",
      "total steps: 5105\n",
      "last 10 episodes average reward: 49.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 101\n",
      "total steps: 5553\n",
      "last 10 episodes average reward: 43.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 111\n",
      "total steps: 5975\n",
      "last 10 episodes average reward: 41.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 121\n",
      "total steps: 6395\n",
      "last 10 episodes average reward: 41.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 131\n",
      "total steps: 6804\n",
      "last 10 episodes average reward: 39.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 141\n",
      "total steps: 7202\n",
      "last 10 episodes average reward: 38.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 151\n",
      "total steps: 7593\n",
      "last 10 episodes average reward: 38.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 161\n",
      "total steps: 8001\n",
      "last 10 episodes average reward: 39.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 171\n",
      "total steps: 8390\n",
      "last 10 episodes average reward: 37.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 181\n",
      "total steps: 8799\n",
      "last 10 episodes average reward: 39.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 191\n",
      "total steps: 9180\n",
      "last 10 episodes average reward: 37.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 201\n",
      "total steps: 9563\n",
      "last 10 episodes average reward: 37.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 211\n",
      "total steps: 9948\n",
      "last 10 episodes average reward: 37.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 221\n",
      "total steps: 10311\n",
      "last 10 episodes average reward: 35.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 231\n",
      "total steps: 10720\n",
      "last 10 episodes average reward: 39.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 241\n",
      "total steps: 11082\n",
      "last 10 episodes average reward: 35.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 251\n",
      "total steps: 11461\n",
      "last 10 episodes average reward: 36.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 261\n",
      "total steps: 11832\n",
      "last 10 episodes average reward: 36.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 271\n",
      "total steps: 12081\n",
      "last 10 episodes average reward: 23.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 281\n",
      "total steps: 12281\n",
      "last 10 episodes average reward: 19.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 291\n",
      "total steps: 12484\n",
      "last 10 episodes average reward: 19.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 301\n",
      "total steps: 12690\n",
      "last 10 episodes average reward: 19.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 311\n",
      "total steps: 12901\n",
      "last 10 episodes average reward: 20.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 321\n",
      "total steps: 13100\n",
      "last 10 episodes average reward: 18.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 331\n",
      "total steps: 13366\n",
      "last 10 episodes average reward: 25.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 341\n",
      "total steps: 13702\n",
      "last 10 episodes average reward: 32.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 351\n",
      "total steps: 14024\n",
      "last 10 episodes average reward: 31.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 361\n",
      "total steps: 14371\n",
      "last 10 episodes average reward: 33.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 371\n",
      "total steps: 14689\n",
      "last 10 episodes average reward: 30.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 381\n",
      "total steps: 15025\n",
      "last 10 episodes average reward: 32.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 391\n",
      "total steps: 15358\n",
      "last 10 episodes average reward: 32.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 401\n",
      "total steps: 15723\n",
      "last 10 episodes average reward: 35.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 411\n",
      "total steps: 16064\n",
      "last 10 episodes average reward: 33.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 421\n",
      "total steps: 16401\n",
      "last 10 episodes average reward: 32.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 431\n",
      "total steps: 16740\n",
      "last 10 episodes average reward: 32.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 441\n",
      "total steps: 17071\n",
      "last 10 episodes average reward: 32.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 451\n",
      "total steps: 17408\n",
      "last 10 episodes average reward: 32.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 461\n",
      "total steps: 17730\n",
      "last 10 episodes average reward: 31.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 471\n",
      "total steps: 18042\n",
      "last 10 episodes average reward: 30.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 481\n",
      "total steps: 18377\n",
      "last 10 episodes average reward: 32.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 491\n",
      "total steps: 18681\n",
      "last 10 episodes average reward: 29.4\n",
      "|------------------------|\n",
      "model saved\n",
      "|------------------------|\n",
      "total episodes: 501\n",
      "total steps: 18981\n",
      "last 10 episodes average reward: 29.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 511\n",
      "total steps: 19151\n",
      "last 10 episodes average reward: 16.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 521\n",
      "total steps: 19331\n",
      "last 10 episodes average reward: 17.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 531\n",
      "total steps: 19562\n",
      "last 10 episodes average reward: 22.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 541\n",
      "total steps: 19873\n",
      "last 10 episodes average reward: 30.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 551\n",
      "total steps: 20178\n",
      "last 10 episodes average reward: 29.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 561\n",
      "total steps: 20471\n",
      "last 10 episodes average reward: 28.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 571\n",
      "total steps: 20768\n",
      "last 10 episodes average reward: 28.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 581\n",
      "total steps: 21075\n",
      "last 10 episodes average reward: 29.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 591\n",
      "total steps: 21379\n",
      "last 10 episodes average reward: 29.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 601\n",
      "total steps: 21698\n",
      "last 10 episodes average reward: 30.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 611\n",
      "total steps: 21987\n",
      "last 10 episodes average reward: 27.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 621\n",
      "total steps: 22284\n",
      "last 10 episodes average reward: 28.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 631\n",
      "total steps: 22571\n",
      "last 10 episodes average reward: 27.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 641\n",
      "total steps: 22860\n",
      "last 10 episodes average reward: 27.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 651\n",
      "total steps: 23131\n",
      "last 10 episodes average reward: 26.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 661\n",
      "total steps: 23429\n",
      "last 10 episodes average reward: 28.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 671\n",
      "total steps: 23701\n",
      "last 10 episodes average reward: 26.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 681\n",
      "total steps: 23983\n",
      "last 10 episodes average reward: 27.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 691\n",
      "total steps: 24257\n",
      "last 10 episodes average reward: 26.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 701\n",
      "total steps: 24529\n",
      "last 10 episodes average reward: 26.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 711\n",
      "total steps: 24828\n",
      "last 10 episodes average reward: 28.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 721\n",
      "total steps: 25098\n",
      "last 10 episodes average reward: 26.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 731\n",
      "total steps: 25388\n",
      "last 10 episodes average reward: 28.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 741\n",
      "total steps: 25659\n",
      "last 10 episodes average reward: 26.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 751\n",
      "total steps: 25929\n",
      "last 10 episodes average reward: 26.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 761\n",
      "total steps: 26199\n",
      "last 10 episodes average reward: 26.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 771\n",
      "total steps: 26456\n",
      "last 10 episodes average reward: 24.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 781\n",
      "total steps: 26728\n",
      "last 10 episodes average reward: 26.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 791\n",
      "total steps: 27017\n",
      "last 10 episodes average reward: 27.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 801\n",
      "total steps: 27297\n",
      "last 10 episodes average reward: 27.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 811\n",
      "total steps: 27540\n",
      "last 10 episodes average reward: 23.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 821\n",
      "total steps: 27681\n",
      "last 10 episodes average reward: 13.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 831\n",
      "total steps: 27824\n",
      "last 10 episodes average reward: 13.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 841\n",
      "total steps: 27973\n",
      "last 10 episodes average reward: 13.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 851\n",
      "total steps: 28117\n",
      "last 10 episodes average reward: 13.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 861\n",
      "total steps: 28259\n",
      "last 10 episodes average reward: 13.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 871\n",
      "total steps: 28399\n",
      "last 10 episodes average reward: 13.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 881\n",
      "total steps: 28554\n",
      "last 10 episodes average reward: 14.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 891\n",
      "total steps: 28702\n",
      "last 10 episodes average reward: 13.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 901\n",
      "total steps: 28925\n",
      "last 10 episodes average reward: 21.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 911\n",
      "total steps: 29206\n",
      "last 10 episodes average reward: 27.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 921\n",
      "total steps: 29455\n",
      "last 10 episodes average reward: 23.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 931\n",
      "total steps: 29731\n",
      "last 10 episodes average reward: 26.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 941\n",
      "total steps: 30022\n",
      "last 10 episodes average reward: 28.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 951\n",
      "total steps: 30290\n",
      "last 10 episodes average reward: 25.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 961\n",
      "total steps: 30540\n",
      "last 10 episodes average reward: 24.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 971\n",
      "total steps: 30800\n",
      "last 10 episodes average reward: 25.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 981\n",
      "total steps: 31045\n",
      "last 10 episodes average reward: 23.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 991\n",
      "total steps: 31310\n",
      "last 10 episodes average reward: 25.5\n",
      "|------------------------|\n",
      "model saved\n",
      "|------------------------|\n",
      "total episodes: 1001\n",
      "total steps: 31556\n",
      "last 10 episodes average reward: 23.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1011\n",
      "total steps: 31815\n",
      "last 10 episodes average reward: 24.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1021\n",
      "total steps: 32089\n",
      "last 10 episodes average reward: 26.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1031\n",
      "total steps: 32341\n",
      "last 10 episodes average reward: 24.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1041\n",
      "total steps: 32581\n",
      "last 10 episodes average reward: 23.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1051\n",
      "total steps: 32851\n",
      "last 10 episodes average reward: 26.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1061\n",
      "total steps: 33123\n",
      "last 10 episodes average reward: 26.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1071\n",
      "total steps: 33402\n",
      "last 10 episodes average reward: 26.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1081\n",
      "total steps: 33676\n",
      "last 10 episodes average reward: 26.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1091\n",
      "total steps: 33861\n",
      "last 10 episodes average reward: 17.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1101\n",
      "total steps: 34008\n",
      "last 10 episodes average reward: 13.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1111\n",
      "total steps: 34146\n",
      "last 10 episodes average reward: 12.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1121\n",
      "total steps: 34271\n",
      "last 10 episodes average reward: 11.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1131\n",
      "total steps: 34421\n",
      "last 10 episodes average reward: 14.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1141\n",
      "total steps: 34560\n",
      "last 10 episodes average reward: 12.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1151\n",
      "total steps: 34701\n",
      "last 10 episodes average reward: 13.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1161\n",
      "total steps: 34839\n",
      "last 10 episodes average reward: 12.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1171\n",
      "total steps: 34961\n",
      "last 10 episodes average reward: 11.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1181\n",
      "total steps: 35119\n",
      "last 10 episodes average reward: 14.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1191\n",
      "total steps: 35381\n",
      "last 10 episodes average reward: 25.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1201\n",
      "total steps: 35639\n",
      "last 10 episodes average reward: 24.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1211\n",
      "total steps: 35889\n",
      "last 10 episodes average reward: 24.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1221\n",
      "total steps: 36135\n",
      "last 10 episodes average reward: 23.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1231\n",
      "total steps: 36373\n",
      "last 10 episodes average reward: 22.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1241\n",
      "total steps: 36628\n",
      "last 10 episodes average reward: 24.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1251\n",
      "total steps: 36896\n",
      "last 10 episodes average reward: 25.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1261\n",
      "total steps: 37211\n",
      "last 10 episodes average reward: 30.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1271\n",
      "total steps: 37482\n",
      "last 10 episodes average reward: 26.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1281\n",
      "total steps: 37761\n",
      "last 10 episodes average reward: 26.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1291\n",
      "total steps: 38031\n",
      "last 10 episodes average reward: 26.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1301\n",
      "total steps: 38274\n",
      "last 10 episodes average reward: 23.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1311\n",
      "total steps: 38514\n",
      "last 10 episodes average reward: 23.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1321\n",
      "total steps: 38770\n",
      "last 10 episodes average reward: 24.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1331\n",
      "total steps: 39030\n",
      "last 10 episodes average reward: 25.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1341\n",
      "total steps: 39308\n",
      "last 10 episodes average reward: 26.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1351\n",
      "total steps: 39542\n",
      "last 10 episodes average reward: 22.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1361\n",
      "total steps: 39806\n",
      "last 10 episodes average reward: 25.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1371\n",
      "total steps: 40056\n",
      "last 10 episodes average reward: 24.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1381\n",
      "total steps: 40290\n",
      "last 10 episodes average reward: 22.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1391\n",
      "total steps: 40563\n",
      "last 10 episodes average reward: 26.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1401\n",
      "total steps: 40829\n",
      "last 10 episodes average reward: 25.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1411\n",
      "total steps: 41086\n",
      "last 10 episodes average reward: 24.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1421\n",
      "total steps: 41362\n",
      "last 10 episodes average reward: 26.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1431\n",
      "total steps: 41621\n",
      "last 10 episodes average reward: 24.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1441\n",
      "total steps: 41851\n",
      "last 10 episodes average reward: 22.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1451\n",
      "total steps: 42118\n",
      "last 10 episodes average reward: 25.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1461\n",
      "total steps: 42353\n",
      "last 10 episodes average reward: 22.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1471\n",
      "total steps: 42593\n",
      "last 10 episodes average reward: 23.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1481\n",
      "total steps: 42861\n",
      "last 10 episodes average reward: 25.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1491\n",
      "total steps: 43108\n",
      "last 10 episodes average reward: 23.7\n",
      "|------------------------|\n",
      "model saved\n",
      "|------------------------|\n",
      "total episodes: 1501\n",
      "total steps: 43340\n",
      "last 10 episodes average reward: 22.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1511\n",
      "total steps: 43591\n",
      "last 10 episodes average reward: 24.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1521\n",
      "total steps: 43836\n",
      "last 10 episodes average reward: 23.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1531\n",
      "total steps: 44071\n",
      "last 10 episodes average reward: 22.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1541\n",
      "total steps: 44312\n",
      "last 10 episodes average reward: 23.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1551\n",
      "total steps: 44559\n",
      "last 10 episodes average reward: 23.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1561\n",
      "total steps: 44821\n",
      "last 10 episodes average reward: 25.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1571\n",
      "total steps: 45060\n",
      "last 10 episodes average reward: 22.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1581\n",
      "total steps: 45266\n",
      "last 10 episodes average reward: 19.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1591\n",
      "total steps: 45470\n",
      "last 10 episodes average reward: 19.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1601\n",
      "total steps: 45681\n",
      "last 10 episodes average reward: 20.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1611\n",
      "total steps: 45880\n",
      "last 10 episodes average reward: 18.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1621\n",
      "total steps: 46076\n",
      "last 10 episodes average reward: 18.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1631\n",
      "total steps: 46281\n",
      "last 10 episodes average reward: 19.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1641\n",
      "total steps: 46481\n",
      "last 10 episodes average reward: 19.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1651\n",
      "total steps: 46711\n",
      "last 10 episodes average reward: 22.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1661\n",
      "total steps: 46921\n",
      "last 10 episodes average reward: 20.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1671\n",
      "total steps: 47140\n",
      "last 10 episodes average reward: 20.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1681\n",
      "total steps: 47369\n",
      "last 10 episodes average reward: 21.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1691\n",
      "total steps: 47615\n",
      "last 10 episodes average reward: 23.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1701\n",
      "total steps: 47839\n",
      "last 10 episodes average reward: 21.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1711\n",
      "total steps: 48060\n",
      "last 10 episodes average reward: 21.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1721\n",
      "total steps: 48261\n",
      "last 10 episodes average reward: 19.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1731\n",
      "total steps: 48462\n",
      "last 10 episodes average reward: 19.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1741\n",
      "total steps: 48693\n",
      "last 10 episodes average reward: 22.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1751\n",
      "total steps: 48912\n",
      "last 10 episodes average reward: 20.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1761\n",
      "total steps: 49150\n",
      "last 10 episodes average reward: 22.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1771\n",
      "total steps: 49379\n",
      "last 10 episodes average reward: 21.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1781\n",
      "total steps: 49581\n",
      "last 10 episodes average reward: 19.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1791\n",
      "total steps: 49781\n",
      "last 10 episodes average reward: 19.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1801\n",
      "total steps: 50057\n",
      "last 10 episodes average reward: 26.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1811\n",
      "total steps: 50304\n",
      "last 10 episodes average reward: 23.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1821\n",
      "total steps: 50534\n",
      "last 10 episodes average reward: 22.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1831\n",
      "total steps: 50777\n",
      "last 10 episodes average reward: 23.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1841\n",
      "total steps: 51017\n",
      "last 10 episodes average reward: 23.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1851\n",
      "total steps: 51279\n",
      "last 10 episodes average reward: 25.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1861\n",
      "total steps: 51551\n",
      "last 10 episodes average reward: 26.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1871\n",
      "total steps: 51795\n",
      "last 10 episodes average reward: 23.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1881\n",
      "total steps: 52054\n",
      "last 10 episodes average reward: 24.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1891\n",
      "total steps: 52286\n",
      "last 10 episodes average reward: 22.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1901\n",
      "total steps: 52511\n",
      "last 10 episodes average reward: 21.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1911\n",
      "total steps: 52711\n",
      "last 10 episodes average reward: 19.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1921\n",
      "total steps: 52920\n",
      "last 10 episodes average reward: 19.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1931\n",
      "total steps: 53121\n",
      "last 10 episodes average reward: 19.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1941\n",
      "total steps: 53349\n",
      "last 10 episodes average reward: 21.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1951\n",
      "total steps: 53561\n",
      "last 10 episodes average reward: 20.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1961\n",
      "total steps: 53757\n",
      "last 10 episodes average reward: 18.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1971\n",
      "total steps: 53841\n",
      "last 10 episodes average reward: 7.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1981\n",
      "total steps: 53910\n",
      "last 10 episodes average reward: 5.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 1991\n",
      "total steps: 53987\n",
      "last 10 episodes average reward: 6.7\n",
      "|------------------------|\n",
      "model saved\n",
      "|------------------------|\n",
      "total episodes: 2001\n",
      "total steps: 54061\n",
      "last 10 episodes average reward: 6.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2011\n",
      "total steps: 54281\n",
      "last 10 episodes average reward: 21.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2021\n",
      "total steps: 54501\n",
      "last 10 episodes average reward: 21.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2031\n",
      "total steps: 54701\n",
      "last 10 episodes average reward: 19.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2041\n",
      "total steps: 54907\n",
      "last 10 episodes average reward: 19.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2051\n",
      "total steps: 55113\n",
      "last 10 episodes average reward: 19.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2061\n",
      "total steps: 55329\n",
      "last 10 episodes average reward: 20.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2071\n",
      "total steps: 55551\n",
      "last 10 episodes average reward: 21.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2081\n",
      "total steps: 55777\n",
      "last 10 episodes average reward: 21.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2091\n",
      "total steps: 55979\n",
      "last 10 episodes average reward: 19.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2101\n",
      "total steps: 56183\n",
      "last 10 episodes average reward: 19.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2111\n",
      "total steps: 56391\n",
      "last 10 episodes average reward: 19.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2121\n",
      "total steps: 56594\n",
      "last 10 episodes average reward: 19.3\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2131\n",
      "total steps: 56811\n",
      "last 10 episodes average reward: 20.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2141\n",
      "total steps: 57021\n",
      "last 10 episodes average reward: 20.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2151\n",
      "total steps: 57231\n",
      "last 10 episodes average reward: 20.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2161\n",
      "total steps: 57462\n",
      "last 10 episodes average reward: 22.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2171\n",
      "total steps: 57671\n",
      "last 10 episodes average reward: 19.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2181\n",
      "total steps: 57901\n",
      "last 10 episodes average reward: 22.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2191\n",
      "total steps: 58132\n",
      "last 10 episodes average reward: 22.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2201\n",
      "total steps: 58341\n",
      "last 10 episodes average reward: 19.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2211\n",
      "total steps: 58541\n",
      "last 10 episodes average reward: 19.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2221\n",
      "total steps: 58741\n",
      "last 10 episodes average reward: 19.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2231\n",
      "total steps: 58941\n",
      "last 10 episodes average reward: 19.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2241\n",
      "total steps: 59141\n",
      "last 10 episodes average reward: 19.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2251\n",
      "total steps: 59381\n",
      "last 10 episodes average reward: 23.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2261\n",
      "total steps: 59591\n",
      "last 10 episodes average reward: 20.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2271\n",
      "total steps: 59811\n",
      "last 10 episodes average reward: 21.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2281\n",
      "total steps: 60033\n",
      "last 10 episodes average reward: 21.2\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2291\n",
      "total steps: 60261\n",
      "last 10 episodes average reward: 21.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2301\n",
      "total steps: 60466\n",
      "last 10 episodes average reward: 19.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2311\n",
      "total steps: 60681\n",
      "last 10 episodes average reward: 20.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2321\n",
      "total steps: 60911\n",
      "last 10 episodes average reward: 22.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2331\n",
      "total steps: 61131\n",
      "last 10 episodes average reward: 21.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2341\n",
      "total steps: 61360\n",
      "last 10 episodes average reward: 21.9\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2351\n",
      "total steps: 61574\n",
      "last 10 episodes average reward: 20.4\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2361\n",
      "total steps: 61801\n",
      "last 10 episodes average reward: 21.7\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2371\n",
      "total steps: 62021\n",
      "last 10 episodes average reward: 21.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2381\n",
      "total steps: 62231\n",
      "last 10 episodes average reward: 20.0\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2391\n",
      "total steps: 62309\n",
      "last 10 episodes average reward: 6.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2401\n",
      "total steps: 62385\n",
      "last 10 episodes average reward: 6.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2411\n",
      "total steps: 62461\n",
      "last 10 episodes average reward: 6.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2421\n",
      "total steps: 62542\n",
      "last 10 episodes average reward: 7.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2431\n",
      "total steps: 62613\n",
      "last 10 episodes average reward: 6.1\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2441\n",
      "total steps: 62691\n",
      "last 10 episodes average reward: 6.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2451\n",
      "total steps: 62769\n",
      "last 10 episodes average reward: 6.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2461\n",
      "total steps: 62837\n",
      "last 10 episodes average reward: 5.8\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2471\n",
      "total steps: 62903\n",
      "last 10 episodes average reward: 5.6\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2481\n",
      "total steps: 62968\n",
      "last 10 episodes average reward: 5.5\n",
      "|------------------------|\n",
      "|------------------------|\n",
      "total episodes: 2491\n",
      "total steps: 63040\n",
      "last 10 episodes average reward: 6.2\n",
      "|------------------------|\n",
      "model saved\n",
      "|------------------------|\n",
      "total episodes: 2501\n",
      "total steps: 63119\n",
      "last 10 episodes average reward: 6.9\n",
      "|------------------------|\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "[Errno 12] Cannot allocate memory",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-cbce62401ac0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m                     \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdqn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mdqn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m                 \u001b[0ms1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m                 \u001b[0mepisode_buffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m                 \u001b[0mepisode_reward\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nemanja/Desktop/TRex_RL_Agent/Game.pyc\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m     49\u001b[0m         \u001b[0;34m\"\"\"Perform action and get new state\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_actions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_screen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0mgame_over\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_game_over\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nemanja/Desktop/TRex_RL_Agent/Game.pyc\u001b[0m in \u001b[0;36m_screen\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     72\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_screen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m         \u001b[0;34m\"\"\"Capture game screen\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m         \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImageGrab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrab\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbbox\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_bbox\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m         \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m         \u001b[0;31m# self.testSight(img)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nemanja/.local/lib/python2.7/site-packages/pyscreenshot/__init__.pyc\u001b[0m in \u001b[0;36mgrab\u001b[0;34m(bbox, childprocess, backend)\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m     \"\"\"\n\u001b[0;32m---> 46\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_grab\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mto_file\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchildprocess\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchildprocess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbbox\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbbox\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nemanja/.local/lib/python2.7/site-packages/pyscreenshot/__init__.pyc\u001b[0m in \u001b[0;36m_grab\u001b[0;34m(to_file, childprocess, backend, bbox, filename)\u001b[0m\n\u001b[1;32m     27\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchildprocess\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m         \u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'running \"%s\" in child process'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mrun_in_childprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_grab_simple\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimcodec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcodec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mto_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbbox\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0m_grab_simple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mto_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbbox\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nemanja/.local/lib/python2.7/site-packages/pyscreenshot/procutil.pyc\u001b[0m in \u001b[0;36mrun_in_childprocess\u001b[0;34m(target, codec, *args, **kwargs)\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0mqueue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mQueue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m     \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mProcess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_wrapper\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcodec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mqueue\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m     \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m     \u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mqueue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m     \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python2.7/multiprocessing/process.pyc\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    128\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m             \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mforking\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPopen\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_popen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m         \u001b[0m_current_process\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_children\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python2.7/multiprocessing/forking.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m    119\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;34m'random'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: [Errno 12] Cannot allocate memory"
     ]
    }
   ],
   "source": [
    "# training\n",
    "total_steps = 0\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()\n",
    "rewards = []\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    try:\n",
    "        if load_model:\n",
    "            saver.restore(sess, model_path)\n",
    "        else:\n",
    "            sess.run(init)\n",
    "\n",
    "        for i in range(num_episodes):\n",
    "            episode_buffer = ExperienceBuffer(10000)\n",
    "            episode_reward = 0.0\n",
    "\n",
    "            s, r, d = env.reset(i==0, i>0)\n",
    "\n",
    "            for j in range(max_ep_len):\n",
    "                if np.random.rand(1) < epsilon or total_steps < pre_train:\n",
    "                    a = np.random.randint(0, 3)\n",
    "                else:\n",
    "                    a = sess.run(dqn.predict, feed_dict={dqn.x: [s]})[0]\n",
    "\n",
    "                s1, r, d = env.step(a)\n",
    "                episode_buffer.add(np.reshape(np.array([s, a, r, s1, d]), [1,5]))\n",
    "                episode_reward += r\n",
    "                s = s1\n",
    "                total_steps += 1\n",
    "\n",
    "                if total_steps > pre_train:\n",
    "                    if epsilon > e_epsilon:\n",
    "                        epsilon -= drop_step\n",
    "\n",
    "                    if total_steps % update_period == 0:\n",
    "                        \n",
    "                        batch = experience_buffer.batch(batch_size)\n",
    "                        Q1 = sess.run(tf.reduce_max(dqn.q_vals, axis=1), feed_dict={dqn.x: np.vstack(batch[:, 3])})\n",
    "                        end_mul = -(batch[:, 4] - 1)\n",
    "                        targetQ = batch[:, 2] + gamma * Q1 * end_mul\n",
    "\n",
    "                        sess.run(dqn.train, feed_dict={dqn.x: np.vstack(batch[:, 0]), \n",
    "                                                       dqn.targetQ: targetQ,\n",
    "                                                       dqn.actions: batch[:, 1]})\n",
    "\n",
    "                if d:\n",
    "                    break\n",
    "            experience_buffer.add(episode_buffer._buffer)\n",
    "            rewards.append(episode_reward)\n",
    "\n",
    "            if i % save_period == 0:\n",
    "                print 'model saved'\n",
    "                saver.save(sess, checkpoints_path, global_step=i)\n",
    "\n",
    "            if i % info_period == 0:\n",
    "                print '|------------------------|'\n",
    "                print ('total episodes: {}\\n'\n",
    "                       'total steps: {}\\n'\n",
    "                       'last {} episodes average reward: {}'.format(i+1, total_steps, info_period, \n",
    "                                                                    np.mean(rewards[-info_period:])))\n",
    "                print '|------------------------|'\n",
    "    except KeyboardInterrupt:\n",
    "        env.exit()\n",
    "        print 'Training stopped'\n",
    "\n",
    "env.exit()\n",
    "print 'Finished'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}