dpg.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center> <h1>Continuous Control with Deep Reinforcement Learning</h1>\n",
    "<h4>Timothy P. Lillicrap, Jonathan J. Hunt, Alexander Pritzel, Nicolas Heess, Tom Erez, Yuval Tassa, David Silver, Daan Wierstra<br />\n",
    "http://arxiv.org/abs/1509.02971</h4>\n",
    "<h4>Single pendulum example - Implemented by Yannis M. Assael </h4>\n",
    "</center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if itorch then\n",
    "    arg = {}\n",
    "end\n",
    "\n",
    "cmd = torch.CmdLine()\n",
    "cmd:text()\n",
    "cmd:text('Options')\n",
    "\n",
    "-- general options:\n",
    "cmd:option('-seed', 1, 'initial random seed')\n",
    "cmd:option('-threads', 8, 'number of threads')\n",
    "\n",
    "-- gpu\n",
    "cmd:option('-cuda', 0, 'cuda')\n",
    "\n",
    "-- experiment\n",
    "cmd:option('-action_size', 1, 'action dimensionality')\n",
    "cmd:option('-action_lim', 10, 'action clipping')\n",
    "cmd:option('-action_repeat', 3, 'frames in history')\n",
    "cmd:option('-exploration_noise', 0.3, 'random exploration noise')\n",
    "cmd:option('-episodes', 1000, 'number of episodes')\n",
    "cmd:option('-episode_length', 30, 'frames of each episode')\n",
    "\n",
    "-- model\n",
    "cmd:option('-gamma', 0.99, 'discount factor')\n",
    "cmd:option('-q_tau', 1e-2, 'soft target updates')\n",
    "cmd:option('-dt', 0.1, 'time interval')\n",
    "cmd:option('-img_size', 64, 'image size')\n",
    "cmd:option('-batch_size', 16, 'batch training size')\n",
    "cmd:option('-replay_buffer', 1e+6, 'replay buffer size')\n",
    "\n",
    "-- training\n",
    "cmd:option('-epochs_per_step', 1, 'epochs per step')\n",
    "cmd:option('-l2_critic', 1e-2, 'l2 weight decay')\n",
    "cmd:option('-l2_actor', 0, 'l2 weight decay')\n",
    "cmd:option('-learningRate_actor', 1e-4, 'actor learning rate')\n",
    "cmd:option('-learningRate_critic', 1e-3, 'critic learning rate')\n",
    "\n",
    "-- get current path\n",
    "require 'sys'\n",
    "dname, fname = sys.fpath()\n",
    "cmd:option('-save', dname, 'save path')\n",
    "cmd:option('-load', 0, 'load pretrained model')\n",
    "\n",
    "cmd:option('-plot', 0, 'display images')\n",
    "cmd:text()\n",
    "\n",
    "opt = cmd:parse(arg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Require"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "require 'hdf5'\n",
    "require 'csvigo'\n",
    "require 'image'\n",
    "require 'nngraph'\n",
    "require 'optim'\n",
    "require 'nn'\n",
    "\n",
    "log = require 'include.log'\n",
    "tools = require 'include.base'\n",
    "require 'include.dynamics'\n",
    "\n",
    "require 'modules/LinearO'\n",
    "require 'modules/GradientClipping'\n",
    "require 'modules/GradientNormalization'\n",
    "\n",
    "torch.manualSeed(opt.seed)\n",
    "torch.setnumthreads(opt.threads)\n",
    "torch.setdefaulttensortype('torch.FloatTensor')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "-- Cuda initialisation\n",
    "if opt.cuda > 0 then\n",
    "    require 'cutorch'\n",
    "    require 'cunn'\n",
    "    require 'cudnn'\n",
    "    cutorch.setDevice(opt.cuda)\n",
    "    opt.dtype = 'torch.CudaTensor'\n",
    "    print(cutorch.getDeviceProperties(opt.cuda))\n",
    "else\n",
    "    opt.dtype = 'torch.FloatTensor'\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pendulum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "game = {}\n",
    "game.init_state = {0, 0} -- 1 angular velocity, 2 angle\n",
    "game.target_state = {0, math.pi} -- 1 angular velocity, 2 angle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "function game.control(s_t, a_t)\n",
    "    -- Control input\n",
    "    local f = {[1] = a_t * opt.action_lim}\n",
    "    -- 1 angular velocity, 2 angle\n",
    "    local s_t1 = control_sp(s_t, opt.dt, f)[1]\n",
    "    -- Correct angle\n",
    "    s_t1[2] = s_t1[2] % (2 * math.pi)\n",
    "    return s_t1\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "function game.reward(s_t, a_t)\n",
    "    -- distance reward\n",
    "    local dist = (math.pi - math.abs(game.target_state[2] - (s_t[2] % (2*math.pi)))) / math.pi\n",
    "    \n",
    "    -- bonus reward if +-10 degrees\n",
    "    local bonus = 0\n",
    "    if dist > 1 - 0.174532925 / math.pi then\n",
    "        bonus = 2\n",
    "    end\n",
    "\n",
    "    -- action cost\n",
    "    local action = 0\n",
    "    if a_t then\n",
    "        action = - a_t:clone():abs():sum()*0.5\n",
    "    end\n",
    "    \n",
    "    return dist + action + bonus\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "function game.generate_dataset(e)\n",
    "    local data = {}\n",
    "\n",
    "    -- Iterate episodes\n",
    "    for episode = 1, e do\n",
    "        \n",
    "        -- Initialise Variables\n",
    "        local s, x, a, r = {}, {}, {}, {}\n",
    "        \n",
    "        -- Receive initial observation state s_1    \n",
    "        s[1], x[1] = {}, torch.Tensor(opt.action_repeat,opt.img_size,opt.img_size)\n",
    "        for i = 1, opt.action_repeat do\n",
    "            s[1][i] = tools.dc(game.init_state)\n",
    "            x[1][i] = draw_sp(s[1][i][2], opt.img_size)\n",
    "        end\n",
    "        \n",
    "        -- Run episode\n",
    "        local err_episode = 0\n",
    "        for t = 1, opt.episode_length do\n",
    "            \n",
    "            -- Pick random action\n",
    "            a[t] = torch.randn(opt.action_size):mul(opt.exploration_noise):clamp(-1, 1)\n",
    "            \n",
    "            -- Execute action a_t and observe reward r_t and observe new state s_{t+1}\n",
    "            s[t+1], x[t+1] = {}, torch.Tensor(opt.action_repeat,opt.img_size,opt.img_size)\n",
    "            \n",
    "            s[t+1][1] = game.control(s[t][opt.action_repeat], a[t])\n",
    "            x[t+1][1] = draw_sp(s[t+1][1][2], opt.img_size)\n",
    "            for i = 2, opt.action_repeat do\n",
    "                s[t+1][i] = game.control(s[t+1][i-1], a[t])\n",
    "                x[t+1][i] = draw_sp(s[t+1][i][2], opt.img_size)\n",
    "            end\n",
    "            \n",
    "            -- Get reward\n",
    "            -- r[t] = game.reward(s[t+1][opt.action_repeat])\n",
    "            r[t] = 0\n",
    "            for i=1,opt.action_repeat do\n",
    "                 r[t] = r[t] + game.reward(s[t+1][i], a[t]) / opt.action_repeat\n",
    "            end\n",
    "            \n",
    "            -- Store transition (s_t, a_t, r_t, s_{t+1}) in R\n",
    "            data[#data+1] = {\n",
    "                                s_t=s[t],\n",
    "                                x_t=x[t],\n",
    "                                a_t=a[t],\n",
    "                                r_t=r[t],\n",
    "                                s_t_next=s[t+1],\n",
    "                                x_t_next=x[t+1]\n",
    "                            }\n",
    "        end    \n",
    "    end\n",
    "            \n",
    "    return data\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Actor / Critic Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "function create_network()\n",
    "        \n",
    "    -- Model Specific parameters\n",
    "    local f_maps_1 = 32\n",
    "    local f_size_1 = 4\n",
    "    local f_stride_1 = 4\n",
    "    \n",
    "    local f_maps_2 = 32\n",
    "    local f_size_2 = 4\n",
    "    local f_stride_2 = 2\n",
    "    \n",
    "    local f_maps_3 = 32\n",
    "    local f_size_3 = 3\n",
    "    local f_stride_3 = 1\n",
    "    \n",
    "    local enc_size = f_maps_3*5*5\n",
    "    local latent_size = 100\n",
    "    \n",
    "    -- Encoder\n",
    "    encoder = nn.Sequential()\n",
    "    encoder:add(nn.View(-1, opt.action_repeat, opt.img_size, opt.img_size))\n",
    "    \n",
    "    encoder:add(nn.SpatialBatchNormalization(opt.action_repeat))\n",
    "    \n",
    "    encoder:add(nn.SpatialConvolution(opt.action_repeat, f_maps_1, f_size_1, f_size_1, f_stride_1, f_stride_1))\n",
    "    encoder:add(nn.GradientClipping())\n",
    "    encoder:add(nn.SpatialBatchNormalization(f_maps_1))\n",
    "    encoder:add(nn.ReLU(true))\n",
    "    \n",
    "    encoder:add(nn.SpatialConvolution(f_maps_1, f_maps_2, f_size_2, f_size_2, f_stride_2, f_stride_2))\n",
    "    encoder:add(nn.GradientClipping())\n",
    "    encoder:add(nn.SpatialBatchNormalization(f_maps_2))\n",
    "    encoder:add(nn.ReLU(true))\n",
    "    \n",
    "    encoder:add(nn.SpatialConvolution(f_maps_2, f_maps_3, f_size_3, f_size_3, f_stride_3, f_stride_3))\n",
    "    encoder:add(nn.GradientClipping())\n",
    "    encoder:add(nn.SpatialBatchNormalization(f_maps_3))\n",
    "    encoder:add(nn.ReLU(true))\n",
    "    \n",
    "    encoder:add(nn.View(-1, enc_size))\n",
    "    \n",
    "    \n",
    "    -- Actor\n",
    "    actor = nn.Sequential()\n",
    "    actor:add(encoder:clone()) -- ('weight', 'bias', 'gradWeight', 'gradBias', 'running_mean', 'running_std')\n",
    "    \n",
    "    actor:add(nn.Linear(enc_size, latent_size))\n",
    "    actor:add(nn.GradientClipping())\n",
    "    actor:add(nn.BatchNormalization(latent_size))\n",
    "    actor:add(nn.ReLU(true))\n",
    "    \n",
    "    actor:add(nn.Linear(latent_size, latent_size))\n",
    "    actor:add(nn.GradientClipping())\n",
    "    actor:add(nn.BatchNormalization(latent_size))\n",
    "    actor:add(nn.ReLU(true))\n",
    "    \n",
    "    actor:add(nn.Linear(latent_size, opt.action_size))\n",
    "    actor:add(nn.GradientClipping())\n",
    "\n",
    "    actor:add(nn.View(-1, opt.action_size))\n",
    "    actor:add(nn.Clamp(-1, 1))\n",
    "    \n",
    "    \n",
    "    -- Critic\n",
    "    critic_encoder = encoder:clone() -- 'weight', 'bias', 'gradWeight', 'gradBias', 'running_mean', 'running_std'\n",
    "    critic_encoder:add(nn.Linear(enc_size, latent_size))\n",
    "    critic_encoder:add(nn.GradientClipping())\n",
    "    critic_encoder:add(nn.BatchNormalization(latent_size))\n",
    "    \n",
    "    critic_action = nn.Sequential()\n",
    "    critic_action:add(nn.View(-1, opt.action_size))\n",
    "    critic_action:add(nn.Linear(opt.action_size, latent_size))\n",
    "    critic_action:add(nn.GradientClipping())\n",
    "    \n",
    "    critic_split = nn.ParallelTable()\n",
    "    critic_split:add(critic_encoder)\n",
    "    critic_split:add(critic_action)\n",
    "    \n",
    "    critic = nn.Sequential()\n",
    "    critic:add(critic_split)\n",
    "    critic:add(nn.CAddTable())\n",
    "    critic:add(nn.GradientClipping())\n",
    "    critic:add(nn.ReLU(true))\n",
    "\n",
    "    critic:add(nn.Linear(latent_size, latent_size))\n",
    "    critic:add(nn.GradientClipping())\n",
    "    critic:add(nn.ReLU(true))\n",
    "    \n",
    "    critic:add(nn.Linear(latent_size, 1))\n",
    "    critic:add(nn.View(-1, 1))\n",
    "    \n",
    "    \n",
    "    return actor, critic\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Setup Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "function setup()\n",
    "    print(\"Creating network.\")\n",
    "    \n",
    "    model = {}\n",
    "    model.replay_buffer = {}\n",
    "    \n",
    "    model.actor, model.critic = create_network()\n",
    "    \n",
    "    -- Cuda\n",
    "    model.actor = model.actor:type(opt.dtype)\n",
    "    model.critic = model.critic:type(opt.dtype)\n",
    "    if opt.cuda > 0 then\n",
    "        cudnn.convert(model.actor, cudnn)\n",
    "        cudnn.convert(model.critic, cudnn)\n",
    "    end\n",
    "            \n",
    "    -- Initialise last layer parameters\n",
    "    model.actor.modules[#model.actor.modules-3].weight:uniform(-3e-4, 3e-4)\n",
    "    model.actor.modules[#model.actor.modules-3].bias:zero()\n",
    "    model.critic.modules[#model.critic.modules-1].weight:uniform(-3e-4, 3e-4)\n",
    "    model.critic.modules[#model.critic.modules-1].bias:zero()\n",
    "    \n",
    "    -- Target networks\n",
    "    model.actor_target = model.actor:clone('running_mean', 'running_std')\n",
    "    model.critic_target = model.critic:clone('running_mean', 'running_std')\n",
    "    \n",
    "    -- Get parameters\n",
    "    params, gradParams = {}, {}\n",
    "    params.actor, gradParams.actor = model.actor:getParameters()\n",
    "    params.actor_target, _ = model.actor_target:getParameters()\n",
    "    params.critic, gradParams.critic = model.critic:getParameters()\n",
    "    params.critic_target, _ = model.critic_target:getParameters()\n",
    "    \n",
    "    -- Set criterion\n",
    "    criterion = nn.MSECriterion():type(opt.dtype)\n",
    "\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "function save_model()\n",
    "    -- save/log current net\n",
    "    local filename = paths.concat(opt.save, 'model/ddpg.t7')\n",
    "    os.execute('mkdir -p ' .. paths.dirname(filename))\n",
    "    if paths.filep(filename) then\n",
    "        os.execute('mv ' .. filename .. ' ' .. filename .. '.old')\n",
    "    end\n",
    "    -- print('<trainer> saving network to '..filename)\n",
    "    torch.save(filename, {model, opt, optim_config, train_err, test_r})\n",
    "end\n",
    "\n",
    "function load_model()\n",
    "    model, opt, optim_config, train_err, test_r = unpack(torch.load('model/ddpg.t7'))\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialise DDPG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DDPG parameters:\t\n",
       "{\n",
       "  seed : 1\n",
       "  batch_size : 16\n",
       "  img_size : 64\n",
       "  q_tau : 0.01\n",
       "  replay_buffer : 1000000\n",
       "  gamma : 0.99\n",
       "  plot : 0\n",
       "  save : /data/tiger/johael/torch-policy-gradient\n",
       "  learningRate_critic : 0.001\n",
       "  action_lim : 10\n",
       "  load : 0\n",
       "  l2_critic : 0.01\n",
       "  dt : 0.1\n",
       "  exploration_noise : 0.3\n",
       "  learningRate_actor : 0.0001\n",
       "  dtype : torch.FloatTensor\n",
       "  episodes : 1000\n",
       "  episode_length : 30\n",
       "  threads : 8\n",
       "  action_size : 1\n",
       "  action_repeat : 3\n",
       "  epochs_per_step : 1\n",
       "  cuda : 0\n",
       "  l2_actor : 0\n",
       "}\n",
       "Creating network.\t\n"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"DDPG parameters:\")\n",
    "print(opt)\n",
    "\n",
    "setup()\n",
    "\n",
    "-- Training configuration\n",
    "optim_config = {}\n",
    "optim_config.actor = {\n",
    "                        learningRate = opt.learningRate_actor,\n",
    "                        beta2 = 0.9\n",
    "                        }\n",
    "optim_config.critic = {\n",
    "                        learningRate = opt.learningRate_critic,\n",
    "                        beta2 = 0.9\n",
    "                        }\n",
    "\n",
    "\n",
    "-- Performance tables\n",
    "stats = {\n",
    "    train_err = {},\n",
    "    test_r = {}\n",
    "}\n",
    "\n",
    "\n",
    "if opt.load == 1 then\n",
    "    load_model()\n",
    "end\n",
    "\n",
    "epoch = #stats.train_err"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batch Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch = {}\n",
    "batch.s_t = {}\n",
    "batch.x_t = torch.Tensor(opt.batch_size, opt.action_repeat, opt.img_size, opt.img_size):type(opt.dtype):zero()\n",
    "batch.a_t = torch.Tensor(opt.batch_size, opt.action_size):type(opt.dtype):zero()\n",
    "batch.r_t = torch.Tensor(opt.batch_size, 1):type(opt.dtype):zero()\n",
    "batch.s_t_next = {}\n",
    "batch.x_t_next = torch.Tensor(opt.batch_size, opt.action_repeat, opt.img_size, opt.img_size):type(opt.dtype):zero()\n",
    "gradCritic = torch.Tensor(opt.batch_size, 1):type(opt.dtype):fill(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "function train()\n",
    "    \n",
    "    -- Collect garbage\n",
    "    collectgarbage()\n",
    "        \n",
    "    -- Epoch tracker\n",
    "    epoch = epoch or 0\n",
    "    \n",
    "    -- Set model to training mode\n",
    "    model.actor:training()\n",
    "    model.critic:training()\n",
    "    model.actor_target:training()\n",
    "    model.critic_target:training()\n",
    "    \n",
    "    -- Error\n",
    "    local err = {}\n",
    "    \n",
    "    -- Sample a random minibatch of N transitions (si , ai , ri , si+1) from R\n",
    "    local batch_idx = torch.randperm(#model.replay_buffer)[{{1,opt.batch_size}}]:long()\n",
    "    for i=1,opt.batch_size do        \n",
    "        local pair = model.replay_buffer[batch_idx[i]]\n",
    "\n",
    "        batch.s_t[i] = pair.s_t\n",
    "        batch.x_t[i] = pair.x_t:type(opt.dtype)\n",
    "        batch.a_t[i] = pair.a_t:type(opt.dtype)\n",
    "        batch.r_t[i] = pair.r_t\n",
    "        batch.s_t_next[i] = pair.s_t_next\n",
    "        batch.x_t_next[i] = pair.x_t_next:type(opt.dtype)\n",
    "        \n",
    "    end\n",
    "\n",
    "    -- Set y_i = r_i + γ Q'(s_{i+1}, μ'(s_{i+1}|θ^μ') | Θ^Q')\n",
    "    local batch_mu_target = model.actor_target:forward(batch.x_t_next)\n",
    "    local batch_q_target = model.critic_target:forward({batch.x_t_next, batch_mu_target})\n",
    "    local batch_y = batch_q_target:clone():mul(opt.gamma):add(batch.r_t)\n",
    "        \n",
    "    -- Update critic MSE: L = 1/N(y_i − Q(s_i, a_i|θ^Q))^2\n",
    "    local feval_critic = function(x)\n",
    "        \n",
    "        -- Zero gradients\n",
    "        model.critic:zeroGradParameters()\n",
    "        \n",
    "        -- Forward pass\n",
    "        local pred_q = model.critic:forward({batch.x_t, batch.a_t})\n",
    "        err.critic = criterion:forward(pred_q, batch_y)\n",
    "        \n",
    "        -- Backward pass\n",
    "        local d_err_q = criterion:backward(pred_q, batch_y)\n",
    "        model.critic:backward({batch.x_t, batch.a_t}, d_err_q)\n",
    "        \n",
    "        -- L2 Weight decay\n",
    "        if opt.l2_critic > 0 then\n",
    "            local l2 = opt.l2_critic * params.critic:norm(2)^2 / 2\n",
    "            gradParams.critic:add(opt.l2_critic, params.critic)\n",
    "            err.critic = err.critic + l2\n",
    "        end\n",
    "        \n",
    "        -- Clip gradients\n",
    "        gradParams.critic:clamp(-5, 5)\n",
    "                        \n",
    "        return err.critic, gradParams.critic\n",
    "    end\n",
    "    \n",
    "    optim.adam(feval_critic, params.critic, optim_config.critic)\n",
    "\n",
    "    \n",
    "    -- Update the actor policy using the sampled gradient\n",
    "    local feval_actor = function(x)\n",
    "        \n",
    "        -- Zero gradients\n",
    "        model.actor:zeroGradParameters()\n",
    "        \n",
    "        -- Forward pass\n",
    "        local pred_mu = model.actor:forward(batch.x_t)\n",
    "        local pred_q = model.critic:forward({batch.x_t, pred_mu})\n",
    "        \n",
    "        -- Backward pass\n",
    "        local d_a_critic = model.critic:updateGradInput({batch.x_t, pred_mu}, gradCritic)[2]\n",
    "        model.actor:backward(batch.x_t, d_a_critic)\n",
    "        \n",
    "        -- Normalise\n",
    "        gradParams.actor:div(opt.batch_size)\n",
    "                \n",
    "        -- L2 Weight decay\n",
    "        if opt.l2_actor > 0 then\n",
    "            local l2 = opt.l2_actor * params.actor:norm(2)^2 / 2\n",
    "            gradParams.actor:add(opt.l2_actor, params.actor)\n",
    "        end\n",
    "        \n",
    "        -- Clip gradients\n",
    "        gradParams.actor:clamp(-5, 5)\n",
    "        \n",
    "        return 0, gradParams.actor\n",
    "    end\n",
    "\n",
    "    optim.adam(feval_actor, params.actor, optim_config.actor)\n",
    "    \n",
    "    \n",
    "    -- Update target networks\n",
    "    params.actor_target:mul(1 - opt.q_tau):add(opt.q_tau, params.actor)\n",
    "    params.critic_target:mul(1 - opt.q_tau):add(opt.q_tau, params.critic)\n",
    "    \n",
    "    epoch = epoch + 1\n",
    "\n",
    "    return err.critic\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "function test(v)\n",
    "    \n",
    "    -- Set model to evaluate mode (for modules that differ in training and testing, like Dropout)\n",
    "    model.actor:evaluate()\n",
    "    model.critic:evaluate()\n",
    "    model.actor_target:evaluate()\n",
    "    model.critic_target:evaluate()\n",
    "    \n",
    "    local total_reward = torch.Tensor(opt.episode_length)\n",
    "    \n",
    "    -- Receive initial observation state s_1\n",
    "    local s, x, a, r = {}, {}, {}, {}\n",
    "    \n",
    "    s[1], x[1] = {}, torch.Tensor(opt.action_repeat,opt.img_size,opt.img_size)\n",
    "    for i = 1, opt.action_repeat do\n",
    "        s[1][i] = tools.dc(game.init_state)\n",
    "        x[1][i] = draw_sp(s[1][i][2], opt.img_size)\n",
    "    end\n",
    "\n",
    "    -- Run episode\n",
    "    for t = 1, opt.episode_length do\n",
    "        \n",
    "        a[t] = model.actor_target:forward(x[t]:type(opt.dtype))[1]:clone()\n",
    "        -- a[t] = model.actor:forward(x[t]:type(opt.dtype))[1]:clone()\n",
    "        \n",
    "        -- Execute action a_t and observe reward r_t and observe new state s_{t+1}\n",
    "        s[t+1], x[t+1] = {}, torch.Tensor(opt.action_repeat,opt.img_size,opt.img_size)   \n",
    "        s[t+1][1] = game.control(s[t][opt.action_repeat], a[t])\n",
    "        x[t+1][1] = draw_sp(s[t+1][1][2], opt.img_size)\n",
    "        for i = 2, opt.action_repeat do\n",
    "            s[t+1][i] = game.control(s[t+1][i-1], a[t])\n",
    "            x[t+1][i] = draw_sp(s[t+1][i][2], opt.img_size)\n",
    "        end\n",
    "                \n",
    "        -- Get reward\n",
    "        r[t] = 0\n",
    "        for i=1,opt.action_repeat do\n",
    "             r[t] = r[t] + game.reward(s[t+1][i], a[t]) / opt.action_repeat\n",
    "        end\n",
    "        \n",
    "        -- Accumulate rewards\n",
    "        total_reward[t] = r[t] \n",
    "        \n",
    "        if v then\n",
    "            itorch.image(x[t+1])\n",
    "            log.infof('step=%d, q=%.3f, r=%.2f, a=%.2f',\n",
    "                t,\n",
    "                critic:forward({x[t]:type(opt.dtype), a[t]})[1][1],\n",
    "                r[t],\n",
    "                a[t][1])\n",
    "        end\n",
    "    end\n",
    "\n",
    "    \n",
    "    return total_reward\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate and Train on initial data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.replay_buffer = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "episodes_random = 20\n",
    "\n",
    "-- Generate random episodes\n",
    "model.replay_buffer = game.generate_dataset(episodes_random)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:51:36]\u001b[0m [string \"...\"]:8: pre-training iter=100, train_mse=0.653\t\n"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:51:40]\u001b[0m [string \"...\"]:8: pre-training iter=200, train_mse=0.631\t\n"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:51:44]\u001b[0m [string \"...\"]:8: pre-training iter=300, train_mse=0.490\t\n"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:51:49]\u001b[0m [string \"...\"]:8: pre-training iter=400, train_mse=0.400\t\n"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:51:53]\u001b[0m [string \"...\"]:8: pre-training iter=500, train_mse=0.385\t\n"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:51:58]\u001b[0m [string \"...\"]:8: pre-training iter=600, train_mse=0.617\t\n"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "-- Train for episodes_random * opt.episode_length epochs\n",
    "for i = 1, episodes_random * opt.episode_length do\n",
    "    \n",
    "    local err = train()\n",
    "    \n",
    "    if i % 100 == 0 then\n",
    "        log.infof('pre-training iter=%d, train_mse=%.3f', i, err)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Learn Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:52:35]\u001b[0m [string \"...\"]:90: e=10, train_mse=0.522, train_mse_avg=0.513, test_r=0.118, test_r_avg=0.129, t/e=3.74 sec, t=0 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:53:15]\u001b[0m [string \"...\"]:90: e=20, train_mse=0.430, train_mse_avg=0.491, test_r=0.509, test_r_avg=0.212, t/e=3.65 sec, t=1 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:53:53]\u001b[0m [string \"...\"]:90: e=30, train_mse=0.424, train_mse_avg=0.474, test_r=0.630, test_r_avg=0.341, t/e=3.82 sec, t=1 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:54:31]\u001b[0m [string \"...\"]:90: e=40, train_mse=0.418, train_mse_avg=0.456, test_r=0.518, test_r_avg=0.440, t/e=3.89 sec, t=2 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:55:11]\u001b[0m [string \"...\"]:90: e=50, train_mse=0.400, train_mse_avg=0.435, test_r=0.598, test_r_avg=0.531, t/e=4.01 sec, t=3 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:55:52]\u001b[0m [string \"...\"]:90: e=60, train_mse=0.354, train_mse_avg=0.410, test_r=0.610, test_r_avg=0.543, t/e=4.14 sec, t=3 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:56:33]\u001b[0m [string \"...\"]:90: e=70, train_mse=0.379, train_mse_avg=0.394, test_r=0.544, test_r_avg=0.553, t/e=4.04 sec, t=4 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:57:14]\u001b[0m [string \"...\"]:90: e=80, train_mse=0.380, train_mse_avg=0.397, test_r=0.574, test_r_avg=0.553, t/e=4.09 sec, t=5 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:57:54]\u001b[0m [string \"...\"]:90: e=90, train_mse=0.378, train_mse_avg=0.386, test_r=0.455, test_r_avg=0.549, t/e=4.14 sec, t=5 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:58:35]\u001b[0m [string \"...\"]:90: e=100, train_mse=0.387, train_mse_avg=0.374, test_r=0.481, test_r_avg=0.568, t/e=4.02 sec, t=6 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  04:59:17]\u001b[0m [string \"...\"]:90: e=110, train_mse=0.331, train_mse_avg=0.361, test_r=0.547, test_r_avg=0.576, t/e=4.18 sec, t=7 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:00:01]\u001b[0m [string \"...\"]:90: e=120, train_mse=0.334, train_mse_avg=0.358, test_r=0.585, test_r_avg=0.575, t/e=4.21 sec, t=8 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:00:44]\u001b[0m [string \"...\"]:90: e=130, train_mse=0.329, train_mse_avg=0.355, test_r=0.543, test_r_avg=0.572, t/e=4.22 sec, t=8 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:01:27]\u001b[0m [string \"...\"]:90: e=140, train_mse=0.295, train_mse_avg=0.348, test_r=0.541, test_r_avg=0.580, t/e=4.21 sec, t=9 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:02:10]\u001b[0m [string \"...\"]:90: e=150, train_mse=0.326, train_mse_avg=0.346, test_r=0.700, test_r_avg=0.589, t/e=4.62 sec, t=10 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:02:54]\u001b[0m [string \"...\"]:90: e=160, train_mse=0.339, train_mse_avg=0.351, test_r=0.624, test_r_avg=0.611, t/e=4.29 sec, t=10 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:03:38]\u001b[0m [string \"...\"]:90: e=170, train_mse=0.355, train_mse_avg=0.356, test_r=0.591, test_r_avg=0.610, t/e=4.20 sec, t=11 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:04:26]\u001b[0m [string \"...\"]:90: e=180, train_mse=0.363, train_mse_avg=0.362, test_r=0.743, test_r_avg=0.659, t/e=5.38 sec, t=12 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:05:12]\u001b[0m [string \"...\"]:90: e=190, train_mse=0.347, train_mse_avg=0.363, test_r=0.789, test_r_avg=0.668, t/e=4.46 sec, t=13 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:05:57]\u001b[0m [string \"...\"]:90: e=200, train_mse=0.386, train_mse_avg=0.375, test_r=0.640, test_r_avg=0.645, t/e=4.56 sec, t=13 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:06:44]\u001b[0m [string \"...\"]:90: e=210, train_mse=0.351, train_mse_avg=0.371, test_r=0.623, test_r_avg=0.641, t/e=4.63 sec, t=14 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:07:33]\u001b[0m [string \"...\"]:90: e=220, train_mse=0.375, train_mse_avg=0.374, test_r=0.764, test_r_avg=0.645, t/e=4.59 sec, t=15 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:08:19]\u001b[0m [string \"...\"]:90: e=230, train_mse=0.419, train_mse_avg=0.383, test_r=0.646, test_r_avg=0.674, t/e=4.74 sec, t=16 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:09:06]\u001b[0m [string \"...\"]:90: e=240, train_mse=0.444, train_mse_avg=0.394, test_r=0.753, test_r_avg=0.676, t/e=4.69 sec, t=17 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:09:53]\u001b[0m [string \"...\"]:90: e=250, train_mse=0.401, train_mse_avg=0.401, test_r=0.942, test_r_avg=0.707, t/e=4.57 sec, t=17 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:10:46]\u001b[0m [string \"...\"]:90: e=260, train_mse=0.366, train_mse_avg=0.394, test_r=0.649, test_r_avg=0.698, t/e=4.70 sec, t=18 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:11:35]\u001b[0m [string \"...\"]:90: e=270, train_mse=0.382, train_mse_avg=0.392, test_r=0.661, test_r_avg=0.716, t/e=5.73 sec, t=19 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:12:29]\u001b[0m [string \"...\"]:90: e=280, train_mse=0.406, train_mse_avg=0.396, test_r=0.842, test_r_avg=0.753, t/e=5.94 sec, t=20 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:13:25]\u001b[0m [string \"...\"]:90: e=290, train_mse=0.405, train_mse_avg=0.401, test_r=0.741, test_r_avg=0.738, t/e=5.89 sec, t=21 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:14:16]\u001b[0m [string \"...\"]:90: e=300, train_mse=0.433, train_mse_avg=0.405, test_r=0.770, test_r_avg=0.762, t/e=4.94 sec, t=22 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:15:09]\u001b[0m [string \"...\"]:90: e=310, train_mse=0.416, train_mse_avg=0.408, test_r=0.973, test_r_avg=0.811, t/e=5.92 sec, t=23 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:16:03]\u001b[0m [string \"...\"]:90: e=320, train_mse=0.439, train_mse_avg=0.415, test_r=0.926, test_r_avg=0.810, t/e=5.94 sec, t=24 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:17:02]\u001b[0m [string \"...\"]:90: e=330, train_mse=0.401, train_mse_avg=0.413, test_r=0.770, test_r_avg=0.830, t/e=5.48 sec, t=25 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:17:50]\u001b[0m [string \"...\"]:90: e=340, train_mse=0.400, train_mse_avg=0.413, test_r=0.896, test_r_avg=0.845, t/e=4.83 sec, t=25 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:18:48]\u001b[0m [string \"...\"]:90: e=350, train_mse=0.417, train_mse_avg=0.415, test_r=0.893, test_r_avg=0.832, t/e=5.48 sec, t=26 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:19:40]\u001b[0m [string \"...\"]:90: e=360, train_mse=0.429, train_mse_avg=0.423, test_r=1.014, test_r_avg=0.909, t/e=5.08 sec, t=27 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:20:33]\u001b[0m [string \"...\"]:90: e=370, train_mse=0.436, train_mse_avg=0.431, test_r=0.686, test_r_avg=0.897, t/e=4.87 sec, t=28 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:21:21]\u001b[0m [string \"...\"]:90: e=380, train_mse=0.435, train_mse_avg=0.433, test_r=0.882, test_r_avg=0.903, t/e=4.88 sec, t=29 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:22:11]\u001b[0m [string \"...\"]:90: e=390, train_mse=0.417, train_mse_avg=0.433, test_r=0.659, test_r_avg=0.897, t/e=4.97 sec, t=30 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:23:01]\u001b[0m [string \"...\"]:90: e=400, train_mse=0.461, train_mse_avg=0.430, test_r=1.013, test_r_avg=0.882, t/e=5.74 sec, t=31 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:24:00]\u001b[0m [string \"...\"]:90: e=410, train_mse=0.456, train_mse_avg=0.441, test_r=0.943, test_r_avg=0.932, t/e=5.56 sec, t=32 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:24:57]\u001b[0m [string \"...\"]:90: e=420, train_mse=0.473, train_mse_avg=0.446, test_r=0.786, test_r_avg=0.894, t/e=6.17 sec, t=32 min.\t"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:25:48]\u001b[0m [string \"...\"]:90: e=430, train_mse=0.470, train_mse_avg=0.455, test_r=0.878, test_r_avg=0.894, t/e=4.83 sec, t=33 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:26:41]\u001b[0m [string \"...\"]:90: e=440, train_mse=0.535, train_mse_avg=0.469, test_r=1.094, test_r_avg=0.967, t/e=4.88 sec, t=34 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:27:35]\u001b[0m [string \"...\"]:90: e=450, train_mse=0.426, train_mse_avg=0.460, test_r=0.779, test_r_avg=0.924, t/e=5.12 sec, t=35 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:28:26]\u001b[0m [string \"...\"]:90: e=460, train_mse=0.463, train_mse_avg=0.455, test_r=0.757, test_r_avg=0.896, t/e=5.23 sec, t=36 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:29:17]\u001b[0m [string \"...\"]:90: e=470, train_mse=0.425, train_mse_avg=0.461, test_r=0.813, test_r_avg=0.893, t/e=5.09 sec, t=37 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:30:20]\u001b[0m [string \"...\"]:90: e=480, train_mse=0.500, train_mse_avg=0.477, test_r=0.950, test_r_avg=0.882, t/e=6.16 sec, t=38 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:31:16]\u001b[0m [string \"...\"]:90: e=490, train_mse=0.449, train_mse_avg=0.472, test_r=1.192, test_r_avg=0.915, t/e=5.05 sec, t=39 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:32:15]\u001b[0m [string \"...\"]:90: e=500, train_mse=0.467, train_mse_avg=0.466, test_r=0.784, test_r_avg=0.904, t/e=6.52 sec, t=40 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:33:18]\u001b[0m [string \"...\"]:90: e=510, train_mse=0.451, train_mse_avg=0.471, test_r=0.782, test_r_avg=0.913, t/e=5.25 sec, t=41 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:34:11]\u001b[0m [string \"...\"]:90: e=520, train_mse=0.459, train_mse_avg=0.477, test_r=0.637, test_r_avg=0.932, t/e=5.31 sec, t=42 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:35:06]\u001b[0m [string \"...\"]:90: e=530, train_mse=0.525, train_mse_avg=0.498, test_r=1.076, test_r_avg=0.970, t/e=5.02 sec, t=43 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:35:56]\u001b[0m [string \"...\"]:90: e=540, train_mse=0.523, train_mse_avg=0.506, test_r=1.206, test_r_avg=1.030, t/e=4.95 sec, t=43 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:36:48]\u001b[0m [string \"...\"]:90: e=550, train_mse=0.562, train_mse_avg=0.512, test_r=0.966, test_r_avg=1.065, t/e=5.13 sec, t=44 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:37:47]\u001b[0m [string \"...\"]:90: e=560, train_mse=0.560, train_mse_avg=0.516, test_r=0.938, test_r_avg=1.082, t/e=6.45 sec, t=45 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:38:43]\u001b[0m [string \"...\"]:90: e=570, train_mse=0.466, train_mse_avg=0.521, test_r=1.179, test_r_avg=1.073, t/e=4.97 sec, t=46 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:39:34]\u001b[0m [string \"...\"]:90: e=580, train_mse=0.547, train_mse_avg=0.528, test_r=1.444, test_r_avg=1.120, t/e=5.19 sec, t=47 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:40:27]\u001b[0m [string \"...\"]:90: e=590, train_mse=0.510, train_mse_avg=0.526, test_r=0.890, test_r_avg=1.090, t/e=5.29 sec, t=48 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:41:19]\u001b[0m [string \"...\"]:90: e=600, train_mse=0.594, train_mse_avg=0.543, test_r=1.163, test_r_avg=1.135, t/e=5.23 sec, t=49 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:42:16]\u001b[0m [string \"...\"]:90: e=610, train_mse=0.618, train_mse_avg=0.557, test_r=1.326, test_r_avg=1.171, t/e=5.10 sec, t=50 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:43:08]\u001b[0m [string \"...\"]:90: e=620, train_mse=0.580, train_mse_avg=0.559, test_r=1.242, test_r_avg=1.167, t/e=5.44 sec, t=51 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:44:12]\u001b[0m [string \"...\"]:90: e=630, train_mse=0.604, train_mse_avg=0.566, test_r=1.579, test_r_avg=1.251, t/e=6.47 sec, t=52 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:45:09]\u001b[0m [string \"...\"]:90: e=640, train_mse=0.615, train_mse_avg=0.584, test_r=1.368, test_r_avg=1.361, t/e=5.12 sec, t=53 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:46:00]\u001b[0m [string \"...\"]:90: e=650, train_mse=0.593, train_mse_avg=0.581, test_r=1.472, test_r_avg=1.401, t/e=5.11 sec, t=54 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:46:53]\u001b[0m [string \"...\"]:90: e=660, train_mse=0.575, train_mse_avg=0.583, test_r=1.612, test_r_avg=1.428, t/e=5.13 sec, t=54 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:47:53]\u001b[0m [string \"...\"]:90: e=670, train_mse=0.562, train_mse_avg=0.586, test_r=1.479, test_r_avg=1.390, t/e=6.43 sec, t=55 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:48:58]\u001b[0m [string \"...\"]:90: e=680, train_mse=0.653, train_mse_avg=0.603, test_r=1.219, test_r_avg=1.409, t/e=5.36 sec, t=57 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:50:02]\u001b[0m [string \"...\"]:90: e=690, train_mse=0.563, train_mse_avg=0.604, test_r=1.206, test_r_avg=1.411, t/e=5.65 sec, t=58 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:51:01]\u001b[0m [string \"...\"]:90: e=700, train_mse=0.618, train_mse_avg=0.618, test_r=0.943, test_r_avg=1.358, t/e=5.47 sec, t=59 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:51:59]\u001b[0m [string \"...\"]:90: e=710, train_mse=0.721, train_mse_avg=0.615, test_r=1.529, test_r_avg=1.418, t/e=5.33 sec, t=60 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:53:04]\u001b[0m [string \"...\"]:90: e=720, train_mse=0.531, train_mse_avg=0.608, test_r=1.254, test_r_avg=1.412, t/e=6.11 sec, t=61 min.\t"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:54:09]\u001b[0m [string \"...\"]:90: e=730, train_mse=0.663, train_mse_avg=0.619, test_r=1.343, test_r_avg=1.390, t/e=6.78 sec, t=62 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:55:13]\u001b[0m [string \"...\"]:90: e=740, train_mse=0.599, train_mse_avg=0.614, test_r=1.667, test_r_avg=1.462, t/e=6.56 sec, t=63 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:56:19]\u001b[0m [string \"...\"]:90: e=750, train_mse=0.642, train_mse_avg=0.618, test_r=1.747, test_r_avg=1.463, t/e=6.81 sec, t=64 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:57:25]\u001b[0m [string \"...\"]:90: e=760, train_mse=0.712, train_mse_avg=0.630, test_r=1.502, test_r_avg=1.369, t/e=5.49 sec, t=65 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:58:19]\u001b[0m [string \"...\"]:90: e=770, train_mse=0.611, train_mse_avg=0.627, test_r=2.152, test_r_avg=1.431, t/e=5.21 sec, t=66 min.\t"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  05:59:13]\u001b[0m [string \"...\"]:90: e=780, train_mse=0.576, train_mse_avg=0.626, test_r=1.150, test_r_avg=1.498, t/e=5.50 sec, t=67 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:00:13]\u001b[0m [string \"...\"]:90: e=790, train_mse=0.613, train_mse_avg=0.643, test_r=1.133, test_r_avg=1.453, t/e=6.33 sec, t=68 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:01:14]\u001b[0m [string \"...\"]:90: e=800, train_mse=0.557, train_mse_avg=0.635, test_r=2.295, test_r_avg=1.509, t/e=6.70 sec, t=69 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:02:20]\u001b[0m [string \"...\"]:90: e=810, train_mse=0.596, train_mse_avg=0.632, test_r=1.999, test_r_avg=1.599, t/e=5.41 sec, t=70 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:03:16]\u001b[0m [string \"...\"]:90: e=820, train_mse=0.774, train_mse_avg=0.641, test_r=1.482, test_r_avg=1.618, t/e=6.43 sec, t=71 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:04:26]\u001b[0m [string \"...\"]:90: e=830, train_mse=0.664, train_mse_avg=0.638, test_r=1.818, test_r_avg=1.578, t/e=6.86 sec, t=72 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:05:27]\u001b[0m [string \"...\"]:90: e=840, train_mse=0.666, train_mse_avg=0.653, test_r=1.220, test_r_avg=1.555, t/e=5.63 sec, t=73 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:06:25]\u001b[0m [string \"...\"]:90: e=850, train_mse=0.612, train_mse_avg=0.647, test_r=2.353, test_r_avg=1.631, t/e=5.35 sec, t=74 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:07:21]\u001b[0m [string \"...\"]:90: e=860, train_mse=0.676, train_mse_avg=0.673, test_r=2.070, test_r_avg=1.633, t/e=5.34 sec, t=75 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:08:16]\u001b[0m [string \"...\"]:90: e=870, train_mse=0.623, train_mse_avg=0.671, test_r=1.711, test_r_avg=1.662, t/e=5.44 sec, t=76 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:09:11]\u001b[0m [string \"...\"]:90: e=880, train_mse=0.719, train_mse_avg=0.689, test_r=1.817, test_r_avg=1.645, t/e=5.46 sec, t=77 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:10:07]\u001b[0m [string \"...\"]:90: e=890, train_mse=0.679, train_mse_avg=0.689, test_r=1.265, test_r_avg=1.614, t/e=5.71 sec, t=78 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:11:02]\u001b[0m [string \"...\"]:90: e=900, train_mse=0.671, train_mse_avg=0.687, test_r=1.735, test_r_avg=1.606, t/e=5.43 sec, t=79 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:12:03]\u001b[0m [string \"...\"]:90: e=910, train_mse=0.634, train_mse_avg=0.675, test_r=1.770, test_r_avg=1.662, t/e=5.38 sec, t=80 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:12:59]\u001b[0m [string \"...\"]:90: e=920, train_mse=0.617, train_mse_avg=0.670, test_r=2.005, test_r_avg=1.675, t/e=5.54 sec, t=81 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:13:57]\u001b[0m [string \"...\"]:90: e=930, train_mse=0.714, train_mse_avg=0.678, test_r=2.608, test_r_avg=1.816, t/e=6.67 sec, t=81 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:14:58]\u001b[0m [string \"...\"]:90: e=940, train_mse=0.785, train_mse_avg=0.685, test_r=1.849, test_r_avg=1.979, t/e=5.57 sec, t=83 min.\t"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:15:54]\u001b[0m [string \"...\"]:90: e=950, train_mse=0.666, train_mse_avg=0.685, test_r=1.811, test_r_avg=1.982, t/e=6.86 sec, t=83 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:17:02]\u001b[0m [string \"...\"]:90: e=960, train_mse=0.676, train_mse_avg=0.686, test_r=1.777, test_r_avg=1.838, t/e=6.75 sec, t=85 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:18:03]\u001b[0m [string \"...\"]:90: e=970, train_mse=0.688, train_mse_avg=0.684, test_r=2.132, test_r_avg=1.773, t/e=5.42 sec, t=86 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:19:01]\u001b[0m [string \"...\"]:90: e=980, train_mse=0.648, train_mse_avg=0.678, test_r=1.534, test_r_avg=1.561, t/e=5.69 sec, t=87 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:20:03]\u001b[0m [string \"...\"]:90: e=990, train_mse=0.714, train_mse_avg=0.685, test_r=1.255, test_r_avg=1.512, t/e=7.29 sec, t=88 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  06:21:04]\u001b[0m [string \"...\"]:90: e=1000, train_mse=0.708, train_mse_avg=0.680, test_r=1.863, test_r_avg=1.538, t/e=5.53 sec, t=89 min.\t\n"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "-- start time\n",
    "local beginning_time = torch.tic()\n",
    "\n",
    "-- Iterate episodes\n",
    "for episode = 1, opt.episodes do\n",
    "    \n",
    "    -- Initialise Variables\n",
    "    local time = sys.clock()\n",
    "    local s, x, a, r = {}, {}, {}, {}    \n",
    "    \n",
    "    -- Initialise a random process N for action exploration\n",
    "    local N_t = torch.randn(opt.episode_length, opt.action_size):mul(opt.exploration_noise):type(opt.dtype)\n",
    "    \n",
    "    -- Receive initial observation state s_1    \n",
    "    s[1], x[1] = {}, torch.Tensor(opt.action_repeat,opt.img_size,opt.img_size)\n",
    "    for i = 1, opt.action_repeat do\n",
    "        s[1][i] = tools.dc(game.init_state)\n",
    "        x[1][i] = draw_sp(s[1][i][2], opt.img_size)\n",
    "    end\n",
    "    \n",
    "    -- Run episode\n",
    "    local err_episode = 0\n",
    "    for t = 1, opt.episode_length do\n",
    "        \n",
    "        -- Select action a_t = μ(s_t|θ^μ) + N_t according to the current policy and exploration noise\n",
    "        model.actor:evaluate()\n",
    "        a[t] = model.actor:forward(x[t]:type(opt.dtype))[1]:clone():add(N_t[t]):clamp(-1, 1)\n",
    "        \n",
    "        -- Execute action a_t and observe reward r_t and observe new state s_{t+1}\n",
    "        s[t+1], x[t+1] = {}, torch.Tensor(opt.action_repeat,opt.img_size,opt.img_size)\n",
    "        s[t+1][1] = game.control(s[t][opt.action_repeat], a[t])\n",
    "        x[t+1][1] = draw_sp(s[t+1][1][2], opt.img_size)\n",
    "        for i = 2, opt.action_repeat do\n",
    "            s[t+1][i] = game.control(s[t+1][i-1], a[t])\n",
    "            x[t+1][i] = draw_sp(s[t+1][i][2], opt.img_size)\n",
    "        end\n",
    "        \n",
    "        -- Get reward\n",
    "        r[t] = 0\n",
    "        for i=1,opt.action_repeat do\n",
    "             r[t] = r[t] + game.reward(s[t+1][i], a[t]) / opt.action_repeat\n",
    "        end\n",
    "        \n",
    "        -- Store transition (s_t, a_t, r_t, s_{t+1}) in R\n",
    "        local pair = {\n",
    "                        s_t=s[t],\n",
    "                        x_t=x[t],\n",
    "                        a_t=a[t],\n",
    "                        r_t=r[t],\n",
    "                        s_t_next=s[t+1],\n",
    "                        x_t_next=x[t+1]\n",
    "                      }\n",
    "        \n",
    "        -- Or replace existing if size exceeded\n",
    "        if #model.replay_buffer < opt.replay_buffer then\n",
    "            model.replay_buffer[#model.replay_buffer+1] = pair\n",
    "        else\n",
    "            local idx_rand = math.round(torch.uniform(1,#model.replay_buffer))\n",
    "            model.replay_buffer[idx_rand] = pair\n",
    "        end\n",
    "        \n",
    "        -- Train with experience replay\n",
    "        if #model.replay_buffer >= opt.batch_size * opt.epochs_per_step then\n",
    "            for e=1, opt.epochs_per_step do\n",
    "                local err_train = train()\n",
    "                err_episode = err_episode + err_train / (opt.episode_length * opt.epochs_per_step)\n",
    "            end\n",
    "        end\n",
    "        \n",
    "    end\n",
    "    \n",
    "    -- Test\n",
    "    local rewards_test = test()\n",
    "    \n",
    "    -- Update stats\n",
    "    table.insert(stats.train_err, err_episode)\n",
    "    table.insert(stats.test_r, rewards_test)\n",
    "    \n",
    "    if episode == 1 then\n",
    "        stats.train_err_avg = err_episode\n",
    "        stats.test_r_avg = rewards_test:mean()\n",
    "    else\n",
    "        stats.train_err_avg = stats.train_err_avg * 0.95 + 0.05 * err_episode\n",
    "        stats.test_r_avg = stats.test_r_avg * 0.95 + 0.05 * rewards_test:mean()\n",
    "    end\n",
    "    \n",
    "    -- Statistics\n",
    "    if episode % 10 == 0 then        \n",
    "        log.infof('e=%d, train_mse=%.3f, train_mse_avg=%.3f, test_r=%.3f, test_r_avg=%.3f, t/e=%.2f sec, t=%d min.',\n",
    "            #stats.train_err,\n",
    "            stats.train_err[#stats.train_err],\n",
    "            stats.train_err_avg,\n",
    "            stats.test_r[#stats.test_r]:mean(),\n",
    "            stats.test_r_avg,\n",
    "            sys.clock() - time,\n",
    "            torch.toc(beginning_time) / 60)\n",
    "        \n",
    "        collectgarbage()\n",
    "    end\n",
    "    \n",
    "    -- Save model\n",
    "    if episode % 100 == 0 then\n",
    "        save_model()\n",
    "    end    \n",
    "    \n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script type=\"text/javascript\">\n",
       "$(function() {\n",
       "    if (typeof (window._bokeh_onload_callbacks) === \"undefined\"){\n",
       "  window._bokeh_onload_callbacks = [];\n",
       "    }\n",
       "    function load_lib(url, callback){\n",
       "  window._bokeh_onload_callbacks.push(callback);\n",
       "  if (window._bokeh_is_loading){\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", new Date());\n",
       "      return null;\n",
       "  }\n",
       "  console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", new Date());\n",
       "  window._bokeh_is_loading = true;\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = function(){\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh-0.7.0.min.css\");\n",
       "      window._bokeh_onload_callbacks.forEach(function(callback){callback()});\n",
       "  };\n",
       "  s.onerror = function(){\n",
       "      console.warn(\"failed to load library \" + url);\n",
       "  };\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "\n",
       "    bokehjs_url = \"https://cdn.pydata.org/bokeh-0.7.0.min.js\"\n",
       "\n",
       "    var elt = document.getElementById(\"bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\");\n",
       "    if(elt==null) {\n",
       "  console.log(\"Bokeh: ERROR: autoload.js configured with elementid 'bdf7a075-d2b5-41e2-c151-7bb1aac9ef46'\"\n",
       "        + \"but no matching script tag was found. \")\n",
       "  return false;\n",
       "    }\n",
       "\n",
       "    if(typeof(Bokeh) !== \"undefined\") {\n",
       "  console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "  var modelid = \"b2fe02a8-48f5-49ca-c913-e90b878cf298\";\n",
       "  var modeltype = \"Plot\";\n",
       "  var all_models = [{\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\",\"type\":\"ColumnDataSource\",\"attributes\":{\"data\":{\"y\":[3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3],\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000]},\"column_names\":[\"y\",\"x\"],\"cont_ranges\":{},\"discrete_ranges\":{},\"selected\":[],\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\",\"doc\":null,\"tags\":[]}},{\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"#000000\"},\"line_color\":{\"value\":\"#000000\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"#000000\"},\"line_color\":{\"value\":\"#000000\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\",\"type\":\"GlyphRenderer\",\"attributes\":{\"name\":null,\"nonselection_glyph\":{\"type\":\"Line\",\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\"},\"doc\":null,\"server_data_source\":null,\"data_source\":{\"type\":\"ColumnDataSource\",\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\"},\"glyph\":{\"type\":\"Line\",\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\"},\"selection_glyph\":null,\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\",\"tags\":[]}},{\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"x\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\"}}],\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\",\"tags\":[],\"doc\":null}},{\"id\":\"005b3e91-330a-4b54-cbd5-09603e8fd2c5\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"y\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\"}}],\"id\":\"005b3e91-330a-4b54-cbd5-09603e8fd2c5\",\"tags\":[],\"doc\":null}},{\"id\":\"1f48dc87-0cdf-4e86-ce55-ef7445ca8630\",\"type\":\"ToolEvents\",\"attributes\":{\"tags\":[],\"id\":\"1f48dc87-0cdf-4e86-ce55-ef7445ca8630\",\"geometries\":[],\"doc\":null}},{\"id\":\"6b6406e7-a779-4d6b-cd40-f518ad55f8af\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"6b6406e7-a779-4d6b-cd40-f518ad55f8af\",\"tags\":[],\"doc\":null}},{\"id\":\"fa209698-eee6-4e37-ce91-622fb23d561a\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"fa209698-eee6-4e37-ce91-622fb23d561a\",\"tags\":[],\"doc\":null}},{\"id\":\"a556f613-de57-4527-c5e7-c6d5e05c96d0\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"6b6406e7-a779-4d6b-cd40-f518ad55f8af\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"fa209698-eee6-4e37-ce91-622fb23d561a\"},\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"a556f613-de57-4527-c5e7-c6d5e05c96d0\",\"doc\":null,\"tags\":[]}},{\"id\":\"54733f11-fb69-4ee3-ce8d-4aac9274e62c\",\"type\":\"Grid\",\"attributes\":{\"dimension\":0,\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"fa209698-eee6-4e37-ce91-622fb23d561a\"},\"id\":\"54733f11-fb69-4ee3-ce8d-4aac9274e62c\",\"doc\":null,\"tags\":[]}},{\"id\":\"24d6e89d-16b0-4fad-cfcd-938e07a048b2\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"24d6e89d-16b0-4fad-cfcd-938e07a048b2\",\"tags\":[],\"doc\":null}},{\"id\":\"6496e80c-2930-4771-c7c3-4d76b5a99164\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"6496e80c-2930-4771-c7c3-4d76b5a99164\",\"tags\":[],\"doc\":null}},{\"id\":\"b307a2f6-0b71-44c9-c475-b84cd872bf05\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"24d6e89d-16b0-4fad-cfcd-938e07a048b2\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"6496e80c-2930-4771-c7c3-4d76b5a99164\"},\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"b307a2f6-0b71-44c9-c475-b84cd872bf05\",\"doc\":null,\"tags\":[]}},{\"id\":\"11e9f07e-a9b9-4695-c63e-d2e0ea283649\",\"type\":\"Grid\",\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"6496e80c-2930-4771-c7c3-4d76b5a99164\"},\"id\":\"11e9f07e-a9b9-4695-c63e-d2e0ea283649\",\"doc\":null,\"tags\":[]}},{\"id\":\"83863e46-7f84-4408-c40b-e3f1c979c535\",\"type\":\"PanTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"83863e46-7f84-4408-c40b-e3f1c979c535\",\"doc\":null,\"tags\":[]}},{\"id\":\"cd2db6ca-9f16-483e-c809-02a50c7a87cf\",\"type\":\"WheelZoomTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"cd2db6ca-9f16-483e-c809-02a50c7a87cf\",\"doc\":null,\"tags\":[]}},{\"id\":\"fafa68b0-7eef-453d-c55c-a9df41aef3e6\",\"type\":\"BoxZoomTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"fafa68b0-7eef-453d-c55c-a9df41aef3e6\",\"tags\":[],\"doc\":null}},{\"id\":\"e60871d9-a5a5-4b46-cbe4-21f392c55cc4\",\"type\":\"PreviewSaveTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"e60871d9-a5a5-4b46-cbe4-21f392c55cc4\",\"tags\":[],\"doc\":null}},{\"id\":\"2d5f0fa2-45de-489c-c80f-3a284921e719\",\"type\":\"ResizeTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"2d5f0fa2-45de-489c-c80f-3a284921e719\",\"tags\":[],\"doc\":null}},{\"id\":\"946b547e-c9ec-47d7-c70c-f8dba3f7bb6e\",\"type\":\"ResetTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"946b547e-c9ec-47d7-c70c-f8dba3f7bb6e\",\"tags\":[],\"doc\":null}},{\"id\":\"19dfee05-f41d-4712-c72b-28b63b83a61a\",\"type\":\"Legend\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"legends\":[[\"Optimal\",[{\"type\":\"GlyphRenderer\",\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\"}]]],\"id\":\"19dfee05-f41d-4712-c72b-28b63b83a61a\",\"doc\":null,\"tags\":[]}},{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"attributes\":{\"x_range\":{\"type\":\"DataRange1d\",\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\"},\"tool_events\":{\"type\":\"ToolEvents\",\"id\":\"1f48dc87-0cdf-4e86-ce55-ef7445ca8630\"},\"below\":[{\"type\":\"LinearAxis\",\"id\":\"a556f613-de57-4527-c5e7-c6d5e05c96d0\"}],\"renderers\":[{\"type\":\"GlyphRenderer\",\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\"},{\"type\":\"LinearAxis\",\"id\":\"a556f613-de57-4527-c5e7-c6d5e05c96d0\"},{\"type\":\"Grid\",\"id\":\"54733f11-fb69-4ee3-ce8d-4aac9274e62c\"},{\"type\":\"LinearAxis\",\"id\":\"b307a2f6-0b71-44c9-c475-b84cd872bf05\"},{\"type\":\"Grid\",\"id\":\"11e9f07e-a9b9-4695-c63e-d2e0ea283649\"},{\"type\":\"Legend\",\"id\":\"19dfee05-f41d-4712-c72b-28b63b83a61a\"}],\"above\":[],\"tools\":[{\"type\":\"PanTool\",\"id\":\"83863e46-7f84-4408-c40b-e3f1c979c535\"},{\"type\":\"WheelZoomTool\",\"id\":\"cd2db6ca-9f16-483e-c809-02a50c7a87cf\"},{\"type\":\"BoxZoomTool\",\"id\":\"fafa68b0-7eef-453d-c55c-a9df41aef3e6\"},{\"type\":\"PreviewSaveTool\",\"id\":\"e60871d9-a5a5-4b46-cbe4-21f392c55cc4\"},{\"type\":\"ResizeTool\",\"id\":\"2d5f0fa2-45de-489c-c80f-3a284921e719\"},{\"type\":\"ResetTool\",\"id\":\"946b547e-c9ec-47d7-c70c-f8dba3f7bb6e\"}],\"doc\":null,\"right\":[],\"title\":\"DDPG Performance\",\"extra_x_ranges\":{},\"left\":[{\"type\":\"LinearAxis\",\"id\":\"b307a2f6-0b71-44c9-c475-b84cd872bf05\"}],\"y_range\":{\"type\":\"DataRange1d\",\"id\":\"005b3e91-330a-4b54-cbd5-09603e8fd2c5\"},\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"extra_y_ranges\":{},\"tags\":[]}}];\n",
       "  Bokeh.load_models(all_models);\n",
       "  var model = Bokeh.Collections(modeltype).get(modelid);\n",
       "  $(\"#bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\").html(''); // clear any previous plot in window_id\n",
       "  var view = new model.default_view({model: model, el: \"#bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\"});\n",
       "    } else {\n",
       "  load_lib(bokehjs_url, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", new Date())\n",
       "      var modelid = \"b2fe02a8-48f5-49ca-c913-e90b878cf298\";\n",
       "      var modeltype = \"Plot\";\n",
       "      var all_models = [{\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\",\"type\":\"ColumnDataSource\",\"attributes\":{\"data\":{\"y\":[3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3],\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000]},\"column_names\":[\"y\",\"x\"],\"cont_ranges\":{},\"discrete_ranges\":{},\"selected\":[],\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\",\"doc\":null,\"tags\":[]}},{\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"#000000\"},\"line_color\":{\"value\":\"#000000\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"#000000\"},\"line_color\":{\"value\":\"#000000\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\",\"type\":\"GlyphRenderer\",\"attributes\":{\"name\":null,\"nonselection_glyph\":{\"type\":\"Line\",\"id\":\"9202e3a1-2ff5-46c3-c61f-42bfb64d2041\"},\"doc\":null,\"server_data_source\":null,\"data_source\":{\"type\":\"ColumnDataSource\",\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\"},\"glyph\":{\"type\":\"Line\",\"id\":\"4db058ec-f32a-423f-c2e3-4150464d3f26\"},\"selection_glyph\":null,\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\",\"tags\":[]}},{\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"x\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\"}}],\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\",\"tags\":[],\"doc\":null}},{\"id\":\"005b3e91-330a-4b54-cbd5-09603e8fd2c5\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"y\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"fdba9e03-761a-46c5-c616-a4bb30bf6ab3\"}}],\"id\":\"005b3e91-330a-4b54-cbd5-09603e8fd2c5\",\"tags\":[],\"doc\":null}},{\"id\":\"1f48dc87-0cdf-4e86-ce55-ef7445ca8630\",\"type\":\"ToolEvents\",\"attributes\":{\"tags\":[],\"id\":\"1f48dc87-0cdf-4e86-ce55-ef7445ca8630\",\"geometries\":[],\"doc\":null}},{\"id\":\"6b6406e7-a779-4d6b-cd40-f518ad55f8af\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"6b6406e7-a779-4d6b-cd40-f518ad55f8af\",\"tags\":[],\"doc\":null}},{\"id\":\"fa209698-eee6-4e37-ce91-622fb23d561a\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"fa209698-eee6-4e37-ce91-622fb23d561a\",\"tags\":[],\"doc\":null}},{\"id\":\"a556f613-de57-4527-c5e7-c6d5e05c96d0\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"6b6406e7-a779-4d6b-cd40-f518ad55f8af\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"fa209698-eee6-4e37-ce91-622fb23d561a\"},\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"a556f613-de57-4527-c5e7-c6d5e05c96d0\",\"doc\":null,\"tags\":[]}},{\"id\":\"54733f11-fb69-4ee3-ce8d-4aac9274e62c\",\"type\":\"Grid\",\"attributes\":{\"dimension\":0,\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"fa209698-eee6-4e37-ce91-622fb23d561a\"},\"id\":\"54733f11-fb69-4ee3-ce8d-4aac9274e62c\",\"doc\":null,\"tags\":[]}},{\"id\":\"24d6e89d-16b0-4fad-cfcd-938e07a048b2\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"24d6e89d-16b0-4fad-cfcd-938e07a048b2\",\"tags\":[],\"doc\":null}},{\"id\":\"6496e80c-2930-4771-c7c3-4d76b5a99164\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"6496e80c-2930-4771-c7c3-4d76b5a99164\",\"tags\":[],\"doc\":null}},{\"id\":\"b307a2f6-0b71-44c9-c475-b84cd872bf05\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"24d6e89d-16b0-4fad-cfcd-938e07a048b2\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"6496e80c-2930-4771-c7c3-4d76b5a99164\"},\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"b307a2f6-0b71-44c9-c475-b84cd872bf05\",\"doc\":null,\"tags\":[]}},{\"id\":\"11e9f07e-a9b9-4695-c63e-d2e0ea283649\",\"type\":\"Grid\",\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"6496e80c-2930-4771-c7c3-4d76b5a99164\"},\"id\":\"11e9f07e-a9b9-4695-c63e-d2e0ea283649\",\"doc\":null,\"tags\":[]}},{\"id\":\"83863e46-7f84-4408-c40b-e3f1c979c535\",\"type\":\"PanTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"83863e46-7f84-4408-c40b-e3f1c979c535\",\"doc\":null,\"tags\":[]}},{\"id\":\"cd2db6ca-9f16-483e-c809-02a50c7a87cf\",\"type\":\"WheelZoomTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"cd2db6ca-9f16-483e-c809-02a50c7a87cf\",\"doc\":null,\"tags\":[]}},{\"id\":\"fafa68b0-7eef-453d-c55c-a9df41aef3e6\",\"type\":\"BoxZoomTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"fafa68b0-7eef-453d-c55c-a9df41aef3e6\",\"tags\":[],\"doc\":null}},{\"id\":\"e60871d9-a5a5-4b46-cbe4-21f392c55cc4\",\"type\":\"PreviewSaveTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"e60871d9-a5a5-4b46-cbe4-21f392c55cc4\",\"tags\":[],\"doc\":null}},{\"id\":\"2d5f0fa2-45de-489c-c80f-3a284921e719\",\"type\":\"ResizeTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"2d5f0fa2-45de-489c-c80f-3a284921e719\",\"tags\":[],\"doc\":null}},{\"id\":\"946b547e-c9ec-47d7-c70c-f8dba3f7bb6e\",\"type\":\"ResetTool\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"946b547e-c9ec-47d7-c70c-f8dba3f7bb6e\",\"tags\":[],\"doc\":null}},{\"id\":\"19dfee05-f41d-4712-c72b-28b63b83a61a\",\"type\":\"Legend\",\"attributes\":{\"plot\":{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"legends\":[[\"Optimal\",[{\"type\":\"GlyphRenderer\",\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\"}]]],\"id\":\"19dfee05-f41d-4712-c72b-28b63b83a61a\",\"doc\":null,\"tags\":[]}},{\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"type\":\"Plot\",\"attributes\":{\"x_range\":{\"type\":\"DataRange1d\",\"id\":\"23560b0b-3267-4c1a-ccf3-960d3e751db7\"},\"tool_events\":{\"type\":\"ToolEvents\",\"id\":\"1f48dc87-0cdf-4e86-ce55-ef7445ca8630\"},\"below\":[{\"type\":\"LinearAxis\",\"id\":\"a556f613-de57-4527-c5e7-c6d5e05c96d0\"}],\"renderers\":[{\"type\":\"GlyphRenderer\",\"id\":\"4dcfb421-12de-42e9-c218-08dbaf70abd6\"},{\"type\":\"LinearAxis\",\"id\":\"a556f613-de57-4527-c5e7-c6d5e05c96d0\"},{\"type\":\"Grid\",\"id\":\"54733f11-fb69-4ee3-ce8d-4aac9274e62c\"},{\"type\":\"LinearAxis\",\"id\":\"b307a2f6-0b71-44c9-c475-b84cd872bf05\"},{\"type\":\"Grid\",\"id\":\"11e9f07e-a9b9-4695-c63e-d2e0ea283649\"},{\"type\":\"Legend\",\"id\":\"19dfee05-f41d-4712-c72b-28b63b83a61a\"}],\"above\":[],\"tools\":[{\"type\":\"PanTool\",\"id\":\"83863e46-7f84-4408-c40b-e3f1c979c535\"},{\"type\":\"WheelZoomTool\",\"id\":\"cd2db6ca-9f16-483e-c809-02a50c7a87cf\"},{\"type\":\"BoxZoomTool\",\"id\":\"fafa68b0-7eef-453d-c55c-a9df41aef3e6\"},{\"type\":\"PreviewSaveTool\",\"id\":\"e60871d9-a5a5-4b46-cbe4-21f392c55cc4\"},{\"type\":\"ResizeTool\",\"id\":\"2d5f0fa2-45de-489c-c80f-3a284921e719\"},{\"type\":\"ResetTool\",\"id\":\"946b547e-c9ec-47d7-c70c-f8dba3f7bb6e\"}],\"doc\":null,\"right\":[],\"title\":\"DDPG Performance\",\"extra_x_ranges\":{},\"left\":[{\"type\":\"LinearAxis\",\"id\":\"b307a2f6-0b71-44c9-c475-b84cd872bf05\"}],\"y_range\":{\"type\":\"DataRange1d\",\"id\":\"005b3e91-330a-4b54-cbd5-09603e8fd2c5\"},\"id\":\"b2fe02a8-48f5-49ca-c913-e90b878cf298\",\"extra_y_ranges\":{},\"tags\":[]}}];\n",
       "      Bokeh.load_models(all_models);\n",
       "      var model = Bokeh.Collections(modeltype).get(modelid);\n",
       "      $(\"#bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\").html(''); // clear any previous plot in window_id\n",
       "      var view = new model.default_view({model: model, el: \"#bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\"});\n",
       "  });\n",
       "    }\n",
       "});\n",
       "</script>\n",
       "<div class=\"plotdiv\" id=\"bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<script type=\"text/javascript\">\n",
       "$(function() {\n",
       "    if (typeof (window._bokeh_onload_callbacks) === \"undefined\"){\n",
       "  window._bokeh_onload_callbacks = [];\n",
       "    }\n",
       "    function load_lib(url, callback){\n",
       "  window._bokeh_onload_callbacks.push(callback);\n",
       "  if (window._bokeh_is_loading){\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", new Date());\n",
       "      return null;\n",
       "  }\n",
       "  console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", new Date());\n",
       "  window._bokeh_is_loading = true;\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = function(){\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh-0.7.0.min.css\");\n",
       "      window._bokeh_onload_callbacks.forEach(function(callback){callback()});\n",
       "  };\n",
       "  s.onerror = function(){\n",
       "      console.warn(\"failed to load library \" + url);\n",
       "  };\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "\n",
       "    bokehjs_url = \"https://cdn.pydata.org/bokeh-0.7.0.min.js\"\n",
       "\n",
       "    var elt = document.getElementById(\"bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\");\n",
       "    if(elt==null) {\n",
       "  console.log(\"Bokeh: ERROR: autoload.js configured with elementid 'bdf7a075-d2b5-41e2-c151-7bb1aac9ef46'\"\n",
       "        + \"but no matching script tag was found. \")\n",
       "  return false;\n",
       "    }\n",
       "\n",
       "    if(typeof(Bokeh) !== \"undefined\") {\n",
       "  console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "  var modelid = \"8de6f979-f1dd-44f2-c48e-67e2322f1c48\";\n",
       "  var modeltype = \"Plot\";\n",
       "  var all_models = [{\"id\":\"b03c92e6-75d7-4ba7-cb45-7bf209d0fa0c\",\"type\":\"ColumnDataSource\",\"attributes\":{\"data\":{\"y\":[3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3],\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000]},\"column_names\":[\"y\",\"x\"],\"cont_ranges\":{},\"discrete_ranges\":{},\"selected\":[],\"id\":\"b03c92e6-75d7-4ba7-cb45-7bf209d0fa0c\",\"doc\":null,\"tags\":[]}},{\"id\":\"73e92ee0-dc65-4137-c18d-34a3381c4052\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"#000000\"},\"line_color\":{\"value\":\"#000000\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"73e92ee0-dc65-4137-c18d-34a3381c4052\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"72d1e3bd-3fd3-4a3c-cac3-aedcf9b0550e\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"#000000\"},\"line_color\":{\"value\":\"#000000\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"72d1e3bd-3fd3-4a3c-cac3-aedcf9b0550e\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"6d4bb37b-d0fb-47cc-c704-2b7ff6bfc2cb\",\"type\":\"GlyphRenderer\",\"attributes\":{\"name\":null,\"nonselection_glyph\":{\"type\":\"Line\",\"id\":\"72d1e3bd-3fd3-4a3c-cac3-aedcf9b0550e\"},\"doc\":null,\"server_data_source\":null,\"data_source\":{\"type\":\"ColumnDataSource\",\"id\":\"b03c92e6-75d7-4ba7-cb45-7bf209d0fa0c\"},\"glyph\":{\"type\":\"Line\",\"id\":\"73e92ee0-dc65-4137-c18d-34a3381c4052\"},\"selection_glyph\":null,\"id\":\"6d4bb37b-d0fb-47cc-c704-2b7ff6bfc2cb\",\"tags\":[]}},{\"id\":\"442e3b75-f8a9-401d-c09a-f29a9fd2bd54\",\"type\":\"ColumnDataSource\",\"attributes\":{\"data\":{\"y\":[0.1452022343874,0.11627835780382,0.12151810526848,0.1232678592205,0.11941703408957,0.12565785646439,0.054540328681469,0.086176387965679,0.06445100158453,0.11767222732306,0.07691416144371,0.21331083774567,0.10325336456299,0.18877494335175,0.39335876703262,0.34627988934517,0.35911333560944,0.41099599003792,0.53969812393188,0.50940239429474,0.43573418259621,0.57178807258606,0.53875172138214,0.40858808159828,0.44315341114998,0.52144736051559,0.56556636095047,0.58056455850601,0.58089780807495,0.62991279363632,0.62747114896774,0.65029710531235,0.69231879711151,0.6623820066452,0.68375104665756,0.67262881994247,0.5755021572113,0.49282014369965,0.39693796634674,0.51786035299301,0.5884056687355,0.87440901994705,0.69735944271088,0.67561632394791,0.67832404375076,0.7175931930542,0.70014882087708,0.57986110448837,0.62984424829483,0.59821969270706,0.63930314779282,0.62633126974106,0.62515592575073,0.53446358442307,0.62766647338867,0.52762788534164,0.44868528842926,0.53710353374481,0.47086328268051,0.61008322238922,0.49282866716385,0.6289439201355,0.55514061450958,0.54795438051224,0.57919061183929,0.57458859682083,0.59715729951859,0.54260671138763,0.60736685991287,0.54437249898911,0.56367856264114,0.51122719049454,0.50561928749084,0.58165001869202,0.56121307611465,0.5913947224617,0.50838762521744,0.58306795358658,0.5309391617775,0.57408857345581,0.64673608541489,0.63517111539841,0.58226144313812,0.55478471517563,0.56920599937439,0.59645920991898,0.51629334688187,0.43189117312431,0.52474772930145,0.45482948422432,0.63670861721039,0.60948956012726,0.60038894414902,0.57098394632339,0.610955119133,0.55260396003723,0.65204477310181,0.58869290351868,0.69162207841873,0.48130622506142,0.52701729536057,0.56631779670715,0.60040771961212,0.58208340406418,0.63646310567856,0.58819055557251,0.5544159412384,0.64056843519211,0.62904387712479,0.54707419872284,0.50399523973465,0.54172348976135,0.60203301906586,0.56978380680084,0.64801210165024,0.55965501070023,0.61910092830658,0.50317525863647,0.59097599983215,0.58495926856995,0.5205482840538,0.53229641914368,0.60052514076233,0.63249760866165,0.58158081769943,0.55282521247864,0.60298782587051,0.55073463916779,0.56411308050156,0.54331719875336,0.58939307928085,0.63218104839325,0.59134942293167,0.65499603748322,0.59003490209579,0.59417253732681,0.67103171348572,0.58376711606979,0.49588960409164,0.54131942987442,0.64106202125549,0.53420746326447,0.53746992349625,0.60748136043549,0.52507209777832,0.61526072025299,0.61725354194641,0.60878604650497,0.60640043020248,0.7002791762352,0.67332178354263,0.54250866174698,0.68109798431396,0.80152797698975,0.65507304668427,0.67837053537369,0.56909000873566,0.64316350221634,0.59624868631363,0.62407499551773,0.55809277296066,0.57640427350998,0.64851695299149,0.60916638374329,0.60473853349686,0.60422194004059,0.67193627357483,0.65681093931198,0.55662852525711,0.59135586023331,0.87479829788208,0.64038306474686,0.83460223674774,0.70956712961197,0.69555377960205,0.64904111623764,0.75341862440109,0.85492038726807,0.59165042638779,0.74270939826965,0.64929693937302,0.66478830575943,0.706698179245,0.65124225616455,0.67475014925003,0.59543889760971,0.73233807086945,0.54096275568008,0.76812076568604,0.78942662477493,0.59604603052139,0.60798996686935,0.69596147537231,0.5777536034584,0.67894625663757,0.57372725009918,0.52820628881454,0.58320516347885,0.64292007684708,0.64014482498169,0.65855139493942,0.62022709846497,0.66327089071274,0.55190658569336,0.60613483190536,0.56507402658463,0.68031668663025,0.6621687412262,0.69323098659515,0.6233885884285,0.62512958049774,0.74882590770721,0.67780488729477,0.65386855602264,0.61661511659622,0.56966042518616,0.58683180809021,0.66113328933716,0.61708527803421,0.76428157091141,0.61531776189804,0.67506402730942,1.0763465166092,0.65361261367798,0.91432613134384,0.65616023540497,0.60983216762543,0.67210847139359,0.70262765884399,0.64577800035477,0.69020199775696,0.61878633499146,0.64076900482178,0.62830686569214,0.69944179058075,0.60106199979782,0.54255598783493,0.89452636241913,0.68022590875626,0.75307637453079,0.71926379203796,0.57169830799103,0.73270952701569,0.61359304189682,0.66080188751221,0.77736580371857,0.67904436588287,0.63912373781204,1.0422577857971,0.9418773651123,0.66505789756775,0.66283404827118,0.6312363743782,0.6707706451416,0.61629593372345,0.88781416416168,0.71504390239716,0.64858543872833,0.69947928190231,0.64865112304688,0.69178831577301,0.69918411970139,0.63160794973373,0.65058553218842,0.61663842201233,0.83678597211838,0.7702648639679,0.83210223913193,0.96323579549789,0.66060703992844,0.73190480470657,0.72676920890808,0.82822185754776,0.70516192913055,0.91049551963806,0.84945672750473,0.70241981744766,1.0074692964554,0.72384393215179,0.84201031923294,0.66348564624786,0.69835501909256,0.75529372692108,0.71969360113144,0.71761512756348,0.74221587181091,0.61761772632599,0.72458529472351,0.75140726566315,0.74123126268387,0.89277732372284,0.79760521650314,0.7689808011055,0.65523862838745,0.63871079683304,0.75596755743027,0.85214364528656,0.91460829973221,0.91141402721405,0.76981514692307,1.08212018013,0.91913282871246,0.95193755626678,0.78260260820389,0.75486469268799,0.90562868118286,0.64427077770233,1.0731525421143,0.77875781059265,0.97275608778,0.86457169055939,0.7191476225853,0.70069164037704,0.89726346731186,0.73183673620224,0.81895500421524,0.86312913894653,0.8096998333931,0.7354719042778,0.92632669210434,0.88626205921173,1.1652189493179,0.79572415351868,0.79976469278336,0.87632894515991,0.59785097837448,0.75139975547791,0.92733800411224,1.0500625371933,0.7696944475174,0.62658834457397,0.70204263925552,0.71904712915421,0.64442098140717,1.0912376642227,0.69399082660675,1.1672251224518,0.8189777135849,1.1429187059402,0.89581364393234,0.64953124523163,1.1141538619995,0.96965265274048,0.77611726522446,0.79408317804337,0.67355245351791,0.78807801008224,0.8763182759285,0.63229292631149,0.89321339130402,0.7798947095871,0.82040339708328,0.85014033317566,0.96043068170547,1.1538344621658,1.0829594135284,1.2710630893707,1.098455786705,1.0561771392822,1.0138514041901,0.98727196455002,0.76210987567902,0.95083332061768,0.84509056806564,0.76710623502731,1.0660499334335,0.88079208135605,1.1715178489685,0.72887015342712,0.68582379817963,0.89298892021179,0.86327069997787,1.0282936096191,0.932397544384,0.97691971063614,1.0369780063629,0.83920884132385,0.88697850704193,0.82472217082977,0.88218462467194,0.84190714359283,0.65081149339676,0.91445362567902,0.84395903348923,0.96435737609863,1.0984205007553,0.7677766084671,1.029820561409,1.0610002279282,0.65939021110535,0.83892667293549,0.67698550224304,0.82219803333282,0.78961426019669,0.85550075769424,0.93165254592896,0.80497014522552,0.64872586727142,1.1152740716934,1.012598991394,0.94260495901108,0.90957456827164,1.0606503486633,0.88942456245422,0.95096862316132,1.0942577123642,1.0693191289902,1.0674315690994,1.0862023830414,0.94319659471512,0.95653259754181,0.74607104063034,0.63439434766769,1.1916078329086,0.58567297458649,0.86374044418335,0.9687939286232,0.76139026880264,0.88832288980484,0.7861356139183,0.85508799552917,0.58087939023972,0.91544741392136,1.0759904384613,1.0275074243546,0.8956264257431,0.55858409404755,1.1407730579376,0.95685803890228,0.87830376625061,1.0094784498215,0.91745442152023,0.87592649459839,1.1565687656403,1.2144404649734,1.1573051214218,1.0067210197449,1.1738752126694,1.0790157318115,1.0937875509262,0.83302700519562,1.1437908411026,1.1136255264282,0.98053556680679,0.69028204679489,0.78602415323257,0.77032446861267,0.78950649499893,0.84357225894928,0.77887845039368,0.79703360795975,0.80217480659485,0.8600560426712,0.98359817266464,0.85038906335831,0.76095259189606,0.83954387903214,0.67496293783188,1.1985043287277,0.75735694169998,0.7220510840416,0.76116096973419,0.75816601514816,0.73420703411102,0.77419352531433,0.93907725811005,1.0750155448914,1.0583482980728,1.087545633316,0.81348145008087,0.93936336040497,0.81517815589905,0.84553039073944,0.86122298240662,0.82546204328537,0.79336076974869,0.89371740818024,0.93356144428253,0.79874753952026,0.94986522197723,0.86065810918808,1.1030675172806,0.92198485136032,0.93831604719162,0.73740154504776,1.0889797210693,0.83094841241837,1.0753296613693,0.83413457870483,1.1916743516922,0.75837802886963,1.0745668411255,0.76628214120865,0.79275017976761,0.94876897335052,0.77698028087616,0.72552651166916,1.14905834198,1.0568577051163,0.78395658731461,0.73836439847946,0.57513254880905,0.95350217819214,1.0572596788406,0.82675021886826,0.80843168497086,1.1006346940994,1.1698687076569,1.1136993169785,0.78229206800461,1.2000012397766,0.81665796041489,1.1547582149506,1.2327213287354,0.69584977626801,0.92005807161331,0.8698365688324,1.1065264940262,1.0890151262283,0.63684815168381,1.1126059293747,0.66536045074463,1.1645638942719,0.98705691099167,1.0030151605606,1.0642470121384,1.2158358097076,0.71157693862915,1.2112889289856,1.0761188268661,1.1510169506073,1.2630207538605,1.1903929710388,0.9413206577301,1.2656391859055,1.1923191547394,0.92332077026367,1.2589302062988,0.84969955682755,1.2061839103699,1.3042341470718,1.2033878564835,1.08316385746,0.95195996761322,1.4791369438171,1.0451848506927,1.1767209768295,1.1471594572067,0.93449884653091,0.96595633029938,1.0097881555557,0.89585304260254,1.0409153699875,1.0506455898285,1.2431737184525,1.2177177667618,1.1367125511169,1.3085988759995,1.1635460853577,0.93756401538849,1.1475546360016,0.85820883512497,0.82280099391937,0.87765085697174,1.1952151060104,0.79925549030304,1.2567739486694,1.2406039237976,1.1037355661392,1.1787724494934,1.0471999645233,1.3226636648178,0.90229767560959,1.0482059717178,1.131908416748,1.1459876298904,1.1863869428635,1.2375094890594,1.2847805023193,1.4439768791199,1.2419325113297,1.3250359296799,1.0246162414551,0.81471693515778,1.3136330842972,0.80461740493774,1.1588053703308,0.87278538942337,1.1292772293091,0.89002299308777,1.2566843032837,1.194228887558,0.99039697647095,1.375864982605,1.2315472364426,1.3571403026581,1.3148744106293,1.0391068458557,1.1247373819351,1.1628968715668,1.2774765491486,1.1802004575729,0.96559023857117,1.0816307067871,1.0847978591919,1.2926717996597,1.1839392185211,1.2882776260376,1.440339922905,1.325984954834,1.0694131851196,1.1063982248306,1.2887151241302,1.0086168050766,1.2562565803528,0.97993940114975,1.3298500776291,1.1611683368683,1.1303870677948,1.2423943281174,0.91838556528091,1.2238975763321,1.3701069355011,1.4628766775131,1.3882297277451,1.4231206178665,1.2252702713013,1.496129155159,1.4813536405563,1.5786159038544,1.6794850826263,1.5281442403793,1.7961578369141,1.5226570367813,1.681009888649,1.407208442688,1.3171232938766,1.5397200584412,1.5442078113556,1.3682622909546,1.182120680809,1.3316655158997,1.8376263380051,1.3470443487167,1.6013431549072,1.5312165021896,1.3359662294388,1.2211793661118,1.6969156265259,1.4718585014343,1.3456215858459,1.4112508296967,1.4297339916229,1.233887553215,1.4173140525818,1.8813115358353,1.2239533662796,1.7003704309464,1.3252873420715,1.6116651296616,1.1041036844254,1.153752207756,1.5345325469971,1.562894821167,1.0128148794174,1.3168244361877,1.591472029686,1.4309965372086,1.0767061710358,1.4788085222244,1.8424981832504,1.5660690069199,1.4164085388184,1.465228676796,1.5064542293549,1.3323475122452,1.5880290269852,1.5389310121536,1.1053144931793,1.2191852331161,1.360875248909,1.346181511879,1.8398958444595,1.3014913797379,1.3971838951111,1.5485363006592,1.3125579357147,1.3120362758636,1.5752868652344,1.2062079906464,1.2275605201721,1.3841038942337,1.0963809490204,1.3539242744446,1.3647441864014,1.4838548898697,1.110355257988,1.524983048439,1.3384205102921,0.94321459531784,1.0760469436646,0.88620781898499,1.3218929767609,1.1921083927155,1.4212509393692,2.1025786399841,1.4743032455444,1.7764747142792,1.9197858572006,1.5286051034927,1.4680963754654,1.6220995187759,1.5465126037598,1.3993220329285,1.6328763961792,1.3016562461853,1.5235607624054,1.3752479553223,1.0778654813766,1.2544348239899,1.4932141304016,1.4343048334122,1.4653215408325,1.1662670373917,1.4765610694885,1.0985797643661,1.4672635793686,1.4940588474274,1.2011618614197,1.3428165912628,1.3932489156723,1.7481383085251,1.3393362760544,1.8508709669113,1.4970986843109,1.1761165857315,1.6942420005798,1.7538660764694,1.5180196762085,1.6667125225067,1.35888671875,1.2786378860474,1.4820936918259,1.1308473348618,1.5222866535187,1.3941330909729,1.6612186431885,1.6179130077362,1.3005647659302,1.7472367286682,1.3453847169876,1.0027928352356,1.0422744750977,1.0249646902084,1.0016592741013,1.1928569078445,1.0528798103333,1.2236078977585,1.7047555446625,1.5016071796417,1.9791238307953,1.9296927452087,1.6525857448578,1.0732080936432,1.9577649831772,1.3992519378662,1.1985028982162,0.98192667961121,1.0755985975266,2.1517448425293,1.4449954032898,2.1665072441101,1.9110794067383,2.0523526668549,2.1116948127747,0.98506134748459,1.2164725065231,1.5379217863083,1.6957613229752,1.1499508619308,1.0762499570847,1.3295881748199,1.659229516983,1.3569928407669,1.289494395256,1.080927491188,1.5398008823395,1.2721623182297,2.0339231491089,1.1331685781479,1.2914298772812,1.0073283910751,1.4904673099518,1.6891660690308,1.5343341827393,1.6589248180389,1.1142619848251,1.6565127372742,1.8268450498581,2.2952880859375,1.9121998548508,1.4958305358887,1.5254690647125,2.1004023551941,1.3247292041779,2.0941646099091,1.4541617631912,1.8724514245987,1.5146670341492,1.9992687702179,1.7828208208084,1.4893035888672,1.8231176137924,1.2839140892029,2.2504134178162,2.1098246574402,1.901765704155,1.282545208931,1.2019698619843,1.4819278717041,1.0509465932846,1.5591523647308,1.5206170082092,1.3829181194305,1.236954331398,1.2967925071716,1.7221531867981,1.3117214441299,2.0308265686035,1.8183161020279,2.2635748386383,1.5701133012772,1.1394877433777,2.0769131183624,1.4202675819397,1.0293002128601,1.5572972297668,1.8508529663086,1.306432723999,1.2195662260056,1.4378818273544,1.515585899353,1.6536300182343,1.6753197908401,1.6793996095657,1.7607321739197,1.962974190712,1.3360667228699,1.8220907449722,2.353200674057,1.2742609977722,1.7515349388123,1.7403109073639,1.5733277797699,1.7494155168533,1.449945807457,1.3106330633163,1.2527709007263,2.0239486694336,2.0696723461151,1.2897975444794,1.9978283643723,1.6143078804016,1.8798394203186,1.7828289270401,1.4428902864456,1.640863776207,1.8705677986145,1.7706897258759,1.7105292081833,1.7958766222,1.728045463562,1.6011823415756,1.69899559021,1.6917471885681,1.5185176134109,1.3742475509644,1.5970497131348,1.4478988647461,1.8166614770889,1.3759965896606,1.8146399259567,1.3575502634048,2.203033208847,1.0840753316879,1.9340724945068,1.3059742450714,1.6679085493088,1.7269117832184,1.2645924091339,1.3088563680649,1.5329682826996,1.9148380756378,1.275958776474,1.6040170192719,1.4699258804321,1.8459672927856,1.4679863452911,1.6780132055283,1.7349985837936,1.9525405168533,1.7879681587219,2.0974202156067,2.4326357841492,1.3748066425323,2.0709097385406,1.2540044784546,1.2728310823441,1.6870158910751,1.7704585790634,1.4026651382446,1.4917454719543,1.4615346193314,1.5329290628433,1.4238270521164,1.4546440839767,2.2301554679871,1.5760939121246,2.0382468700409,2.0051789283752,1.8716418743134,1.8662232160568,2.1270451545715,1.5002548694611,1.3810645341873,2.1510956287384,1.8499354124069,2.3694107532501,2.2167212963104,2.6075127124786,2.3605029582977,2.427255153656,1.873238325119,2.6338391304016,2.5504095554352,1.3575179576874,2.7427670955658,1.9512524604797,2.5922474861145,1.8487793207169,1.9915332794189,1.2642180919647,1.9645494222641,2.5299105644226,2.5060911178589,2.318642616272,1.5794062614441,2.0401864051819,1.8757908344269,1.8106182813644,1.8057956695557,1.5287480354309,1.6433643102646,1.7893639802933,1.4387691020966,1.8401030302048,1.6109600067139,1.4426708221436,1.4104764461517,1.7765171527863,1.5158184766769,1.3200393915176,1.6753798723221,1.4117214679718,1.428870677948,1.4760152101517,1.1964383125305,1.5353670120239,2.6861822605133,2.1316108703613,1.5258768796921,1.2438675165176,1.4655103683472,0.9033243060112,1.3437737226486,1.0767545700073,1.0059639215469,1.1776745319366,1.2056657075882,1.5337153673172,1.3081291913986,1.9964346885681,1.6591212749481,1.3358269929886,1.6240762472153,1.5454148054123,1.2204133272171,1.4266442060471,1.2104421854019,1.255139708519,1.2998020648956,1.5885033607483,1.50734603405,1.7922949790955,1.1678071022034,1.6651309728622,1.7218272686005,1.4160262346268,1.6131038665771,1.8626878261566],\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000]},\"column_names\":[\"y\",\"x\"],\"cont_ranges\":{},\"discrete_ranges\":{},\"selected\":[],\"id\":\"442e3b75-f8a9-401d-c09a-f29a9fd2bd54\",\"doc\":null,\"tags\":[]}},{\"id\":\"9ce24391-45bb-4be7-cb2a-059464107515\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"green\"},\"line_color\":{\"value\":\"green\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"9ce24391-45bb-4be7-cb2a-059464107515\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"b05b74bc-be85-4ff9-cfd0-eac697af0391\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"green\"},\"line_color\":{\"value\":\"green\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"b05b74bc-be85-4ff9-cfd0-eac697af0391\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"81e7c1c2-2756-4c06-cc7d-afc5e76081be\",\"type\":\"GlyphRenderer\",\"attributes\":{\"name\":null,\"nonselection_glyph\":{\"type\":\"Line\",\"id\":\"b05b74bc-be85-4ff9-cfd0-eac697af0391\"},\"doc\":null,\"server_data_source\":null,\"data_source\":{\"type\":\"ColumnDataSource\",\"id\":\"442e3b75-f8a9-401d-c09a-f29a9fd2bd54\"},\"glyph\":{\"type\":\"Line\",\"id\":\"9ce24391-45bb-4be7-cb2a-059464107515\"},\"selection_glyph\":null,\"id\":\"81e7c1c2-2756-4c06-cc7d-afc5e76081be\",\"tags\":[]}},{\"id\":\"b4f24390-ccf7-440f-c4d8-d750fec3b4cc\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"x\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"b03c92e6-75d7-4ba7-cb45-7bf209d0fa0c\"}},{\"columns\":[\"x\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"442e3b75-f8a9-401d-c09a-f29a9fd2bd54\"}}],\"id\":\"b4f24390-ccf7-440f-c4d8-d750fec3b4cc\",\"tags\":[],\"doc\":null}},{\"id\":\"67d1d42a-5835-4fac-cf39-5639ed59f3a5\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"y\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"b03c92e6-75d7-4ba7-cb45-7bf209d0fa0c\"}},{\"columns\":[\"y\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"442e3b75-f8a9-401d-c09a-f29a9fd2bd54\"}}],\"id\":\"67d1d42a-5835-4fac-cf39-5639ed59f3a5\",\"tags\":[],\"doc\":null}},{\"id\":\"2ec5695e-0b33-4846-c837-8246d7857d35\",\"type\":\"ToolEvents\",\"attributes\":{\"tags\":[],\"id\":\"2ec5695e-0b33-4846-c837-8246d7857d35\",\"geometries\":[],\"doc\":null}},{\"id\":\"df53d30f-15e2-48ed-c856-9e298e3679ba\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"df53d30f-15e2-48ed-c856-9e298e3679ba\",\"tags\":[],\"doc\":null}},{\"id\":\"320898a3-5b5a-4053-c07b-fd9abc6e8b57\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"320898a3-5b5a-4053-c07b-fd9abc6e8b57\",\"tags\":[],\"doc\":null}},{\"id\":\"73463324-11ff-4df5-cd22-77221d2d2b08\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"df53d30f-15e2-48ed-c856-9e298e3679ba\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"320898a3-5b5a-4053-c07b-fd9abc6e8b57\"},\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"73463324-11ff-4df5-cd22-77221d2d2b08\",\"doc\":null,\"tags\":[]}},{\"id\":\"d81c44e7-a25b-4575-c503-b2b26e019215\",\"type\":\"Grid\",\"attributes\":{\"dimension\":0,\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"320898a3-5b5a-4053-c07b-fd9abc6e8b57\"},\"id\":\"d81c44e7-a25b-4575-c503-b2b26e019215\",\"doc\":null,\"tags\":[]}},{\"id\":\"dfcb2093-dab2-406a-c037-c5acb2c9087c\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"dfcb2093-dab2-406a-c037-c5acb2c9087c\",\"tags\":[],\"doc\":null}},{\"id\":\"ee107d99-5faa-4003-c0d0-10512a9fa066\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"ee107d99-5faa-4003-c0d0-10512a9fa066\",\"tags\":[],\"doc\":null}},{\"id\":\"d58eb0af-36cc-4ca7-cc91-07b7c5b4236c\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"dfcb2093-dab2-406a-c037-c5acb2c9087c\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"ee107d99-5faa-4003-c0d0-10512a9fa066\"},\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"d58eb0af-36cc-4ca7-cc91-07b7c5b4236c\",\"doc\":null,\"tags\":[]}},{\"id\":\"9c238f85-9a92-4dd3-cdf4-7247f041e224\",\"type\":\"Grid\",\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"ee107d99-5faa-4003-c0d0-10512a9fa066\"},\"id\":\"9c238f85-9a92-4dd3-cdf4-7247f041e224\",\"doc\":null,\"tags\":[]}},{\"id\":\"c3235607-3964-4d5b-cd91-990ba11ba7d3\",\"type\":\"PanTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"c3235607-3964-4d5b-cd91-990ba11ba7d3\",\"doc\":null,\"tags\":[]}},{\"id\":\"bba3dbfc-b056-41ce-c1f0-ccdcfa933979\",\"type\":\"WheelZoomTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"bba3dbfc-b056-41ce-c1f0-ccdcfa933979\",\"doc\":null,\"tags\":[]}},{\"id\":\"cf9a1998-bd72-4233-c264-64b58a1ea865\",\"type\":\"BoxZoomTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"cf9a1998-bd72-4233-c264-64b58a1ea865\",\"tags\":[],\"doc\":null}},{\"id\":\"6bdc5456-5054-4820-c896-0bda603d1aa3\",\"type\":\"PreviewSaveTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"6bdc5456-5054-4820-c896-0bda603d1aa3\",\"tags\":[],\"doc\":null}},{\"id\":\"253666db-35a1-4a2a-ca06-081bfb1ffcf9\",\"type\":\"ResizeTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"253666db-35a1-4a2a-ca06-081bfb1ffcf9\",\"tags\":[],\"doc\":null}},{\"id\":\"132547f9-471b-4e67-cec2-103187494a64\",\"type\":\"ResetTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"132547f9-471b-4e67-cec2-103187494a64\",\"tags\":[],\"doc\":null}},{\"id\":\"b7fefa12-fb3a-4be1-cbdb-f623be640eba\",\"type\":\"Legend\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"legends\":[[\"Optimal\",[{\"type\":\"GlyphRenderer\",\"id\":\"6d4bb37b-d0fb-47cc-c704-2b7ff6bfc2cb\"}]],[\"Test Reward\",[{\"type\":\"GlyphRenderer\",\"id\":\"81e7c1c2-2756-4c06-cc7d-afc5e76081be\"}]]],\"id\":\"b7fefa12-fb3a-4be1-cbdb-f623be640eba\",\"doc\":null,\"tags\":[]}},{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"attributes\":{\"x_range\":{\"type\":\"DataRange1d\",\"id\":\"b4f24390-ccf7-440f-c4d8-d750fec3b4cc\"},\"tool_events\":{\"type\":\"ToolEvents\",\"id\":\"2ec5695e-0b33-4846-c837-8246d7857d35\"},\"below\":[{\"type\":\"LinearAxis\",\"id\":\"73463324-11ff-4df5-cd22-77221d2d2b08\"}],\"renderers\":[{\"type\":\"GlyphRenderer\",\"id\":\"6d4bb37b-d0fb-47cc-c704-2b7ff6bfc2cb\"},{\"type\":\"GlyphRenderer\",\"id\":\"81e7c1c2-2756-4c06-cc7d-afc5e76081be\"},{\"type\":\"LinearAxis\",\"id\":\"73463324-11ff-4df5-cd22-77221d2d2b08\"},{\"type\":\"Grid\",\"id\":\"d81c44e7-a25b-4575-c503-b2b26e019215\"},{\"type\":\"LinearAxis\",\"id\":\"d58eb0af-36cc-4ca7-cc91-07b7c5b4236c\"},{\"type\":\"Grid\",\"id\":\"9c238f85-9a92-4dd3-cdf4-7247f041e224\"},{\"type\":\"Legend\",\"id\":\"b7fefa12-fb3a-4be1-cbdb-f623be640eba\"}],\"above\":[],\"tools\":[{\"type\":\"PanTool\",\"id\":\"c3235607-3964-4d5b-cd91-990ba11ba7d3\"},{\"type\":\"WheelZoomTool\",\"id\":\"bba3dbfc-b056-41ce-c1f0-ccdcfa933979\"},{\"type\":\"BoxZoomTool\",\"id\":\"cf9a1998-bd72-4233-c264-64b58a1ea865\"},{\"type\":\"PreviewSaveTool\",\"id\":\"6bdc5456-5054-4820-c896-0bda603d1aa3\"},{\"type\":\"ResizeTool\",\"id\":\"253666db-35a1-4a2a-ca06-081bfb1ffcf9\"},{\"type\":\"ResetTool\",\"id\":\"132547f9-471b-4e67-cec2-103187494a64\"}],\"doc\":null,\"right\":[],\"title\":\"DDPG Performance\",\"extra_x_ranges\":{},\"left\":[{\"type\":\"LinearAxis\",\"id\":\"d58eb0af-36cc-4ca7-cc91-07b7c5b4236c\"}],\"y_range\":{\"type\":\"DataRange1d\",\"id\":\"67d1d42a-5835-4fac-cf39-5639ed59f3a5\"},\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"extra_y_ranges\":{},\"tags\":[]}}];\n",
       "  Bokeh.load_models(all_models);\n",
       "  var model = Bokeh.Collections(modeltype).get(modelid);\n",
       "  $(\"#bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\").html(''); // clear any previous plot in window_id\n",
       "  var view = new model.default_view({model: model, el: \"#bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\"});\n",
       "    } else {\n",
       "  load_lib(bokehjs_url, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", new Date())\n",
       "      var modelid = \"8de6f979-f1dd-44f2-c48e-67e2322f1c48\";\n",
       "      var modeltype = \"Plot\";\n",
       "      var all_models = [{\"id\":\"b03c92e6-75d7-4ba7-cb45-7bf209d0fa0c\",\"type\":\"ColumnDataSource\",\"attributes\":{\"data\":{\"y\":[3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3],\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000]},\"column_names\":[\"y\",\"x\"],\"cont_ranges\":{},\"discrete_ranges\":{},\"selected\":[],\"id\":\"b03c92e6-75d7-4ba7-cb45-7bf209d0fa0c\",\"doc\":null,\"tags\":[]}},{\"id\":\"73e92ee0-dc65-4137-c18d-34a3381c4052\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"#000000\"},\"line_color\":{\"value\":\"#000000\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"73e92ee0-dc65-4137-c18d-34a3381c4052\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"72d1e3bd-3fd3-4a3c-cac3-aedcf9b0550e\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"#000000\"},\"line_color\":{\"value\":\"#000000\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"72d1e3bd-3fd3-4a3c-cac3-aedcf9b0550e\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"6d4bb37b-d0fb-47cc-c704-2b7ff6bfc2cb\",\"type\":\"GlyphRenderer\",\"attributes\":{\"name\":null,\"nonselection_glyph\":{\"type\":\"Line\",\"id\":\"72d1e3bd-3fd3-4a3c-cac3-aedcf9b0550e\"},\"doc\":null,\"server_data_source\":null,\"data_source\":{\"type\":\"ColumnDataSource\",\"id\":\"b03c92e6-75d7-4ba7-cb45-7bf209d0fa0c\"},\"glyph\":{\"type\":\"Line\",\"id\":\"73e92ee0-dc65-4137-c18d-34a3381c4052\"},\"selection_glyph\":null,\"id\":\"6d4bb37b-d0fb-47cc-c704-2b7ff6bfc2cb\",\"tags\":[]}},{\"id\":\"442e3b75-f8a9-401d-c09a-f29a9fd2bd54\",\"type\":\"ColumnDataSource\",\"attributes\":{\"data\":{\"y\":[0.1452022343874,0.11627835780382,0.12151810526848,0.1232678592205,0.11941703408957,0.12565785646439,0.054540328681469,0.086176387965679,0.06445100158453,0.11767222732306,0.07691416144371,0.21331083774567,0.10325336456299,0.18877494335175,0.39335876703262,0.34627988934517,0.35911333560944,0.41099599003792,0.53969812393188,0.50940239429474,0.43573418259621,0.57178807258606,0.53875172138214,0.40858808159828,0.44315341114998,0.52144736051559,0.56556636095047,0.58056455850601,0.58089780807495,0.62991279363632,0.62747114896774,0.65029710531235,0.69231879711151,0.6623820066452,0.68375104665756,0.67262881994247,0.5755021572113,0.49282014369965,0.39693796634674,0.51786035299301,0.5884056687355,0.87440901994705,0.69735944271088,0.67561632394791,0.67832404375076,0.7175931930542,0.70014882087708,0.57986110448837,0.62984424829483,0.59821969270706,0.63930314779282,0.62633126974106,0.62515592575073,0.53446358442307,0.62766647338867,0.52762788534164,0.44868528842926,0.53710353374481,0.47086328268051,0.61008322238922,0.49282866716385,0.6289439201355,0.55514061450958,0.54795438051224,0.57919061183929,0.57458859682083,0.59715729951859,0.54260671138763,0.60736685991287,0.54437249898911,0.56367856264114,0.51122719049454,0.50561928749084,0.58165001869202,0.56121307611465,0.5913947224617,0.50838762521744,0.58306795358658,0.5309391617775,0.57408857345581,0.64673608541489,0.63517111539841,0.58226144313812,0.55478471517563,0.56920599937439,0.59645920991898,0.51629334688187,0.43189117312431,0.52474772930145,0.45482948422432,0.63670861721039,0.60948956012726,0.60038894414902,0.57098394632339,0.610955119133,0.55260396003723,0.65204477310181,0.58869290351868,0.69162207841873,0.48130622506142,0.52701729536057,0.56631779670715,0.60040771961212,0.58208340406418,0.63646310567856,0.58819055557251,0.5544159412384,0.64056843519211,0.62904387712479,0.54707419872284,0.50399523973465,0.54172348976135,0.60203301906586,0.56978380680084,0.64801210165024,0.55965501070023,0.61910092830658,0.50317525863647,0.59097599983215,0.58495926856995,0.5205482840538,0.53229641914368,0.60052514076233,0.63249760866165,0.58158081769943,0.55282521247864,0.60298782587051,0.55073463916779,0.56411308050156,0.54331719875336,0.58939307928085,0.63218104839325,0.59134942293167,0.65499603748322,0.59003490209579,0.59417253732681,0.67103171348572,0.58376711606979,0.49588960409164,0.54131942987442,0.64106202125549,0.53420746326447,0.53746992349625,0.60748136043549,0.52507209777832,0.61526072025299,0.61725354194641,0.60878604650497,0.60640043020248,0.7002791762352,0.67332178354263,0.54250866174698,0.68109798431396,0.80152797698975,0.65507304668427,0.67837053537369,0.56909000873566,0.64316350221634,0.59624868631363,0.62407499551773,0.55809277296066,0.57640427350998,0.64851695299149,0.60916638374329,0.60473853349686,0.60422194004059,0.67193627357483,0.65681093931198,0.55662852525711,0.59135586023331,0.87479829788208,0.64038306474686,0.83460223674774,0.70956712961197,0.69555377960205,0.64904111623764,0.75341862440109,0.85492038726807,0.59165042638779,0.74270939826965,0.64929693937302,0.66478830575943,0.706698179245,0.65124225616455,0.67475014925003,0.59543889760971,0.73233807086945,0.54096275568008,0.76812076568604,0.78942662477493,0.59604603052139,0.60798996686935,0.69596147537231,0.5777536034584,0.67894625663757,0.57372725009918,0.52820628881454,0.58320516347885,0.64292007684708,0.64014482498169,0.65855139493942,0.62022709846497,0.66327089071274,0.55190658569336,0.60613483190536,0.56507402658463,0.68031668663025,0.6621687412262,0.69323098659515,0.6233885884285,0.62512958049774,0.74882590770721,0.67780488729477,0.65386855602264,0.61661511659622,0.56966042518616,0.58683180809021,0.66113328933716,0.61708527803421,0.76428157091141,0.61531776189804,0.67506402730942,1.0763465166092,0.65361261367798,0.91432613134384,0.65616023540497,0.60983216762543,0.67210847139359,0.70262765884399,0.64577800035477,0.69020199775696,0.61878633499146,0.64076900482178,0.62830686569214,0.69944179058075,0.60106199979782,0.54255598783493,0.89452636241913,0.68022590875626,0.75307637453079,0.71926379203796,0.57169830799103,0.73270952701569,0.61359304189682,0.66080188751221,0.77736580371857,0.67904436588287,0.63912373781204,1.0422577857971,0.9418773651123,0.66505789756775,0.66283404827118,0.6312363743782,0.6707706451416,0.61629593372345,0.88781416416168,0.71504390239716,0.64858543872833,0.69947928190231,0.64865112304688,0.69178831577301,0.69918411970139,0.63160794973373,0.65058553218842,0.61663842201233,0.83678597211838,0.7702648639679,0.83210223913193,0.96323579549789,0.66060703992844,0.73190480470657,0.72676920890808,0.82822185754776,0.70516192913055,0.91049551963806,0.84945672750473,0.70241981744766,1.0074692964554,0.72384393215179,0.84201031923294,0.66348564624786,0.69835501909256,0.75529372692108,0.71969360113144,0.71761512756348,0.74221587181091,0.61761772632599,0.72458529472351,0.75140726566315,0.74123126268387,0.89277732372284,0.79760521650314,0.7689808011055,0.65523862838745,0.63871079683304,0.75596755743027,0.85214364528656,0.91460829973221,0.91141402721405,0.76981514692307,1.08212018013,0.91913282871246,0.95193755626678,0.78260260820389,0.75486469268799,0.90562868118286,0.64427077770233,1.0731525421143,0.77875781059265,0.97275608778,0.86457169055939,0.7191476225853,0.70069164037704,0.89726346731186,0.73183673620224,0.81895500421524,0.86312913894653,0.8096998333931,0.7354719042778,0.92632669210434,0.88626205921173,1.1652189493179,0.79572415351868,0.79976469278336,0.87632894515991,0.59785097837448,0.75139975547791,0.92733800411224,1.0500625371933,0.7696944475174,0.62658834457397,0.70204263925552,0.71904712915421,0.64442098140717,1.0912376642227,0.69399082660675,1.1672251224518,0.8189777135849,1.1429187059402,0.89581364393234,0.64953124523163,1.1141538619995,0.96965265274048,0.77611726522446,0.79408317804337,0.67355245351791,0.78807801008224,0.8763182759285,0.63229292631149,0.89321339130402,0.7798947095871,0.82040339708328,0.85014033317566,0.96043068170547,1.1538344621658,1.0829594135284,1.2710630893707,1.098455786705,1.0561771392822,1.0138514041901,0.98727196455002,0.76210987567902,0.95083332061768,0.84509056806564,0.76710623502731,1.0660499334335,0.88079208135605,1.1715178489685,0.72887015342712,0.68582379817963,0.89298892021179,0.86327069997787,1.0282936096191,0.932397544384,0.97691971063614,1.0369780063629,0.83920884132385,0.88697850704193,0.82472217082977,0.88218462467194,0.84190714359283,0.65081149339676,0.91445362567902,0.84395903348923,0.96435737609863,1.0984205007553,0.7677766084671,1.029820561409,1.0610002279282,0.65939021110535,0.83892667293549,0.67698550224304,0.82219803333282,0.78961426019669,0.85550075769424,0.93165254592896,0.80497014522552,0.64872586727142,1.1152740716934,1.012598991394,0.94260495901108,0.90957456827164,1.0606503486633,0.88942456245422,0.95096862316132,1.0942577123642,1.0693191289902,1.0674315690994,1.0862023830414,0.94319659471512,0.95653259754181,0.74607104063034,0.63439434766769,1.1916078329086,0.58567297458649,0.86374044418335,0.9687939286232,0.76139026880264,0.88832288980484,0.7861356139183,0.85508799552917,0.58087939023972,0.91544741392136,1.0759904384613,1.0275074243546,0.8956264257431,0.55858409404755,1.1407730579376,0.95685803890228,0.87830376625061,1.0094784498215,0.91745442152023,0.87592649459839,1.1565687656403,1.2144404649734,1.1573051214218,1.0067210197449,1.1738752126694,1.0790157318115,1.0937875509262,0.83302700519562,1.1437908411026,1.1136255264282,0.98053556680679,0.69028204679489,0.78602415323257,0.77032446861267,0.78950649499893,0.84357225894928,0.77887845039368,0.79703360795975,0.80217480659485,0.8600560426712,0.98359817266464,0.85038906335831,0.76095259189606,0.83954387903214,0.67496293783188,1.1985043287277,0.75735694169998,0.7220510840416,0.76116096973419,0.75816601514816,0.73420703411102,0.77419352531433,0.93907725811005,1.0750155448914,1.0583482980728,1.087545633316,0.81348145008087,0.93936336040497,0.81517815589905,0.84553039073944,0.86122298240662,0.82546204328537,0.79336076974869,0.89371740818024,0.93356144428253,0.79874753952026,0.94986522197723,0.86065810918808,1.1030675172806,0.92198485136032,0.93831604719162,0.73740154504776,1.0889797210693,0.83094841241837,1.0753296613693,0.83413457870483,1.1916743516922,0.75837802886963,1.0745668411255,0.76628214120865,0.79275017976761,0.94876897335052,0.77698028087616,0.72552651166916,1.14905834198,1.0568577051163,0.78395658731461,0.73836439847946,0.57513254880905,0.95350217819214,1.0572596788406,0.82675021886826,0.80843168497086,1.1006346940994,1.1698687076569,1.1136993169785,0.78229206800461,1.2000012397766,0.81665796041489,1.1547582149506,1.2327213287354,0.69584977626801,0.92005807161331,0.8698365688324,1.1065264940262,1.0890151262283,0.63684815168381,1.1126059293747,0.66536045074463,1.1645638942719,0.98705691099167,1.0030151605606,1.0642470121384,1.2158358097076,0.71157693862915,1.2112889289856,1.0761188268661,1.1510169506073,1.2630207538605,1.1903929710388,0.9413206577301,1.2656391859055,1.1923191547394,0.92332077026367,1.2589302062988,0.84969955682755,1.2061839103699,1.3042341470718,1.2033878564835,1.08316385746,0.95195996761322,1.4791369438171,1.0451848506927,1.1767209768295,1.1471594572067,0.93449884653091,0.96595633029938,1.0097881555557,0.89585304260254,1.0409153699875,1.0506455898285,1.2431737184525,1.2177177667618,1.1367125511169,1.3085988759995,1.1635460853577,0.93756401538849,1.1475546360016,0.85820883512497,0.82280099391937,0.87765085697174,1.1952151060104,0.79925549030304,1.2567739486694,1.2406039237976,1.1037355661392,1.1787724494934,1.0471999645233,1.3226636648178,0.90229767560959,1.0482059717178,1.131908416748,1.1459876298904,1.1863869428635,1.2375094890594,1.2847805023193,1.4439768791199,1.2419325113297,1.3250359296799,1.0246162414551,0.81471693515778,1.3136330842972,0.80461740493774,1.1588053703308,0.87278538942337,1.1292772293091,0.89002299308777,1.2566843032837,1.194228887558,0.99039697647095,1.375864982605,1.2315472364426,1.3571403026581,1.3148744106293,1.0391068458557,1.1247373819351,1.1628968715668,1.2774765491486,1.1802004575729,0.96559023857117,1.0816307067871,1.0847978591919,1.2926717996597,1.1839392185211,1.2882776260376,1.440339922905,1.325984954834,1.0694131851196,1.1063982248306,1.2887151241302,1.0086168050766,1.2562565803528,0.97993940114975,1.3298500776291,1.1611683368683,1.1303870677948,1.2423943281174,0.91838556528091,1.2238975763321,1.3701069355011,1.4628766775131,1.3882297277451,1.4231206178665,1.2252702713013,1.496129155159,1.4813536405563,1.5786159038544,1.6794850826263,1.5281442403793,1.7961578369141,1.5226570367813,1.681009888649,1.407208442688,1.3171232938766,1.5397200584412,1.5442078113556,1.3682622909546,1.182120680809,1.3316655158997,1.8376263380051,1.3470443487167,1.6013431549072,1.5312165021896,1.3359662294388,1.2211793661118,1.6969156265259,1.4718585014343,1.3456215858459,1.4112508296967,1.4297339916229,1.233887553215,1.4173140525818,1.8813115358353,1.2239533662796,1.7003704309464,1.3252873420715,1.6116651296616,1.1041036844254,1.153752207756,1.5345325469971,1.562894821167,1.0128148794174,1.3168244361877,1.591472029686,1.4309965372086,1.0767061710358,1.4788085222244,1.8424981832504,1.5660690069199,1.4164085388184,1.465228676796,1.5064542293549,1.3323475122452,1.5880290269852,1.5389310121536,1.1053144931793,1.2191852331161,1.360875248909,1.346181511879,1.8398958444595,1.3014913797379,1.3971838951111,1.5485363006592,1.3125579357147,1.3120362758636,1.5752868652344,1.2062079906464,1.2275605201721,1.3841038942337,1.0963809490204,1.3539242744446,1.3647441864014,1.4838548898697,1.110355257988,1.524983048439,1.3384205102921,0.94321459531784,1.0760469436646,0.88620781898499,1.3218929767609,1.1921083927155,1.4212509393692,2.1025786399841,1.4743032455444,1.7764747142792,1.9197858572006,1.5286051034927,1.4680963754654,1.6220995187759,1.5465126037598,1.3993220329285,1.6328763961792,1.3016562461853,1.5235607624054,1.3752479553223,1.0778654813766,1.2544348239899,1.4932141304016,1.4343048334122,1.4653215408325,1.1662670373917,1.4765610694885,1.0985797643661,1.4672635793686,1.4940588474274,1.2011618614197,1.3428165912628,1.3932489156723,1.7481383085251,1.3393362760544,1.8508709669113,1.4970986843109,1.1761165857315,1.6942420005798,1.7538660764694,1.5180196762085,1.6667125225067,1.35888671875,1.2786378860474,1.4820936918259,1.1308473348618,1.5222866535187,1.3941330909729,1.6612186431885,1.6179130077362,1.3005647659302,1.7472367286682,1.3453847169876,1.0027928352356,1.0422744750977,1.0249646902084,1.0016592741013,1.1928569078445,1.0528798103333,1.2236078977585,1.7047555446625,1.5016071796417,1.9791238307953,1.9296927452087,1.6525857448578,1.0732080936432,1.9577649831772,1.3992519378662,1.1985028982162,0.98192667961121,1.0755985975266,2.1517448425293,1.4449954032898,2.1665072441101,1.9110794067383,2.0523526668549,2.1116948127747,0.98506134748459,1.2164725065231,1.5379217863083,1.6957613229752,1.1499508619308,1.0762499570847,1.3295881748199,1.659229516983,1.3569928407669,1.289494395256,1.080927491188,1.5398008823395,1.2721623182297,2.0339231491089,1.1331685781479,1.2914298772812,1.0073283910751,1.4904673099518,1.6891660690308,1.5343341827393,1.6589248180389,1.1142619848251,1.6565127372742,1.8268450498581,2.2952880859375,1.9121998548508,1.4958305358887,1.5254690647125,2.1004023551941,1.3247292041779,2.0941646099091,1.4541617631912,1.8724514245987,1.5146670341492,1.9992687702179,1.7828208208084,1.4893035888672,1.8231176137924,1.2839140892029,2.2504134178162,2.1098246574402,1.901765704155,1.282545208931,1.2019698619843,1.4819278717041,1.0509465932846,1.5591523647308,1.5206170082092,1.3829181194305,1.236954331398,1.2967925071716,1.7221531867981,1.3117214441299,2.0308265686035,1.8183161020279,2.2635748386383,1.5701133012772,1.1394877433777,2.0769131183624,1.4202675819397,1.0293002128601,1.5572972297668,1.8508529663086,1.306432723999,1.2195662260056,1.4378818273544,1.515585899353,1.6536300182343,1.6753197908401,1.6793996095657,1.7607321739197,1.962974190712,1.3360667228699,1.8220907449722,2.353200674057,1.2742609977722,1.7515349388123,1.7403109073639,1.5733277797699,1.7494155168533,1.449945807457,1.3106330633163,1.2527709007263,2.0239486694336,2.0696723461151,1.2897975444794,1.9978283643723,1.6143078804016,1.8798394203186,1.7828289270401,1.4428902864456,1.640863776207,1.8705677986145,1.7706897258759,1.7105292081833,1.7958766222,1.728045463562,1.6011823415756,1.69899559021,1.6917471885681,1.5185176134109,1.3742475509644,1.5970497131348,1.4478988647461,1.8166614770889,1.3759965896606,1.8146399259567,1.3575502634048,2.203033208847,1.0840753316879,1.9340724945068,1.3059742450714,1.6679085493088,1.7269117832184,1.2645924091339,1.3088563680649,1.5329682826996,1.9148380756378,1.275958776474,1.6040170192719,1.4699258804321,1.8459672927856,1.4679863452911,1.6780132055283,1.7349985837936,1.9525405168533,1.7879681587219,2.0974202156067,2.4326357841492,1.3748066425323,2.0709097385406,1.2540044784546,1.2728310823441,1.6870158910751,1.7704585790634,1.4026651382446,1.4917454719543,1.4615346193314,1.5329290628433,1.4238270521164,1.4546440839767,2.2301554679871,1.5760939121246,2.0382468700409,2.0051789283752,1.8716418743134,1.8662232160568,2.1270451545715,1.5002548694611,1.3810645341873,2.1510956287384,1.8499354124069,2.3694107532501,2.2167212963104,2.6075127124786,2.3605029582977,2.427255153656,1.873238325119,2.6338391304016,2.5504095554352,1.3575179576874,2.7427670955658,1.9512524604797,2.5922474861145,1.8487793207169,1.9915332794189,1.2642180919647,1.9645494222641,2.5299105644226,2.5060911178589,2.318642616272,1.5794062614441,2.0401864051819,1.8757908344269,1.8106182813644,1.8057956695557,1.5287480354309,1.6433643102646,1.7893639802933,1.4387691020966,1.8401030302048,1.6109600067139,1.4426708221436,1.4104764461517,1.7765171527863,1.5158184766769,1.3200393915176,1.6753798723221,1.4117214679718,1.428870677948,1.4760152101517,1.1964383125305,1.5353670120239,2.6861822605133,2.1316108703613,1.5258768796921,1.2438675165176,1.4655103683472,0.9033243060112,1.3437737226486,1.0767545700073,1.0059639215469,1.1776745319366,1.2056657075882,1.5337153673172,1.3081291913986,1.9964346885681,1.6591212749481,1.3358269929886,1.6240762472153,1.5454148054123,1.2204133272171,1.4266442060471,1.2104421854019,1.255139708519,1.2998020648956,1.5885033607483,1.50734603405,1.7922949790955,1.1678071022034,1.6651309728622,1.7218272686005,1.4160262346268,1.6131038665771,1.8626878261566],\"x\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000]},\"column_names\":[\"y\",\"x\"],\"cont_ranges\":{},\"discrete_ranges\":{},\"selected\":[],\"id\":\"442e3b75-f8a9-401d-c09a-f29a9fd2bd54\",\"doc\":null,\"tags\":[]}},{\"id\":\"9ce24391-45bb-4be7-cb2a-059464107515\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"green\"},\"line_color\":{\"value\":\"green\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"9ce24391-45bb-4be7-cb2a-059464107515\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"b05b74bc-be85-4ff9-cfd0-eac697af0391\",\"type\":\"Line\",\"attributes\":{\"fill_alpha\":{\"units\":\"data\",\"value\":0.2},\"line_alpha\":{\"units\":\"data\",\"value\":1},\"doc\":null,\"size\":{\"units\":\"screen\",\"value\":10},\"fill_color\":{\"value\":\"green\"},\"line_color\":{\"value\":\"green\"},\"x\":{\"units\":\"data\",\"field\":\"x\"},\"id\":\"b05b74bc-be85-4ff9-cfd0-eac697af0391\",\"y\":{\"units\":\"data\",\"field\":\"y\"},\"tags\":[]}},{\"id\":\"81e7c1c2-2756-4c06-cc7d-afc5e76081be\",\"type\":\"GlyphRenderer\",\"attributes\":{\"name\":null,\"nonselection_glyph\":{\"type\":\"Line\",\"id\":\"b05b74bc-be85-4ff9-cfd0-eac697af0391\"},\"doc\":null,\"server_data_source\":null,\"data_source\":{\"type\":\"ColumnDataSource\",\"id\":\"442e3b75-f8a9-401d-c09a-f29a9fd2bd54\"},\"glyph\":{\"type\":\"Line\",\"id\":\"9ce24391-45bb-4be7-cb2a-059464107515\"},\"selection_glyph\":null,\"id\":\"81e7c1c2-2756-4c06-cc7d-afc5e76081be\",\"tags\":[]}},{\"id\":\"b4f24390-ccf7-440f-c4d8-d750fec3b4cc\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"x\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"b03c92e6-75d7-4ba7-cb45-7bf209d0fa0c\"}},{\"columns\":[\"x\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"442e3b75-f8a9-401d-c09a-f29a9fd2bd54\"}}],\"id\":\"b4f24390-ccf7-440f-c4d8-d750fec3b4cc\",\"tags\":[],\"doc\":null}},{\"id\":\"67d1d42a-5835-4fac-cf39-5639ed59f3a5\",\"type\":\"DataRange1d\",\"attributes\":{\"sources\":[{\"columns\":[\"y\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"b03c92e6-75d7-4ba7-cb45-7bf209d0fa0c\"}},{\"columns\":[\"y\"],\"source\":{\"type\":\"ColumnDataSource\",\"id\":\"442e3b75-f8a9-401d-c09a-f29a9fd2bd54\"}}],\"id\":\"67d1d42a-5835-4fac-cf39-5639ed59f3a5\",\"tags\":[],\"doc\":null}},{\"id\":\"2ec5695e-0b33-4846-c837-8246d7857d35\",\"type\":\"ToolEvents\",\"attributes\":{\"tags\":[],\"id\":\"2ec5695e-0b33-4846-c837-8246d7857d35\",\"geometries\":[],\"doc\":null}},{\"id\":\"df53d30f-15e2-48ed-c856-9e298e3679ba\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"df53d30f-15e2-48ed-c856-9e298e3679ba\",\"tags\":[],\"doc\":null}},{\"id\":\"320898a3-5b5a-4053-c07b-fd9abc6e8b57\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"320898a3-5b5a-4053-c07b-fd9abc6e8b57\",\"tags\":[],\"doc\":null}},{\"id\":\"73463324-11ff-4df5-cd22-77221d2d2b08\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"df53d30f-15e2-48ed-c856-9e298e3679ba\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"320898a3-5b5a-4053-c07b-fd9abc6e8b57\"},\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"73463324-11ff-4df5-cd22-77221d2d2b08\",\"doc\":null,\"tags\":[]}},{\"id\":\"d81c44e7-a25b-4575-c503-b2b26e019215\",\"type\":\"Grid\",\"attributes\":{\"dimension\":0,\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"320898a3-5b5a-4053-c07b-fd9abc6e8b57\"},\"id\":\"d81c44e7-a25b-4575-c503-b2b26e019215\",\"doc\":null,\"tags\":[]}},{\"id\":\"dfcb2093-dab2-406a-c037-c5acb2c9087c\",\"type\":\"BasicTickFormatter\",\"attributes\":{\"id\":\"dfcb2093-dab2-406a-c037-c5acb2c9087c\",\"tags\":[],\"doc\":null}},{\"id\":\"ee107d99-5faa-4003-c0d0-10512a9fa066\",\"type\":\"BasicTicker\",\"attributes\":{\"num_minor_ticks\":5,\"id\":\"ee107d99-5faa-4003-c0d0-10512a9fa066\",\"tags\":[],\"doc\":null}},{\"id\":\"d58eb0af-36cc-4ca7-cc91-07b7c5b4236c\",\"type\":\"LinearAxis\",\"attributes\":{\"formatter\":{\"type\":\"BasicTickFormatter\",\"id\":\"dfcb2093-dab2-406a-c037-c5acb2c9087c\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"ee107d99-5faa-4003-c0d0-10512a9fa066\"},\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"axis_label\":null,\"id\":\"d58eb0af-36cc-4ca7-cc91-07b7c5b4236c\",\"doc\":null,\"tags\":[]}},{\"id\":\"9c238f85-9a92-4dd3-cdf4-7247f041e224\",\"type\":\"Grid\",\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"ticker\":{\"type\":\"BasicTicker\",\"id\":\"ee107d99-5faa-4003-c0d0-10512a9fa066\"},\"id\":\"9c238f85-9a92-4dd3-cdf4-7247f041e224\",\"doc\":null,\"tags\":[]}},{\"id\":\"c3235607-3964-4d5b-cd91-990ba11ba7d3\",\"type\":\"PanTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"c3235607-3964-4d5b-cd91-990ba11ba7d3\",\"doc\":null,\"tags\":[]}},{\"id\":\"bba3dbfc-b056-41ce-c1f0-ccdcfa933979\",\"type\":\"WheelZoomTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"dimensions\":[\"width\",\"height\"],\"id\":\"bba3dbfc-b056-41ce-c1f0-ccdcfa933979\",\"doc\":null,\"tags\":[]}},{\"id\":\"cf9a1998-bd72-4233-c264-64b58a1ea865\",\"type\":\"BoxZoomTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"cf9a1998-bd72-4233-c264-64b58a1ea865\",\"tags\":[],\"doc\":null}},{\"id\":\"6bdc5456-5054-4820-c896-0bda603d1aa3\",\"type\":\"PreviewSaveTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"6bdc5456-5054-4820-c896-0bda603d1aa3\",\"tags\":[],\"doc\":null}},{\"id\":\"253666db-35a1-4a2a-ca06-081bfb1ffcf9\",\"type\":\"ResizeTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"253666db-35a1-4a2a-ca06-081bfb1ffcf9\",\"tags\":[],\"doc\":null}},{\"id\":\"132547f9-471b-4e67-cec2-103187494a64\",\"type\":\"ResetTool\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"id\":\"132547f9-471b-4e67-cec2-103187494a64\",\"tags\":[],\"doc\":null}},{\"id\":\"b7fefa12-fb3a-4be1-cbdb-f623be640eba\",\"type\":\"Legend\",\"attributes\":{\"plot\":{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"subtype\":\"Figure\"},\"legends\":[[\"Optimal\",[{\"type\":\"GlyphRenderer\",\"id\":\"6d4bb37b-d0fb-47cc-c704-2b7ff6bfc2cb\"}]],[\"Test Reward\",[{\"type\":\"GlyphRenderer\",\"id\":\"81e7c1c2-2756-4c06-cc7d-afc5e76081be\"}]]],\"id\":\"b7fefa12-fb3a-4be1-cbdb-f623be640eba\",\"doc\":null,\"tags\":[]}},{\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"type\":\"Plot\",\"attributes\":{\"x_range\":{\"type\":\"DataRange1d\",\"id\":\"b4f24390-ccf7-440f-c4d8-d750fec3b4cc\"},\"tool_events\":{\"type\":\"ToolEvents\",\"id\":\"2ec5695e-0b33-4846-c837-8246d7857d35\"},\"below\":[{\"type\":\"LinearAxis\",\"id\":\"73463324-11ff-4df5-cd22-77221d2d2b08\"}],\"renderers\":[{\"type\":\"GlyphRenderer\",\"id\":\"6d4bb37b-d0fb-47cc-c704-2b7ff6bfc2cb\"},{\"type\":\"GlyphRenderer\",\"id\":\"81e7c1c2-2756-4c06-cc7d-afc5e76081be\"},{\"type\":\"LinearAxis\",\"id\":\"73463324-11ff-4df5-cd22-77221d2d2b08\"},{\"type\":\"Grid\",\"id\":\"d81c44e7-a25b-4575-c503-b2b26e019215\"},{\"type\":\"LinearAxis\",\"id\":\"d58eb0af-36cc-4ca7-cc91-07b7c5b4236c\"},{\"type\":\"Grid\",\"id\":\"9c238f85-9a92-4dd3-cdf4-7247f041e224\"},{\"type\":\"Legend\",\"id\":\"b7fefa12-fb3a-4be1-cbdb-f623be640eba\"}],\"above\":[],\"tools\":[{\"type\":\"PanTool\",\"id\":\"c3235607-3964-4d5b-cd91-990ba11ba7d3\"},{\"type\":\"WheelZoomTool\",\"id\":\"bba3dbfc-b056-41ce-c1f0-ccdcfa933979\"},{\"type\":\"BoxZoomTool\",\"id\":\"cf9a1998-bd72-4233-c264-64b58a1ea865\"},{\"type\":\"PreviewSaveTool\",\"id\":\"6bdc5456-5054-4820-c896-0bda603d1aa3\"},{\"type\":\"ResizeTool\",\"id\":\"253666db-35a1-4a2a-ca06-081bfb1ffcf9\"},{\"type\":\"ResetTool\",\"id\":\"132547f9-471b-4e67-cec2-103187494a64\"}],\"doc\":null,\"right\":[],\"title\":\"DDPG Performance\",\"extra_x_ranges\":{},\"left\":[{\"type\":\"LinearAxis\",\"id\":\"d58eb0af-36cc-4ca7-cc91-07b7c5b4236c\"}],\"y_range\":{\"type\":\"DataRange1d\",\"id\":\"67d1d42a-5835-4fac-cf39-5639ed59f3a5\"},\"id\":\"8de6f979-f1dd-44f2-c48e-67e2322f1c48\",\"extra_y_ranges\":{},\"tags\":[]}}];\n",
       "      Bokeh.load_models(all_models);\n",
       "      var model = Bokeh.Collections(modeltype).get(modelid);\n",
       "      $(\"#bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\").html(''); // clear any previous plot in window_id\n",
       "      var view = new model.default_view({model: model, el: \"#bdf7a075-d2b5-41e2-c151-7bb1aac9ef46\"});\n",
       "  });\n",
       "    }\n",
       "});\n",
       "</script>\n",
       "<div class=\"plotdiv\" id=\"19bf6371-4bcf-41c4-c1db-9581a31a9e58\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot = itorch.Plot()\n",
    "plot_colors = {'blue', 'green', 'red', 'purple', 'orange', 'magenta', 'cyan'}\n",
    "plot:title('DDPG Performance')\n",
    "plot:legend(true)\n",
    "\n",
    "plot:line(torch.range(1, #stats.test_r), torch.ones(#stats.test_r):add(2), '#000000', 'Optimal'):redraw()\n",
    "\n",
    "local line_test = torch.Tensor(#stats.test_r)\n",
    "for i=1,#stats.test_r, 1 do\n",
    "    line_test[i] = stats.test_r[i]:mean()\n",
    "end\n",
    "\n",
    "plot:line(torch.range(1, #stats.test_r), line_test, plot_colors[2], 'Test Reward'):redraw()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAB4klEQVRoge2YwXGkMBBFX9k++CZnIGeAM2AzUAhsBhMCIeAMcAZTGwF73D1RjmB83RPOoH3wIMR4VMbUgFhXv+KgEkh8Sd2fLkBRFEVRFEVRFEVRFEVRFEVR1uZ6yckd3MK/JV+xCAZKOGSt5FWDSS3nK+RQg4Cwk0ZERKS1eWpZn2KggMNROoIVOumkEymltFs+gwxq6Abp79e+MVKIIC2SWmKMAppT3XRQ4yxCLoggOU1qoSdYqM5sOQcowGCOoVQIUiGwT63Y42D/QbdADUGWVsfuskMMAmU6xe8Y2I0SdNjykhOPzIMscMdmkUg3I08cXXtw50e0/SPNkB9JPNQFWsIErcBGB5XBg3YYlsRDq7H05vNAsEFil9hgLUmwgyeSTRrhI6aFIBcSeqj7wuG74LQygF3gUEtzE+n/NXUCA099+xGeAe76jpeZqtbE58thODMfUBG32g6B8YdivYVtvg71SscFg1/Uthkbv8cGMbVhQuPfje5swUMnMDb+EH8w1SpCruYMcvCjb/88vek99HXO1CtgguA5t8lpy7gJ1GeMP8RX3tOKkPXJ+k2OfKX+Ew+NqM+iub0Us5KYaK103zdWy+C5C4jw0Dd+X3beOJf/N3oPr/AH/l58akVRviFvYM7pp0ctYXoAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:39]\u001b[0m [string \"...\"]:47: step=1, q=70.076, r=-0.30, a=-1.00\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABhklEQVRoge3Y0W2DMBSF4V/qS988At2AbsAIHsEjMAIjMIJHiDoBIzACI1id4PaBkhriJJSmNVT3E28o0j05NiYBpZQ6MANF7hk2MtBAgC73JBsVINNV5R5mIz8F+A8llLmGePrBZ9/hBV4BeIa3hwz0OyyY5I24hN0+jhwI9NcydFMA/8dzrRN/xX1yqVe7L8FFE4Zkhp2XADgIUYblcz8uIb3OdqCMMgi4xe1zCU2W6dYpYbiWwUYF7bYEwEAfZZit+eEIJXAjgztICSMfZfg6Io5SwiiRIS7hEOoowzAeEcO159QfaWGAE9Qr3zCXx5yLAuUQ789xpPthbHzMGWzIWUKYB1iE6aBJ/gCbHXMNLmsJJdRwuhlGLsOU5/oMEqiz7oSzAhz4+RGcDNOCNZh+KkEEPz6edmNVGEPf0YqxEsyYwSRWXH4GLLQXm/7z8oi0ItKJ4Lvcw96TDuMLEenCgMm9Db6rgmZ8vXZVk+3PigfZ/zudUkoppZRSSimllFJKKbXBB8RV8gRspYh4AAAAAElFTkSuQmCC",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:40]\u001b[0m [string \"...\"]:47: step=2, q=71.125, r=0.69, a=-0.07\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABcElEQVRoge2YsW3DMBBFH5L01gbmBvEG0iYeRSMlG1jegNmAKlNFLlNdClIGDCdIc9+A4XsQxFNzX193pChBEDw2z+4Ze1i2/fcukTr45OSuIGMEA6MfR8MMOzDKRZ8cc5U12Dkm/Q+FgZJacEsjTlg99rWFjDe5omcFgFyHQl+Dzjf9LzgbKG2cUgt63/zXaCpwbv+7q8BUh+PSDOySb/5rRC2U1wUo+ea/xtnADAsAHWx9U/+BswHaNMiU+vAHtQ+RgYXCUK+Tu8IFMgNtIZIvQ/4GCsAH7dHLdxP+Bo51mNZ7H9wVLvA3QOuiQnsPvyokzkgMlPWUQD0NZBVYoM6AdIcGJiBPNAPaeaxrIbjJF43EwFzfZG1DkRQSZyQGgJxLDZJ4QyozsP5NGfSfBBL2bMzMzN7MpEKqChROdSVNwEYkIsYOX7UI9MIt9Ysu9Tu5K2kqZbt0M7NOKAiCIAiCIAiCIAiCIAiCIAiCB+MHbwlhKOXhNuAAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:40]\u001b[0m [string \"...\"]:47: step=3, q=71.438, r=2.09, a=0.41\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABYUlEQVRoge3XwW3DMAwF0I+id3kDa4NoA2mTjtAROkpH6AjWBmE3UCZIPAF7UIIkh1wCUoGL/0480f4QSNkAEdnaYZcxv/otnvOVda+qqrtl2DPfLJul1lIv0rAjMA0gh9aLNEUEy86PmQaoEFQASChIlp0fsz0BNDQAKIibDLBCTtLLOG8xACByDlBStO38iHGAtcqpV6mMWUTGASCr4AT0MRgRwTwABAKMW0TWAQ5ofZNOg+bYOgAgtfUixo0GkNaLUqJ58yE+cFRVVf3RAVNsfwK3cxyR7fvfcwjwC+kfFBElRPv+9xwCAO1yH0f/+9glgFwDFI/+7sLn3MdYv4+vfpfn5Msi2us86s/G2KJLj5CdF5HLDOD2Osu+97FXgNYuc+z8QeF3AuM2qYuQw3kRLfrqd3lOwFHPq2j2/LN592q8orY6TZNIRQBWr+d42uYFQERERERERERERET0//0BpvmCk0dKFJcAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:40]\u001b[0m [string \"...\"]:47: step=4, q=71.813, r=2.17, a=-0.22\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABOklEQVRoge2YwW2DQBBFX6LcTQemA7sD6MylmXSw7mDTAVRADrAgrFjJYT4Syn8S0hzQX33NzM4AGPO/eQtXPMH1Ql/x4EyqGYAMX+EHSWhghPHOfYTxRjPCCDfdie+xcn0Jqljd1wQbeJTgGqv7mmADTEnoAM5AGy7/TLyBVII6XPon4g3kEuxTRRoDCaY+bsPln9EY6OHYGQD2ukk1TZxgKh99FuINDPMD7JGFeAPMYwCgUahvkRiAuRUqpnkmRJaBDFML1IoTViQGlpWuVqhvkRhIzCZqjpuBBMc1sCzVtUJ9i+oWyqUPLuJRIDNQtupKPIxVBpavAvUyoTKwfhwftITS5xy0By2h3X5PqAyUBBy2B2Bt5JNyKf3QSXd9Jndkqj4Pv79ujDHGGGOMMcYYY4wxxhjzR74BFiM7LerCtQsAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:40]\u001b[0m [string \"...\"]:47: step=5, q=71.803, r=2.92, a=-0.12\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAA90lEQVRoge3WwXHCMBCF4Zc0gDqwOog6MJ1RGk4HpoN1B6IC5SA0ucRJLs8ZJv938Tsw2lnYNZKA/+3Fc+wkZUmntyhpk1RX3e6eUh4XqUltvl5aU2u6zq5Kr66Dj0IDX4v+SCn3UKunjruBUnIP681ThxH6e94Rkorn/E+mBrb+yEo9hKeM3COURwhbCXZg16qph9n4/TsbqEcMkHuEziO820qwA7uWEc62EhK/wHemEWw3Ucn6FsqPkFZbDXn/Bx73oCJnB84dSCM4Z4gl3hXjJp2fdAe2MUH5Sd9CB6GBfRGxLKG61Dr9/GkAAAAAAAAAAIDf+gAlQTHaUeWZ0QAAAABJRU5ErkJggg==",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:40]\u001b[0m [string \"...\"]:47: step=6, q=72.094, r=2.85, a=0.27\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABdklEQVRoge3YwW3DMAyF4R9F7/YG0gb2BtYmGaUjtRvEG4SdIPIEcSdgD46LAk0uAenAKL+TTqQeLAmEIYRgqyMNDNA8eyMPeENPR1XVTmHYpueLZbGe2s/LoiNbVr7PNIBQEYCedpcBRmQNUCiWle8z/wIzQCHTWlbezHBGVVX12Omz9/KQ5ojqWVX1oNBt0NH0CMHXyEwFyKRNTpFxAGS9x4W8yT32CFBheYiycfFbrANM1FkAWnLKxsVvsQ4AInVZ5L43L/6XQ4BxWhalbzcY6ewDfAkzI0Ah4f4R7AMgCNeRLvvfY4cA0/qSthT/e+wQAOooyyLn4lH/N5cAIvOyyGWfI11zuI50+r7PkY6OyzLSnTWRXFu5HCE+14Eik5vs0mLlEwBkvcelL04tFl4B6s9A4XyP3b5AXV/STSYie82Qrg/R6fLsvTzqck2grs/Qq1/pUT7a3IpUUmKa/Bq52ePf0RBCCCGEEEIIIYQQQgjhP/gGAzGCpSewthwAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:40]\u001b[0m [string \"...\"]:47: step=7, q=72.162, r=2.12, a=-0.31\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABQElEQVRoge2YwY3CMBRE3672jjvAHSQdQOcbOggdpIRQQfaQ/AgFsRf+RELMu/jrG9kaM+MoAWM+my/Fog0UAPoTt7l1mXsXxXb5/MIEE5ymqO56uXt95y73nDaKPnddpYAmiv4QnoLwVBYSAcshF85zMa7H36XvJREwbhv16czL7JOBGkVyABAJGKKoy9g+zKSxj4DyMJPGPhY6R/EmFlqjWgCOd+3kOxSRgGsULUgTzC4WEiaYXQQIE4xOwGqXkzLB6ASMrOYvcfT5j2G0Fqrz0IYAycuASsCwjMe6aWSjFlDrppGN+hY6RyFJMNoQA5SyaWSjvkbbeIx1oo3UFqpRvNs/ENQorv/86BWkFmpYUqxKMDoBN9jBP4gtJPwYsfKjW3oYOrpuoBzGMf9FxhhjjDHGGGOMMcYYY4wxn8sfgr1A/ch2AasAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:40]\u001b[0m [string \"...\"]:47: step=8, q=71.845, r=2.92, a=-0.10\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABcElEQVRoge2YvW3DMBCFvySNO2kDcwNzA7NMl1Eygkdxmy1Ml+6YCSyX6eQuHVNQzE8TBAGfDMP3QYCuuuPjvcMJAsO4be5kmdc12AMswUGCs6xga3J9ANhAZpc5btjC8qIn+yM/BWwhkzP5mQxduzL37VL9iquWSo19NJeAQChBIjVNPIOArljeAyOciU2z6wTEGngcQIBy+VfXAQgsoQcSI7w2zT2LgL4OwNjYP8wkwONLEBv7h9kEhBIM7TugYzMtsm7TlRWWP7daS/QdUPqHOQQ4rX9mFXCdHQjVQunKOjBd98r5HoBx4CQooxMwlpd35fzEthu4IrbQ+usjNGoqiAX0dQBGyQSjFtB9WwJ7TQmdgAEIHnBAHFRldAJOgHcUAWmQ1VGSd7nA06WP8i9W0/HzUfgfRTjEq2kBMGhWWEEoIIRpl6Uk/JH1oEu94LHvWbynlxgPhzddIcMwDMMwDMMwDMMwDMMwDMMwbowPrh1rynZMsbIAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:40]\u001b[0m [string \"...\"]:47: step=9, q=71.499, r=2.01, a=0.56\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABxElEQVRoge2X4VHCQBCFP20gdAAdkA5iB9IBV4Id0IIdGDvQCowVBCsgVmCoYP0RNkYNg5pd/eF+A5M3c5nsPe7ecoEg+N+ceT24UPHoVcGeDdQgsAREP86cGz4rh1zFKebwAjed2SlYGtiq+IKBFcwgwbXhBKZS6K554OQW2unw+hcneIpsOO1+hiN7pHf6Mr2o5RbaQ6t63qiafb4xqbibXtTSAF+KQQYr1QYBMDZQqThuYKWLsoWn6RW9VuDi6C1XKkrb0jbM+3huVG1GxwUyk4rGK/CsYgbzkfHhz783qWhsgEEMFiODSYVB/+mwN3A8Bpca3wburcq5GWi0ES36kaSiNC9qybILacHu7VzB+/iOpeOn2K9A39sX7xpNUlENom6AvQHG/86SitK2louBRoUaKDQKLdza1nIxsD18yemaUdIRs+7py+G4LNQCMjxlT33/+j06AyIga519/deT+g61Giikdn35cskAXY5bYJmTazd1CYCXAc1xWtBlubQ6vX3Ay0B1uCbtpKVTIS8ykIe1iIgIxc6vkNcK7KFh27Z3VUXWlk5VfMnerjYvX0EQBEEQBEEQBEEQBEEQBEEQ2PAKEpCHq99x3zMAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:40]\u001b[0m [string \"...\"]:47: step=10, q=71.924, r=1.10, a=-1.00\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABR0lEQVRoge3Z0W2DMBSF4b8TeAS6ASMwQkaADTJCR6AboEwQdQJGoJ2AdAK6we1DFcuYPsRQ4t7ofuKFF+sgceRrAGOMMcYYY8wvHLjcGdY7wARd7hhrOGhBrtchd540JQxB+hGq3JESHIPoAmdFHXDQB9EnqHNHSvDTV59+gCJ3pFtFfRV4yR0pgfU1k2Vfj7kjJagWfS1zR0oQ9bXNnSdB1NdJXV8nvX0979zXp79ecGYseG4K+ATeoYGPjSte4LQ9183qyokMIkMrbtbf1Vd/x/QAVd+JVCLIiJR7PMC+rxDFkeb1evMFDbxtWvBy31cIWMyamrZdr4QxeAatB91wAhoUbQahbj4+a5qDvFrvydGLRlJNhzAvmu46jZV4wFpPVutsrNb/Q3T+0fSFLtQ+WK2V/R7wSnWfvJZUFsAYY4wxxiT4Bpbb2VVz9pgpAAAAAElFTkSuQmCC",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:41]\u001b[0m [string \"...\"]:47: step=11, q=70.102, r=0.40, a=0.19\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAB3UlEQVRoge2Y33GCQBCHv2R81w6SDrADTAWWoKlAOyAdaCpAO7AD7UBSgWcFYgXkQQ6QED2BdXRmv+Fhj/v3W47bPQBFURRFURRFURRFURRFURRFuTcvAmP6APShBwaWAlPk1HOgC30ABoDV2rM3bZMN/nzD8sPAvqHMWryBD0MIIIA1rOEAidO1TYIkSZJDMiGQVNk5L3owt8+1NhFhTN+AgZi42WDXKDnQc1MfQwRABDGYVCs/AAGM0zqmsCy+WO1TcsCc26fihlTPSevF93kEX6kZL+AboNdQ44340K3Z1SvsgnVe2rWqTwyvsMO36UPI3Hl4urC1Yg/5Ej6JAyX1Xl6ztrd9sclfWxhjXshg4zQUPQ+zwosyKleGtmYiNn+zFRjB1NqLilOPsYZcJG3ggA8La6/gs6JJloXvnAocqAqaf/HzrCBFrRXowso+1RgGcGxTkzD/B83Kto+XCsKCKIfw/mAOhJeCZiXZTnkTlubAqKB+5trpDsnYjWFBfXhDv8yBoZg0N3bXg2Ylge0n9GHpHEYHEIF5uKDZud7kxB4G8H6zemMNoQ/LWxLZsc5J01hD6DTRxnH6ItLHIYk/c2WyLCYxmfgKSOO8iRtw+oUUg/d0n2uKoojzC8/qqc72dHmUAAAAAElFTkSuQmCC",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:41]\u001b[0m [string \"...\"]:47: step=12, q=70.329, r=0.16, a=-0.17\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABmklEQVRoge2XTZmEMBBEn4NIiAQkRMJIiAQkIAEJSBgJSIgEJERC72EWpvmZXXYOk/BtvxuBQxVUdQgYhmFcGA8tTBBLK3mTCAICqbSSN3GzAYGmtJgzeLhtlobZwFBE0V8IkCFv3nWYDWRwpaSdI6nAr6RO843Kq9xAPsxLO6+OpaSdJqrStsuqrrIvqO4c/eHgWarcF5R2mrTvra5y/XhVhmfsK6vyL1kOKkjdY6mrqMoOJpj2u5amUx5ugK+oylrbCOHVc3eVfK+vS1e5VSFfbBz86ziV/ATuNl9Mn9d8oK1be3hsX9twNOvbT0M/pe9zeDXhReVj9Sehd7e4uL6X0nyAV+leMt9pG4vL7GmqqfKGAOPOxvfMd2p3m0bcar7WRVAxXwob0btb5F5NlV8RdzYShGUISaarqcqv6PbTticISI8IQS5wTHN7GyODeC9CFpxUf0wDNtPWIRmRcRBxSfS5oXaeNhpEoohkkdBXXOVDvqdt60RynpJrrvMFNAFS59sr5N8wDMMwDMMwDMMwDMMwDMMw/h1f8xvzL2AgmkIAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:41]\u001b[0m [string \"...\"]:47: step=13, q=70.765, r=0.72, a=0.12\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABc0lEQVRoge3YwXGDMBCF4X8yuUMH0AHqAHWSElJCSkgplJB1BZYrQK7AuILNATtxLsmMZyUPk/1OOi16A3pgg3PO2ADjo/dwh47XiY9ZddIBffRu7jAwKaqznvQVhaHSdZ+sBh2QBci0hA7orQb/wSwASAIEiAEIdoN/ZRjgIEAG+ki3xQAkhAwQCNUeIUsNb4qqqur7JosIxhnVWVX3Q60isnyEICXWYxAilYrINsBZSAhAYKxzjm0DkEjXcxw3GWB3U0RNbzu7km5/KaLTWKWIjO8AHIWFDLSRoUIRmQeo/TqzDyDIWkSxyjm2D3AkLRmgJXZbDAC7lNdFH/qmwPwfSgRAdrIuYogl5t8qE+D7dRZKF1GRAF9FFImli6hIgDOSZV3GsfA5LhIAzpLzugqh7H8shQKQrrcg9LV+XNoaeVk/iWadHr2XOw06X77qmpJF9Fxs8kHS0uckSdqFc7HLOOecc84555xzzjnnnHPu//kELcCHijwO/GoAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:41]\u001b[0m [string \"...\"]:47: step=14, q=71.613, r=1.47, a=0.27\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABbUlEQVRoge3WwXGDMBSE4X/SgNKBSlAJuAPRAe6AUijB6QBKoAOcCjSpQEkFm4M8uduWxTB534kT7wErZsEYY/DBj+PgmCHuvcv95oQ0Shq84NJ4+tvzt1ivwAqceuD0/A1bCwMSUs6zQBD23uheniSkWVJ0grHl8AoR4ovlCiwcN0XdiOQlpVuKDseThZQkBSfomk2uESH+UrQC54OmKI5IUdI2C7a913mAR3KSJHkErs3YShGipOgHrkAfaZaieg8Ay0r5mR71GLiA1ElSliPtvc5DtkQ5BkMU+AYTa0YI+FgoKeoPmiIfbtVaeYdqXceWQklRjLnBuMoRAtblE76Bvn8/YLWGEJAuknJqXa2rSXkoKerCvPcuD5kuvjzANB2xWkOMt2qdUtNqXVPOU/kIIUwvHVT/L1Qsy1ou+vPpRSNea4i3ar2ldtW6JgfK27alaby0KUX1uQO+d2OMMcYYY4wxxhhjjDHmX/gF7vrFvaHLc5QAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:41]\u001b[0m [string \"...\"]:47: step=15, q=71.243, r=0.73, a=-0.36\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABcklEQVRoge2Yu23DMBBAH5L00gZimU7cwNrMo5kbhJogdJmOmuBSkHYaI0CCOwVC7gECWN3p6T40DI7zv3nWDzlBnOGVbYQP/fh2nOHtjMhpFuEscNkh6ZNirAix9gNBMfB3aArk/pSbQFQMvgcnkAkRLiK8C8hfv9EPGUBABBFBmsBgnVSzhTYotC5KMzCzQxdpCtAE7nM86sZ+jLJAos9yABaOV4FMq0BdaBUwr4KBQAJqZKc2Uha4Qq1AHmEKHK+FgLz2QwzqsR+gL5CAkmkffz5iBYCy0ZbQeLQZ4Osuq4EmYXsZ6wustE2aAwwj1nOsLwCk1A/R/veoiUCutDZaAq2N7LARuLXRDpvURgAolb5Jg0WKOyYCK/0qCDCMwSLFHRMBIOd+iItRho6ZQO0G8YhDDOS0tcNxK0BfRCMwGWWxReQiIiICJ7ssL3ahcyXkkmuZqFe7NHaY/6PiOI7jOI7jOI7jOI7jOI7jOM7v+AQ4sW0GJqerzAAAAABJRU5ErkJggg==",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:41]\u001b[0m [string \"...\"]:47: step=16, q=71.838, r=2.27, a=-0.04\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABFklEQVRoge3WwW3DMAyFYabIPdog2sDewN4so6UjqBOE3UCegD1IKlwgQS/9CwR534mHgC+CJFpmIq/tQDSdzlPKycyKl+1zIyJY18s1IiJiuSx01hsdQNMC7ileWrHmlei/hyygeu1VJtr/oCN0j5u3IvNbwCygeCvynIn+e8wR+v52JaT9HnOJR8H/f+YpYWYR0QMOVESDTaH3UcCPCY3RB4r1j/FiExTRUAuoddzkxN5k7AiVUcxUQkMtwMcszfAs5RbQtyDDW4AdIR9FphIa7BJ7L1KGEjp8jM62QhEN9yGrv//kLzz9GKWc7BzdDQ0Cn4r/8yB9+sfckWvtxSy5u58m2z64HBEREREREREREREReTVf6a9Ea6/scvMAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:41]\u001b[0m [string \"...\"]:47: step=17, q=72.006, r=2.97, a=0.02\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAA/UlEQVRoge3YwQ3CMAyF4Qfi3o6QDegG7eZlhLBB2SBMEA7FFUj0+Ecg3nfyyZYVxw1IZv/tQCQ9d+oHSedc+vtVUpbuRCHKPKpW1TqPc5WqNHK1jlzqNtzAJ5cIJiL7O+YEcgQDkv4V08C2cnok/SukgYYHwDRQIuAPgNtCWxMdVWGFNLDoOUZJ+BghDdwiSET2d/6Q7Wh1BbAGcqzS4RfvgNRulfoO7LhEMEEFAj9CiaqwwhrY3kOJqrDCtlAE9HsO/w7Q7zlvoR1LzFCaoApPVAO3+FGWoALBI7SvzXvuxKXOJaeiZVHqyvWn/lk0MzMzMzMzMzMzM7Pv9gDMNi5u4dFm9wAAAABJRU5ErkJggg==",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:41]\u001b[0m [string \"...\"]:47: step=18, q=72.135, r=2.93, a=-0.12\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAw0lEQVRoge3VsQ3CMBSE4QPRhxHYAG/gbI7ZIGzgEcwEpsHpSIMulpX/q16BfDrxHEvAsZ1sJ0dNikHSU0XPly3HpirWWmutqg9jzNl49i4o0JuvQG7D3RYh7VLgaouQWKFNcxuSMcRYYGpD8WU4C6TwHebFliHuQH8U+Gm9uqO+A+vVDVu/+hsr1JuxwLr7gz5kS9v9MOhD9m7DqF+hnVCgN2uB1Iboy+Af6O3iOzpnpVRuWkq2PmUAAAAAAAAAAOBoPuqoIgxFkdvoAAAAAElFTkSuQmCC",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:41]\u001b[0m [string \"...\"]:47: step=19, q=72.287, r=2.90, a=0.18\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABcElEQVRoge2YMVIDMQxF3wB9fIP4BnFJt74B3IgjcYQ4J8CcAKekwimpRGFChoE0jGSGQa9ZVV/7RuPV7oLj/G8u1RNXC9ed8MqzerQxa+7uF3kRuZc7trO6XulFBdotAIlE0MudyCKIiIg8rOW37+VnbBF5ERG5EVhNaXmhmlaoVIDEQlKNPoeuQKXRATKRqBp9Dm2BjwmkPymwp/QOEEjrqBp9Dl0BaLWOIuWoHP092gL70kYRUlwrZ89hYTt2wVaWGf20J0Cl0gAymY12+lfUBQ601kYZNxNeKNQFoNY2ipQm7DIDgd2HQP6bE6CcDOwPgYXArh93wYT3IQsB6nGb5Wj+KDUROBwFyOYjMBE4jSAl6xHYCOxrH0XIwfi7xkaAXSmjiCnZGhgJQCvjGoy3mZVAOZ2CbNRiYCXQjwIhzvk2Vme1kXeeTPso/tj6zOGxF2qv1MoKDlZ9HMdxHMdxHMdxHMdxHMdxHMf5f7wB8/N3tDqB1coAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:41]\u001b[0m [string \"...\"]:47: step=20, q=72.443, r=1.54, a=-0.15\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABh0lEQVRoge3YzXHCMBCG4XeS3O0OrA5QB1YJ6YiSkg4wFURUgOhAHDltDjIMCRNy2WXGwz4XND6s5pu1fgw499xe9UuOUEdOASqc9OsbGeBjzV74kA1rAYHxAfO+aRXq4b3MA6JW1f+9aBXaQS0APdMQ2rOkVfwOtQBA3s6DuMQOABOQgRxp739SLP4X1Q4ABagJCIqF79EOkIGcaOv4gW+Slv2ICCKy2rSd1JxmB4Bc2m85r+OVbv1b2gEOUIESexhop4Ip5QATbRlc1nHQrX9LuwNc7aSJ5QU4Qs6012iMLC8A8zrOQAwsM8CPC4X5ItY3gnwhInsRRvOjQL8DW9qFggBdADr1Ka7pBwCmTNtRo/2FwiRAnuZBsr8N2QQozBvR+TiwYxPgALUCab5QGDIJsIOaM9DDEJLFFBcmAYCcaxvEZDTDzCrAdL5YJ+MPZLMOlEMbxLDAwxjoQGQvIiJi+lmj9sfWL0con6X2NU+1GzgejOZxzjnnnHPOOeecc84559wT+gZz4XOKhQUIgwAAAABJRU5ErkJggg==",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:42]\u001b[0m [string \"...\"]:47: step=21, q=71.787, r=1.51, a=-0.21\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAA1UlEQVRoge3VsQ2DMBSE4UuUPh7BG8AGsDlkA2cDZwOYwCmIaSJS5aDg/xqMhHw64QcScG4X076dpEYKzUNBSZqTNJuyLIpUBpUyqBR1ZWnkcDXtuxsKHI0CG8bPta/Dm0xBu70B1zeUI3Q0CmxYZ7Y3BVSuApNp3y/WI9TURfZlWAuEusi+DIZ4wzrErSmgsg9x+PXUH3CEjmYtEOsi+zJcBbK0FIjrnYerwKsuoimgYgaORoFt67/s7suQbr6tU8pRY1Ybp+npiwEAAAAAAAAAAKfzBhMdI20X9kjNAAAAAElFTkSuQmCC",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:42]\u001b[0m [string \"...\"]:47: step=22, q=71.945, r=2.97, a=0.04\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAyElEQVRoge3WsRHCMBBE0YMGcAdWB7gDuzOVhujAdHDuQB2IAC5E2aLA/0UbaefmpBmZAed2UR4+3+a0JDPz6sfrUFZp5DW31lpr+ZFVHVfVwR8pQlU1aAfY0jfsu6pCvIEgW8C/BtBhgNHEjzhCkVWIN7BFeKoauEKjMUDXFkH2BNhAzz2C7CNk2gGmCLqfEFeoZ41QdB1sYDzlACmCC0uEA0wpkutKlAMsEYqugzcwHgP85u6lFKulVte1AAAAAAAAAACA83kDVb8leozk9jgAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:42]\u001b[0m [string \"...\"]:47: step=23, q=72.343, r=2.93, a=0.11\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABlElEQVRoge3YwW3CMBTG8b/a3pMNkg3Isbdkg7IBI3QEVugmwAR4g7oTNDn2hDPB68GmqlShqug9EOr7XWwu7+XDjkMA5/63e/2SDVXHI7QzNXzoNzDRwAo2rJ8QekHWCOztGz8o1VnCC0BKsANolQr/5k6pTihjHMqkVSp8QQcQGkEWvSAHBF6vfU1/sgEBQZ4RRPIHc1pbCNiW8WsX9Xq1T1MMEMqYhjKp9WqfphhgghEg1OXL7/Rqn6YYgLIICRgC5SCy3ke6AbYAERjgJk/S6niSyl6qfJLe1grMEJnyfLi5mzgLAJGYd1GfB0M2ARIpX7n9IqgH2EFO0UFzgZNUPQAQSHkyXOAgsgkQiZADXORppmxBJb2IyLsczN9pLFbgjTnvoZa6oTLo8I1FACAcf9otrfeQVYDxeB+3rU2HI6MA2ynmya0GmBhHAGq6hU2LwigAjKFM6qVVC1vV6l2yfXPtazlPtS8BpLc8SbX+2PppJsSRekwh1Q3zZNbIkPEzzDnnnHPOOeecc84555xz5/kEWpt7X7R58YkAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:42]\u001b[0m [string \"...\"]:47: step=24, q=72.410, r=2.26, a=0.05\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABoElEQVRoge3Y0W2DMBSF4V9dwN3A2cDdoNkgbEA2YISMkBHCBskGYQPTCUgmQJ3g9AHz1oeiGBDq/Z4QUux742MjAGOMIQSogBNEEIS1K/qzCroOiYPAX0EgKBcr4O3F3+9g1wIUQNGOtz9eHHY5AVQi0Qvi57gC97XrmqJzSEOKHF5jDwt5NUJA/Q03gCPf7B/j7e3sYz+mSMJd7wvv4wwr8IS2SddF0eCGy93rI/9FhgaA+gktwJGWYri3zzLyQhyoGlLk/XWIUL92URNdfdoGlXrc0INfYN48EQJuv6RoicdZvgaAGuCDJizYQE6XlKLDWcJt7XkMHEARKXQS5Qb3MdCns0ghnUVu7YomGlN0v6QUfa5d0UQhpejcpxSd5p4x2yk0+IJHDbTvcCjY3kEEnD1SkHSRcN3a5Uw3pkiSXDn7Ps4cIeAL2hpogGL+FOVvAKhvwIOhAbefY4p5eVCsxhRdZp1rlhV4QlOnLxTHYoMHEVD6tI+jtLnHMQyvODHG2FfVHbfEi0F+bpP/vDHGGGOMMcYYY4wxxhjzD/wAmhi7psPqV+wAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:42]\u001b[0m [string \"...\"]:47: step=25, q=71.184, r=0.66, a=-0.51\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABRUlEQVRoge2YwVHFMAxEF4b7TwffHSQdJB1QEiXQCh3EdJAS/DtwKjAHRXBguGk942HfRTs+SFmP7CgBhPjfPJHyzsAEADhmnBOAO46E8wBOUsVgdqABDVgv9Y7VFt5iCz3HpvubxUWOzcs3kDxUE5VeMYSfFrLQmi8E06eFVhdHeOo+BiYX8f3DMlAsrHZ4Fz+6ObwQ2cC1+cuv5TA6t1AJT002kCxs/uSf4RU6GLgDtvWUN0CHFkou4u9QdDGwuRjKQLawYbMTnDFuC7HGOKOngcLI3uEWmgB7+AejAtuAj3GPTKrAbqHkgvUZwDJQAdwAJJ8iKHcol7aitb21hr0Br6Qq7BYifgkYbAPfd2j8GGcwDSzzJUrhFWEamLx/RjVAHuOMF17qUnLNH3VLt1oG+Z8ohBBCCCGEEEIIIYQQQogh+AJPqk5RqABiTgAAAABJRU5ErkJggg==",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:42]\u001b[0m [string \"...\"]:47: step=26, q=72.158, r=2.91, a=0.11\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABQUlEQVRoge2YwVHDMBBFHwx31AHqwOrA7pgSolQQpQNRAaYCcbBFTsxw0DeTyX8X72k3z9q1dgLGPDZP4vzz/pzOBDiLq41mggYNLlx4b3w2TjPTf/+svzN3gRONS6M12u1QxvA8NNsvvAEkYIXRfXSIQOxvvVBG5z5EIJG2oN6pQLgJ1NG5DxFYiFuQyaNzH3QCyxas99lC095A1Ctfo5PrBSbhBHOEQLh3gaULCC4BDj0BwQQjFojAWyJQgTXzIaghF0gRqEBZJTXkLZTiHuSrJL9YYOpXmGCJ2BALhL5EaL6hqAVef7aggqaDxAIp7EHWTDBqgaVvQSWrSohPIO5BraoSWoEY96Ao7jBAKzDftiDdH0JCgblPsOgO3hAKJP0EAy+61GuuJecUqGvVVTHGGGOMMcYYY4wxxhhjzOPxDc/TT56mUHj9AAAAAElFTkSuQmCC",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:42]\u001b[0m [string \"...\"]:47: step=27, q=72.117, r=2.12, a=0.35\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABgElEQVRoge3XzW2EMBCG4VdJ7tABdIA7wB0kHaSELSGl0MJ2gNOBtwKWCmCPOU0OJgqHSFFWHq9WmeeAfJrhkz38gDH/26NG0R5aWOFDo7qOCl4ZBqa3cUFAoL/1Pf1JB8IiyDQK0pQL8JCpzglWItAGGjyQLupyBQACIS08Ll/V3+QPEL8C+Hyly+ioBJFeJhEExlvf0BUWJqlERDrpkCItMx4hIBC5QMkxyB+A3Rh0Wav/KHeAALDi0wjXWasXUS2IDGkM4KDfMO8OwCUQOQN4+hI7kDsAx+8ArsQkZw8Q9gHucQdm4nkFqHFdm7t6GQOSHAq8y7LvAOkUrZDmuFFosKcVIAK84GkVGuxpBJgJaQzA9dpzrBEAYohp4bz2k1QnwClsi9bXlUoHbU0/bk+iUfnXWGcHmInntKq9bgKlALzHY1o455VaKOuetzO03OOvJUA1LluEXvNl9qRW+UKMsW3XGFYamNUaKdL+jDDGGGOMMcYYY4wxxhhjzFU+AdYZj4k99vQVAAAAAElFTkSuQmCC",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:42]\u001b[0m [string \"...\"]:47: step=28, q=72.090, r=1.50, a=-0.22\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABkElEQVRoge3Y0W2DMBDG8b+qvpMNYAPYAG+QbpAROkJG6AgdodmgzgQhE+BMQDLB1wfz0EqtqlZnIqT7iQjkB59P9l0Q4Jyz1kJ/7zX8UQ27F95ObCdBvlblGTQi8aqRNifQLhP6wWSWCAwAgUTIY43JzL+ySeAM1wjQENuQxzqTmX9lkwAQY76nMK98bQkcznAFUmigZrEjZKYGvSMxSeyWa0RmO3CBFAE2pD7ksSUakVkCfFMGjeHkP7FM4HDM9/jUQcUydWy6A/MvdVAF1pfADYYIJCAE1neEgDgAR+YEFvorsNRXSEjjqEXfiCzphPQuqe4F29LhjI8QczONLHWK7BM4RD7V8QrLoK2RekmTBKd7L+dfphFJktq++BuR/RFiLoMEhI7SjahIAl/LoCkRoqy6RtpL0iTYF41VZAcuF9I1AWzo27KNqEgCQIwpP4TQFApR1vZ5bkSn17V9I8qqFmmUpFFVyUb0WGje25khDZuUYhw2XG+FwhRV3XsBzjnnnHPOOeecc84555z73gfxwZpw2jDjhQAAAABJRU5ErkJggg==",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:42]\u001b[0m [string \"...\"]:47: step=29, q=71.758, r=1.47, a=-0.25\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABR0lEQVRoge2XwW3DMAxFf4verQ3KDeIN7M0yWp0N5A3kCaJOwB4kFYYvAQp9F0H+u5gH41MfNEUaEOK1eWOITtMEICFut2+GPpcrvtzd3a8+nZDu/YQcVPobiIjIADBixtBd/kh/AxkZEQACgLG7/BGGgViCcMLxGQZW5PYJBVh3+SOMJk6tBuOTGtjq+RFOuEcp12hOuUb2ydDfQzJQa2BmDP09FAN1EADGbwLOJG4VwGgU/R2cCmw1mENg6O/h9ACWFtJnGcVAAsooMAAXRgY+XndqB3kW0Ndp9j7KMrDEtk7M3BLQKpDz43d6wDKQkEowki8imoEllSDM3FHAa+JUHux/AloT/35CwUgpCrQK3FoT89chCgNQBtnd/b/P8lf83obxQJxmHzzpHJdkIacMM6wrL5EQQgghhBBCCCGEEEIIIV6MH0bbX1jE9RgaAAAAAElFTkSuQmCC",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 64,
       "width": 64
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m[INFO  11:14:42]\u001b[0m [string \"...\"]:47: step=30, q=71.986, r=2.92, a=0.10\t\n"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_ = test(true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "iTorch",
   "language": "lua",
   "name": "itorch"
  },
  "language_info": {
   "name": "lua",
   "version": "5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}