QR-DQN_cartpole.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "import random\n",
    "import math\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from collections import deque, namedtuple\n",
    "import time\n",
    "import gym\n",
    "\n",
    "\n",
    "def weight_init(layers):\n",
    "    for layer in layers:\n",
    "        torch.nn.init.kaiming_normal_(layer.weight, nonlinearity='relu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QR_DQN(nn.Module):\n",
    "    def __init__(self, state_size, action_size,layer_size, n_step, seed, N, layer_type=\"ff\"):\n",
    "        super(QR_DQN, self).__init__()\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        self.input_shape = state_size\n",
    "        self.action_size = action_size\n",
    "        self.N = N\n",
    "\n",
    "        self.head_1 = nn.Linear(self.input_shape[0], layer_size)\n",
    "        self.ff_1 = nn.Linear(layer_size, layer_size)\n",
    "        self.ff_2 = nn.Linear(layer_size, action_size*N)\n",
    "        weight_init([self.head_1, self.ff_1])\n",
    "\n",
    "\n",
    "    \n",
    "    def forward(self, input):\n",
    "        \"\"\"\n",
    "        \n",
    "        \"\"\"\n",
    "        x = torch.relu(self.head_1(input))\n",
    "        x = torch.relu(self.ff_1(x))\n",
    "        out = self.ff_2(x)\n",
    "        \n",
    "        return out.view(input.shape[0], self.N, self.action_size)\n",
    "    def get_action(self,input):\n",
    "        x = self.forward(input)\n",
    "        return x.mean(dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    \"\"\"Fixed-size buffer to store experience tuples.\"\"\"\n",
    "\n",
    "    def __init__(self, buffer_size, batch_size, device, seed, gamma, n_step=1):\n",
    "        \"\"\"Initialize a ReplayBuffer object.\n",
    "        Params\n",
    "        ======\n",
    "            buffer_size (int): maximum size of buffer\n",
    "            batch_size (int): size of each training batch\n",
    "            seed (int): random seed\n",
    "        \"\"\"\n",
    "        self.device = device\n",
    "        self.memory = deque(maxlen=buffer_size)  \n",
    "        self.batch_size = batch_size\n",
    "        self.experience = namedtuple(\"Experience\", field_names=[\"state\", \"action\", \"reward\", \"next_state\", \"done\"])\n",
    "        self.seed = random.seed(seed)\n",
    "        self.gamma = gamma\n",
    "        self.n_step = n_step\n",
    "        self.n_step_buffer = deque(maxlen=self.n_step)\n",
    "    \n",
    "    def add(self, state, action, reward, next_state, done):\n",
    "        \"\"\"Add a new experience to memory.\"\"\"\n",
    "        #print(\"before:\", state,action,reward,next_state, done)\n",
    "        self.n_step_buffer.append((state, action, reward, next_state, done))\n",
    "        if len(self.n_step_buffer) == self.n_step:\n",
    "            state, action, reward, next_state, done = self.calc_multistep_return()\n",
    "            #print(\"after:\",state,action,reward,next_state, done)\n",
    "            e = self.experience(state, action, reward, next_state, done)\n",
    "            self.memory.append(e)\n",
    "    \n",
    "    def calc_multistep_return(self):\n",
    "        Return = 0\n",
    "        for idx in range(self.n_step):\n",
    "            Return += self.gamma**idx * self.n_step_buffer[idx][2]\n",
    "        \n",
    "        return self.n_step_buffer[0][0], self.n_step_buffer[0][1], Return, self.n_step_buffer[-1][3], self.n_step_buffer[-1][4]\n",
    "        \n",
    "    \n",
    "    \n",
    "    def sample(self):\n",
    "        \"\"\"Randomly sample a batch of experiences from memory.\"\"\"\n",
    "        experiences = random.sample(self.memory, k=self.batch_size)\n",
    "\n",
    "        states = torch.from_numpy(np.stack([e.state for e in experiences if e is not None])).float().to(self.device)\n",
    "        actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(self.device)\n",
    "        rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(self.device)\n",
    "        next_states = torch.from_numpy(np.stack([e.next_state for e in experiences if e is not None])).float().to(self.device)\n",
    "        dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(self.device)\n",
    "  \n",
    "        return (states, actions, rewards, next_states, dones)\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"Return the current size of internal memory.\"\"\"\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN_Agent():\n",
    "    \"\"\"Interacts with and learns from the environment.\"\"\"\n",
    "\n",
    "    def __init__(self,\n",
    "                 state_size,\n",
    "                 action_size,\n",
    "                 Network,\n",
    "                 layer_size,\n",
    "                 n_step,\n",
    "                 BATCH_SIZE,\n",
    "                 BUFFER_SIZE,\n",
    "                 LR,\n",
    "                 TAU,\n",
    "                 GAMMA,\n",
    "                 UPDATE_EVERY,\n",
    "                 device,\n",
    "                 seed):\n",
    "        \"\"\"Initialize an Agent object.\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "            state_size (int): dimension of each state\n",
    "            action_size (int): dimension of each action\n",
    "            Network (str): dqn network type\n",
    "            layer_size (int): size of the hidden layer\n",
    "            BATCH_SIZE (int): size of the training batch\n",
    "            BUFFER_SIZE (int): size of the replay memory\n",
    "            LR (float): learning rate\n",
    "            TAU (float): tau for soft updating the network weights\n",
    "            GAMMA (float): discount factor\n",
    "            UPDATE_EVERY (int): update frequency\n",
    "            device (str): device that is used for the compute\n",
    "            seed (int): random seed\n",
    "        \"\"\"\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.seed = random.seed(seed)\n",
    "        self.device = device\n",
    "        self.TAU = TAU\n",
    "        self.GAMMA = GAMMA\n",
    "        self.UPDATE_EVERY = UPDATE_EVERY\n",
    "        self.BATCH_SIZE = BATCH_SIZE\n",
    "        self.Q_updates = 0\n",
    "        self.n_step = n_step\n",
    "        self.N = 32\n",
    "        self.quantile_tau = torch.FloatTensor([i/self.N for i in range(1,self.N+1)]).to(device)\n",
    "\n",
    "        self.action_step = 4\n",
    "        self.last_action = None\n",
    "\n",
    "        # Q-Network\n",
    "        self.qnetwork_local = QR_DQN(state_size, action_size,layer_size, n_step, seed, self.N).to(device)\n",
    "        self.qnetwork_target = QR_DQN(state_size, action_size,layer_size, n_step, seed, self.N).to(device)\n",
    "\n",
    "        self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)\n",
    "        print(self.qnetwork_local)\n",
    "        \n",
    "        # Replay memory\n",
    "        self.memory = ReplayBuffer(BUFFER_SIZE, BATCH_SIZE, self.device, seed, self.GAMMA, n_step)\n",
    "        \n",
    "        # Initialize time step (for updating every UPDATE_EVERY steps)\n",
    "        self.t_step = 0\n",
    "    \n",
    "    def step(self, state, action, reward, next_state, done, writer):\n",
    "        # Save experience in replay memory\n",
    "        self.memory.add(state, action, reward, next_state, done)\n",
    "        \n",
    "        # Learn every UPDATE_EVERY time steps.\n",
    "        self.t_step = (self.t_step + 1) % self.UPDATE_EVERY\n",
    "        if self.t_step == 0:\n",
    "            # If enough samples are available in memory, get random subset and learn\n",
    "            if len(self.memory) > self.BATCH_SIZE:\n",
    "                experiences = self.memory.sample()\n",
    "                loss = self.learn(experiences)\n",
    "                self.Q_updates += 1\n",
    "                writer.add_scalar(\"Q_loss\", loss, self.Q_updates)\n",
    "\n",
    "    def act(self, state, eps=0.):\n",
    "        \"\"\"Returns actions for given state as per current policy. Acting only every 4 frames!\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "            frame: to adjust epsilon\n",
    "            state (array_like): current state\n",
    "            \n",
    "        \"\"\"\n",
    "\n",
    "        if self.action_step == 4:\n",
    "            state = np.array(state)\n",
    "\n",
    "            state = torch.from_numpy(state).float().unsqueeze(0).to(self.device)\n",
    "            self.qnetwork_local.eval()\n",
    "            with torch.no_grad():\n",
    "                action_values = self.qnetwork_local.get_action(state)\n",
    "            self.qnetwork_local.train()\n",
    "\n",
    "            # Epsilon-greedy action selection\n",
    "            if random.random() > eps: # select greedy action if random number is higher than epsilon or noisy network is used!\n",
    "                action = np.argmax(action_values.cpu().data.numpy())\n",
    "                self.last_action = action\n",
    "                return action\n",
    "            else:\n",
    "                action = random.choice(np.arange(self.action_size))\n",
    "                self.last_action = action \n",
    "                return action\n",
    "            #self.action_step = 0\n",
    "        else:\n",
    "            self.action_step += 1\n",
    "            return self.last_action\n",
    "\n",
    "    def learn(self, experiences):\n",
    "        \"\"\"Update value parameters using given batch of experience tuples.\n",
    "        Params\n",
    "        ======\n",
    "            experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples \n",
    "            gamma (float): discount factor\n",
    "        \"\"\"\n",
    "        self.optimizer.zero_grad()\n",
    "        states, actions, rewards, next_states, dones = experiences\n",
    "        # Get max predicted Q values (for next states) from target model\n",
    "        Q_targets_next = self.qnetwork_target(next_states).detach().cpu() #.max(2)[0].unsqueeze(1) #(batch_size, 1, N)\n",
    "        action_indx = torch.argmax(Q_targets_next.mean(dim=1), dim=1, keepdim=True)\n",
    "\n",
    "        Q_targets_next = Q_targets_next.gather(2, action_indx.unsqueeze(-1).expand(self.BATCH_SIZE, self.N, 1)).transpose(1,2)\n",
    "\n",
    "        assert Q_targets_next.shape == (self.BATCH_SIZE,1, self.N)\n",
    "        # Compute Q targets for current states \n",
    "        Q_targets = rewards.unsqueeze(-1) + (self.GAMMA**self.n_step * Q_targets_next.to(self.device) * (1 - dones.unsqueeze(-1)))\n",
    "        # Get expected Q values from local model\n",
    "        Q_expected = self.qnetwork_local(states).gather(2, actions.unsqueeze(-1).expand(self.BATCH_SIZE, self.N, 1))\n",
    "        # Compute loss\n",
    "        td_error = Q_targets - Q_expected\n",
    "        assert td_error.shape == (self.BATCH_SIZE, self.N, self.N), \"wrong td error shape\"\n",
    "        huber_l = calculate_huber_loss(td_error, 1.0)\n",
    "        quantil_l = abs(self.quantile_tau -(td_error.detach() < 0).float()) * huber_l / 1.0\n",
    "\n",
    "        loss = quantil_l.sum(dim=1).mean(dim=1) # , keepdim=True if per weights get multipl\n",
    "        loss = loss.mean()\n",
    "        # Minimize the loss\n",
    "        loss.backward()\n",
    "        #clip_grad_norm_(self.qnetwork_local.parameters(),1)\n",
    "        self.optimizer.step()\n",
    "\n",
    "        # ------------------- update target network ------------------- #\n",
    "        self.soft_update(self.qnetwork_local, self.qnetwork_target)\n",
    "        return loss.detach().cpu().numpy()            \n",
    "\n",
    "    def soft_update(self, local_model, target_model):\n",
    "        \"\"\"Soft update model parameters.\n",
    "        θ_target = τ*θ_local + (1 - τ)*θ_target\n",
    "        Params\n",
    "        ======\n",
    "            local_model (PyTorch model): weights will be copied from\n",
    "            target_model (PyTorch model): weights will be copied to\n",
    "            tau (float): interpolation parameter \n",
    "        \"\"\"\n",
    "        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):\n",
    "            target_param.data.copy_(self.TAU*local_param.data + (1.0-self.TAU)*target_param.data)\n",
    "            \n",
    "def calculate_huber_loss(td_errors, k=1.0):\n",
    "    \"\"\"\n",
    "    Calculate huber loss element-wisely depending on kappa k.\n",
    "    \"\"\"\n",
    "    loss = torch.where(td_errors.abs() <= k, 0.5 * td_errors.pow(2), k * (td_errors.abs() - 0.5 * k))\n",
    "    assert loss.shape == (td_errors.shape[0], 32, 32), \"huber loss has wrong shape\"\n",
    "    return loss\n",
    "\n",
    "def eval_runs(eps, frame):\n",
    "    \"\"\"\n",
    "    Makes an evaluation run with the current epsilon\n",
    "    \"\"\"\n",
    "    env = gym.make(\"CartPole-v0\")\n",
    "    reward_batch = []\n",
    "    for i in range(5):\n",
    "        state = env.reset()\n",
    "        rewards = 0\n",
    "        while True:\n",
    "            action = agent.act(state, eps)\n",
    "            state, reward, done, _ = env.step(action)\n",
    "            rewards += reward\n",
    "            if done:\n",
    "                break\n",
    "        reward_batch.append(rewards)\n",
    "        \n",
    "    writer.add_scalar(\"Reward\", np.mean(reward_batch), frame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                              | 46/45000 [00:00<01:39, 452.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using  cpu\n",
      "QR_DQN(\n",
      "  (head_1): Linear(in_features=4, out_features=256, bias=True)\n",
      "  (ff_1): Linear(in_features=256, out_features=256, bias=True)\n",
      "  (ff_2): Linear(in_features=256, out_features=64, bias=True)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|▏                                                                             | 92/45000 [00:00<03:02, 246.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 2\tFrame 93 \tAverage Score: 46.50"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|▏                                                                            | 122/45000 [00:00<03:34, 209.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 3\tFrame 109 \tAverage Score: 36.33"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|▎                                                                            | 169/45000 [00:00<03:40, 202.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 4\tFrame 152 \tAverage Score: 38.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|▎                                                                            | 213/45000 [00:01<03:49, 194.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 5\tFrame 193 \tAverage Score: 38.60"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▍                                                                            | 253/45000 [00:01<03:58, 187.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 8\tFrame 268 \tAverage Score: 33.50"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▌                                                                            | 315/45000 [00:01<03:51, 192.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 11\tFrame 327 \tAverage Score: 29.73"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▌                                                                            | 335/45000 [00:01<04:15, 174.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 12\tFrame 341 \tAverage Score: 28.42"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▋                                                                            | 391/45000 [00:02<04:24, 168.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 13\tFrame 363 \tAverage Score: 27.92"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▋                                                                            | 429/45000 [00:02<04:22, 169.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 15\tFrame 412 \tAverage Score: 27.47"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▊                                                                            | 469/45000 [00:02<04:08, 179.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 17\tFrame 470 \tAverage Score: 27.65"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▊                                                                            | 507/45000 [00:02<04:05, 181.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 18\tFrame 494 \tAverage Score: 27.44"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▉                                                                            | 563/45000 [00:03<04:14, 174.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 19\tFrame 547 \tAverage Score: 28.79"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█▏                                                                           | 710/45000 [00:03<04:07, 178.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 21\tFrame 716 \tAverage Score: 34.10"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█▎                                                                           | 763/45000 [00:04<04:27, 165.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 22\tFrame 737 \tAverage Score: 33.50"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█▍                                                                           | 813/45000 [00:04<04:40, 157.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 25\tFrame 827 \tAverage Score: 33.08"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█▍                                                                           | 844/45000 [00:04<05:04, 144.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 26\tFrame 839 \tAverage Score: 32.27"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█▌                                                                           | 891/45000 [00:05<04:53, 150.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 28\tFrame 893 \tAverage Score: 31.89"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█▌                                                                           | 924/45000 [00:05<04:47, 153.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 30\tFrame 928 \tAverage Score: 30.93"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█▋                                                                           | 975/45000 [00:05<04:42, 155.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 32\tFrame 983 \tAverage Score: 30.72"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█▋                                                                          | 1022/45000 [00:05<05:21, 136.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 34\tFrame 1031 \tAverage Score: 30.32"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█▊                                                                          | 1069/45000 [00:06<05:04, 144.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 35\tFrame 1056 \tAverage Score: 30.17"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█▉                                                                          | 1115/45000 [00:06<04:54, 148.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 38\tFrame 1110 \tAverage Score: 29.21"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|█▉                                                                          | 1147/45000 [00:06<04:44, 153.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 40\tFrame 1151 \tAverage Score: 28.77"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|█▉                                                                          | 1179/45000 [00:06<04:54, 148.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 42\tFrame 1175 \tAverage Score: 27.98"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██                                                                          | 1211/45000 [00:07<04:55, 148.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 43\tFrame 1199 \tAverage Score: 27.88"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▏                                                                         | 1272/45000 [00:07<04:56, 147.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 44\tFrame 1249 \tAverage Score: 28.39"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▏                                                                         | 1305/45000 [00:07<04:48, 151.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 46\tFrame 1305 \tAverage Score: 28.37"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▎                                                                         | 1370/45000 [00:08<04:49, 150.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 48\tFrame 1355 \tAverage Score: 28.23"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▍                                                                         | 1439/45000 [00:08<04:34, 158.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 50\tFrame 1443 \tAverage Score: 28.86"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▍                                                                         | 1471/45000 [00:08<04:44, 152.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 51\tFrame 1457 \tAverage Score: 28.57"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▌                                                                         | 1553/45000 [00:09<04:56, 146.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 52\tFrame 1532 \tAverage Score: 29.46"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|██▋                                                                         | 1599/45000 [00:09<04:57, 145.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 54\tFrame 1601 \tAverage Score: 29.65"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|██▊                                                                         | 1668/45000 [00:10<04:37, 156.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 56\tFrame 1665 \tAverage Score: 29.73"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|██▉                                                                         | 1714/45000 [00:10<05:09, 139.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 59\tFrame 1723 \tAverage Score: 29.20"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|███                                                                         | 1786/45000 [00:11<05:23, 133.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 61\tFrame 1791 \tAverage Score: 29.36"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|███                                                                         | 1800/45000 [00:11<05:23, 133.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 62\tFrame 1809 \tAverage Score: 29.18"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|███                                                                         | 1843/45000 [00:11<05:29, 130.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 64\tFrame 1846 \tAverage Score: 28.84"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|███▏                                                                        | 1897/45000 [00:12<05:36, 128.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 66\tFrame 1892 \tAverage Score: 28.67"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|███▏                                                                        | 1924/45000 [00:12<05:29, 130.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 67\tFrame 1907 \tAverage Score: 28.46"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|███▎                                                                        | 1952/45000 [00:12<05:28, 131.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 68\tFrame 1941 \tAverage Score: 28.54"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|███▎                                                                        | 1992/45000 [00:12<05:51, 122.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 69\tFrame 1986 \tAverage Score: 28.78"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▍                                                                        | 2030/45000 [00:13<06:41, 107.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 70\tFrame 2012 \tAverage Score: 28.74"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▋                                                                        | 2170/45000 [00:14<06:01, 118.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 72\tFrame 2165 \tAverage Score: 30.07"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▊                                                                        | 2226/45000 [00:14<05:32, 128.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 73\tFrame 2208 \tAverage Score: 30.25"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▊                                                                        | 2252/45000 [00:15<05:34, 127.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 75\tFrame 2256 \tAverage Score: 30.08"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▊                                                                        | 2292/45000 [00:15<05:39, 125.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 76\tFrame 2269 \tAverage Score: 29.86"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▉                                                                        | 2333/45000 [00:15<05:36, 126.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 78\tFrame 2333 \tAverage Score: 29.91"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▉                                                                        | 2359/45000 [00:15<06:14, 113.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 80\tFrame 2366 \tAverage Score: 29.57"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|████                                                                        | 2400/45000 [00:16<05:53, 120.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 81\tFrame 2388 \tAverage Score: 29.48"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|████▏                                                                       | 2467/45000 [00:16<05:43, 123.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 83\tFrame 2457 \tAverage Score: 29.60"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▏                                                                       | 2494/45000 [00:16<05:40, 124.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 84\tFrame 2473 \tAverage Score: 29.44"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▍                                                                       | 2610/45000 [00:17<05:22, 131.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 86\tFrame 2605 \tAverage Score: 30.29"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▍                                                                       | 2639/45000 [00:18<05:22, 131.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 88\tFrame 2639 \tAverage Score: 29.99"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▌                                                                       | 2680/45000 [00:18<05:35, 126.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 90\tFrame 2678 \tAverage Score: 29.76"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▋                                                                       | 2749/45000 [00:18<05:31, 127.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 92\tFrame 2744 \tAverage Score: 29.83"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▋                                                                       | 2791/45000 [00:19<05:24, 130.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 94\tFrame 2798 \tAverage Score: 29.77"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▊                                                                       | 2821/45000 [00:19<05:35, 125.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 96\tFrame 2826 \tAverage Score: 29.44"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▉                                                                       | 2898/45000 [00:20<06:31, 107.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 97\tFrame 2887 \tAverage Score: 29.76"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|█████                                                                       | 3026/45000 [00:21<06:34, 106.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 99\tFrame 3028 \tAverage Score: 30.59"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|█████▏                                                                      | 3078/45000 [00:21<05:50, 119.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tFrame 3054\tAverage Score: 30.544\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|█████▌                                                                      | 3262/45000 [00:23<05:45, 120.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 101\tFrame 3244 \tAverage Score: 31.80"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|█████▋                                                                      | 3343/45000 [00:23<05:42, 121.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 102\tFrame 3329 \tAverage Score: 32.36"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|█████▉                                                                      | 3499/45000 [00:25<05:11, 133.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 103\tFrame 3481 \tAverage Score: 33.72"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|██████                                                                      | 3597/45000 [00:25<05:10, 133.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 104\tFrame 3581 \tAverage Score: 34.29"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|██████▎                                                                     | 3746/45000 [00:27<05:19, 129.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 105\tFrame 3738 \tAverage Score: 35.45"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▋                                                                     | 3944/45000 [00:28<05:08, 133.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 106\tFrame 3922 \tAverage Score: 36.82"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▋                                                                     | 3971/45000 [00:28<05:22, 127.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 108\tFrame 3964 \tAverage Score: 36.96"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▊                                                                     | 4038/45000 [00:29<06:40, 102.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 109\tFrame 4019 \tAverage Score: 37.30"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▊                                                                     | 4062/45000 [00:29<06:13, 109.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 110\tFrame 4047 \tAverage Score: 37.32"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▉                                                                     | 4141/45000 [00:30<05:37, 121.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 111\tFrame 4120 \tAverage Score: 37.93"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|███████                                                                     | 4198/45000 [00:30<05:23, 126.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 112\tFrame 4175 \tAverage Score: 38.34"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|███████▎                                                                    | 4321/45000 [00:31<05:17, 127.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 113\tFrame 4304 \tAverage Score: 39.41"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|███████▍                                                                    | 4429/45000 [00:32<05:19, 126.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 114\tFrame 4411 \tAverage Score: 40.12"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|███████▌                                                                    | 4472/45000 [00:32<05:10, 130.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 115\tFrame 4460 \tAverage Score: 40.48"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|███████▋                                                                    | 4559/45000 [00:33<05:13, 128.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 116\tFrame 4538 \tAverage Score: 40.91"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|███████▊                                                                    | 4642/45000 [00:34<05:12, 129.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 117\tFrame 4624 \tAverage Score: 41.54"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|████████▏                                                                   | 4811/45000 [00:35<05:20, 125.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 118\tFrame 4797 \tAverage Score: 43.03"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|████████▎                                                                   | 4900/45000 [00:36<04:59, 133.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 119\tFrame 4883 \tAverage Score: 43.36"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|████████▌                                                                   | 5094/45000 [00:37<05:13, 127.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 120\tFrame 5073 \tAverage Score: 43.89"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|████████▋                                                                   | 5177/45000 [00:38<05:15, 126.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 121\tFrame 5162 \tAverage Score: 44.46"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|████████▉                                                                   | 5263/45000 [00:39<04:55, 134.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 122\tFrame 5240 \tAverage Score: 45.03"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█████████▏                                                                  | 5460/45000 [00:40<05:03, 130.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 123\tFrame 5440 \tAverage Score: 46.40"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█████████▍                                                                  | 5555/45000 [00:41<05:07, 128.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 124\tFrame 5544 \tAverage Score: 47.29"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█████████▌                                                                  | 5653/45000 [00:42<05:15, 124.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 125\tFrame 5630 \tAverage Score: 48.03"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█████████▋                                                                  | 5723/45000 [00:42<05:07, 127.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 126\tFrame 5704 \tAverage Score: 48.65"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█████████▋                                                                  | 5765/45000 [00:43<05:11, 125.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 127\tFrame 5753 \tAverage Score: 48.84"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█████████▉                                                                  | 5863/45000 [00:43<05:01, 129.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 128\tFrame 5840 \tAverage Score: 49.47"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|██████████                                                                  | 5958/45000 [00:44<05:07, 126.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 129\tFrame 5938 \tAverage Score: 50.32"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|██████████▏                                                                 | 6026/45000 [00:45<06:16, 103.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 130\tFrame 6016 \tAverage Score: 50.88"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|██████████▎                                                                 | 6120/45000 [00:46<05:12, 124.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 131\tFrame 6105 \tAverage Score: 51.52"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|██████████▍                                                                 | 6189/45000 [00:46<04:56, 131.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 132\tFrame 6166 \tAverage Score: 51.83"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|██████████▌                                                                 | 6230/45000 [00:46<05:05, 126.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 133\tFrame 6218 \tAverage Score: 52.13"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|██████████▋                                                                 | 6336/45000 [00:47<05:13, 123.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 134\tFrame 6316 \tAverage Score: 52.85"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|██████████▊                                                                 | 6413/45000 [00:48<05:41, 113.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 135\tFrame 6397 \tAverage Score: 53.41"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|███████████                                                                 | 6514/45000 [00:49<05:23, 119.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 136\tFrame 6490 \tAverage Score: 54.06"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|███████████▏                                                                | 6591/45000 [00:50<05:31, 115.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 137\tFrame 6577 \tAverage Score: 54.80"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|███████████▎                                                                | 6699/45000 [00:50<05:12, 122.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 138\tFrame 6678 \tAverage Score: 55.68"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|███████████▍                                                                | 6776/45000 [00:51<05:26, 117.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 139\tFrame 6763 \tAverage Score: 56.39"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|███████████▌                                                                | 6858/45000 [00:52<05:07, 124.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 140\tFrame 6841 \tAverage Score: 56.90"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|███████████▋                                                                | 6939/45000 [00:52<05:02, 125.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 141\tFrame 6920 \tAverage Score: 57.58"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|███████████▊                                                                | 6993/45000 [00:53<05:01, 126.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 142\tFrame 6999 \tAverage Score: 58.24"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|███████████▉                                                                | 7104/45000 [00:54<05:01, 125.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 143\tFrame 7083 \tAverage Score: 58.84"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|████████████▏                                                               | 7197/45000 [00:55<05:00, 125.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 144\tFrame 7173 \tAverage Score: 59.24"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|████████████▎                                                               | 7273/45000 [00:55<05:35, 112.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 145\tFrame 7250 \tAverage Score: 59.65"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|████████████▍                                                               | 7343/45000 [00:56<06:03, 103.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 146\tFrame 7334 \tAverage Score: 60.29"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|████████████▌                                                               | 7434/45000 [00:57<05:39, 110.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 147\tFrame 7415 \tAverage Score: 60.72"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|████████████▋                                                               | 7527/45000 [00:57<05:03, 123.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 149\tFrame 7520 \tAverage Score: 61.01"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|████████████▊                                                               | 7621/45000 [00:58<04:55, 126.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 150\tFrame 7610 \tAverage Score: 61.67"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█████████████                                                               | 7721/45000 [00:59<04:41, 132.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 151\tFrame 7704 \tAverage Score: 62.47"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█████████████▏                                                              | 7792/45000 [00:59<04:39, 132.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 152\tFrame 7782 \tAverage Score: 62.50"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████▎                                                              | 7875/45000 [01:00<04:46, 129.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 153\tFrame 7862 \tAverage Score: 62.77"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████▍                                                              | 7953/45000 [01:01<05:09, 119.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 154\tFrame 7941 \tAverage Score: 63.40"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████▌                                                              | 8045/45000 [01:02<05:24, 113.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 155\tFrame 8031 \tAverage Score: 63.82"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████▋                                                              | 8139/45000 [01:02<04:48, 127.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 156\tFrame 8116 \tAverage Score: 64.51"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████▊                                                              | 8208/45000 [01:03<04:46, 128.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 157\tFrame 8193 \tAverage Score: 64.92"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|██████████████                                                              | 8305/45000 [01:04<04:35, 133.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 158\tFrame 8290 \tAverage Score: 65.76"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|██████████████▏                                                             | 8387/45000 [01:04<04:48, 127.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 159\tFrame 8370 \tAverage Score: 66.47"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|██████████████▎                                                             | 8480/45000 [01:05<04:45, 127.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 160\tFrame 8455 \tAverage Score: 66.89"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|██████████████▍                                                             | 8558/45000 [01:06<04:54, 123.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 161\tFrame 8542 \tAverage Score: 67.51"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|██████████████▌                                                             | 8639/45000 [01:06<04:48, 126.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 162\tFrame 8620 \tAverage Score: 68.11"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|██████████████▋                                                             | 8731/45000 [01:07<04:44, 127.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 163\tFrame 8709 \tAverage Score: 68.87"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██████████████▊                                                             | 8800/45000 [01:08<04:36, 130.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 164\tFrame 8789 \tAverage Score: 69.43"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|███████████████                                                             | 8898/45000 [01:08<04:35, 131.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 165\tFrame 8872 \tAverage Score: 69.94"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|███████████████▏                                                            | 8969/45000 [01:09<04:28, 134.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 166\tFrame 8953 \tAverage Score: 70.61"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|███████████████▎                                                            | 9064/45000 [01:10<05:11, 115.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 167\tFrame 9044 \tAverage Score: 71.37"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|███████████████▍                                                            | 9145/45000 [01:10<04:38, 128.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 168\tFrame 9130 \tAverage Score: 71.89"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|███████████████▌                                                            | 9226/45000 [01:11<04:38, 128.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 169\tFrame 9213 \tAverage Score: 72.27"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|███████████████▋                                                            | 9281/45000 [01:11<04:36, 129.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 170\tFrame 9265 \tAverage Score: 72.53"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|███████████████▊                                                            | 9375/45000 [01:12<04:39, 127.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 171\tFrame 9352 \tAverage Score: 72.02"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|███████████████▉                                                            | 9452/45000 [01:13<04:59, 118.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 172\tFrame 9437 \tAverage Score: 72.72"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|████████████████                                                            | 9524/45000 [01:14<05:53, 100.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 173\tFrame 9515 \tAverage Score: 73.07"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|████████████████▏                                                           | 9606/45000 [01:14<05:22, 109.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 174\tFrame 9599 \tAverage Score: 73.60"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|████████████████▎                                                           | 9689/45000 [01:15<05:19, 110.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 175\tFrame 9677 \tAverage Score: 74.21"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|████████████████▌                                                           | 9774/45000 [01:16<05:12, 112.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 176\tFrame 9761 \tAverage Score: 74.92"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|████████████████▋                                                           | 9859/45000 [01:17<05:18, 110.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 177\tFrame 9846 \tAverage Score: 75.29"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|████████████████▊                                                           | 9957/45000 [01:17<05:09, 113.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 178\tFrame 9944 \tAverage Score: 76.11"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|████████████████▊                                                          | 10061/45000 [01:18<04:53, 118.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 179\tFrame 10037 \tAverage Score: 76.85"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|████████████████▉                                                          | 10139/45000 [01:19<04:47, 121.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 180\tFrame 10120 \tAverage Score: 77.54"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|█████████████████                                                          | 10219/45000 [01:20<04:28, 129.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 181\tFrame 10200 \tAverage Score: 78.12"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|█████████████████▏                                                         | 10298/45000 [01:20<04:42, 122.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 182\tFrame 10283 \tAverage Score: 78.38"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|█████████████████▎                                                         | 10379/45000 [01:21<04:28, 129.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 183\tFrame 10369 \tAverage Score: 79.12"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|█████████████████▍                                                         | 10475/45000 [01:22<04:31, 127.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 184\tFrame 10458 \tAverage Score: 79.85"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|█████████████████▌                                                         | 10554/45000 [01:22<04:33, 125.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 185\tFrame 10530 \tAverage Score: 79.42"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|█████████████████▋                                                         | 10646/45000 [01:23<04:29, 127.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 186\tFrame 10623 \tAverage Score: 80.18"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|█████████████████▊                                                         | 10724/45000 [01:24<04:43, 121.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 187\tFrame 10708 \tAverage Score: 80.89"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██████████████████                                                         | 10812/45000 [01:24<04:51, 117.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 188\tFrame 10799 \tAverage Score: 81.60"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██████████████████▏                                                        | 10903/45000 [01:25<04:51, 117.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 189\tFrame 10891 \tAverage Score: 82.31"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██████████████████▎                                                        | 10990/45000 [01:26<04:43, 119.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 190\tFrame 10982 \tAverage Score: 83.04"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██████████████████▍                                                        | 11079/45000 [01:27<04:53, 115.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 191\tFrame 11062 \tAverage Score: 83.33"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██████████████████▌                                                        | 11167/45000 [01:28<04:46, 118.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 192\tFrame 11149 \tAverage Score: 84.05"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██████████████████▊                                                        | 11256/45000 [01:28<04:41, 119.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 193\tFrame 11245 \tAverage Score: 84.62"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██████████████████▉                                                        | 11355/45000 [01:29<04:48, 116.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 194\tFrame 11338 \tAverage Score: 85.40"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|███████████████████                                                        | 11463/45000 [01:30<04:59, 111.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 195\tFrame 11452 \tAverage Score: 86.42"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|███████████████████▎                                                       | 11564/45000 [01:31<04:43, 117.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 196\tFrame 11546 \tAverage Score: 87.20"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|███████████████████▍                                                       | 11651/45000 [01:32<04:34, 121.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 197\tFrame 11635 \tAverage Score: 87.48"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|███████████████████▌                                                       | 11740/45000 [01:32<04:41, 118.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 198\tFrame 11725 \tAverage Score: 87.14"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|███████████████████▋                                                       | 11831/45000 [01:33<04:24, 125.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 199\tFrame 11813 \tAverage Score: 87.85"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|███████████████████▉                                                       | 11934/45000 [01:34<04:45, 115.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 200\tFrame 11916\tAverage Score: 88.622\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|████████████████████                                                       | 12033/45000 [01:35<05:14, 104.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 201\tFrame 12014 \tAverage Score: 87.70"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|████████████████████▏                                                      | 12133/45000 [01:36<04:29, 122.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 202\tFrame 12116 \tAverage Score: 87.87"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|████████████████████▍                                                      | 12234/45000 [01:37<04:54, 111.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 203\tFrame 12221 \tAverage Score: 87.40"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|████████████████████▌                                                      | 12335/45000 [01:38<05:08, 105.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 204\tFrame 12317 \tAverage Score: 87.36"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|████████████████████▋                                                      | 12436/45000 [01:39<05:25, 100.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 205\tFrame 12422 \tAverage Score: 86.84"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|████████████████████▉                                                      | 12542/45000 [01:40<04:43, 114.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 206\tFrame 12527 \tAverage Score: 86.05"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|█████████████████████                                                      | 12650/45000 [01:41<04:49, 111.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 207\tFrame 12638 \tAverage Score: 86.86"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|█████████████████████▎                                                     | 12769/45000 [01:42<04:54, 109.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 208\tFrame 12751 \tAverage Score: 87.87"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|█████████████████████▍                                                     | 12872/45000 [01:43<04:56, 108.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 209\tFrame 12859 \tAverage Score: 88.40"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|█████████████████████▊                                                     | 13072/45000 [01:45<04:58, 107.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 210\tFrame 13059 \tAverage Score: 90.12"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|█████████████████████▉                                                     | 13180/45000 [01:46<04:44, 112.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 211\tFrame 13169 \tAverage Score: 90.49"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██████████████████████▏                                                    | 13312/45000 [01:47<04:40, 113.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 212\tFrame 13302 \tAverage Score: 91.27"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██████████████████████▍                                                    | 13482/45000 [01:48<04:34, 114.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 213\tFrame 13473 \tAverage Score: 91.69"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██████████████████████▊                                                    | 13686/45000 [01:50<04:39, 112.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 214\tFrame 13673 \tAverage Score: 92.62"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███████████████████████▏                                                   | 13887/45000 [01:52<05:07, 101.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 215\tFrame 13873 \tAverage Score: 94.13"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███████████████████████▍                                                   | 14090/45000 [01:54<05:04, 101.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 216\tFrame 14073 \tAverage Score: 95.35"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███████████████████████▊                                                   | 14267/45000 [01:56<04:32, 112.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 217\tFrame 14249 \tAverage Score: 96.25"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|████████████████████████                                                   | 14462/45000 [01:58<04:35, 110.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 218\tFrame 14449 \tAverage Score: 96.52"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|████████████████████████▍                                                  | 14641/45000 [01:59<04:44, 106.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 219\tFrame 14628 \tAverage Score: 97.45"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|████████████████████████▋                                                  | 14841/45000 [02:01<04:33, 110.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 220\tFrame 14828 \tAverage Score: 97.55"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|█████████████████████████▍                                                  | 15040/45000 [02:03<05:16, 94.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 221\tFrame 15028 \tAverage Score: 98.66"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|█████████████████████████▍                                                 | 15238/45000 [02:05<04:15, 116.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 222\tFrame 15228 \tAverage Score: 99.88"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|█████████████████████████▋                                                 | 15439/45000 [02:07<04:27, 110.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 223\tFrame 15428 \tAverage Score: 99.88"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|██████████████████████████                                                 | 15645/45000 [02:08<04:23, 111.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 224\tFrame 15628 \tAverage Score: 100.84"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|██████████████████████████▍                                                | 15838/45000 [02:10<04:29, 108.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 225\tFrame 15828 \tAverage Score: 101.98"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███████████████████████████                                                 | 16046/45000 [02:12<05:00, 96.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 226\tFrame 16028 \tAverage Score: 103.24"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███████████████████████████                                                | 16240/45000 [02:14<04:13, 113.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 227\tFrame 16228 \tAverage Score: 104.75"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███████████████████████████▍                                               | 16449/45000 [02:16<04:02, 117.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 228\tFrame 16428 \tAverage Score: 105.88"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███████████████████████████▋                                               | 16646/45000 [02:17<04:02, 116.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 229\tFrame 16628 \tAverage Score: 106.90"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|████████████████████████████                                               | 16843/45000 [02:19<04:07, 113.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 230\tFrame 16828 \tAverage Score: 108.12"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|████████████████████████████▊                                               | 17037/45000 [02:21<05:17, 88.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 231\tFrame 17028 \tAverage Score: 109.23"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|████████████████████████████▋                                              | 17245/45000 [02:23<04:09, 111.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 232\tFrame 17228 \tAverage Score: 110.62"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|█████████████████████████████                                              | 17441/45000 [02:25<04:00, 114.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 233\tFrame 17428 \tAverage Score: 112.10"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|█████████████████████████████▍                                             | 17641/45000 [02:26<03:55, 116.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 234\tFrame 17628 \tAverage Score: 113.12"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|█████████████████████████████▋                                             | 17847/45000 [02:28<03:45, 120.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 235\tFrame 17828 \tAverage Score: 114.31"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|██████████████████████████████▍                                             | 18048/45000 [02:30<04:55, 91.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 236\tFrame 18028 \tAverage Score: 115.38"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|██████████████████████████████▍                                            | 18246/45000 [02:32<03:48, 117.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 237\tFrame 18228 \tAverage Score: 116.51"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|██████████████████████████████▋                                            | 18443/45000 [02:34<03:45, 117.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 238\tFrame 18428 \tAverage Score: 117.50"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|███████████████████████████████                                            | 18645/45000 [02:35<03:43, 118.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 239\tFrame 18628 \tAverage Score: 118.65"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|███████████████████████████████▍                                           | 18848/45000 [02:37<03:40, 118.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 240\tFrame 18828 \tAverage Score: 119.87"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████████████████████████████████▏                                           | 19046/45000 [02:39<04:49, 89.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 241\tFrame 19028 \tAverage Score: 121.08"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████████████████████████████████                                           | 19251/45000 [02:41<03:39, 117.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 242\tFrame 19228 \tAverage Score: 122.29"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████████████████████████████████▍                                          | 19448/45000 [02:42<03:36, 118.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 243\tFrame 19428 \tAverage Score: 123.45"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████████████████████████████████▋                                          | 19647/45000 [02:44<03:34, 118.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 244\tFrame 19628 \tAverage Score: 124.55"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|█████████████████████████████████                                          | 19850/45000 [02:46<03:33, 117.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 245\tFrame 19828 \tAverage Score: 125.78"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|█████████████████████████████████▊                                          | 20041/45000 [02:48<04:24, 94.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 246\tFrame 20028 \tAverage Score: 126.94"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|█████████████████████████████████▋                                         | 20241/45000 [02:49<03:23, 121.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 247\tFrame 20228 \tAverage Score: 128.13"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|██████████████████████████████████                                         | 20445/45000 [02:51<03:27, 118.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 248\tFrame 20428 \tAverage Score: 129.24"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|██████████████████████████████████▍                                        | 20649/45000 [02:53<03:23, 119.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 249\tFrame 20628 \tAverage Score: 131.08"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|██████████████████████████████████▋                                        | 20841/45000 [02:55<03:24, 118.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 250\tFrame 20828 \tAverage Score: 132.18"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|███████████████████████████████████▌                                        | 21044/45000 [02:57<04:13, 94.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 251\tFrame 21028 \tAverage Score: 133.24"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|███████████████████████████████████▍                                       | 21239/45000 [02:58<03:46, 105.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 252\tFrame 21228 \tAverage Score: 134.46"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|███████████████████████████████████▋                                       | 21437/45000 [03:00<03:43, 105.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 253\tFrame 21428 \tAverage Score: 135.66"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████████████████████████████████████                                       | 21645/45000 [03:02<03:28, 111.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 254\tFrame 21628 \tAverage Score: 136.87"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████████████████████████████████████▍                                      | 21845/45000 [03:04<03:16, 118.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 255\tFrame 21828 \tAverage Score: 137.97"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|█████████████████████████████████████▏                                      | 22038/45000 [03:06<04:16, 89.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 256\tFrame 22028 \tAverage Score: 139.12"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|█████████████████████████████████████                                      | 22251/45000 [03:08<03:12, 118.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 257\tFrame 22228 \tAverage Score: 140.35"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████████████████████████████████████▍                                     | 22444/45000 [03:10<03:10, 118.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 258\tFrame 22428 \tAverage Score: 141.38"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████████████████████████████████████▋                                     | 22649/45000 [03:11<03:20, 111.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 259\tFrame 22628 \tAverage Score: 142.58"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|██████████████████████████████████████                                     | 22843/45000 [03:13<03:07, 117.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 260\tFrame 22828 \tAverage Score: 143.73"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|██████████████████████████████████████▉                                     | 23047/45000 [03:15<03:56, 92.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 261\tFrame 23028 \tAverage Score: 144.86"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|██████████████████████████████████████▋                                    | 23246/45000 [03:17<03:07, 116.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 262\tFrame 23228 \tAverage Score: 146.08"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|███████████████████████████████████████                                    | 23446/45000 [03:19<03:07, 115.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 263\tFrame 23428 \tAverage Score: 147.19"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|███████████████████████████████████████▍                                   | 23643/45000 [03:20<03:03, 116.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 264\tFrame 23628 \tAverage Score: 148.39"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|███████████████████████████████████████▋                                   | 23845/45000 [03:22<03:03, 115.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 265\tFrame 23828 \tAverage Score: 149.56"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|████████████████████████████████████████▌                                   | 24047/45000 [03:24<03:50, 90.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 266\tFrame 24028 \tAverage Score: 150.75"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|████████████████████████████████████████▍                                  | 24240/45000 [03:26<03:00, 115.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 267\tFrame 24228 \tAverage Score: 151.84"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|████████████████████████████████████████▋                                  | 24448/45000 [03:28<03:00, 113.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 268\tFrame 24428 \tAverage Score: 152.98"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████████████████████████████████████████                                  | 24646/45000 [03:29<02:51, 118.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 269\tFrame 24628 \tAverage Score: 154.15"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████████████████████████████████████████▍                                 | 24840/45000 [03:31<02:56, 114.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 270\tFrame 24828 \tAverage Score: 155.63"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|██████████████████████████████████████████▎                                 | 25043/45000 [03:33<03:26, 96.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 271\tFrame 25028 \tAverage Score: 156.76"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|██████████████████████████████████████████                                 | 25248/45000 [03:35<02:51, 115.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 272\tFrame 25228 \tAverage Score: 157.91"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|██████████████████████████████████████████▍                                | 25447/45000 [03:37<02:51, 114.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 273\tFrame 25428 \tAverage Score: 159.13"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|██████████████████████████████████████████▋                                | 25636/45000 [03:38<02:57, 108.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 274\tFrame 25628 \tAverage Score: 160.29"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|███████████████████████████████████████████                                | 25842/45000 [03:40<02:48, 113.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 275\tFrame 25828 \tAverage Score: 161.51"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|███████████████████████████████████████████▉                                | 26035/45000 [03:42<03:29, 90.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 276\tFrame 26028 \tAverage Score: 162.67"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|███████████████████████████████████████████▋                               | 26246/45000 [03:44<02:43, 114.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 277\tFrame 26228 \tAverage Score: 163.82"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|████████████████████████████████████████████                               | 26437/45000 [03:46<02:44, 112.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 278\tFrame 26428 \tAverage Score: 164.84"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|████████████████████████████████████████████▍                              | 26649/45000 [03:48<02:39, 114.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 279\tFrame 26628 \tAverage Score: 165.91"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|████████████████████████████████████████████▋                              | 26840/45000 [03:49<02:34, 117.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 280\tFrame 26828 \tAverage Score: 167.08"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████████████████████████████████████████████▋                              | 27041/45000 [03:51<03:19, 90.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 281\tFrame 27028 \tAverage Score: 168.28"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|█████████████████████████████████████████████▍                             | 27239/45000 [03:53<02:35, 114.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 282\tFrame 27228 \tAverage Score: 169.45"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|█████████████████████████████████████████████▋                             | 27442/45000 [03:55<02:30, 116.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 283\tFrame 27428 \tAverage Score: 170.59"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████████████████████████████████████████████                             | 27645/45000 [03:56<02:32, 113.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 284\tFrame 27628 \tAverage Score: 171.70"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████████████████████████████████████████████▍                            | 27845/45000 [03:58<02:28, 115.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 285\tFrame 27828 \tAverage Score: 172.98"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|███████████████████████████████████████████████▎                            | 28039/45000 [04:00<03:11, 88.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 286\tFrame 28028 \tAverage Score: 174.05"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|███████████████████████████████████████████████                            | 28240/45000 [04:02<02:27, 113.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 287\tFrame 28228 \tAverage Score: 175.20"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|███████████████████████████████████████████████▍                           | 28443/45000 [04:04<02:22, 115.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 288\tFrame 28428 \tAverage Score: 176.29"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|███████████████████████████████████████████████▋                           | 28640/45000 [04:06<02:39, 102.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 289\tFrame 28628 \tAverage Score: 177.37"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|████████████████████████████████████████████████                           | 28850/45000 [04:08<02:20, 115.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 290\tFrame 28828 \tAverage Score: 178.46"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|█████████████████████████████████████████████████                           | 29045/45000 [04:10<02:54, 91.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 291\tFrame 29028 \tAverage Score: 179.66"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|████████████████████████████████████████████████▋                          | 29246/45000 [04:11<02:21, 111.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 292\tFrame 29228 \tAverage Score: 180.79"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|█████████████████████████████████████████████████                          | 29439/45000 [04:13<02:15, 115.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 293\tFrame 29428 \tAverage Score: 181.83"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|█████████████████████████████████████████████████▍                         | 29641/45000 [04:15<02:20, 109.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 294\tFrame 29628 \tAverage Score: 182.90"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|█████████████████████████████████████████████████▋                         | 29839/45000 [04:17<02:15, 111.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 295\tFrame 29828 \tAverage Score: 183.76"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████████████████████████████████████████████████▊                         | 30052/45000 [04:19<02:33, 97.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 296\tFrame 30028 \tAverage Score: 184.82"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████████████████████████████████████████████████▍                        | 30240/45000 [04:20<02:04, 118.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 297\tFrame 30228 \tAverage Score: 185.93"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████████████████████████████████████████████████▋                        | 30441/45000 [04:22<02:11, 110.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 298\tFrame 30428 \tAverage Score: 187.03"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|███████████████████████████████████████████████████                        | 30640/45000 [04:24<02:06, 113.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 299\tFrame 30628 \tAverage Score: 188.15"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|███████████████████████████████████████████████████▍                       | 30841/45000 [04:26<02:01, 116.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 300\tFrame 30828\tAverage Score: 189.122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|████████████████████████████████████████████████████▍                       | 31043/45000 [04:28<02:32, 91.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 301\tFrame 31028 \tAverage Score: 190.14"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|████████████████████████████████████████████████████                       | 31242/45000 [04:29<02:00, 113.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 302\tFrame 31228 \tAverage Score: 191.12"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|████████████████████████████████████████████████████▍                      | 31440/45000 [04:31<01:59, 113.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 303\tFrame 31428 \tAverage Score: 192.07"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|████████████████████████████████████████████████████▋                      | 31644/45000 [04:33<01:56, 114.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 304\tFrame 31628 \tAverage Score: 193.11"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|█████████████████████████████████████████████████████                      | 31848/45000 [04:35<01:55, 114.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 305\tFrame 31828 \tAverage Score: 194.06"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|██████████████████████████████████████████████████████                      | 32038/45000 [04:37<02:41, 80.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 306\tFrame 32028 \tAverage Score: 195.01"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|██████████████████████████████████████████████████████▍                     | 32238/45000 [04:39<02:21, 89.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 307\tFrame 32228 \tAverage Score: 195.90"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|██████████████████████████████████████████████████████                     | 32444/45000 [04:41<02:01, 103.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 308\tFrame 32428 \tAverage Score: 196.77"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|██████████████████████████████████████████████████████▍                    | 32644/45000 [04:43<01:54, 108.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 309\tFrame 32628 \tAverage Score: 197.69"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|██████████████████████████████████████████████████████▋                    | 32845/45000 [04:45<01:46, 114.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 310\tFrame 32828 \tAverage Score: 197.69"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████████████████████████████████████████████████████▊                    | 33046/45000 [04:47<02:09, 92.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 311\tFrame 33028 \tAverage Score: 198.59"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████████████████████████████████████████████████████▍                   | 33240/45000 [04:48<01:43, 113.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 312\tFrame 33228 \tAverage Score: 199.26"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████████████████████████████████████████████████████▋                   | 33442/45000 [04:50<01:40, 115.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 313\tFrame 33428 \tAverage Score: 199.55"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|████████████████████████████████████████████████████████                   | 33640/45000 [04:52<01:40, 112.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 314\tFrame 33628 \tAverage Score: 199.55"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|████████████████████████████████████████████████████████▍                  | 33846/45000 [04:54<01:37, 114.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 315\tFrame 33828 \tAverage Score: 199.55"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|█████████████████████████████████████████████████████████▍                  | 34044/45000 [04:56<01:57, 93.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 316\tFrame 34028 \tAverage Score: 199.55"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|█████████████████████████████████████████████████████████                  | 34246/45000 [04:57<01:29, 120.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 317\tFrame 34228 \tAverage Score: 199.79"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|█████████████████████████████████████████████████████████▍                 | 34443/45000 [04:59<01:32, 114.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 318\tFrame 34428 \tAverage Score: 199.79"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|█████████████████████████████████████████████████████████▋                 | 34647/45000 [05:01<01:28, 116.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 319\tFrame 34628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|██████████████████████████████████████████████████████████                 | 34840/45000 [05:03<01:29, 113.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 320\tFrame 34828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████████████████████████████████████████████████████████▏                | 35040/45000 [05:05<01:51, 89.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 321\tFrame 35028 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|██████████████████████████████████████████████████████████▋                | 35242/45000 [05:06<01:23, 116.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 322\tFrame 35228 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████████████████████████████████████████████████████████                | 35447/45000 [05:08<01:22, 115.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 323\tFrame 35428 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████████████████████████████████████████████████████████▍               | 35639/45000 [05:10<01:29, 104.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 324\tFrame 35628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████████████████████████████████████████████████████████▋               | 35841/45000 [05:12<01:23, 109.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 325\tFrame 35828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████████████████████████████████████████████████████████▊               | 36039/45000 [05:14<01:55, 77.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 326\tFrame 36028 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████████████████████████████████████████████████████████▍              | 36242/45000 [05:16<01:23, 105.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 327\tFrame 36228 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████████████████████████████████████████████████████████▋              | 36446/45000 [05:18<01:12, 117.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 328\tFrame 36428 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|█████████████████████████████████████████████████████████████              | 36645/45000 [05:20<01:15, 111.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 329\tFrame 36628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|█████████████████████████████████████████████████████████████▍             | 36848/45000 [05:22<01:12, 112.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 330\tFrame 36828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|██████████████████████████████████████████████████████████████▌             | 37039/45000 [05:24<01:29, 89.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 331\tFrame 37028 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|██████████████████████████████████████████████████████████████             | 37243/45000 [05:25<01:05, 118.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 332\tFrame 37228 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|██████████████████████████████████████████████████████████████▍            | 37447/45000 [05:27<01:03, 118.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 333\tFrame 37428 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|██████████████████████████████████████████████████████████████▊            | 37650/45000 [05:29<01:03, 116.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 334\tFrame 37628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|███████████████████████████████████████████████████████████████            | 37852/45000 [05:31<00:59, 119.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 335\tFrame 37828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████████████████████████████████████████████████████████████▏           | 38042/45000 [05:33<01:18, 89.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 336\tFrame 38028 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|███████████████████████████████████████████████████████████████▋           | 38237/45000 [05:34<00:57, 118.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 337\tFrame 38228 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████████████████████████████████████████████████████████████           | 38438/45000 [05:36<00:55, 118.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 338\tFrame 38428 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████████████████████████████████████████████████████████████▍          | 38644/45000 [05:38<00:55, 114.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 339\tFrame 38628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████████████████████████████████████████████████████████████▋          | 38845/45000 [05:40<00:52, 116.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 340\tFrame 38828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|█████████████████████████████████████████████████████████████████▉          | 39042/45000 [05:42<01:08, 86.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 341\tFrame 39028 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|█████████████████████████████████████████████████████████████████▍         | 39245/45000 [05:43<00:48, 119.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 342\tFrame 39228 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|█████████████████████████████████████████████████████████████████▋         | 39442/45000 [05:45<00:51, 108.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 343\tFrame 39428 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|██████████████████████████████████████████████████████████████████         | 39642/45000 [05:47<00:44, 120.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 344\tFrame 39628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|██████████████████████████████████████████████████████████████████▍        | 39843/45000 [05:49<00:46, 111.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 345\tFrame 39828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|███████████████████████████████████████████████████████████████████▋        | 40048/45000 [05:51<00:52, 94.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 346\tFrame 40028 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|███████████████████████████████████████████████████████████████████        | 40245/45000 [05:52<00:41, 114.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 347\tFrame 40228 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████████▍       | 40447/45000 [05:54<00:38, 118.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 348\tFrame 40428 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████████▋       | 40640/45000 [05:56<00:37, 115.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 349\tFrame 40628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|████████████████████████████████████████████████████████████████████       | 40843/45000 [05:58<00:35, 117.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 350\tFrame 40828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████████████████████████████████████████████████████████████████▎      | 41041/45000 [06:00<00:42, 92.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 351\tFrame 41028 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|████████████████████████████████████████████████████████████████████▋      | 41239/45000 [06:01<00:33, 111.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 352\tFrame 41228 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████████████████████████████████████████████████████████████████      | 41445/45000 [06:03<00:32, 109.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 353\tFrame 41428 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████████████████████████████████████████████████████████████████▍     | 41648/45000 [06:05<00:27, 120.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 354\tFrame 41628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████████████████████████████████████████████████████████████████▋     | 41847/45000 [06:07<00:27, 112.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 355\tFrame 41828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|███████████████████████████████████████████████████████████████████████     | 42049/45000 [06:09<00:31, 94.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 356\tFrame 42028 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|██████████████████████████████████████████████████████████████████████▍    | 42248/45000 [06:11<00:24, 112.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 357\tFrame 42228 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|██████████████████████████████████████████████████████████████████████▋    | 42448/45000 [06:12<00:22, 114.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 358\tFrame 42428 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|███████████████████████████████████████████████████████████████████████    | 42639/45000 [06:14<00:22, 106.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 359\tFrame 42628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|███████████████████████████████████████████████████████████████████████▍   | 42840/45000 [06:16<00:21, 102.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 360\tFrame 42828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|████████████████████████████████████████████████████████████████████████▋   | 43041/45000 [06:18<00:24, 81.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 361\tFrame 43028 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|████████████████████████████████████████████████████████████████████████   | 43235/45000 [06:20<00:15, 112.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 362\tFrame 43228 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|████████████████████████████████████████████████████████████████████████▍  | 43439/45000 [06:22<00:14, 107.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 363\tFrame 43428 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|████████████████████████████████████████████████████████████████████████▋  | 43639/45000 [06:24<00:11, 114.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 364\tFrame 43628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████████████████████████████████████████████████████████████████████  | 43839/45000 [06:25<00:10, 113.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 365\tFrame 43828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|██████████████████████████████████████████████████████████████████████████▍ | 44043/45000 [06:27<00:10, 94.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 366\tFrame 44028 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████████████████████████████████████████████████████████████████████▋ | 44243/45000 [06:29<00:06, 111.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 367\tFrame 44228 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|██████████████████████████████████████████████████████████████████████████ | 44445/45000 [06:31<00:05, 109.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 368\tFrame 44428 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|██████████████████████████████████████████████████████████████████████████▍| 44648/45000 [06:33<00:03, 114.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 369\tFrame 44628 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████▋| 44848/45000 [06:35<00:01, 111.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 370\tFrame 44828 \tAverage Score: 200.00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 45000/45000 [06:36<00:00, 113.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training time: 6.61min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "QR=[]\n",
    "def run(frames=1000, eps_fixed=False, eps_frames=1e6, min_eps=0.01):\n",
    "    \"\"\"Deep Q-Learning.\n",
    "    \n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        eps_start (float): starting value of epsilon, for epsilon-greedy action selection\n",
    "        eps_end (float): minimum value of epsilon\n",
    "        eps_decay (float): multiplicative factor (per episode) for decreasing epsilon\n",
    "    \"\"\"\n",
    "    scores = []                        # list containing scores from each episode\n",
    "    scores_window = deque(maxlen=100)  # last 100 scores\n",
    "    output_history = []\n",
    "    frame = 0\n",
    "    if eps_fixed:\n",
    "        eps = 0\n",
    "    else:\n",
    "        eps = 1\n",
    "    eps_start = 1\n",
    "    i_episode = 1\n",
    "    state = env.reset()\n",
    "    score = 0                  \n",
    "    for frame in tqdm(range(1, frames+1)):\n",
    "\n",
    "        action = agent.act(state, eps)\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        agent.step(state, action, reward, next_state, done, writer)\n",
    "        state = next_state\n",
    "        score += reward\n",
    "        # linear annealing to the min epsilon value until eps_frames and from there slowly decease epsilon to 0 until the end of training\n",
    "        if eps_fixed == False:\n",
    "            if frame < eps_frames:\n",
    "                eps = max(eps_start - (frame*(1/eps_frames)), min_eps)\n",
    "            else:\n",
    "                eps = max(min_eps - min_eps*((frame-eps_frames)/(frames-eps_frames)), 0.001)\n",
    "\n",
    "        # evaluation runs\n",
    "        if frame % 1000 == 0:\n",
    "            eval_runs(eps, frame)\n",
    "        \n",
    "        if done:\n",
    "            scores_window.append(score)       # save most recent score\n",
    "            scores.append(score)              # save most recent score\n",
    "            writer.add_scalar(\"Average100\", np.mean(scores_window), frame)\n",
    "            output_history.append(np.mean(scores_window))\n",
    "            print('\\rEpisode {}\\tFrame {} \\tAverage Score: {:.2f}'.format(i_episode, frame, np.mean(scores_window)), end=\"\")\n",
    "            if i_episode % 100 == 0:\n",
    "                print('\\rEpisode {}\\tFrame {}\\tAverage Score: {:.2f}'.format(i_episode,frame, np.mean(scores_window)))\n",
    "            i_episode +=1 \n",
    "            state = env.reset()\n",
    "            score = 0  \n",
    "            QR.append([i_episode,frame,np.mean(scores_window)])\n",
    "\n",
    "    return output_history\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    writer = SummaryWriter(\"runs/\"+\"DQN_LL_new_1\")\n",
    "    seed = 1\n",
    "    BUFFER_SIZE = 100000\n",
    "    BATCH_SIZE = 32\n",
    "    GAMMA = 0.99\n",
    "    TAU = 1e-2\n",
    "    LR = 1e-3\n",
    "    UPDATE_EVERY = 1\n",
    "    n_step = 1\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(\"Using \", device)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    np.random.seed(seed)\n",
    "    env = gym.make(\"CartPole-v0\")\n",
    "\n",
    "    env.seed(seed)\n",
    "    action_size     = env.action_space.n\n",
    "    state_size = env.observation_space.shape\n",
    "\n",
    "    agent = DQN_Agent(state_size=state_size,    \n",
    "                        action_size=action_size,\n",
    "                        Network=\"DDQN\",\n",
    "                        layer_size=256,\n",
    "                        n_step=n_step,\n",
    "                        BATCH_SIZE=BATCH_SIZE, \n",
    "                        BUFFER_SIZE=BUFFER_SIZE, \n",
    "                        LR=LR, \n",
    "                        TAU=TAU, \n",
    "                        GAMMA=GAMMA, \n",
    "                        UPDATE_EVERY=UPDATE_EVERY, \n",
    "                        device=device, \n",
    "                        seed=seed)\n",
    "\n",
    "\n",
    "\n",
    "    # set epsilon frames to 0 so no epsilon exploration\n",
    "    eps_fixed = False\n",
    "\n",
    "    t0 = time.time()\n",
    "    final_average100 = run(frames = 45000, eps_fixed=eps_fixed, eps_frames=5000, min_eps=0.025)\n",
    "    t1 = time.time()\n",
    "    \n",
    "    print(\"Training time: {}min\".format(round((t1-t0)/60,2)))\n",
    "    torch.save(agent.qnetwork_local.state_dict(), \"DQN\"+\".pth\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[2, 64, 64.0],\n",
       " [3, 93, 46.5],\n",
       " [4, 109, 36.333333333333336],\n",
       " [5, 152, 38.0],\n",
       " [6, 193, 38.6],\n",
       " [7, 240, 40.0],\n",
       " [8, 256, 36.57142857142857],\n",
       " [9, 268, 33.5],\n",
       " [10, 289, 32.111111111111114],\n",
       " [11, 315, 31.5],\n",
       " [12, 327, 29.727272727272727],\n",
       " [13, 341, 28.416666666666668],\n",
       " [14, 363, 27.923076923076923],\n",
       " [15, 399, 28.5],\n",
       " [16, 412, 27.466666666666665],\n",
       " [17, 447, 27.9375],\n",
       " [18, 470, 27.647058823529413],\n",
       " [19, 494, 27.444444444444443],\n",
       " [20, 547, 28.789473684210527],\n",
       " [21, 684, 34.2],\n",
       " [22, 716, 34.095238095238095],\n",
       " [23, 737, 33.5],\n",
       " [24, 800, 34.78260869565217],\n",
       " [25, 815, 33.958333333333336],\n",
       " [26, 827, 33.08],\n",
       " [27, 839, 32.26923076923077],\n",
       " [28, 869, 32.18518518518518],\n",
       " [29, 893, 31.892857142857142],\n",
       " [30, 906, 31.24137931034483],\n",
       " [31, 928, 30.933333333333334],\n",
       " [32, 953, 30.741935483870968],\n",
       " [33, 983, 30.71875],\n",
       " [34, 1005, 30.454545454545453],\n",
       " [35, 1031, 30.323529411764707],\n",
       " [36, 1056, 30.17142857142857],\n",
       " [37, 1084, 30.11111111111111],\n",
       " [38, 1097, 29.64864864864865],\n",
       " [39, 1110, 29.210526315789473],\n",
       " [40, 1124, 28.82051282051282],\n",
       " [41, 1151, 28.775],\n",
       " [42, 1162, 28.341463414634145],\n",
       " [43, 1175, 27.976190476190474],\n",
       " [44, 1199, 27.88372093023256],\n",
       " [45, 1249, 28.386363636363637],\n",
       " [46, 1285, 28.555555555555557],\n",
       " [47, 1305, 28.369565217391305],\n",
       " [48, 1343, 28.574468085106382],\n",
       " [49, 1355, 28.229166666666668],\n",
       " [50, 1419, 28.959183673469386],\n",
       " [51, 1443, 28.86],\n",
       " [52, 1457, 28.568627450980394],\n",
       " [53, 1532, 29.46153846153846],\n",
       " [54, 1585, 29.90566037735849],\n",
       " [55, 1601, 29.64814814814815],\n",
       " [56, 1649, 29.98181818181818],\n",
       " [57, 1665, 29.732142857142858],\n",
       " [58, 1701, 29.842105263157894],\n",
       " [59, 1714, 29.551724137931036],\n",
       " [60, 1723, 29.203389830508474],\n",
       " [61, 1766, 29.433333333333334],\n",
       " [62, 1791, 29.360655737704917],\n",
       " [63, 1809, 29.177419354838708],\n",
       " [64, 1822, 28.92063492063492],\n",
       " [65, 1846, 28.84375],\n",
       " [66, 1878, 28.892307692307693],\n",
       " [67, 1892, 28.666666666666668],\n",
       " [68, 1907, 28.46268656716418],\n",
       " [69, 1941, 28.544117647058822],\n",
       " [70, 1986, 28.782608695652176],\n",
       " [71, 2012, 28.742857142857144],\n",
       " [72, 2150, 30.281690140845072],\n",
       " [73, 2165, 30.069444444444443],\n",
       " [74, 2208, 30.246575342465754],\n",
       " [75, 2239, 30.256756756756758],\n",
       " [76, 2256, 30.08],\n",
       " [77, 2269, 29.855263157894736],\n",
       " [78, 2317, 30.09090909090909],\n",
       " [79, 2333, 29.91025641025641],\n",
       " [80, 2352, 29.772151898734176],\n",
       " [81, 2366, 29.575],\n",
       " [82, 2388, 29.48148148148148],\n",
       " [83, 2445, 29.817073170731707],\n",
       " [84, 2457, 29.602409638554217],\n",
       " [85, 2473, 29.44047619047619],\n",
       " [86, 2588, 30.44705882352941],\n",
       " [87, 2605, 30.290697674418606],\n",
       " [88, 2619, 30.103448275862068],\n",
       " [89, 2639, 29.988636363636363],\n",
       " [90, 2660, 29.8876404494382],\n",
       " [91, 2678, 29.755555555555556],\n",
       " [92, 2729, 29.98901098901099],\n",
       " [93, 2744, 29.82608695652174],\n",
       " [94, 2783, 29.9247311827957],\n",
       " [95, 2798, 29.76595744680851],\n",
       " [96, 2810, 29.57894736842105],\n",
       " [97, 2826, 29.4375],\n",
       " [98, 2887, 29.762886597938145],\n",
       " [99, 3011, 30.724489795918366],\n",
       " [100, 3028, 30.585858585858585],\n",
       " [101, 3054, 30.54],\n",
       " [102, 3244, 31.8],\n",
       " [103, 3329, 32.36],\n",
       " [104, 3481, 33.72],\n",
       " [105, 3581, 34.29],\n",
       " [106, 3738, 35.45],\n",
       " [107, 3922, 36.82],\n",
       " [108, 3952, 36.96],\n",
       " [109, 3964, 36.96],\n",
       " [110, 4019, 37.3],\n",
       " [111, 4047, 37.32],\n",
       " [112, 4120, 37.93],\n",
       " [113, 4175, 38.34],\n",
       " [114, 4304, 39.41],\n",
       " [115, 4411, 40.12],\n",
       " [116, 4460, 40.48],\n",
       " [117, 4538, 40.91],\n",
       " [118, 4624, 41.54],\n",
       " [119, 4797, 43.03],\n",
       " [120, 4883, 43.36],\n",
       " [121, 5073, 43.89],\n",
       " [122, 5162, 44.46],\n",
       " [123, 5240, 45.03],\n",
       " [124, 5440, 46.4],\n",
       " [125, 5544, 47.29],\n",
       " [126, 5630, 48.03],\n",
       " [127, 5704, 48.65],\n",
       " [128, 5753, 48.84],\n",
       " [129, 5840, 49.47],\n",
       " [130, 5938, 50.32],\n",
       " [131, 6016, 50.88],\n",
       " [132, 6105, 51.52],\n",
       " [133, 6166, 51.83],\n",
       " [134, 6218, 52.13],\n",
       " [135, 6316, 52.85],\n",
       " [136, 6397, 53.41],\n",
       " [137, 6490, 54.06],\n",
       " [138, 6577, 54.8],\n",
       " [139, 6678, 55.68],\n",
       " [140, 6763, 56.39],\n",
       " [141, 6841, 56.9],\n",
       " [142, 6920, 57.58],\n",
       " [143, 6999, 58.24],\n",
       " [144, 7083, 58.84],\n",
       " [145, 7173, 59.24],\n",
       " [146, 7250, 59.65],\n",
       " [147, 7334, 60.29],\n",
       " [148, 7415, 60.72],\n",
       " [149, 7504, 61.49],\n",
       " [150, 7520, 61.01],\n",
       " [151, 7610, 61.67],\n",
       " [152, 7704, 62.47],\n",
       " [153, 7782, 62.5],\n",
       " [154, 7862, 62.77],\n",
       " [155, 7941, 63.4],\n",
       " [156, 8031, 63.82],\n",
       " [157, 8116, 64.51],\n",
       " [158, 8193, 64.92],\n",
       " [159, 8290, 65.76],\n",
       " [160, 8370, 66.47],\n",
       " [161, 8455, 66.89],\n",
       " [162, 8542, 67.51],\n",
       " [163, 8620, 68.11],\n",
       " [164, 8709, 68.87],\n",
       " [165, 8789, 69.43],\n",
       " [166, 8872, 69.94],\n",
       " [167, 8953, 70.61],\n",
       " [168, 9044, 71.37],\n",
       " [169, 9130, 71.89],\n",
       " [170, 9213, 72.27],\n",
       " [171, 9265, 72.53],\n",
       " [172, 9352, 72.02],\n",
       " [173, 9437, 72.72],\n",
       " [174, 9515, 73.07],\n",
       " [175, 9599, 73.6],\n",
       " [176, 9677, 74.21],\n",
       " [177, 9761, 74.92],\n",
       " [178, 9846, 75.29],\n",
       " [179, 9944, 76.11],\n",
       " [180, 10037, 76.85],\n",
       " [181, 10120, 77.54],\n",
       " [182, 10200, 78.12],\n",
       " [183, 10283, 78.38],\n",
       " [184, 10369, 79.12],\n",
       " [185, 10458, 79.85],\n",
       " [186, 10530, 79.42],\n",
       " [187, 10623, 80.18],\n",
       " [188, 10708, 80.89],\n",
       " [189, 10799, 81.6],\n",
       " [190, 10891, 82.31],\n",
       " [191, 10982, 83.04],\n",
       " [192, 11062, 83.33],\n",
       " [193, 11149, 84.05],\n",
       " [194, 11245, 84.62],\n",
       " [195, 11338, 85.4],\n",
       " [196, 11452, 86.42],\n",
       " [197, 11546, 87.2],\n",
       " [198, 11635, 87.48],\n",
       " [199, 11725, 87.14],\n",
       " [200, 11813, 87.85],\n",
       " [201, 11916, 88.62],\n",
       " [202, 12014, 87.7],\n",
       " [203, 12116, 87.87],\n",
       " [204, 12221, 87.4],\n",
       " [205, 12317, 87.36],\n",
       " [206, 12422, 86.84],\n",
       " [207, 12527, 86.05],\n",
       " [208, 12638, 86.86],\n",
       " [209, 12751, 87.87],\n",
       " [210, 12859, 88.4],\n",
       " [211, 13059, 90.12],\n",
       " [212, 13169, 90.49],\n",
       " [213, 13302, 91.27],\n",
       " [214, 13473, 91.69],\n",
       " [215, 13673, 92.62],\n",
       " [216, 13873, 94.13],\n",
       " [217, 14073, 95.35],\n",
       " [218, 14249, 96.25],\n",
       " [219, 14449, 96.52],\n",
       " [220, 14628, 97.45],\n",
       " [221, 14828, 97.55],\n",
       " [222, 15028, 98.66],\n",
       " [223, 15228, 99.88],\n",
       " [224, 15428, 99.88],\n",
       " [225, 15628, 100.84],\n",
       " [226, 15828, 101.98],\n",
       " [227, 16028, 103.24],\n",
       " [228, 16228, 104.75],\n",
       " [229, 16428, 105.88],\n",
       " [230, 16628, 106.9],\n",
       " [231, 16828, 108.12],\n",
       " [232, 17028, 109.23],\n",
       " [233, 17228, 110.62],\n",
       " [234, 17428, 112.1],\n",
       " [235, 17628, 113.12],\n",
       " [236, 17828, 114.31],\n",
       " [237, 18028, 115.38],\n",
       " [238, 18228, 116.51],\n",
       " [239, 18428, 117.5],\n",
       " [240, 18628, 118.65],\n",
       " [241, 18828, 119.87],\n",
       " [242, 19028, 121.08],\n",
       " [243, 19228, 122.29],\n",
       " [244, 19428, 123.45],\n",
       " [245, 19628, 124.55],\n",
       " [246, 19828, 125.78],\n",
       " [247, 20028, 126.94],\n",
       " [248, 20228, 128.13],\n",
       " [249, 20428, 129.24],\n",
       " [250, 20628, 131.08],\n",
       " [251, 20828, 132.18],\n",
       " [252, 21028, 133.24],\n",
       " [253, 21228, 134.46],\n",
       " [254, 21428, 135.66],\n",
       " [255, 21628, 136.87],\n",
       " [256, 21828, 137.97],\n",
       " [257, 22028, 139.12],\n",
       " [258, 22228, 140.35],\n",
       " [259, 22428, 141.38],\n",
       " [260, 22628, 142.58],\n",
       " [261, 22828, 143.73],\n",
       " [262, 23028, 144.86],\n",
       " [263, 23228, 146.08],\n",
       " [264, 23428, 147.19],\n",
       " [265, 23628, 148.39],\n",
       " [266, 23828, 149.56],\n",
       " [267, 24028, 150.75],\n",
       " [268, 24228, 151.84],\n",
       " [269, 24428, 152.98],\n",
       " [270, 24628, 154.15],\n",
       " [271, 24828, 155.63],\n",
       " [272, 25028, 156.76],\n",
       " [273, 25228, 157.91],\n",
       " [274, 25428, 159.13],\n",
       " [275, 25628, 160.29],\n",
       " [276, 25828, 161.51],\n",
       " [277, 26028, 162.67],\n",
       " [278, 26228, 163.82],\n",
       " [279, 26428, 164.84],\n",
       " [280, 26628, 165.91],\n",
       " [281, 26828, 167.08],\n",
       " [282, 27028, 168.28],\n",
       " [283, 27228, 169.45],\n",
       " [284, 27428, 170.59],\n",
       " [285, 27628, 171.7],\n",
       " [286, 27828, 172.98],\n",
       " [287, 28028, 174.05],\n",
       " [288, 28228, 175.2],\n",
       " [289, 28428, 176.29],\n",
       " [290, 28628, 177.37],\n",
       " [291, 28828, 178.46],\n",
       " [292, 29028, 179.66],\n",
       " [293, 29228, 180.79],\n",
       " [294, 29428, 181.83],\n",
       " [295, 29628, 182.9],\n",
       " [296, 29828, 183.76],\n",
       " [297, 30028, 184.82],\n",
       " [298, 30228, 185.93],\n",
       " [299, 30428, 187.03],\n",
       " [300, 30628, 188.15],\n",
       " [301, 30828, 189.12],\n",
       " [302, 31028, 190.14],\n",
       " [303, 31228, 191.12],\n",
       " [304, 31428, 192.07],\n",
       " [305, 31628, 193.11],\n",
       " [306, 31828, 194.06],\n",
       " [307, 32028, 195.01],\n",
       " [308, 32228, 195.9],\n",
       " [309, 32428, 196.77],\n",
       " [310, 32628, 197.69],\n",
       " [311, 32828, 197.69],\n",
       " [312, 33028, 198.59],\n",
       " [313, 33228, 199.26],\n",
       " [314, 33428, 199.55],\n",
       " [315, 33628, 199.55],\n",
       " [316, 33828, 199.55],\n",
       " [317, 34028, 199.55],\n",
       " [318, 34228, 199.79],\n",
       " [319, 34428, 199.79],\n",
       " [320, 34628, 200.0],\n",
       " [321, 34828, 200.0],\n",
       " [322, 35028, 200.0],\n",
       " [323, 35228, 200.0],\n",
       " [324, 35428, 200.0],\n",
       " [325, 35628, 200.0],\n",
       " [326, 35828, 200.0],\n",
       " [327, 36028, 200.0],\n",
       " [328, 36228, 200.0],\n",
       " [329, 36428, 200.0],\n",
       " [330, 36628, 200.0],\n",
       " [331, 36828, 200.0],\n",
       " [332, 37028, 200.0],\n",
       " [333, 37228, 200.0],\n",
       " [334, 37428, 200.0],\n",
       " [335, 37628, 200.0],\n",
       " [336, 37828, 200.0],\n",
       " [337, 38028, 200.0],\n",
       " [338, 38228, 200.0],\n",
       " [339, 38428, 200.0],\n",
       " [340, 38628, 200.0],\n",
       " [341, 38828, 200.0],\n",
       " [342, 39028, 200.0],\n",
       " [343, 39228, 200.0],\n",
       " [344, 39428, 200.0],\n",
       " [345, 39628, 200.0],\n",
       " [346, 39828, 200.0],\n",
       " [347, 40028, 200.0],\n",
       " [348, 40228, 200.0],\n",
       " [349, 40428, 200.0],\n",
       " [350, 40628, 200.0],\n",
       " [351, 40828, 200.0],\n",
       " [352, 41028, 200.0],\n",
       " [353, 41228, 200.0],\n",
       " [354, 41428, 200.0],\n",
       " [355, 41628, 200.0],\n",
       " [356, 41828, 200.0],\n",
       " [357, 42028, 200.0],\n",
       " [358, 42228, 200.0],\n",
       " [359, 42428, 200.0],\n",
       " [360, 42628, 200.0],\n",
       " [361, 42828, 200.0],\n",
       " [362, 43028, 200.0],\n",
       " [363, 43228, 200.0],\n",
       " [364, 43428, 200.0],\n",
       " [365, 43628, 200.0],\n",
       " [366, 43828, 200.0],\n",
       " [367, 44028, 200.0],\n",
       " [368, 44228, 200.0],\n",
       " [369, 44428, 200.0],\n",
       " [370, 44628, 200.0],\n",
       " [371, 44828, 200.0]]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "QR"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlIAAABUCAIAAADGe04QAAAgAElEQVR4Ae1dPUgjTRhOYbFFiilSTGHhQgoXLFywcMHGBQsXLAxYuGBxLFcc4QoJVxzB5ggWEiw+whVHsBByxUEsDtZCiI2QK4RYCHuFsBYWW1hskWKKFPMxsz/ZJLsxuTMxnq8ISTaT2dln3pln3vl5nxQd9Y80PsraV3vU5JAOEAAEAAFAABCYPQRSYxTpvqIuGmZ7jF9AUkAAEAAEAAFAYKYQGIf2qFPdwtqJM1MPAIUBBAABQAAQAARGR2As2qP2sYK2a+7o2UNKQAAQAAQAAUBglhAYj/bouYGzhWZnlp4AygIIAAKAACAACIyMwJi0d1kQsWEC7Y2MLyQEBAABQAAQmCkExqQ98PZmqvagMIAAIAAIAAJjIjAe7cHa3pjwQnJAABAABACB2UJgLNqDnZyzVXlQGkAAEAAEAIFxERiH9u4rahbO7Y2LMKQHBAABQAAQmCEERqc9iNIyQ9UGRQEEAAFAABD4MwRGp70/yx9+BQgAAoAAIAAIzBACQHszVBlQFEAAEAAEAIFJIwC0N2mEIX9AABAABACBGUIAaG+GKgOKAggAAoAAIDBpBCZIe843DW1VnyOAp1vdQto3CIE9aWOA/AEBQAAQ+PcRGJH27PK6kIr8CXv1J7Ehv83quUWeTPd0gj+mPdI81tUVEc2lxP1G5D5O40CTsCBkJPVT3Q5irZHbqrEmojTCK3rl+jkKHrnlGG9/6igCdUpQKw9j/PrZkj6ahQ1ZzAgpAYlrevkqHMA45pecPI+ENBLXjMp1eJ3f2anr8ylhoxIOUuwfBXURCWksb5caj89WumfMyD3NoZSgHr8eLUm3VXmvihlBQFhazZUueqvgGaH5o6zIVUlbEVGaGY68UzID6yU3VWNdRGlWau2zGba7oTexK5soldbrTzTS+BY9NGf48u0iMAbtKUcWaRP/P7DCqSD357TX+lGtXTQr2yhKe85pDmf12j2hjplfQup/vL/rtIrLSDlouh3SOtZwNt94KWXBnzrChukGUJMXIuC23fplOY+EtJ3WiS7O63Xeu7o/dJzRyjeEUrd5oKDFaGhy13yvKGtil/buympGyp87lNi1XRHvzqB8h1vbQWI2Uuap2PRf3MQ132Fxu9J0CG071mW1ehmOMf4i12f86UOreWu7LiGPVv2DjDarrHwdq7QiSB9Mp0PJfU1fQLkRJMyck5yypuCQ9hIaaXyLfsYngqz+LQTGob3B4fCZjpZy+V1VWZWlFb38Kxh1/q5oi6KIBaF3ktO9LOVWRJRBeEFSDxp+d06s2kdVwghlROVdpRWQDRsbrmKEsLRdzK+Hk5zE/lHQljBKI3FV73c1EuqmvhelPUai0kHLS2sfq8JamfHer4KEcn6vTExjHhvnL8Q3P3U0n2/EDCzs8jrKHdYKm7KUFfGiXnModczitiJixIDayNd+8zKzelG1JYxX8sWPCkZYPfKfl9xU8xsMbJRV86ej+eId1zo1xLRSvmOY2f+pwnrF94yui1I6AI1S9yKv7FQbh0pIe/aR0pMY5Wqz5vC16zqWCiclBWm8e6a0bRrzUuFXYEx3ZSXjFzsRvQsDL+Wrx7qyJIrzovKFod36qqtLDGm0IOuHjaBtcL8kw/H/pEuhnknHaXzJyQtIQE+5xZ1WcQnpP2KNk1invHUghJfUwk+PDknrq6Es9FoIr8kYc6I08RmpW3snSTvVMZzitm3uy4I3MOo0DIzzFx6qpLYjSJ99mwyAHnh1avqqUf9ZEEPai2+kCS16ID+4AAh4CPwt7QlzUp7PsbhnurhYaEYaY//aXqdVWkHaN95q2nbzyvbSWocyWi2y6a92q7yJpX2eRziso9Q5M6Q5wVvbI1cFCSvFC4dQYn83pNF8sh7a6zQLi0Luu2v/KBZPWu65gbmmhHuqCUvF1mOz8qlkPrBJXWWQ5qdjNcNoTxCWjCrnNvJgMwQfzMpJw3Yp7biNzwpaLVuU0jMdLRdb7VZxWZC/WOS6KC8VmVzUY91YwNpR0+1QclvJLcjFm+GPxEbofMJVEMP+7qGmr2jla9e7o7gV9IPtZmFVLd9RK0J75juE3zfcm2rhoG4/1nJpqXg9/I7T/pb8NPBCvtFmXBJIKBPzvSh9anpFsY8UvMOHQ0PQuzBwGqsHDZcNVlz7jnFc67Rcv3EIpeSuqi9g4yezd+YrL+jVO0LbVnkTpQLasw4VtJyv3xHacRr7sjjMLXbrexitGJWfTbt3DOGeG+K8Vr5ibEceGnXuBZLLgoRV5p37FlKy/BEVM/J+cxryjNSpbAjCaokZ2JN/TlXzDGcOq8ct3tJJ81BVPtRtQsnvqp6VCleRziImQ7f+Ts6dOvSqS3vxjTShRcdkCZcAAY7AGLQnpBEO/jRvYvBMR0G7pcxDioyRKe2nPWqVVpCyb1qPUXO3SqtIO/GHwqwP8saGtyU5rVX9Vs06X057xHyPxQ+Bm8ioUcxfPl2TvbTXyC8g/cwx34t4p2Zf8kZFqPNVZU36pqxgqXjlVDcZYTyd9SRS/NTRnBBiLe7VA7xYPyUfJpfqd0nJ8FWQM51vJmKjYOYWPFY1n9pzjA4DP7LxXhzpGdt242ux9MMfptCObX5SsZBKzaWERb3KXUBKSfOzohywDi5Ce6S+K4j7Dfskh7OG+VjX0Uj1NQlQE/JkFoU5wo2PYiihTC7ybAzHgLLKq1j/weyTLQEmoXdh4IxeDyYqBu5F6ntIYRXH3oSESsLmwzpulPseOIT3FXVYbpS2rfqhwdbPBCRuFHwXn5L6Lhp0oZr7oveArFSPVQ35XjulMeY07BkHnurJC8Rp1r4Uq2w+nP251xV9GaXmUqk5pByE7m98Nu55Xt7gkwoR2otvpJ34Fh2fL1wFBCgdg/aUL03nwfH+Xa+Fn+loJRj9saaLjfMuqAO0R8ltrbCrSvNshsc44TNsvMEbP4Nf/SpIGa7ndxUV9nOrm94kp1vdFISMKGb9f4z9QXTw+/jXXtrzvL2ASkJv74R7e5GB8Et6e1iv3ftQO25QVNZPodxp+JE/rNusfNDkRQ7IAhLSOUaSZzrvwd3aNtLPKHVrOczokBFSGofoifM47ILjgQuvsmqSS9w1tI4UYcngfonb+poTs4bpUnpbUpb91dAI7VHP2/Ozac+et0dYj5k7ZXzDhlwZPZjlbhSYO0Lp75Iy7/PZMPTYJGd3POE9r31W1NclD22MBJnNqzNLVo6CacKrgj/J2TF1JKD5qGHn/BnXsApi3zxatfcSWuMuPrUr3bWAbmozQrS0Yxo4HJvGmNOwZ+xmOc6725KSzbOBKps6xqo30/BgFlaxepw8gGs3CsuKZ2893l5sI2XGKeS+B+0iaNHjlBLSvi0ExqG9wUm/M11YCFahCF8jCVdEYry9EFk2Pyn649kEb+93SUG5mj98tstrgbf3Dst84STMa5Q3PbRHE1YCGOMGy1SzvLYXNm/+5Gwsv12xPKBuSzIaRnvuiYYieyxHgc5PwwbUAmdcz4HzJwAp8R049zTXs9M3lUohRrSzvrZ3VRDnUkLwl5pD+pnfezb3JWm/0TqU8TvTuzQMvQsDRxxBBppT1TJygc3GM0qt7yJOe0O8vT/1g5nd6lz5eWRv77dXq5z2es1p2DP6pjDmC5u2Ucv3bGDE5m+CzTfWoRwuAMfkeFdW+uxpTmSjkPhGmtCiY/KFS4AAQ2Ac2juySPBHPa/oTBfmcO6ETddb//Hdj8GQizX8/nN7duMHX4XyluvmDa87aX1ha3vNYG1P/MjnMNlCIGYz+2yNvaSkg7W9y4KUzfk7WVy7+cMMt8DE12eHUsK6A5ZtULb4fV/hamKHWC++k3PIlpbefsp8H6yGUrfxSR7u7dHHur4gGacWW4IirnVZM/0eMAY897JS/t60HJc8WuaBitJKmSdmDsFS3nxgi1b2qS4itXLf8/Oot0fvyiqa3Z2czU9SpPNlWzpDkmM9bFZVl8TuzqYh6A3S3n1ZzQS9/F01hz1vz1/bY7uIiVXZwkKwRtA6VMSNUoOhSslDyzxrBjOePdjyD455WKpd2U6bkker9o55e57/6P40RKyV/LW9ZrC2lxcx33nrre2tlIJZ7hjaG2oho25paZ2UqpfMcNz7Rnlb9Le0uHV9HmvHLfZcj83iGuquVrB9UjlpyfC2Cvc/cGSSkyY00vgW3Z8RfAYEfATGoL2es2TrvKWxrRN6YU9mGy6Xc+FOzsY+n9vBQsqbUlsusH2JHau6p/BNhAgva0V/mxmlxKp9UMUMEjKislfhzYIVjp+ik+RVRd3O6/7aHrvMdnIu8w1y85K6V2kmLqiwTBofcaTYAl9fYYzcPbe3n3RuL7nbmbTxDNvSgrqTOV4x7mr5dUlaUdUNLX9gDPf2+BjC28kpoIwobxrV28SHIVfl3JqEkSAISFzJFX8GU3PEqu1r7Is0q8fCWXA9yKmH9ihl5/ay7NyetF1qBIP9IO2LvrItkYL6tVsm93sOBaMx1sMupVILfIIuKGawy3EAvUHao27zKCcvysqGqu0WC1uYe3uB7WUQzqqFzz07Oc0vOt9vifCikvsyZOnLZadR2WZmdm5P3irU/eVVVr3WSZ7tc0YIL4Y7Od3WfyxnASFxPV+7DUZ/3px57yhqqIWMuqXF38KaFthhza1w6ZG6l2V9VURssyrbsx0lduuLnBLCtfwAbu81Snt+txAerg0baXyL7s0IPgECPgIj0l4CXmc6GnFnV0IGcBkQeLsIXBh4JdxX+XZhgCcHBKaMANDelAGH271xBOzmBZ9k7ji1PXZMInS+3jgu8PiAwNQQANqbGtRwI0CA0k6rtCGxc0DzfEp/6BQ94AUIAAKTQODvaG8SJYI8AQFAABAABACBiSEAtDcxaCFjQAAQAAQAgdlDYIK0N3CA4Y+fnp3LAeGhP4YPfggIAAKAACAQIjAi7bE4RpGTAKnXITz0YJb2WFAYtgV/3aiEkbKjBxhAeCi0hf43SbJNQbTitCBgKfefF2vDru/nlEV23gEvdYNcu1dlg6sXCUhUdiMHGB4bpR2Zp5a0SBX0F2Eqn0F46Jlh7ljVd5qSRUJKYEGC/L94C0lSKQp+1f8aRqLHi10zownmxMKhrXKpo9EDr/ffED7/gwiMQXuvT3jotlo8rDV/O65rm58UdiSL7yCIP9waOQkLwkPc0kmsbBM/0Yy1w4b9SNyHVvOGn3vrNCufyvVr23Vd67shpf0IWPaPUumkYT247kOztIHRlhemgwVTxttVq03JvZlfDrSfXqZ9gfDQs+NumV9r5lXVWIjQXoKF0FiVoqQSPVRUJOqnLDysc56X/BAKCeZEGvksUg9bboc6VyU1I89aGPSkp4Trk0ZgHNqLC072KoSHGIj3FTXtRZVMCGUUr2kyafzj8h9+XH3qwkO9od1YzO4nIpwNHAP3HpJ8z/mh7Fios66uE1cy8uOMxMEx4WsgPBTVsXpG4aFOs5DlIWEHKzDWQqIqRYM/8a54Qlfe9leuZMSCACeZ001J7qpikdpOTJzupPvA9X8bgb+lvVchPMSiv58ZeIF7e37gWhAeGkV4iBl/H+2xcGjvS8VNCSEkrkRV14OW8lDVMlIxEp2Vf0GanyS0zSQC+/upY1WID8YWZDjJVxAe6tGxei7hIVbLybTXZyExKkUJVd6xKluKfmKRDnHO8lLWqD8mmxOjPS3QNXZr24KwE4qZJOQPl98GAmPQ3usVHqIPdWNR1L9703HxMiXxmiYvYgQzJjzUS3ssFLWQUQrnNum4bDbYG0yEQHXs6pYofeyPreVeFuQFreKHAOWzUjveJGc9vyIInuxGmMn03oDwUI8MwnMKDyXRXoKF9KkUJZgAsU7zSoaJXqWQnD/zAsslmFN3kpM4l0U10696nXALuPzvIzAG7b1W4aHHRmFV1HytSz4IjZMpcWM1TV7EAH7qaJaEh3ppj2kJCZtVPxhiu66jUMiG0o5TfydJuzU7kPTz8CPXJTWrFK/CCIosKmppm21pwUta8VAXs4GOx5QBB+GhXh2r5xQeiqW9BAvxqz1UKUoyA7YSoXi6Fmy5bl4u/uJRbhLMyf3lbWlhqoSljzLaBW8vCdm3dX0c2otb25t14SG3UVwT1cNoCKjktT0QHooz/j7aYz3jVkB7pK5nQtpzzA+yuF3lOordjMhNWcvKhYsI53W/ZO/sY4WL4vZenc4nEB7qDUX9nMJDMbQXbyHdqg5VirqXet6xKZlQ4JPGa0EnmBPTnuwqHfbkCh/eHALj0N6rEx5ym8U1UfnccDy9pMAFeXInJwgP+e0gTraJKcpmlOKlw+L9/xdOcjrmR1ncLLdcD2vqSVOR24qWlY0z21esCqqA7d/77bht1zovqVg0zhNJcaItEoSH+gU9hogr0VGFh1iVdShpM41G/TshfqXHW0i8SpFf6wN3vGLenjdt4N5UNIx0T5I+wZzc26b14BLXbhznxPlc7WGi1gSZvxoExqC9nnN7r0J46KeOooWew/kLr2LiZUq41NGgpsnU63L4Ts7e4TmdmPBQsmwTsb7n1UUuPBRuaXmsaj0HO1Oezk5zX4zWQMoTf6eUXpfVRSywQ35a/rTPP5wW4AP7CUF46FmEh1jc0eVItXsKsQkWkqRSxI1gUOqItL4aahajNEILSu4wULJKMCf7my5hQUjzY7vXLzO0mpY1w33GQGBE2kvIEYSHEoCBy4DA0wiA8NDTGEEKQOD5EQDae35MIUdAIBkBEB5Kxga+AQSmggDQ3lRghpsAAh4CIDwElgAIvDQCf0d7L116uD8gAAgAAoAAIDAWAkB7Y8EFiQEBQAAQAAReNwITpD0QHnrdpgGlBwQAAUDgX0RgRNp7ncJDj2aBq96kBCSuRaNHRg4wRFRveg8w8OgPL1LlfecuBLXyIueNEtGLFR6i1G2Wd1ngFZRVImcSHPNLTp5npx3ENaMSbiK/qxe2FQkLKUHzRBleBGnvpiA89MzgxwsP0QQZoAQLiS1Tpx49kyT4UVdI6ySvrYg4LQjzsva5HgQJipc6is0YLr4pBMagvdcnPNS2W78s55GQttM60cV5vc6P7jx5XB2Eh1gbSEAvXniIEvODiDcrVpu6V0UlIxdvWB7uDx1ntPINYax4oKDFQtM7vPzQqJ6YjbOCnH5x2gPhoWfv8eKEh7oRMntkgBItJLZQnbqekUvXhLT5vz80dc2jYuW8ZT+6znUltyAoRzzQaJLUUWzOcPEtITAO7cUFJ3sdwkMd1zo1xLRSvmNdcXULSQctr5btY1VY46o3IDyUZPc96CUIDxHTwDh/4fVDLBCU9LnJAo/9pwrrFdvL2VONiR4aZsGoXpr2QHhoOsJDCTJAT1hIn016tHfbd7XnY+MDDrzAyPWB0ASR7+Dtm0Pgb2lv5oWHrNKKFz5EEHeqrP8F4aGjptuh5HYU4aEB9CiNFx66KytptXLnmIeFypXb/CwJ21zy5aGmr2jla5d23MZnRdziVRC2shmgPRAempLwUJIM0HALCU3Fe8MmOZG4KIoLkrJTrP0eWInoWOVVpAwO0PukjvqyhY9vDIExaO8VCw+17cbXYukHE2XmYm9IP3PM9yLeqdmXBTGt1wlzYoTVknVTVrBUvIqPcjsl25gx4SEaRY8mCA/xTq360CwsYfXYah3KwkaFS+vZ5icVC0wpRljUq8zbjvy9PO2B8NC0hIe6k5y9MkCdoRYSMRb+1m78aLR+2/ZNo7IrCdl8w5Oc9ZOR1qEqrpVafWyYIHXUnzd8fjMIjEF7r1V4yKtL5uRxdXXf2wtaxrmBsWF2KAgPDbP5ED2aIDzkeXv3fh6twNuzjhRhyajfEdpxW19zYtYwZ2qSE4SHpig8FCsD9ISFDDHKdl3PhFF2WTBR61tOXM6bngZf+MPhUkdhMnjzlhAYh/YGpw7O9FkXHgrrssPiwedYI09e2wPhoRCuvjdd9Gi88NDg2t6nJuWuobjPFvnYH6nrSMxf+p/Yy4t7eyA81BvZfMLCQ2HVhzJAbPJgmIWEvxh8w2nP8IPLU/tEl5aMejDwCpI/JXUUpIPXN4XAOLT32oSH3MtK+XvTclzyaJkHKkorZa7u/eROThAeYjt/EtBLEB4ibNJ4YCcn48ilvPlAKCX2qS4itRJ2TB1C2NSoWrknnkrR9BseCA9NUXiIxsoADbOQAakjclOvXfC92Y5V31fQgj95YJ/q0mKu+tuXtwrMKV7qaPpmBnecNQTGoL2IlEhKeA3CQ+SqnFuTMBIEAYkrueJPf0ch0/Y+0JgiSUZS98NTPrT33F50Mm66tTYbwkPJ6MUJDzGe5Of20gJaUIyTQEuIWLV9jdVBGuFlrXAWVMFDRY0KFSG9HkrxTQ3sgd19IDw0QeEhSuNlgJIshJlBv/AQuSyqixgJqVQaSxv5KjsYQym1SqtRY0r568oJUkdTsy+40cwiMCLtJZQfhIcSgIHLgMDTCIDw0NMYQQpA4PkRANp7fkwhR0AgGQEQHkrGBr4BBKaCANDeVGCGmwACHgIgPASWAAi8NAJ/R3svXXq4PyAACAACgAAgMBYCQHtjwQWJAQFAABAABF43AhOkPRAeet2mAaUHBAABD4FOs5CVCr9Gg2OsxKNl+USq6d/xiQJN4OvbksLjijxL1iPS3j8mPORBZ1c2USodbJ2P1zR5FpDHzGRGhIeCUpPLvDQnyF96olhRSlpfFGGue/w8XlaGnwcPjr4I6lceQoO10uCa97pcbE3/AEPwgCA8FCDxTK8TEx5ioZR6DAflfnhnGIj9o6AtY3ZWKauWrryI871p2Q+Rcf5nz+g0T2vNvvgviTmNlTgxl5gv7isqytV64rF5qSZ2x8FCjNVPPpilPVWaR+wI2bpR+eWdCksSihq8WeSK26qfNILzT5Hrf/R2DNr7Z4SHPKCck5yypuAo7cVomvwRqH/5o586wobphuoqQRy1v8z2z35OmoU1RVkaoL3bsrquyOmA9roRF3tkZehVQczmG+GzhNxGgqdr29VtJH/xBTH+rIx/9ysQHvo7/GJ+PTnhodBsiHtVkLDO451TclmQsFI4s5y269w2Wl5IBN/GWsUVQT22mVYRedGmFAPUmJcSaW/MfP4mebz2U0KOt9XiYa3523Fd2/ykoHnDZJydIBSVkMckLo9De3HByV6h8BCH0anpq0b9J49D7fXFI2iaTKICYvIcflz9sFaIKsU4ZnFbETFCiB3g9WPSn+loSdWWMF7JFz8qGGH1yOcVclPNb0gseVaNiMHGlIJfIq0DVT1qVDZ7aa9jVzaV4mWtG2wsQVaG016gsRd7k4eKmvEEoWK/nvxFEB6KmhMNNIRjLMStvZMkT8ZklGphPj3Sz4KkCRYynvBQkBmlpPFRFN+bnMfc+i4emI0IkzIVEX+aIbx2YeClfPVYV5ZEcV5U+Kir9VVXl1jLQAuyftgIw1WYHyUxi5HQM8nZ/CSJO4XCjspyWM3XAyHo2MT03MDLRvFjTl2VxKxavPTzZvExVjFCWN4p5teR9i3Znew0i6uiuICEOQFnRTErKgd+zL+YOzpVLSOrGyKaVwsHhoyRuB0onzw2y3uKmBEQlrSIwjbXCMux6G7hY4dY9b35437yvqKmeVTk3gy7QlFJvVanWVzjz55hwZODP7u8hrT9kr6pyFlR3qta3a+CJMmvf0t7r094iGHh1t/JuVOHdcpRb2+4pkkyiM/8zTDaE4Qlo0cp5sGsnDRsl3rKPmi1zOYiz3S0XGy1W8VlRlfkuigvFZm+62PdWMDa6MJDtyV1tdgi/XoU9jdN/mC60RibXIHBG3pT6ta2BWGnzrokhjBmSjGLsvax2hpoVNahgjZfUl8dhId6zGmYhfTHTHnC7GNoT4uxkLGEh8JbunV9XipccdZj0Xaw/qWUW+YUslU0AxLiyRNoL43Vg4bL+krXvmN22Tot128cFkbvrqovYONnxDUcWDxjYe0WjDrjKUa6EotAG/wNJKbnBhLkwhW7i/1VRZ7AZ6dVWkHKZ6YC5pznJUEYRnte3kneXt8dnarGogCSxgeRBaxpm/msyqVG7eoWlnarlstiKpXWsfpfd9bQ+iKnBK36GDxF0uuT2k8JP3TPDLzgeXuRFFGhqKRey0vev7Znl9cEtFW1O5S2G/nF3vqK3CH27Ri0948ID1HqnuflDa59GqU9OlzTJBa9yVycEeGhjlXekLlyrFuNensPtdyyXnMoZQ1gcJKzV1bmoVk/b1n3tnVZza8gvMt+1/1jwg5Y/z5Aht0Uk34HwkM9S7ZsmTOyztp4LyZ7UU9VTR/tdafBey1kPOEh/6bOiYaWS/56MI+TLiwZbKqjbVV3RbQdNbME2svo9ZhFMi9/Ut9DymEEmT5eobT5ScLvTS+1/Z+KPHVJ7/NAYubteeNOSul1UZ7nXsvvkoJCmmHx1Z6T9vjuD/tIET82KLXKa9xV9WRSgjEBC8XnqYP5oI748kf95EPdWBT17z2tn+0PiApFneloq+pyqQD9B6GPVS26hyWG9lDu1Os6SG0HqYOTkckPNAbt/SPCQ+1GYVkp3XBIemgvAlK/pknkqym8ZWt7eu3ecR74vxuOOu3yOuIiEpFCuM3KB01eZPMebA4knWNO1pnO2yGLc89mmdxaDrOdOyzsbxqzlN7/fO8oNZIrG5Z+06Q9PpylUdpz63uS9pUPEqO0R2msrEw0S3KZF3sX5Mm5IQ4OAKO/mfR7EB56lcJDbIKrqyXLpcS6LHWZF3tmwxJob6l/F5V9VtTXJa9pYCTIB5H15gEmY5Oc+w3PPJ2vKu+vA2MdSMxob6Xks+htSfZ686uC2O3WmSzMc9IeZ1b7WJVYIW1Ge1d86kVA3hwp7y4wXueTQ0HBx34dsZ98bBRWRe24TwZxQCgqodfySxVHe8EUOqnvIuWo67k++SDj0N4gnb5G4aG7stK3w2tOZDYR/evVNIl+M504dNYAAAPESURBVI33wyY5UV/I/Oa+iLcrljduvS3JaBjtjSUrY75HPZvmUilhs+p2msXF/svdHsdHJ5SV6UWLDTJywTQXmx2q7SAplCXqTTulTyA89BqFh66LckaLzIw71U2kHAXO2Yi0F3FqmbHx9bDCBZvk5IJZaOK098fe3uDkSB/ROlUtlvbuysowH3f8NjdKP+k2imuietgMB+/ebWKEomaU9v4V4aFu9Ua8vSRNk27iqb0bh/bM91ja90zKbXySh3t79LGuL0jGqcWWNIhrXdZMrsT01JNFvb1I2j5v77ZpPbjEtRvHOXE+V+NzKfZ5zby23TZxf5ulDYy2Ip0VW6iIWeKO3GDib0F4qG8UNdRCxtnS0qGkzRQu9e+EBHsN/l54iBsE28yCd3v2XjjfNLycN+8JJXZtT0TbETOjCd5eH+3dl9WQSu+qOTx5b89b2ztosoW2i4I8ytqeW8tl5OKvPgahdETao3Z1U1T2TbtNaYc4t2b9sush2f+NtKVlaD85YCEu242ifG44niRUYAnxQlGzSXvRQf4rFx4K+tMo7cVrmgQpp/k6Du3Ru1p+XZJWVHVDyx8Yw729obIyQ55wJNqLlZVpHWsy236WEjKi8q7cjCyYW19kwVvbH3LniX4FwkPr/ZMHQy1k5C0tnVZpOdJbBLMpsRZCxxEeYubg1nUsGud9Xb9jfub6VghLI25p6aM96jaPcvKirGyo2m6xsIV9b++uorEVBIzmBDQvilkp941RRfwkZ0Li+ElOypXOVjHKYHmnVNhA2rdBP66vAZDmoSbNY7wgKp/5JprYOyZ5e5TSx2b5nb/xW1yJCIFROuKWlgTtJ6+cAxbSdwR5DueZLHCCUFQC7bnfdclbwfF2sS6qfImKTXRPepKzD/3gIwgPBUjAKyAwNgIgPDQ2ZP/qD9iyPdvHAX9TQWDEtb2EsgDtJQADlwGBBARAeCgBmLd32b1pNB8Y1ZHrkoK1arDH8u0hMe0nBtqbNuJwvzeNAAgPvenq73l454chL2A8j8WlnvnGnkTwYQII/B3tTaBAkCUgAAgAAoAAIDA5BID2Joct5AwIAAKAACAwcwgA7c1clUCBAAFAABAABCaHANDe5LCFnAEBQAAQAARmDgGgvZmrEigQIAAIAAKAwOQQANqbHLaQMyAACAACgMDMIQC0N3NVAgUCBAABQAAQmBwCQHuTwxZyBgQAAUAAEJg5BID2Zq5KoECAACAACAACk0MAaG9y2ELOgAAgAAgAAjOHANDezFUJFAgQAAQAAUBgcgj8D2PnX4VeuNdfAAAAAElFTkSuQmCC"
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df=pd.DataFrame(QR)\n",
    "df.to_csv('QR_CartPole_Result.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       64.000000\n",
       "1       46.500000\n",
       "2       36.333333\n",
       "3       38.000000\n",
       "4       38.600000\n",
       "          ...    \n",
       "365    200.000000\n",
       "366    200.000000\n",
       "367    200.000000\n",
       "368    200.000000\n",
       "369    200.000000\n",
       "Name: 2, Length: 370, dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\Miniconda3\\lib\\site-packages\\ipykernel_launcher.py:16: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAasAAAEWCAYAAADYRbjGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAA3I0lEQVR4nO3dd3wUdfrA8c+TSkJCj6E36SAgREWwHlZseOdZ78RyYv3p3empWLFgPct53ulZODuKgtgLdjkVCEiHUEMPBEJCCAkp+/z+mAkuMWUD2Z3d7PN+vfa1s99pz87s7rMz853vV1QVY4wxJpzFeB2AMcYYUxdLVsYYY8KeJStjjDFhz5KVMcaYsGfJyhhjTNizZGWMMSbsWbKKUCJytIhkeR2HqZuIPCsid7rDx4nIBq9j8oKI7BKR7l7HEUrRvL8bmiWr/SAi2SJygpcxqOr3qto7WMsXkZNF5DsRKRSRXBH5VkTODNb6GpKI9BOR90WkwI3/KxEZFqJ1XyIiM/zLVPUqVb2vAZb9koiUuj/6eSIyXUT6HOhyQ0VVU1R1dTCWLSIXikimu202i8gnInLUASxPRaSH3+vjRMTnLr9QRLJE5NKGid4EwpJVmBKRWA/XfQ7wNvAK0BFIB+4CztiPZYmIhOxzJiIHA/8DFgLdgPbANGC6iBweqjiC6BFVTQE6ABuBFxt6BSIS19DLDCYR+SvwJPAAzme1M/Bv4Kz9WFZt732Tu+2bAbcAz4tIv3oHbPaPqtqjng8gGzihmvIY4FZgFbAdmAy08hv/NpADFADfAf39xr0EPAN8DBQBJ7jruQlY4M7zFtDEnf44YEOVmKqd1h1/M7AZ2AT8CVCgRzXvQYB1wN9qef/jgdf8Xnd1lxfnvv4GmICTNIpxvtiZVZbxF+B9dzgR+Lu73i3As0DSfu6bV4GPqyl/Bvi6um1XdZ8ChwM/AvnuNnsaSPCbVoGrgBXuNP9yt1tfoASoAHYB+X779v4a9lt7YAqQC6wBrq/lve1djvt6FFAUyLKAJOBlYAew1P08VP383OJ+fvYAccAw4Af3Pc4HjvOb/hJgNVDorusit7wH8C3OZ3Ab8FaV7dbDHW6O82coF1gL3AHE+C17hvuZ2OEu/9Qatklzd1v/vpbtFsj+vNbdn2twvpuK8z3cBZxXw2cmFzgH5/P7JM53a5M7nHig+9seVfaj1wFE4oOak9UNwE84RyOJwH+ASX7jLwNS/T7c8/zGveR+wUfgJL0m7npmuR/wVu6PzFXu9FW/BLVNewpOkuwPJAOvUXOy6uOO61bL+x9P3clqnbu+OPcHpRDo6TfPbOB8d/gJ4H037lTgA+DBGtbd2f3R6VzD+Bzg0mrKjwfK3e1a3Q/P3n0KDMX5oY5z39tS4M9+0yrwIdDCjScXOMUddwkwo8qyX6KaZOXu5zk4R60JQHecBHByDe/NfzlNcRLz/ECWBTyEk0Ra4nw+F1Tz+ZkHdMJJbB1w/nCNcpd9ovs6zV33TqC3O2873D9ewCTgdn75DB9VZbtVJqtXgPfc/d0VWA5c7rcNy4ArgFjgapwkINVsk1Pc/RpXy+c1kP05Hefzl1Q11hr229lujL2Be3G+9we52+cH4L4D3d/2qLIfvQ4gEh/UnKyWAiP9XrdzP9C/+iLh/NAp0Nx9/RLwSjXr+YPf60eAZ93hvV+CAKadiN+PP86/35qS1Qh3XJOq4/ymGU/dyereKvO8BtzlDvfESV7JOEckRcDBftMeCazZz31Tjps4qpRXJuH2VbddbfvUHfdn4F2/18q+P8KTgVvd4UsIPFkdAayrMu044L81xPESzpFbPuDD+Wc+MJBlUeVHEefouurn5zK/17cAr1ZZ3mfAGJxklQ/8jipHwDhJ6DmgYzXxq/vZiwVKgX5+464EvvHbhiv9xiW787atZpkXATn1/IxUtz9/U12sfq+Pc7d5PpCHk9gr/2ytAkb5TXsykH2g+9se+z7smlXD6gK8KyL5IpKPk7wqgHQRiRWRh0RklYjsxPlxAGjjN//6apaZ4ze8G0ipZf01Tdu+yrKrW0+l7e5zu1qmCUTVdbwBXOAOXwhMU9XdOP9Ek4E5ftvtU7d8f2yj+tjb4fwAba9m3D5EpJeIfCgiOe6+eoB99xPUb7/UpAvQvvJ9u+/9NpzrLjX5u6q2wPmDUIzzzz6QZQXyGfAv6wL8vsryjgLaqWoRzqmxq4DNIvKRX0WPm3H+gMwSkcUiclk162kDxOOc/qu0FudortLe7et+TqD6bbwdaFPbtaYA92dt34lKm1S1haq2UtXBqvqmW96+mvfSvpr592d/G5clq4a1Hufcegu/RxNV3YjzA30WzrWo5jg/NuB8sStpkOLajHPqp1KnWqbNwnkfv6tlmiKcBFOpbTXTVH0v04E0ERmMk7TecMu34fzo9vfbZs3VuZC9P74Afl9N+bnAT6q6p2r8bmUW/+T4DLAM57RlM5wfFP/9VJv67MP1OEeQ/p+XVFUdVedKVNfhnHb+h4gkBbCsQD4D/rGvxzmy8l9eU1V9yF3/Z6p6Is6fgGXA8255jqpeoartcY6W/u1fq861DeeMQxe/ss44FUbq60eca2yja5kmkP15IN+9Tfz6vWyqZrr93t/GktWBiBeRJn6POJyKARNEpAuAiKSJyFnu9Kk4X6rtOD+UD4Qw1snApSLSV0SSgTtrmlCdcxN/Be4UkUtFpJmIxIjIUSLynDvZPOAYEeksIs1xTmXUSlXLcCqYPIpzbWC6W+7D+aF7QkQOAhCRDiJy8n6+13uA4SIyQURaiUiqiPwfcCnOtQJwro80EZHTRCQe5+J+ot8yUnGuyexyjxiursf6twAdRSQhgGlnAYUicouIJLlH3wNE5LBAVqSq03F+FMcGsKzJwDgRaSkiHYDr6lj8a8AZ4tzCEOt+xo8TkY4iki4iZ4lIU5zP9C6cU2SIyO9FpDIp7sBJAr4qcVe48Uxw908XnM/ca4G87yrLKsDZr/8SkdEikiwi8SJyqog84k62P/tzC841pUBMAu5wv+9t3Hiqey8HtL+jnSWr/fcxzhFB5WM88A+cigKfi0ghzkXXI9zpX8E5PbARWOKOCwlV/QR4CvgaWOm37j01TP8Ozmmey3B+DLcA9+NcEK/8kXwL5yL9HJzKBoF4A+fI8m1VLfcrv6UyLvc0zRf8cnprH26C3CUinWuIfQXO6apBOKda84H7gLNV9Qt3mgLgGuAFnP1RBPjfuHkTzpFwIU4ifSvA9wfwFbAYyBGRbbVN6P5onw4Mxrn+tM2NqXk91vcozqm3uDqWdS/Oe1yDs33foYb978a2HudMwG04FUjWA3/D+c2IwUkum3Cu3xzLLwngMGCmiOzC+S7coNXfW/V/ONt9NU7Nvzdwrq3Wm6o+5sZzh1+s1+HcsgD7tz/HAy+7p+vOrWPa+4FMnO/DQmCuW1Y1zobY31FLnD/SJpqISF9gEU712vK6po9k7r/8n4C7VbXB70mKVCJyNU4FgWO9jsWYQNiRVZQQkbNFJFFEWgIPAx809kQFoKobgFOBdiKyv9fBIp6ItBOREe4p3d7AjcC7XsdlTKDsyCpKiMinOFXCK3Dut7lGVTd7G5UJFfe60Ec4rXrkA28C41S11Mu4jAmUJStjjDFhz04DGmOMCXsR1WBlVW3atNGuXbt6HYYxxkSUOXPmbFPV/b3x3hMRnay6du1KZmam12EYY0xEEZG1dU8VXuw0oDHGmLBnycoYY0zYs2RljDEm7FmyMsYYE/YsWRljjAl7QUtWItJJRL4WkSVuvzY3uOWtRGS6iKxwn1u65SIiT4nIShFZICJDghWbMcaYyBLMI6ty4EZV7YfTpfS1ItIPuBX4UlV7Al+6r8Fpv62n+xiL0weNMcYYE7z7rNx25za7w4UishSnJ9CzcLp6BngZpwv0W9zyV9z+lH4SkRYi0s7arzPGBFtu4R6ycgpZsbWQHUWR0Vxir7apnD6wug6JG6eQ3BQsIl2BQ4GZQLpfAsrhly6dO7Bv19Ib3LJ9kpWIjMU58qJz52q7NDLGmFq9nbmeJ79YwaaCYgD8m0iVQPuE9tjpA9tbsmpIbrcMU4A/q+pO8fskqKqKSL1a0lXV54DnADIyMqwVXmNMvbzyYzZ3vbeYoV1a8tshHRAgKSGOQZ2a07lVMh1bJnsdoqlGUJOV22X4FOB1VZ3qFm+pPL0nIu2ArW75RqCT3+wd3TJjjDlg67bv5v8mzWX+hgJO6JvOM38YQnysVYiOFMGsDSjAi8BSVX3cb9T7wBh3eAxuV+lu+cVurcBhQIFdrzLGNITi0gqufn0OSzbvZMyRXXjivEGWqCJMMI+sRgB/BBaKyDy37DbgIWCyiFwOrAXOdcd9DIwCVgK7gUuDGJsxJkqoKuOmLmDJ5p28OCaD3/RJr3smE3aCWRtwBlDTpcqR1UyvwLXBiscYE53++79sps3bxI0n9rJEFcHsONgY02j9tHo7Ez5eykn90rn2+B5eh2MOgCUrY0yjtKOolD+/OY/OrZJ57NxBxMRESJ10U62I7nzRGGOqo6rcMmUB24v28O6YEaQ2ifc6JHOA7MjKGNPovDV7PZ8v2cItp/RhQIfmXodjGoAlK2NMo7J2exH3friEET1ac9mIbl6HYxqIJStjTKNR4VP+Onk+sTHCo+fYdarGxK5ZGWMajWe/XcWctTt48rzBtG+R5HU4pgHZkZUxplFYvKmAJ79YzmkD23HW4Ohp4DVaWLIyxkS8krIK/vLWPFomJzBh9AAkUppONwGz04DGmIj398+yWL5lFy9fdjgtkhO8DscEgR1ZGWMi2g+rtvHi/9bwx2FdOLZXmtfhmCCxZGWMiVg7S8q4afJ8urVuyrhRfbwOxwSRnQY0xkQkVeW2qQvZUriHKVcPJznBfs4aMzuyMsZEpBe+X8OHCzZz40m9GNyphdfhmCCzZGWMiTgrthTy6GdZnNw/nauPPdjrcEwIWLIyxkSU8gofN72zgKaJsdw/+hCrph4l7CSvMSaiPP/9Guavz+efFxxKWmqi1+GYELEjK2NMxFiVu4snvljOqQPacvrAdl6HY0LIkpUxJiL4fMq4KQtJio/l3rOslYpoY8nKGBMRJs1ex6zsPG4/ra+d/otCQUtWIjJRRLaKyCK/srdEZJ77yBaReW55VxEp9hv3bLDiMsZEnpyCEh76eBnDD27N74d29Doc44FgVrB4CXgaeKWyQFXPqxwWkceAAr/pV6nq4CDGY4yJUOPfX0xphY8Hf2u1/6JV0I6sVPU7IK+6ceJ82s4FJgVr/caYxmH6ki18ujiHP5/Qiy6tm3odjvGIV9esjga2qOoKv7JuIvKziHwrIkfXNKOIjBWRTBHJzM3NDX6kxhjPFO0p5+73FtE7PZU/HW1d1Eczr5LVBex7VLUZ6KyqhwJ/Bd4QkWbVzaiqz6lqhqpmpKVZC8vGNGZPfrGcTQUlPPDbAcTHWn2waBbyvS8iccBvgbcqy1R1j6pud4fnAKuAXqGOzRgTPhZtLGDi/7K58IjODO3SyutwjMe8+KtyArBMVTdUFohImojEusPdgZ7Aag9iM8aEgQqfcvu7C2mZnMAtJ1vXHya4VdcnAT8CvUVkg4hc7o46n19XrDgGWOBWZX8HuEpVq62cYYxp/F77aS3zNxRw1xn9aJ4c73U4JgwEreq6ql5QQ/kl1ZRNAaYEKxZjTOTIKSjh0c+yOLpnG86wJpWMy65YGmPCyj0fLKaswsf9o61JJfMLS1bGmLDxxZItfLIoh+tH9rR7qsw+LFkZY8LC7tJy7n5/Mb3SU7ji6O5eh2PCjPVnZYwJC09+sYKN+cW8c9WRJMTZ/2izL/tEGGM8N2dtHi98v5oLDu9MRle7p8r8miUrY4ynivaU89fJ82nXPInbRtk9VaZ6dhrQGOOpRz/LYl3ebiZdMYzUJnZPlameHVkZYzyzaGMBr/yYzUVHdGZY99Zeh2PCmCUrY4wnfD7lzvcW0TI5gb+dZKf/TO0sWRljPPH2nPX8vC6fcaP6WpNKpk6WrIwxIbejqJSHPlnGYV1b8rshHbwOx0QAS1bGmJB75LMsdpaUc581qWQCZMnKGBNSP6/bwZuz13HJ8K70aVttH6vG/IolK2NMyFS4lSrSUhL58wk9vQ7HRBBLVsaYkHl95loWbdzJnaf3s3uqTL1YsjLGhERu4R4e/SyLo3q04XTrp8rUkyUrY0xIPPjJUkrKKrjnrP5WqcLUmyUrY0zQzVqTx9S5G7ni6O4cnJbidTgmAlmyMsYEVVmFjzunLaJDiySu+00Pr8MxESpoyUpEJorIVhFZ5Fc2XkQ2isg89zHKb9w4EVkpIlkicnKw4jLGhNbLP2STtaWQu87oR3KCtZ1t9k8wj6xeAk6ppvwJVR3sPj4GEJF+wPlAf3eef4tIbBBjM8aEQE5BCU9MX87xvdM4qV+61+GYCBa0ZKWq3wF5AU5+FvCmqu5R1TXASuDwYMVmjAmN+z9aQrlPuedMa6nCHBgvrlldJyIL3NOELd2yDsB6v2k2uGXGmAg1Y8U2PlywmWuO60Hn1sleh2MiXKiT1TPAwcBgYDPwWH0XICJjRSRTRDJzc3MbODxjTEMoKavgzvcW0bV1Mlce293rcEwjENJkpapbVLVCVX3A8/xyqm8j0Mlv0o5uWXXLeE5VM1Q1Iy0tLbgBG2P2y7+/WcWabUXcP/oQmsTb5Wdz4EKarETE/7b1s4HKmoLvA+eLSKKIdAN6ArNCGZsxpmGsyt3Fs9+s4qzB7TmqZxuvwzGNRNDqkYrIJOA4oI2IbADuBo4TkcGAAtnAlQCqulhEJgNLgHLgWlWtCFZsxpjgUFXunLaIxPgYbj+tr9fhmEYkaMlKVS+opvjFWqafAEwIVjzGmOB7b94mfli1nftGD+Cg1CZeh2MaEWvBwhjTIAqKy7j/oyUM6tSCiw7v7HU4ppGxZGWMaRB//yyLvKJSJoweQEyM3VNlGpYlK2PMAZu/Pp/XZq7l4iO7MqBDc6/DMY2QJStjzAGp8Cl3THN6/73xpF5eh2MaKUtWxpgD8tx3q1m4scB6/zVBZcnKGLPfVm4t5PHpWYw6pK31/muCypKVMWa/qDqn/5IT4rj3LGuo1gSXJStjzH55b94mflqdx82n9KZNSqLX4ZhGzpKVMabe/O+puuAwu6fKBJ8lK2NMvT32ud1TZULLkpUxpl4WbMjn1Z/snioTWpasjDEBq7ynqk1KIn+1e6pMCFmyMsYE7JUfs1mwoYA7TutLM7unyoSQJStjTEA25hfz6GdZHNsrjTMHtfc6HBNlLFkZY+qkqtzx7kIAJpxt91SZ0LNkZYyp0wcLNvN1Vi43ntSbji2TvQ7HRCFLVsaYWqkq//pqJX3apnLJ8K5eh2OilCUrY0yt5q3PJ2tLIWOGdyXW7qkyHrFkZYypkary2OfLSU2Ms4ZqjacsWRljavTevE3MWLmNm0/pbd1/GE/F1TZSRIbUNl5V59Yy70TgdGCrqg5wyx4FzgBKgVXApaqaLyJdgaVAljv7T6p6VaBvwhjT8PJ3l3Lfh0sY3KkFFx7RxetwTJSrNVkBj7nPTYAMYD4gwEAgEziylnlfAp4GXvErmw6MU9VyEXkYGAfc4o5bpaqD6xO8MSZ4HvpkGfnFZbx69iF2rcp4rtbTgKp6vKoeD2wGhqhqhqoOBQ4FNtYx73dAXpWyz1W13H35E9BxvyM3xgTN7Ow83py9nstGdKVf+2Zeh2NMwNeseqvqwsoXqroI6HuA674M+MTvdTcR+VlEvhWRo2uaSUTGikimiGTm5uYeYAjGmKrKKnzc/u5COrRI4i8nWvt/JjzUdRqw0kIReQF4zX19EbBgf1cqIrcD5cDrbtFmoLOqbheRocA0EemvqjurzquqzwHPAWRkZOj+xmCMqd6LM9awfMsunr84g+SEQH8ijAmuQD+JlwBXAze4r78DntmfFYrIJTgVL0aqqgKo6h5gjzs8R0RWAb1wrosZY0JkU34xT325ghP6pnNiv3SvwzFmrzqTlYjEAp+4166eOJCVicgpwM3Asaq62688DchT1QoR6Q70BFYfyLqMMfU34aOlVPiUu8/o53UoxuyjzmtWqloB+ESkXr2sicgk4Eegt4hsEJHLcWoHpgLTRWSeiDzrTn4MsEBE5gHvAFepal51yzXGBMeMFdv4aOFmrj2+B51aWft/JrwEehpwF851q+lAUWWhql5f0wyqekE1xS/WMO0UYEqAsRhjGlhpuY+73l9El9bJjD2mu9fhGPMrgSarqe7DGNMIvThjDatzi5h4SQZN4mO9DseYXwkoWanqy8EOxBjjjextRfzjy+Wc0Ded3/SxShUmPAWUrESkJ/Ag0A+nNQsAVNXOFxgTwSp8yk1vzyc+Nob7Rw/wOhxjahToTcH/xamqXg4cj9OE0mu1zmGMCXsv/ZBN5todjD+jP22bN6l7BmM8EmiySlLVLwFR1bWqOh44LXhhGWOCbX3ebv7+WRa/6XMQvx3SwetwjKlVoBUs9ohIDLBCRK7DaRcwJXhhGWOCSVW5fdoiYgTuGz0AEWuo1oS3QI+sbgCSgeuBocAfgDHBCsoYE1zvzdvEd8tz+dvJvenQIsnrcIypU6BHVnmqugvnfqtLgxiPMSbIcgv3cK/bT9Ufj+zqdTjGBCTQZDVRRDoCs4Hvge/8W2E3xkSO295dSNGech45Z6D1U2UiRqD3WR0rIgnAYcBxwEcikqKqrYIZnDGmYc1Zu4PpS7bwt5N70ys91etwjAlYoPdZHQUc7T5aAB/iHGEZYyLIk18sp1XTBC4Z3tXrUIypl0BPA34DzMG5MfhjVS0NWkTGmKCYszaP71dsY9ypfWiaaP1UmcgS6Ce2DTACp3X060XEB/yoqncGLTJjTIMpr/Bx9/uLaZOSyB+P7OJ1OMbUW6DXrPJFZDXQCegIDAfigxmYMabhvDhjDYs27uTfFw2x3n9NRAr0mtVqYBkwA6fZpUvtVKAxkSF7WxGPT1/OSf3SOXVAW6/DMWa/BPoXq4eq+oIaiTGmwakq46YuJCE2hnvPspYqTOQKtAWLHiLypYgsAhCRgSJyRxDjMsY0gMmZ6/lx9XbGjeprDdWaiBZosnoeGAeUAajqAuD8YAVljDlwxaUVPPpZFod3bcX5h3XyOhxjDkigySpZVWdVKStv6GCMMQ3nrdnr2LarlBtP6kWMtVRhIlygyWqbiBwMKICInANsDlpUxpgDsqOolH98uYJh3VtxeDdraMZEvkCT1bXAf4A+IrIR+DNwVV0zichEEdlaea3LLWslItNFZIX73NItFxF5SkRWisgCERlS/7djjAF49PMsdpaUM/7M/lapwjQKASUrVV2tqicAaUAf4FjgqABmfQk4pUrZrcCXqtoT+NJ9DXAq0NN9jMWpIm+MqafM7DwmzVrHmCO70qdtM6/DMaZB1JqsRKSZiIwTkadF5ERgN04/ViuBc+tauKp+B+RVKT4LeNkdfhkY7Vf+ijp+AlqISLuA34kxUc7nU8ZNXcDv//MjnVom85cTe3odkjENpq77rF4FdgA/AlcAtwMCnK2q8/ZznemqWnm9KwdId4c7AOv9ptvglu1zbUxExuIcedG5c+f9DMGYxufN2euZNGs9Qzq34IHfHkJqE2tkxjQedSWr7qp6CICIvICTODqraklDrFxVVUS0nvM8BzwHkJGRUa95jWmsNuUX89AnSxnWvRWTrhhm16lMo1PXNauyygFVrQA2NECi2lJ5es993uqWb8Rpe7BSR7fMGFOL12eu5aiHv6K4rIL7Rx9iico0SnUlq0EistN9FAIDK4dFZOd+rvN9nOteuM/v+ZVf7NYKHAYU+J0uNMZUUVru4/Z3F3L3e4sZfnAbpl49gh4HpXgdljFBUetpQFWNPZCFi8gknJ6F24jIBuBu4CFgsohcDqzll4oaHwOjcCpv7AYuPZB1G9PYPfDxUl6fuY7zD+vEuFF9aZ5k16hM4xXUvgJU9YIaRo2sZlrFuZ/LGFOHuet28NIP2VwyvCvjz+zvdTjGBF2gNwUbY8KEqvLAR0tJb5bI307u7XU4xoSEJStjIszMNXlkrt3BNcf1sO7pTdSwZGVMBMkt3MOtUxaQ3iyRczOsJXUTPexvmTER4ud1OxgzcRYlZT5ev+IIkhIOqP6TMRHFkpUxEeD1mWt54KOltE5JZOIlh1kVdRN1LFkZE+byikq554MlDO7UgifOG0yHFkleh2RMyNk1K2M8tmZbEWf963+8N28jPt++LYipKv/5bhWl5T7uHz3AEpWJWnZkZYwHyip8TJ27gXnr8/lkUQ75u8u44c153DplIRcd0ZkT+qUzrHtr/vnVSv7z7WrOGNSeXumpXodtjGcsWRkTYhU+5c9vzuOjhU5rYs2T4nnt8iOYviSHdXm7eWHGGl6YsYZzhnZk6twNjB7cnsfPHext0MZ4zJKVMSH2/Per+WjhZm49tQ9nDmoPQPsWSRzVsw0AJWUV/O2dBbwzZwO90lO4b/QAYmKscVoT3SxZGRNCXy/byt8/y+LUAW258pju1baQ3iQ+lsfPHcTh3Vpxcr9065fKGCxZGRMyk2at485pi+jTLpWHzxlYa1ce8bEx/HFYlxBGZ0x4s2RlTJDlFZXy0CdLmZy5gWN7pfHUBYfSzI6WjKkXS1bGBNH2XXu46IWZrN5WxMVHduH20/qSGGctTxhTX5asjAmSykS1ZlsRE8cctrcChTGm/uymYGOCIK+odG+ietESlTEHzI6sjGlAs7PzmPDRUgpLyli/o5gXx2RYojKmAViyMqaBZGbnccnEWVSoogrPX5zB0T3TvA7LmEbBkpUxB6ikrIJbpizgvXmb6NamKW+OHUZKYpx1jGhMAwr5t0lEegNv+RV1B+4CWgBXALlu+W2q+nFoozOmfvJ3l3LZS7P5eX0+Vx93MJcf1Y02KYleh2VMoxPyZKWqWcBgABGJBTYC7wKXAk+o6t9DHZMx+6NoTzmX/Hc2Szbv5N8XDuHUQ9p5HZIxjZbX5ylGAqtUdW1td/MbE27W5+3m5ncWsHBjAc/+YSgn9kv3OiRjGjWvq66fD0zye32diCwQkYki0rK6GURkrIhkikhmbm5udZMYE1Rz1+1g1D++Z+66HTx6zkBLVMaEgKhq3VMFY8UiCcAmoL+qbhGRdGAboMB9QDtVvay2ZWRkZGhmZmbwgzXGNTlzPXdMW0S75k147fIj6NQq2euQjKk3EZmjqhlex1EfXp4GPBWYq6pbACqfAUTkeeBDrwIzplLRnnK+WLqFpglx/PeHNfxv5XZG9GjNU+cfSmurSGFMyHiZrC7A7xSgiLRT1c3uy7OBRZ5EZYzrvXkbufeDJWwvKgUgOSGWO0/vx5gjuxAX6/UZdGOiiyfJSkSaAicCV/oVPyIig3FOA2ZXGWdMSP3r65U8+lkWQzq34J8XHMrP6/MZ0aMNgzu18Do0Y6KSJ8lKVYuA1lXK/uhFLMb48/mUBz9ZyvPfr+HsQzvw6DkDiYuNYXgPazLJGC95XXXdmLBRXFrBX96ax6eLc7j4yC7cdXo/O91nTJiwZGUMkFu4hz+9ksmCDfncdXo/Lh3RtdaefI0xoWXJykS9FVsKufSl2WzfVcp//jCUk/q39TokY0wVlqxMVHv1p7U8/MkykhJieevKYQzs2MLrkIwx1bBkZaLWxBlruPfDJRzdsw0P/W4gHVokeR2SMaYGlqxM1Ckt9/Hst6t4fPpyTunflqcvPNQqUhgT5qLyG7py6y7GTJzF/PX5XodiQmxHUSlXvprJ49OXc9oh7fjHBYMtURkTAaLyW1pSVsG3y3PJ2VnidSgmRFSVH1Zt44THv+W7Fdt44OxD+NdFQ0iMi/U6NGNMAKLyNGBSgvMDVVJW4XEkJhSe+24VT0xfQXFZBV1bJ/P6FUfQp20zr8MyxtRDVCarZDdZ7S61ZNVYFZaUcdPb84kR4ZNFORzbK41TBrTl9IHtSG0S73V4xph6ispklRTvJKtiS1aN0pJNO7np7fksy9mJT2HUIW158rxDSYiLyrPexjQK0Zms3COrYjsN2KhU+JQpczdwz/uLSU6M47k/ZtC7bSodWiQRE2OtURgTyaIyWSXExhAjsLu03OtQTAPZmF/Mla9msmjjToZ2acm/LxpCerMmXodljGkgUZmsRITkhDiKS31eh2IawLKcnYyZOIvdpRU8dcGhnDGwnbXrZ0wjE5XJCqBJfCzFZXZkFem+X5HLNa/PpWlCHG9fdaTV8jOmkYraZJWcEGsVLCLcC9+v5v6PltIrPYX/Xnq4NZdkTCMWtckqKT7Wqq5HsHfmbOD+j5Zy6oC2PH7u4L2VZowxjVP0JquEWKsNGKG+ztrKbVMXMvzg1jx1waHEW3NJxjR6UfstT4q304CR6M1Z67ji5Ux6pqfwzEVDLVEZEyWi9sgqOSGWnJ1lXodhAqSqPPnFCv7x5QqO7tmGpy8cQvMka4nCmGjhWbISkWygEKgAylU1Q0RaAW8BXYFs4FxV3RGM9TexChYR48MFm3j+u9XM31DAOUM78tBvD7GW0o2JMl5/449X1cGqmuG+vhX4UlV7Al+6r4MiOd6uWYW7sgofd05bxHVv/MyuPeXcN3oAj/xuoCUqY6JQuJ0GPAs4zh1+GfgGuCUYK0pOsNqA4cznU66f9DOfLMrhymO6c/MpfYi1JpOMiVpe/kVV4HMRmSMiY92ydFXd7A7nAOlVZxKRsSKSKSKZubm5+73yJlYbMGzlFJRw85QFfLIoh9tH9WXcqL6WqIyJcl4eWR2lqhtF5CBguogs8x+pqioiWnUmVX0OeA4gIyPjV+MDlRwfR2m5jwqf2g9hGJmcuZ7bpi6k3KfcMLInfzq6m9chGWPCgGfJSlU3us9bReRd4HBgi4i0U9XNItIO2Bqs9SclOAeVxWUVpCSG29nQ6PTB/E3c/M4Cju7ZhrtO70fP9FSvQzLGhAlPTgOKSFMRSa0cBk4CFgHvA2PcycYA7wUrhqQEJ0FZy+vhYXZ2HjdOns9hXVvy/MUZlqiMMfvw6pAiHXjXbRk7DnhDVT8VkdnAZBG5HFgLnBusAFISneZ5CkvKOch+Fz1V4VPueHcRbZs34fmLM2gSb00nGWP25UmyUtXVwKBqyrcDI0MRQ4ukBAAKiu3GYC+VV/h4+NNlZG0p5OkLD6VFcoLXIRljwlDUXqxpkey0flCw25KVV7bt2sP1k37mh1XbufCIzowa0M7rkIwxYSqKk5XzDz6/uNTjSKLTuz9vYPz7Sygpq+DRcwby+4xOXodkjAlj0Zus3Hbl8u3IKuQKisu4a9piuqc15eFzBlqHicaYOkVtuzXNLFl5orTcxz0fLKZwTzkP/PYQS1TGmIBE7ZFVbIzQrEkc+bud04A5BSWkpSbaDcJB9PniHB74eCnZ23dz/cie9G/f3OuQjDERImqPrMC5bpVfXMbSzTsZ9uCXvPRDttchNVqv/JjN2FfnkBAXw38vOYy/ntjL65CMMREkqpNVy+R48opKefSzLAC+Xb7/bQ2ams1fn889HyzhhL4H8eH/Hc3xfQ7yOiRjTISJ2tOAAM2TE/jOL0Gtzt3lYTSNz/Zde3hv3iYe+nQZaSmJPPb7wSTERfX/I2PMforqZFXsNrU05sgudGyZzISPl7K1sISDUpt4HFnkWrSxgGe/XcXmghLmrtuBKvymz0FMOHsAzZOtZ19jzP6J6mRV6U9Hd2drYQkAP6/L5+T+bT2OKDKVVfi47o257NhdRq/0FK4+9mCGdG7J8X0OsoorxpgDEtXJ6vFzB7Ny6y46tUomLTWR+Fhh7rod9UpWVovwF2/OWkf29t28cHEGJ/T7VVdkxhiz36I6WXVqlUynVskANImPpX/75vy8Nr/O+VZuLWTq3I2kpSYy4aOlnNQ/nacvGEJMFCesVbm7eOSzLIYf3JqRfa0ChTGmYdnVbj9DOrdkwcZ8SuroQfilH7L59zeruOeDJZT7lI8X5vDo51khijL8bN1Zwtn/+h/xsTFMOPsQ3Nb0jTGmwViy8nNiv3RKynxMmrWu1unW5RXTKz2Fscd05z9/HMqFR3TmmW9W8c8vV7Bhx+4QRRseVJUnvljO7tIKJl95JN3aNPU6JGNMIxTVpwGrOvLg1hzZvTWPfb6c3umpDO/Rptrp1uftpl/7Ztw2qi8AI/scxKb8Yh6bvpzHv1jOZSO6cduovlFxHeuRz7KYNGs9lwzvSo+DUrwOxxjTSFmyquLx8wZx0QszuXXqQr656bhfXYeq8CkbduzmlAG/VMKIi43hxTGHsSxnJ2/MXMeLM9bw6aIcjujeiu5tmnJ0zzQOPiiFlMTQbm5VpbTCR1xMDMVlFTRNiK31FF32tiLKKnx0T0upNdEW7Snnhe/XUFbh4z/fruKcoR258/R+wXgLxhgDWLL6lXbNk7hhZE9ueHMe36/cxrG90vYZv7mgmLIKpbNbMaNSbIzQv31zJpx9CMf2SuPN2ev5YeV23v15I3//fDkAXVonc8PInpx9aIeAr+vMXbeD9+dtIkaEnSVltE5JcKqD9z6IwpIyWiYn1Fix48UZa3jg46U0iY9ld2kFHVok0eOgFLYX7WFwpxYc1rUVzZPiWbChgE6tkhg3dSElZT6S4mPp0y6V/u2bkVdUSpO4WBTo2y6VwZ1a8tIPa/h4YQ4AbVISuOO06DiKNMZ4x5JVNU4Z0JY2KYn86+uVHNPTORVYmVzW5TnXpKomK38n9W/LSW71980FxcxfX8Cq3F18vjiHv06ezxsz13H+4Z05rncabVISAedoJSEuht2lFazcWsjXy3LZtaecV37M3rv+pgmx7NpTjk8hITaG0gofzZrEcVjXVgzq1ILkhFhG9GhDerMmfL8ilwkfL0UVfj+0I22bJzFv/Q5WbNlFerMmvDt3I6/9tO+1uZbJ8Yw/oz9ZWwpZvGkn7/28iZQmcfhUEYR3f964d9pbTunDJcO74lOlaYiPGI0x0cd+ZaqRGBfLDSf05M5pi/jdMz+wKb+E8Wf2o1XTRL7JykUEDk4L7PpMu+ZJtGueBMDVxx7MO3M28NCny7jp7fmkJMZx6YiuLNm0k6+ythIrQrlP95m/b7tmvHnFMCQGUhLiqFDl88Vb+Gn1drq2acrKrYXMXJPHl8u2Vrv+t8YO44jurX9VXl7hY1lOIbmFexjUqQWLNxXQtlkTeqan7p1GVfc5Aty2aw/z1uXToWUSfdtZ1x7GmNARVa17qjCVkZGhmZmZQVl2eYWPm6csYOrcjb8ad/ahHXjivMH7veySsgpWbt3FP75cwfQlW2gSH8OlI7pRVu4jLjaGDTt2c9cZ/ViyaScDOjTfe/RVm+LSCnaWlPFtVi4FxWUc2rkF6c2a7L2PzBhjKonIHFXN8DqO+gh5shKRTsArQDqgwHOq+g8RGQ9cAVS2LHubqn5c27KCmawq7S4t56fV28nK2UWHlkn8tHo71/+mJ22bN0z7gXlFpcTGCM2TrN08Y0xoWLIKZIUi7YB2qjpXRFKBOcBo4Fxgl6r+PdBlhSJZGWNMYxOJySrk16xUdTOw2R0uFJGlQIdQx2GMMSZyeNqChYh0BQ4FZrpF14nIAhGZKCIta5hnrIhkikhmbq51lmiMMdHAs2QlIinAFODPqroTeAY4GBiMc+T1WHXzqepzqpqhqhlpaWnVTWKMMaaR8SRZiUg8TqJ6XVWnAqjqFlWtUFUf8DxwuBexGWOMCT8hT1bi3LjzIrBUVR/3K2/nN9nZwKJQx2aMMSY8eXFT8Ajgj8BCEZnnlt0GXCAig3Gqs2cDV3oQmzHGmDDkRW3AGUB1DcnVek+VMcaY6GX9WRljjAl7Ed3ckojkAmsPYBFtgG0NFE6wWIwNIxJihMiI02JsGF7G2EVVI6o6dUQnqwMlIpnhfhe3xdgwIiFGiIw4LcaGEQkxhhM7DWiMMSbsWbIyxhgT9qI9WT3ndQABsBgbRiTECJERp8XYMCIhxrAR1desjDHGRIZoP7IyxhgTASxZGWOMCXtRmaxE5BQRyRKRlSJyq9fxVBKRbBFZKCLzRCTTLWslItNFZIX7XG3XKUGOa6KIbBWRRX5l1cYljqfcbbtARIZ4GON4Ednobs95IjLKb9w4N8YsETk5RDF2EpGvRWSJiCwWkRvc8rDZlrXEGDbbUkSaiMgsEZnvxniPW95NRGa6sbwlIglueaL7eqU7vmuwY6wjzpdEZI3fthzslnvy3YkYqhpVDyAWWAV0BxKA+UA/r+NyY8sG2lQpewS41R2+FXjYg7iOAYYAi+qKCxgFfILTpNYwYKaHMY4Hbqpm2n7ufk8Eurmfh9gQxNgOGOIOpwLL3VjCZlvWEmPYbEt3e6S4w/E4/eENAyYD57vlzwJXu8PXAM+6w+cDb4XoM1lTnC8B51QzvSffnUh5ROOR1eHASlVdraqlwJvAWR7HVJuzgJfd4ZeB0aEOQFW/A/KqFNcU11nAK+r4CWhRpUX9UMZYk7OAN1V1j6quAVYSgi5pVHWzqs51hwuByl6yw2Zb1hJjTUK+Ld3tsct9Ge8+FPgN8I5bXnU7Vm7fd4CRbu8PQVVLnDXx5LsTKaIxWXUA1vu93kDtX8ZQUuBzEZkjImPdsnRV3ewO5wDp3oT2KzXFFW7bt7repz2PUfbtJTsst6UE1pO3JzGKSKw4vTZsBabjHNHlq2p5NXHsjdEdXwC0DnaM1cWpqpXbcoK7LZ8QkcSqcbq8/u6ElWhMVuHsKFUdApwKXCsix/iPVOdcQdjdaxCucRFg79OhJr/uJXuvcNmW1cQYVttSnY5aBwMdcY7k+ngZT02qxikiA4BxOPEeBrQCbvEuwsgRjclqI9DJ73VHt8xzqrrRfd4KvIvzJdxSeSrAfd7qXYT7qCmusNm+WnPv057FKNX0kk2YbcvqYgzHbenGlQ98DRyJc9qsstsj/zj2xuiObw5sD1WMVeI8xT3Vqqq6B/gvYbItw100JqvZQE+35lACzgXX9z2OCRFpKiKplcPASTi9Jb8PjHEnGwO8502Ev1JTXO8DF7s1m4YBBX6nuEJKau59+n3gfLeWWDegJzArBPFU20s2YbQta4oxnLaliKSJSAt3OAk4Eefa2tfAOe5kVbdj5fY9B/jKPYINqhriXOb3x0Rwrqv5b8uw+O6EIy96CvaUqpaLyHXAZzg1Ayeq6mKPwwLnOsW77nXfOOANVf1URGYDk0XkcpzuUM4NdWAiMgk4DmgjIhuAu4GHaojrY5xaTSuB3cClHsZ4nFTT+7SqLhaRycASoBy4VlUrQhBmTb1kh9O2rFdP3h5ty3bAyyISi/OHe7KqfigiS4A3ReR+4GecpIv7/KqIrMSphHN+kOOrK86vRCQNp9bfPOAqd/oD3t9z5sw5KC4u7gVgAJF7MOIDFpWXl/9p6NChe88kWXNLxhjTSMyfP//9tm3b9k1LS9sZExMTkT/uPp9PcnNzm+fk5CwZNGjQmZXlkZp5jTHG/NqASE5UADExMZqWllaAc3T4S7lH8RhjjGl4MZGcqCq572Gf/GTJyhhjTNizZGWMMabBrFq1Kn7kyJEHd+nSZUDHjh0PufjiizsXFxfLhx9+mJqamjq4T58+/bp169Z/7NixHeuzXEtWxhhjGoTP52P06NE9zjzzzPy1a9cuys7OXlhSUiLXXHNNR4CMjIxdy5YtW7Jw4cIl06dPb/755583DXTZlqyM8SMiFX6tYc+TOlrlF5GrROTiBlhvtoi0OdDlGOOlDz74IDUxMdF3ww03bAeIi4vj2WefXT9lypTWhYWFe/NNSkqK9u/fv3jdunUJgS476u6zMqYOxW7zOAFR1WeDGIsx++1v78zvtDynMLkhl9mrberuR88ZtL6m8QsXLkwaNGjQbv+yVq1a+Tp06FC6YsWKyjYQyc3NjV2zZk3iSSedVBjouu3IypgAuEc+j4jT39gsEenhlo8XkZvc4evF6QdqgYi86Za1EpFpbtlPIjLQLW8tIp+L08/RCzg3iFau6w/uOuaJyH/cm0qNiXiZmZkpvXv37te5c+eBxx9//M7OnTuX1z2Xw46sjNlXkl/LDQAPqupb7nCBqh7invZ7Eji9yry3At1UdU9lMzvAPcDPqjpaRH4DvILTGOzdwAxVvVdETgMuBxCRvsB5wAhVLRORfwMXufMZE7DajoCCZcCAAcXTpk3bp4PYvLy8mG3btsX16dOnJCMjY9fXX3+9ctmyZQkjRozoe+GFF+YNHz68OJBl25GVMfsqVtXBfo+3/MZN8ns+spp5FwCvi8gfcJoeAjgKeBVAVb8CWotIM5zOIl9zyz8CdrjTjwSGArPdpDkSp6NQY8LemWeeWVhSUhLz9NNPtwYoLy/nmmuu6XTZZZdtTU5O3nv/V58+fUqvv/76zQ8++GDbQJdtycqYwGkNw5VOA/6F02PxbL8WwOtDgJf9kmVvVR2/H8sxJuRiYmKYNm3ayqlTp7bs0qXLgJYtWw6OiYnh4Ycfzqk67Y033pg7c+bM1KysrIAqWViyMiZw5/k9/+g/QkRigE6q+jVO/0TNgRTge5zTeIjIccA2t3+o74AL3fJTgcpTJ18C54jIQe64ViLSJXhvyZiG1aNHj7Kvvvpq5dq1axdNnTp1xTfffNNsxowZyaeffnrh119/vbJyupSUFN26deuC3r17lwayXLtmZcy+ql6z+lRVK6uvtxSRBcAe4IIq88UCr4lIc5yjo6dUNV9ExgMT3fl280tXFfcAk0RkMfADsA5AVZeIyB04PUbHAGXAtTitsRsTUU488cSiTZs2LWyIZVmr68YEQESygQxV3eZ1LMbUZP78+dmDBg1qFJ/R+fPntxk0aFDXytd2GtAYYxoPn8/nk7onC2/ue/D5l1myMiYAqtrVjqpMBFiUm5vbPJITVmV/VvzSgzJg16yMMabRKC8v/1NOTs4LOTk5jaKnYP9Cu2ZljDEm7EVq5jXGGBNFLFkZY4wJe5asjDHGhL3/ByljEjfcFZRoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "fig=plt.figure()   \n",
    "    \n",
    "reward='QR_CartPole'\n",
    "df.to_csv(reward+'.csv')\n",
    "plt.xlabel(\"Episode\")\n",
    "plt.ylabel(\"Reward\")\n",
    "plt.plot(df[2],  label='QR')\n",
    "\n",
    "\n",
    "plt.title('Learning Curve : Quantile Regression CartPole')\n",
    "\n",
    "# agent.save('Model_dqn.h5')\n",
    "fig.legend(loc='lower right')\n",
    "fig.savefig(reward+'.png', dpi=100)\n",
    "fig.show()   \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}