InformedGiveUp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import jdc\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "#matplotlib.use('Agg')\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "from scipy.stats import expon\n",
    "import random\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RV : \n",
      " <scipy.stats._distn_infrastructure.rv_frozen object at 0x000001AEE8458518>\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from scipy.stats import expon\n",
    "\n",
    "numargs = expon.numargs\n",
    "[ ] = [0.6, ] * numargs\n",
    "rv = expon( )\n",
    "\n",
    "print (\"RV : \\n\", rv)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rl_glue import RLGlue\n",
    "from agent import BaseAgent\n",
    "from environment import BaseEnvironment  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 1. Environment: the Informed Give-Up Task as an MDP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def isInBounds(HowLongHasTheAgentWaited, MaxTime):\n",
    "    return (HowLongHasTheAgentWaited >= 0) and (HowLongHasTheAgentWaited < MaxTime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def FindIndexOfTime(TimeArray, TargetTimestamp): \n",
    "    FindIndex = np.where(TimeArray == TargetTimestamp)\n",
    "    IndexArray = FindIndex[0]\n",
    "    index = IndexArray[0]\n",
    "    return index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pursuit_init(TimeArray, MinDelay, scale): \n",
    "        PSRewardTimePDF = expon.pdf(TimeArray, MinDelay, scale)\n",
    "        PSRewardTimeCDF = expon.cdf(TimeArray, MinDelay, scale)\n",
    "        return PSRewardTimePDF, PSRewardTimeCDF\n",
    "\n",
    "def background_init(TimeArray, FixedRewardTime): \n",
    "        FixedRewardBin = FindIndexOfTime(TimeArray, FixedRewardTime) \n",
    "        BGRewardTimePDF = np.zeros(len(TimeArray))\n",
    "        BGRewardTimePDF[FixedRewardBin] = 1\n",
    "        return BGRewardTimePDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def FindStateStart(TimeArray, TargetTimestamp):\n",
    "    FindIndex = np.where(TimeArray <= TargetTimestamp)\n",
    "    IndexArray = FindIndex[0]\n",
    "    index = max(IndexArray)\n",
    "    T = TimeArray[index]\n",
    "    return T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def PrecisionOf(dt):\n",
    "    import decimal\n",
    "    d = decimal.Decimal(str(dt))\n",
    "    DecimalDigit = -d.as_tuple().exponent\n",
    "    return DecimalDigit\n",
    "\n",
    "PrecisionOf(0.02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def argmax(array):\n",
    "        \"\"\"argmax with random tie-breaking\n",
    "        Args:\n",
    "            Numpy array: e.g. the array of action-values\n",
    "        Returns:\n",
    "            action (int): e.g. an action with the highest value\n",
    "        \"\"\"\n",
    "        top = float(\"-inf\")\n",
    "        ties = []\n",
    "\n",
    "        for i in range(len(array)):\n",
    "            if array[i] > top:\n",
    "                top = array[i]\n",
    "                ties = []\n",
    "                ties.append(i)\n",
    "\n",
    "            if array[i] == top:\n",
    "                ties.append(i)\n",
    "\n",
    "        return np.random.choice(ties)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.   0.1  0.21 0.33 0.47 0.62 0.79 0.98 1.19 1.42 1.68 1.97 2.29 2.64\n",
      " 3.03 3.47 3.96 4.5 ]\n",
      "18\n"
     ]
    }
   ],
   "source": [
    "# construct dilating states\n",
    "def construct_dilating_states(dur, fdt, gamma):\n",
    "        p = 1 / gamma\n",
    "        \n",
    "        if (p == 1): \n",
    "            TimeArray = np.round(np.arange(0, dur, fdt), PrecisionOf(fdt))\n",
    "        else: \n",
    "            state_num = 0\n",
    "            while ((fdt * (1 - p ** (state_num + 1)) / (1 - p)) < dur): \n",
    "                state_num += 1\n",
    "            increments = np.ones(state_num + 1) * fdt\n",
    "            TimeArray = np.zeros(state_num + 1)\n",
    "            for ii in range(1, len(increments)): \n",
    "                increments[ii] = increments[ii] / (gamma ** (ii))\n",
    "                TimeArray[ii] = np.round(sum(increments[0:ii]), PrecisionOf(fdt) + 1)\n",
    "#         if (len(TimeArray) == 1): \n",
    "#             last_timestamp = np.round(TimeArray[-1] + fdt, PrecisionOf(fdt) + 1)\n",
    "#         else:\n",
    "#             last_timestamp = np.round(TimeArray[-1] + (TimeArray[-1] - TimeArray[-2]) * p, PrecisionOf(fdt) + 1)\n",
    "        \n",
    "#         TimeArray = np.append(TimeArray, last_timestamp)\n",
    "            \n",
    "        return TimeArray\n",
    "\n",
    "x = construct_dilating_states(4.8, 0.1, 0.9)\n",
    "print(x)\n",
    "print(len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unfinished work of programming an informing environment\n",
    "# Create GiveUpEnvironment class.\n",
    "# class GiveUpEnvironment(BaseEnvironment):\n",
    "class InformedGiveUpEnvironment(BaseEnvironment):\n",
    "    def __init__(self, env_info={}):\n",
    "        self.current_state4d = np.zeros(3)\n",
    "        self.current_state1d = 0\n",
    "        \n",
    "        \"\"\"Setup for the environment called when the experiment first starts.\n",
    "        Note:\n",
    "            Initialize a tuple with the reward, first state, boolean\n",
    "            indicating if it's terminal.\n",
    "        \"\"\"\n",
    "        # Note, we can setup the following variables later, in env_start() as it is equivalent. \n",
    "        # Code is left here to adhere to the note above, but these variables are initialized once more\n",
    "        # in env_start() [See the env_start() function below.]\n",
    "        \n",
    "        # reward = None\n",
    "        # state = None # See Aside\n",
    "        # termination = None\n",
    "        # self.reward_state_term = (reward, state, termination)\n",
    "        # self.state = np.zeros(3) # port{background, pursuit}, reward collected in the current port{0, 1}, time waited in the current port\n",
    "        self.StateStruct = env_info.get(\"state representation structure\", \"regular\")\n",
    "        self.PSRewardAmount = env_info.get(\"reward amount in pursuit\", 1)\n",
    "        self.BGRewardAmount = env_info.get(\"reward amount in background\", 1)\n",
    "        \n",
    "        self.dt = env_info.get(\"fundamental timestep\", 0.1)\n",
    "        self.gamma = env_info.get(\"gamma\", 0.9)\n",
    "        self.rand_generator = np.random.RandomState(env_info.get(\"seed\", 0))\n",
    "        self.PS_dur = env_info.get(\"pursuit port total duration\", 9)\n",
    "        self.BG_dur = env_info.get(\"background port total duration\", 2)\n",
    "        self.transit_dur = env_info.get(\"transit duration\", 0.5)\n",
    "        self.consmp_dur = env_info.get(\"consumption duration\", 1)\n",
    "        \n",
    "        # pursuit port: set the default distribution of reward delivery time\n",
    "        self.PSMinDelay = env_info.get(\"first time in pursuit port that could give a reward\", 1.4)\n",
    "        self.PSscale = env_info.get(\"exponential distribution scale\", 4.8)\n",
    "        self.PSMinStay = env_info.get(\"required minimum wait time in pursuit port\", 1.4)\n",
    "        self.PSResetTime = env_info.get(\"time to wait until the trial reset\", 1.4)\n",
    "        \n",
    "        # background port: set the default distribution of reward delivery time\n",
    "        self.BGRewardTime = env_info.get(\"delivery time in the background port\", 1.5)\n",
    "        self.BGMinStay = env_info.get(\"required minimum wait time in background port\", 1.4)\n",
    "        \n",
    "        if (self.StateStruct == \"dilating\"): \n",
    "            self.BGTimeArray = construct_dilating_states(self.BG_dur, self.dt, self.gamma)\n",
    "            self.PSTimeArray = construct_dilating_states(self.PS_dur, self.dt, self.gamma)\n",
    "            self.PSMinDelay = FindStateStart(self.PSTimeArray, self.PSMinDelay)\n",
    "            self.BGRewardTime = FindStateStart(self.BGTimeArray, self.BGRewardTime)\n",
    "            self.TravelTimeArray = construct_dilating_states(self.transit_dur, self.dt, self.gamma)\n",
    "        elif (self.StateStruct == \"regular\"): \n",
    "            self.BGTimeArray = np.round(np.arange(0, self.BG_dur, self.dt), PrecisionOf(self.dt))\n",
    "            self.PSTimeArray = np.round(np.arange(0, self.PS_dur, self.dt), PrecisionOf(self.dt))\n",
    "            self.TravelTimeArray = np.round(np.arange(0, self.transit_dur, self.dt), PrecisionOf(self.dt))\n",
    "        else: \n",
    "            raise Exception(str(self.StateStruct) + ' not in recognized state representation structures [\"regular\", \"dilating\"]!')\n",
    "        \n",
    "        PDFCDF = pursuit_init(self.PSTimeArray, self.PSMinDelay, self.PSscale)\n",
    "        self.PSRewardTimePDF = PDFCDF[0]\n",
    "        self.PSRewardTimeCDF = PDFCDF[1]\n",
    "        self.BGRewardTimePDF = background_init(self.BGTimeArray, self.BGRewardTime)\n",
    "        \n",
    "        self.num_states = len(self.BGTimeArray) - FindIndexOfTime(self.BGTimeArray, self.BGRewardTime)\\\n",
    "        + len(self.TravelTimeArray)\\\n",
    "        + 2* (len(self.PSTimeArray) + len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay)\\\n",
    "              + len(self.TravelTimeArray) + FindIndexOfTime(self.BGTimeArray, self.BGRewardTime))  \n",
    "        \n",
    "    def env_init(self, env_info={}):\n",
    "        \"\"\"Setup for the environment called when the experiment first starts.\n",
    "        Note:\n",
    "            Initialize a tuple with the reward, first state, boolean\n",
    "            indicating if it's terminal.\n",
    "        \"\"\"\n",
    "        # Note, we can setup the following variables later, in env_start() as it is equivalent. \n",
    "        # Code is left here to adhere to the note above, but these variables are initialized once more\n",
    "        # in env_start() [See the env_start() function below.]\n",
    "        \n",
    "        # reward = None\n",
    "        # state = None # See Aside\n",
    "        # termination = None\n",
    "        # self.reward_state_term = (reward, state, termination)\n",
    "        # self.state = np.zeros(3) # port{background, pursuit}, reward collected in the current port{0, 1}, time waited in the current port\n",
    "        self.StateStruct = env_info.get(\"state representation structure\", \"regular\")\n",
    "        self.PSRewardAmount = env_info.get(\"reward amount in pursuit\", 1.8)\n",
    "        self.BGRewardAmount = env_info.get(\"reward amount in background\", [1.2, 2.4])\n",
    "        \n",
    "        self.dt = env_info.get(\"fundamental timestep\", 0.1)\n",
    "        self.gamma = env_info.get(\"gamma\", 0.9)\n",
    "        self.rand_generator = np.random.RandomState(env_info.get(\"seed\", 0))\n",
    "        self.PS_dur = env_info.get(\"pursuit port total duration\", 9)\n",
    "        self.BG_dur = env_info.get(\"background port total duration\", 2)\n",
    "        self.transit_dur = env_info.get(\"transit duration\", 0.5)\n",
    "        self.consmp_dur = env_info.get(\"consumption duration\", 1)\n",
    "        \n",
    "        # pursuit port: set the default distribution of reward delivery time\n",
    "        self.PSMinDelay = env_info.get(\"first time in pursuit port that could give a reward\", 1.4)\n",
    "        self.PSscale = env_info.get(\"exponential distribution scale\", 4.8)\n",
    "        self.PSMinStay = env_info.get(\"required minimum wait time in pursuit port\", 1.4)\n",
    "        self.PSResetTime = env_info.get(\"time to wait until the trial reset\", 1.4)\n",
    "        self.CueTime = env_info.get(\"cue time\", 1.4)\n",
    "        \n",
    "        # background port: set the default distribution of reward delivery time\n",
    "        self.BGRewardTime = env_info.get(\"delivery time in the background port\", 1.5)\n",
    "        self.BGMinStay = env_info.get(\"required minimum wait time in background port\", 1.4)\n",
    "        \n",
    "        if (self.StateStruct == \"dilating\"): \n",
    "            self.BGTimeArray = construct_dilating_states(self.BG_dur, self.dt, self.gamma)\n",
    "            self.PSTimeArray = construct_dilating_states(self.PS_dur, self.dt, self.gamma)\n",
    "            self.PSMinDelay = FindStateStart(self.PSTimeArray, self.PSMinDelay)\n",
    "            self.BGRewardTime = FindStateStart(self.BGTimeArray, self.BGRewardTime)\n",
    "        elif (self.StateStruct == \"regular\"): \n",
    "            self.BGTimeArray = np.round(np.arange(0, self.BG_dur, self.dt), PrecisionOf(self.dt) + 1)\n",
    "            self.PSTimeArray = np.round(np.arange(0, self.PS_dur, self.dt), PrecisionOf(self.dt) + 1)\n",
    "        else: \n",
    "            raise Exception(str(self.StateStruct) + ' not in recognized state representation structures [\"regular\", \"dilating\"]!')\n",
    "        \n",
    "        PDFCDF = pursuit_init(self.PSTimeArray, self.PSMinDelay, self.PSscale)\n",
    "        self.PSRewardTimePDF = PDFCDF[0]\n",
    "        self.PSRewardTimeCDF = PDFCDF[1]\n",
    "        self.BGRewardTimePDF = background_init(self.BGTimeArray, self.BGRewardTime)\n",
    "        \n",
    "        self.num_states = len(self.BGTimeArray) + len(self.PSTimeArray) + len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay)\n",
    "    \n",
    "    \n",
    "    def env_start(self):\n",
    "        \"\"\"The first method called when the episode starts, called before the\n",
    "        agent starts.\n",
    "\n",
    "        Returns:\n",
    "            The first state from the environment.\n",
    "        \"\"\"\n",
    "        reward = 0\n",
    "        # current state of the agent is a 3-dimensional variable of {WhichPort, HowLongHasTheAgentWaited, RewardCollected}\n",
    "        self.current_state4d = (0, 0, 0)\n",
    "        self.current_state1d = self.state1d(self.current_state4d)\n",
    "        termination = False\n",
    "        self.reward_state_term = (reward, self.current_state1d, termination)\n",
    "\n",
    "        return self.reward_state_term[1]\n",
    "\n",
    "    def env_step(self, action):\n",
    "        \"\"\"A step taken by the environment.\n",
    "\n",
    "        Args:\n",
    "            action: The action taken by the agent. \n",
    "            can be 0: wait, or 1: give up\n",
    "\n",
    "        Returns:\n",
    "            (float, state, Boolean): a tuple of the reward, state,\n",
    "                and boolean indicating if it's terminal.\n",
    "        \"\"\"\n",
    "\n",
    "        self.current_state1d = self.state1d(self.current_state4d)\n",
    "        P, T, C, Q = self.current_state4d # Port, Time, Collected reward, Cue\n",
    "        reward = 0 # default reward is 0\n",
    "        num_rewards = 0 # default number of reward is 0. # The number of rewards has nothing to do with reward amount\n",
    "        \n",
    "        # inside the background port\n",
    "        if (P == 0):\n",
    "            \n",
    "            if (action == 0):\n",
    "            # if stay, port does not change\n",
    "                \n",
    "                if isInBounds(T, self.PSTimeArray[-1]):\n",
    "                # within the maximum duration, will follow the reward function\n",
    "                    \n",
    "                    if (C == 0): \n",
    "                        # generate a random number between 0 and 1. \n",
    "                        # when it falls below Pr{rewarded at t|not rewarded prior to t}, then give a reward, otherwise no reward\n",
    "                        Pr_NeverRewarded = 1 - self.PSRewardTimeCDF[FindIndexOfTime(self.PSTimeArray, T)]\n",
    "                        Pr_NeverRewardedNRewardedAtT = self.PSRewardTimePDF[FindIndexOfTime(self.PSTimeArray, T)]\n",
    "                        Pr_RewardedAtTGivenNotYetRewarded = Pr_NeverRewardedNRewardedAtT / Pr_NeverRewarded\n",
    "                        rand = self.rand_generator.uniform(low = 0, high = 1, size = 1) \n",
    "                        if (rand < Pr_RewardedAtTGivenNotYetRewarded): \n",
    "                            reward = self.PSRewardAmount\n",
    "                            num_rewards = 1\n",
    "                        else: \n",
    "                            pass\n",
    "\n",
    "                        NextIndex = FindIndexOfTime(self.PSTimeArray, T) + 1\n",
    "                        T = self.PSTimeArray[NextIndex]\n",
    "\n",
    "                    elif (C == 1): \n",
    "                        # no reward\n",
    "                        NextIndex = FindIndexOfTime(self.PSTimeArray, T) + 1\n",
    "                        T = self.PSTimeArray[NextIndex]\n",
    "                else: \n",
    "                    # if agent stays longer than the maximum time, they will be seen as in the same state\n",
    "                    pass \n",
    "                        \n",
    "\n",
    "                C = C + num_rewards\n",
    "\n",
    "            # if leave/give up, then jump to the first state in the background port\n",
    "            elif (action == 1):\n",
    "                if isInBounds(T, self.PS_dur) and (T >= self.PSMinStay):\n",
    "                    \n",
    "                    P, T, C = 2, 0, 0\n",
    "                elif isInBounds(T, self.PSMinStay): \n",
    "                    P, T, C = 0, 0, 0\n",
    "                else: \n",
    "                    pass\n",
    "\n",
    "            else: \n",
    "                raise Exception(str(action) + \" not in recognized actions [0: Wait, 1: Give up and leave]!\")\n",
    "            \n",
    "            \n",
    "            \n",
    "            \n",
    "            \n",
    "            \n",
    "            \n",
    "\n",
    "        # inside the pursuit port\n",
    "        elif (P == 1): \n",
    "            # if stay/wait, then get a reward at a fixed time point\n",
    "            if (action == 0): \n",
    "                # port doe not change\n",
    "                if isInBounds(T, self.BGTimeArray[-1]):\n",
    "                    \n",
    "                    reward = self.BGRewardTimePDF[FindIndexOfTime(self.BGTimeArray, T)] * self.BGRewardAmount[Q]\n",
    "                    num_rewards = self.BGRewardTimePDF[FindIndexOfTime(self.BGTimeArray, T)] * 1\n",
    "                    NextIndex = FindIndexOfTime(self.BGTimeArray, T) + 1\n",
    "                    T = self.BGTimeArray[NextIndex]\n",
    "                # if agent stays longer than the maximum time, they will be seen as in the same state\n",
    "                if not isInBounds(T, self.BGTimeArray[-1]): \n",
    "                    pass\n",
    "\n",
    "                C = C + num_rewards\n",
    "\n",
    "            # if leave/give up, then jump to the first state in the pursuit port\n",
    "            elif (action == 1): \n",
    "                if isInBounds(T, self.BG_dur) and (T >= self.BGMinStay):\n",
    "                    P, T, C = 3, 0, 0\n",
    "                elif isInBounds(T, self.BGMinStay): \n",
    "                    P, T, C = 1, 0, 0\n",
    "                else: \n",
    "                    pass\n",
    "\n",
    "            else: \n",
    "                raise Exception(str(action) + \" not in recognized actions [0: Wait, 1: Give up and leave]!\")\n",
    "        \n",
    "        # the travel pursuit (from port 0 to port 1)\n",
    "        elif (P == 2): \n",
    "            if isInBounds(T, self.TravelTimeArray[-1]):\n",
    "                NextIndex = FindIndexOfTime(self.TravelTimeArray, T) + 1\n",
    "                T = self.TravelTimeArray[NextIndex]\n",
    "            else: \n",
    "                if (action == 0): \n",
    "                    pass\n",
    "                elif (action == 1): \n",
    "                    P, T, C = 1, 0, 0\n",
    "                else: \n",
    "                    raise Exception(str(action) + \" not in recognized actions [0: Stay, 1: Leave the travel pursuit]!\")\n",
    "            \n",
    "        # the travel pursuit (from port 1 to port 0)\n",
    "        elif (P == 3): \n",
    "            if isInBounds(T, self.TravelTimeArray[-1]):\n",
    "                NextIndex = FindIndexOfTime(self.TravelTimeArray, T) + 1\n",
    "                T = self.TravelTimeArray[NextIndex]\n",
    "            else: \n",
    "                if (action == 0): \n",
    "                    pass\n",
    "                elif (action == 1): \n",
    "                    P, T, C = 0, 0, 0\n",
    "                else: \n",
    "                    raise Exception(str(action) + \" not in recognized actions [0: Stay, 1: Leave the travel pursuit]!\")\n",
    "\n",
    "        # assign the new state to the environment object\n",
    "        self.current_state4d = (P, T, C)\n",
    "        self.current_state1d = self.state1d(self.current_state4d)\n",
    "        termination = False\n",
    "        self.reward_state_term = (reward, self.current_state1d, termination)\n",
    "        return self.reward_state_term\n",
    "\n",
    "\n",
    "    def env_cleanup(self):\n",
    "        \"\"\"Cleanup done after the environment ends\"\"\"\n",
    "        self.current_state4d = (0, 0, 0)\n",
    "        self.current_state1d = 0\n",
    "    \n",
    "    # helper method \n",
    "    def state1d(self, state4d):\n",
    "        P = state4d[0]\n",
    "        T = state4d[1]\n",
    "        C = state4d[2]\n",
    "        if (P == 0):\n",
    "            if (T <= self.BGRewardTime) and (C == 1): \n",
    "                raise Exception(f\"It is not possible that a reward is delivered before {self.BGRewardTime}. Hence this state does not exist. \")\n",
    "            elif (T > self.BGRewardTime) and (C == 0):\n",
    "                raise Exception(f\"It is not possible that no reward is delivered at {self.BGRewardTime}. Hence this state does not exist. \")\n",
    "            else: \n",
    "                return FindIndexOfTime(self.BGTimeArray, T)\n",
    "        \n",
    "        elif (P == 1): \n",
    "            if (C == 0): \n",
    "                return FindIndexOfTime(self.PSTimeArray, T) + len(self.BGTimeArray)\n",
    "            elif (C == 1): \n",
    "                return FindIndexOfTime(self.PSTimeArray, T) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) + len(self.PSTimeArray) + len(self.BGTimeArray)\n",
    "            else: \n",
    "                raise Exception(str(C) + \" not in the possible range of number of collected rewards [0, 1]!\")\n",
    "        \n",
    "        elif (P == 2): \n",
    "            if (C == 0): \n",
    "                return FindIndexOfTime(self.TravelTimeArray, T) + len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) + len(self.PSTimeArray) + len(self.BGTimeArray)\n",
    "            else: \n",
    "                raise Exception(f\"No reward could be delivered during travel time.\")\n",
    "                \n",
    "        elif (P == 3): \n",
    "            if (C == 0): \n",
    "                return FindIndexOfTime(self.TravelTimeArray, T) + len(self.TravelTimeArray) + len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) + len(self.PSTimeArray) + len(self.BGTimeArray)\n",
    "            else: \n",
    "                raise Exception(f\"No reward could be delivered during travel time.\")\n",
    "        \n",
    "        else: \n",
    "            raise Exception(str(P) + \" not in recognized pursuit [0: Background, 1: Pursuit, 2: travel from 0 to 1, 3: travel from 1 to 0]!\")\n",
    "    \n",
    "    def state4d(self, state1d): \n",
    "        s = state1d\n",
    "        if (s < self.BGTimeArray.shape[0]): \n",
    "            P = 0\n",
    "            T = self.BGTimeArray[s]\n",
    "            C = 0\n",
    "            if (T > self.BGRewardTime): \n",
    "                C = 1\n",
    "        elif (s >= self.BGTimeArray.shape[0]) and (s < len(self.PSTimeArray) + len(self.BGTimeArray)):\n",
    "            P = 1\n",
    "            T = self.PSTimeArray[s-self.BGTimeArray.shape[0]]\n",
    "            C = 0\n",
    "        \n",
    "        elif (s >= len(self.PSTimeArray) + len(self.BGTimeArray)) and \\\n",
    "        (s < len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) \\\n",
    "         + len(self.PSTimeArray) + len(self.BGTimeArray)):\n",
    "            P = 1\n",
    "            T = self.PSTimeArray[s-len(self.PSTimeArray)-len(self.BGTimeArray)\\\n",
    "                                 +FindIndexOfTime(self.PSTimeArray, self.PSMinDelay)]\n",
    "            C = 1\n",
    "        \n",
    "        elif (s >= len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) \\\n",
    "              + len(self.PSTimeArray) + len(self.BGTimeArray)) \\\n",
    "        and (s < len(self.TravelTimeArray) + len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) \\\n",
    "             + len(self.PSTimeArray) + len(self.BGTimeArray)): \n",
    "            P = 2\n",
    "            T = self.TravelTimeArray[s-len(self.PSTimeArray)+FindIndexOfTime(self.PSTimeArray, self.PSMinDelay)\\\n",
    "                                       -len(self.PSTimeArray)-len(self.BGTimeArray)]\n",
    "            C = 0\n",
    "        \n",
    "        else: \n",
    "            P = 3\n",
    "            T = self.TravelTimeArray[s-len(self.TravelTimeArray)\\\n",
    "                                     -len(self.PSTimeArray)+FindIndexOfTime(self.PSTimeArray, self.PSMinDelay)\\\n",
    "                                     -len(self.PSTimeArray)-len(self.BGTimeArray)]\n",
    "            C = 0\n",
    "        \n",
    "        \n",
    "        return (P, T, C)\n",
    "    \n",
    "    # function that finds the real optimal policy (that maximizes the global reward rate)\n",
    "    def PolicyMax(self): \n",
    "        PSRewardRate = np.zeros(len(self.PSTimeArray))\n",
    "        # BGLeaveTime = self.BGRewardTime\n",
    "        # csmp = self.consmp_dur\n",
    "        # transit = self.transit_dur\n",
    "        # BGRewardRate = self.BGRewardAmount / BGLeaveTime\n",
    "        GlobalRewardRate = np.zeros(len(self.PSTimeArray))\n",
    "        PS_ExpectedReward = np.zeros(len(self.PSTimeArray))\n",
    "        PS_ExpectedTime = np.zeros(len(self.PSTimeArray))\n",
    "        \n",
    "        for i in range(0, len(self.PSTimeArray)): \n",
    "            t = self.PSTimeArray[i]\n",
    "            if isInBounds(t, self.PSMinStay): \n",
    "                t_b = self.BGRewardTime + self.PSResetTime\n",
    "            else: \n",
    "                t_b = self.BGRewardTime\n",
    "            Pr_RewardedBeforeT = self.PSRewardTimeCDF[i]\n",
    "            Pr_NotRewardedBeforeT = 1 - Pr_RewardedBeforeT\n",
    "            PS_ExpectedReward = Pr_RewardedBeforeT * self.PSRewardAmount + Pr_NotRewardedBeforeT * 0\n",
    "            PS_ExpectedExitTimeIfRewardedBeforeT = np.dot(self.PSTimeArray[0:i], self.PSRewardTimePDF[0:i])\n",
    "            PS_ExpectedTime = PS_ExpectedExitTimeIfRewardedBeforeT + Pr_NotRewardedBeforeT * t\n",
    "            Total_ExpectedReward = self.BGRewardAmount + PS_ExpectedReward\n",
    "            Total_ExpectedTime = t_b + PS_ExpectedTime + self.transit_dur * 2\n",
    "            PSRewardRate[i] = PS_ExpectedReward / PS_ExpectedTime\n",
    "            GlobalRewardRate[i] = Total_ExpectedReward / Total_ExpectedTime\n",
    "        \n",
    "        optimal_giveup_index_rou_g = argmax(GlobalRewardRate)\n",
    "        optimal_giveup_time_rou_g = self.PSTimeArray[optimal_giveup_index_rou_g]\n",
    "        optimal_giveup_index_rou_l = argmax(PSRewardRate)\n",
    "        optimal_giveup_time_rou_l = self.PSTimeArray[optimal_giveup_index_rou_l]\n",
    "        return (GlobalRewardRate, PSRewardRate, optimal_giveup_time_rou_g, optimal_giveup_time_rou_l)\n",
    "    \n",
    "    def test_reward_function(self): \n",
    "        fig, ax = plt.subplots(1, 1)\n",
    "        ax.plot(self.PSTimeArray, self.PSRewardTimePDF)\n",
    "        fig, bx = plt.subplots(1, 1)\n",
    "        bx.plot(self.PSTimeArray, self.PSRewardTimeCDF)\n",
    "        fig, cx = plt.subplots(1, 1)\n",
    "        cx.plot(self.BGTimeArray, self.BGRewardTimePDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5488135] [0.71518937]\n"
     ]
    }
   ],
   "source": [
    "rand_generator = np.random.RandomState(0)\n",
    "a = rand_generator.uniform(low = 0, high = 1, size = 1) \n",
    "b = rand_generator.uniform(low = 0, high = 1, size = 1)\n",
    "print(a, b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create GiveUpEnvironment class.\n",
    "# class GiveUpEnvironment(BaseEnvironment):\n",
    "class GiveUpEnvironment(BaseEnvironment):\n",
    "    def __init__(self, env_info={}):\n",
    "        self.current_state3d = np.zeros(3)\n",
    "        self.current_state1d = 0\n",
    "        \n",
    "        \"\"\"Setup for the environment called when the experiment first starts.\n",
    "        Note:\n",
    "            Initialize a tuple with the reward, first state, boolean\n",
    "            indicating if it's terminal.\n",
    "        \"\"\"\n",
    "        # Note, we can setup the following variables later, in env_start() as it is equivalent. \n",
    "        # Code is left here to adhere to the note above, but these variables are initialized once more\n",
    "        # in env_start() [See the env_start() function below.]\n",
    "        \n",
    "        # reward = None\n",
    "        # state = None # See Aside\n",
    "        # termination = None\n",
    "        # self.reward_state_term = (reward, state, termination)\n",
    "        # self.state = np.zeros(3) # port{background, pursuit}, reward collected in the current port{0, 1}, time waited in the current port\n",
    "        self.StateStruct = env_info.get(\"state representation structure\", \"regular\")\n",
    "        self.PSRewardAmount = env_info.get(\"reward amount in pursuit\", 1)\n",
    "        self.BGRewardAmount = env_info.get(\"reward amount in background\", [0.3, 0.9])\n",
    "        \n",
    "        self.dt = env_info.get(\"fundamental timestep\", 0.1)\n",
    "        self.gamma = env_info.get(\"gamma\", 0.9)\n",
    "        self.rand_generator = np.random.RandomState(env_info.get(\"seed\", 0))\n",
    "        self.PS_dur = env_info.get(\"pursuit port total duration\", 9)\n",
    "        self.BG_dur = env_info.get(\"background port total duration\", 2)\n",
    "        self.transit_dur = env_info.get(\"transit duration\", 0.5)\n",
    "        self.consmp_dur = env_info.get(\"consumption duration\", 1)\n",
    "        \n",
    "        # pursuit port: set the default distribution of reward delivery time\n",
    "        self.PSMinDelay = env_info.get(\"first time in pursuit port that could give a reward\", 1.4)\n",
    "        self.PSscale = env_info.get(\"exponential distribution scale\", 4.8)\n",
    "        self.PSMinStay = env_info.get(\"required minimum wait time in pursuit port\", 1.4)\n",
    "        self.PSResetTime = env_info.get(\"time to wait until the trial reset\", 1.4)\n",
    "        \n",
    "        # background port: set the default distribution of reward delivery time\n",
    "        self.BGRewardTime = env_info.get(\"delivery time in the background port\", 1.5)\n",
    "        self.BGMinStay = env_info.get(\"required minimum wait time in background port\", 1.4)\n",
    "        \n",
    "        if (self.StateStruct == \"dilating\"): \n",
    "            self.BGTimeArray = construct_dilating_states(self.BG_dur, self.dt, self.gamma)\n",
    "            self.PSTimeArray = construct_dilating_states(self.PS_dur, self.dt, self.gamma)\n",
    "            self.PSMinDelay = FindStateStart(self.PSTimeArray, self.PSMinDelay)\n",
    "            self.BGRewardTime = FindStateStart(self.BGTimeArray, self.BGRewardTime)\n",
    "            self.TravelTimeArray = construct_dilating_states(self.transit_dur, self.dt, self.gamma)\n",
    "        elif (self.StateStruct == \"regular\"): \n",
    "            self.BGTimeArray = np.round(np.arange(0, self.BG_dur, self.dt), PrecisionOf(self.dt))\n",
    "            self.PSTimeArray = np.round(np.arange(0, self.PS_dur, self.dt), PrecisionOf(self.dt))\n",
    "            self.TravelTimeArray = np.round(np.arange(0, self.transit_dur, self.dt), PrecisionOf(self.dt))\n",
    "        else: \n",
    "            raise Exception(str(self.StateStruct) + ' not in recognized state representation structures [\"regular\", \"dilating\"]!')\n",
    "        \n",
    "        PDFCDF = pursuit_init(self.PSTimeArray, self.PSMinDelay, self.PSscale)\n",
    "        self.PSRewardTimePDF = PDFCDF[0]\n",
    "        self.PSRewardTimeCDF = PDFCDF[1]\n",
    "        self.BGRewardTimePDF = background_init(self.BGTimeArray, self.BGRewardTime)\n",
    "        \n",
    "        self.num_states = len(self.BGTimeArray) + len(self.PSTimeArray) + len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) + len(self.TravelTimeArray) * 2\n",
    "        \n",
    "    def env_init(self, env_info={}):\n",
    "        \"\"\"Setup for the environment called when the experiment first starts.\n",
    "        Note:\n",
    "            Initialize a tuple with the reward, first state, boolean\n",
    "            indicating if it's terminal.\n",
    "        \"\"\"\n",
    "        # Note, we can setup the following variables later, in env_start() as it is equivalent. \n",
    "        # Code is left here to adhere to the note above, but these variables are initialized once more\n",
    "        # in env_start() [See the env_start() function below.]\n",
    "        \n",
    "        # reward = None\n",
    "        # state = None # See Aside\n",
    "        # termination = None\n",
    "        # self.reward_state_term = (reward, state, termination)\n",
    "        # self.state = np.zeros(3) # port{background, pursuit}, reward collected in the current port{0, 1}, time waited in the current port\n",
    "        self.StateStruct = env_info.get(\"state representation structure\", \"regular\")\n",
    "        self.PSRewardAmount = env_info.get(\"reward amount in pursuit\", 1)\n",
    "        self.BGRewardAmount = env_info.get(\"reward amount in background\", 1)\n",
    "        \n",
    "        self.dt = env_info.get(\"fundamental timestep\", 0.1)\n",
    "        self.gamma = env_info.get(\"gamma\", 0.9)\n",
    "        self.rand_generator = np.random.RandomState(env_info.get(\"seed\", 0))\n",
    "        self.PS_dur = env_info.get(\"pursuit port total duration\", 9)\n",
    "        self.BG_dur = env_info.get(\"background port total duration\", 2)\n",
    "        self.transit_dur = env_info.get(\"transit duration\", 0.5)\n",
    "        self.consmp_dur = env_info.get(\"consumption duration\", 1)\n",
    "        \n",
    "        # pursuit port: set the default distribution of reward delivery time\n",
    "        self.PSMinDelay = env_info.get(\"first time in pursuit port that could give a reward\", 1.4)\n",
    "        self.PSscale = env_info.get(\"exponential distribution scale\", 4.8)\n",
    "        self.PSMinStay = env_info.get(\"required minimum wait time in pursuit port\", 1.4)\n",
    "        self.PSResetTime = env_info.get(\"time to wait until the trial reset\", 1.4)\n",
    "        \n",
    "        # background port: set the default distribution of reward delivery time\n",
    "        self.BGRewardTime = env_info.get(\"delivery time in the background port\", 1.5)\n",
    "        self.BGMinStay = env_info.get(\"required minimum wait time in background port\", 1.4)\n",
    "        \n",
    "        if (self.StateStruct == \"dilating\"): \n",
    "            self.BGTimeArray = construct_dilating_states(self.BG_dur, self.dt, self.gamma)\n",
    "            self.PSTimeArray = construct_dilating_states(self.PS_dur, self.dt, self.gamma)\n",
    "            self.PSMinDelay = FindStateStart(self.PSTimeArray, self.PSMinDelay)\n",
    "            self.BGRewardTime = FindStateStart(self.BGTimeArray, self.BGRewardTime)\n",
    "        elif (self.StateStruct == \"regular\"): \n",
    "            self.BGTimeArray = np.round(np.arange(0, self.BG_dur, self.dt), PrecisionOf(self.dt) + 1)\n",
    "            self.PSTimeArray = np.round(np.arange(0, self.PS_dur, self.dt), PrecisionOf(self.dt) + 1)\n",
    "        else: \n",
    "            raise Exception(str(self.StateStruct) + ' not in recognized state representation structures [\"regular\", \"dilating\"]!')\n",
    "        \n",
    "        PDFCDF = pursuit_init(self.PSTimeArray, self.PSMinDelay, self.PSscale)\n",
    "        self.PSRewardTimePDF = PDFCDF[0]\n",
    "        self.PSRewardTimeCDF = PDFCDF[1]\n",
    "        self.BGRewardTimePDF = background_init(self.BGTimeArray, self.BGRewardTime)\n",
    "        \n",
    "        self.num_states = len(self.BGTimeArray) + len(self.PSTimeArray) + len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay)\n",
    "    \n",
    "    \n",
    "    def env_start(self):\n",
    "        \"\"\"The first method called when the episode starts, called before the\n",
    "        agent starts.\n",
    "\n",
    "        Returns:\n",
    "            The first state from the environment.\n",
    "        \"\"\"\n",
    "        reward = 0\n",
    "        # current state of the agent is a 3-dimensional variable of {WhichPort, HowLongHasTheAgentWaited, RewardCollected}\n",
    "        self.current_state3d = (0, 0, 0)\n",
    "        self.current_state1d = self.state1d(self.current_state3d)\n",
    "        termination = False\n",
    "        self.reward_state_term = (reward, self.current_state1d, termination)\n",
    "\n",
    "        return self.reward_state_term[1]\n",
    "\n",
    "    def env_step(self, action):\n",
    "        \"\"\"A step taken by the environment.\n",
    "\n",
    "        Args:\n",
    "            action: The action taken by the agent. \n",
    "            can be 0: wait, or 1: give up\n",
    "\n",
    "        Returns:\n",
    "            (float, state, Boolean): a tuple of the reward, state,\n",
    "                and boolean indicating if it's terminal.\n",
    "        \"\"\"\n",
    "\n",
    "        self.current_state1d = self.state1d(self.current_state3d)\n",
    "        P, T, C = self.current_state3d # Port, Time, Collected reward\n",
    "        reward = 0 # default reward is 0\n",
    "        num_rewards = 0 # default number of reward is 0. # The number of rewards has nothing to do with reward amount\n",
    "        \n",
    "        # inside the background port\n",
    "        if (P == 0):\n",
    "\n",
    "            # if stay/wait, then get a reward at a fixed time point\n",
    "            if (action == 0): \n",
    "                # port doe not change\n",
    "                if isInBounds(T, self.BGTimeArray[-1]):\n",
    "                    amount_rand = self.rand_generator.uniform(low = 0, high = 1, size = 1)\n",
    "                    if (amount_rand < 0.5):\n",
    "                        reward = self.BGRewardTimePDF[FindIndexOfTime(self.BGTimeArray, T)] * self.BGRewardAmount[0]\n",
    "                    else: \n",
    "                        reward = self.BGRewardTimePDF[FindIndexOfTime(self.BGTimeArray, T)] * self.BGRewardAmount[1]\n",
    "                    num_rewards = self.BGRewardTimePDF[FindIndexOfTime(self.BGTimeArray, T)] * 1\n",
    "                    NextIndex = FindIndexOfTime(self.BGTimeArray, T) + 1\n",
    "                    T = self.BGTimeArray[NextIndex]\n",
    "                # if agent stays longer than the maximum time, they will be seen as in the same state\n",
    "                if not isInBounds(T, self.BGTimeArray[-1]): \n",
    "                    pass\n",
    "\n",
    "                C = C + num_rewards\n",
    "\n",
    "            # if leave/give up, then jump to the first state in the pursuit port\n",
    "            elif (action == 1): \n",
    "                if isInBounds(T, self.BG_dur) and (T >= self.BGMinStay):\n",
    "                    P, T, C = 2, 0, 0\n",
    "                elif isInBounds(T, self.BGMinStay): \n",
    "                    P, T, C = 0, 0, 0\n",
    "                else: \n",
    "                    pass\n",
    "\n",
    "            else: \n",
    "                raise Exception(str(action) + \" not in recognized actions [0: Wait, 1: Give up and leave]!\")\n",
    "\n",
    "        # inside the pursuit port\n",
    "        elif (P == 1): \n",
    "            if (action == 0):\n",
    "            # if stay, port does not change\n",
    "                \n",
    "                if isInBounds(T, self.PSTimeArray[-1]):\n",
    "                # within the maximum duration, will follow the reward function\n",
    "                    \n",
    "                    if (C == 0): \n",
    "                        # generate a random number between 0 and 1. \n",
    "                        # when it falls below Pr{rewarded at t|not rewarded prior to t}, then give a reward, otherwise no reward\n",
    "                        Pr_NeverRewarded = 1 - self.PSRewardTimeCDF[FindIndexOfTime(self.PSTimeArray, T)]\n",
    "                        Pr_NeverRewardedNRewardedAtT = self.PSRewardTimePDF[FindIndexOfTime(self.PSTimeArray, T)]\n",
    "                        Pr_RewardedAtTGivenNotYetRewarded = Pr_NeverRewardedNRewardedAtT / Pr_NeverRewarded\n",
    "                        rand = self.rand_generator.uniform(low = 0, high = 1, size = 1) \n",
    "                        if (rand < Pr_RewardedAtTGivenNotYetRewarded): \n",
    "                            reward = self.PSRewardAmount\n",
    "                            num_rewards = 1\n",
    "                        else: \n",
    "                            pass\n",
    "\n",
    "                        NextIndex = FindIndexOfTime(self.PSTimeArray, T) + 1\n",
    "                        T = self.PSTimeArray[NextIndex]\n",
    "\n",
    "                    elif (C == 1): \n",
    "                        # no reward\n",
    "                        NextIndex = FindIndexOfTime(self.PSTimeArray, T) + 1\n",
    "                        T = self.PSTimeArray[NextIndex]\n",
    "                else: \n",
    "                    # if agent stays longer than the maximum time, they will be seen as in the same state\n",
    "                    pass \n",
    "                        \n",
    "\n",
    "                C = C + num_rewards\n",
    "\n",
    "            # if leave/give up, then jump to the first state in the background port\n",
    "            elif (action == 1):\n",
    "                if isInBounds(T, self.PS_dur) and (T >= self.PSMinStay):\n",
    "                    P, T, C = 3, 0, 0\n",
    "                elif isInBounds(T, self.PSMinStay): \n",
    "                    P, T, C = 1, 0, 0\n",
    "                else: \n",
    "                    pass\n",
    "\n",
    "            else: \n",
    "                raise Exception(str(action) + \" not in recognized actions [0: Wait, 1: Give up and leave]!\")\n",
    "        \n",
    "        # the travel pursuit (from port 0 to port 1)\n",
    "        elif (P == 2): \n",
    "            if isInBounds(T, self.TravelTimeArray[-1]):\n",
    "                NextIndex = FindIndexOfTime(self.TravelTimeArray, T) + 1\n",
    "                T = self.TravelTimeArray[NextIndex]\n",
    "            else: \n",
    "                if (action == 0): \n",
    "                    pass\n",
    "                elif (action == 1): \n",
    "                    P, T, C = 1, 0, 0\n",
    "                else: \n",
    "                    raise Exception(str(action) + \" not in recognized actions [0: Stay, 1: Leave the travel pursuit]!\")\n",
    "            \n",
    "        # the travel pursuit (from port 1 to port 0)\n",
    "        elif (P == 3): \n",
    "            if isInBounds(T, self.TravelTimeArray[-1]):\n",
    "                NextIndex = FindIndexOfTime(self.TravelTimeArray, T) + 1\n",
    "                T = self.TravelTimeArray[NextIndex]\n",
    "            else: \n",
    "                if (action == 0): \n",
    "                    pass\n",
    "                elif (action == 1): \n",
    "                    P, T, C = 0, 0, 0\n",
    "                else: \n",
    "                    raise Exception(str(action) + \" not in recognized actions [0: Stay, 1: Leave the travel pursuit]!\")\n",
    "\n",
    "        # assign the new state to the environment object\n",
    "        self.current_state3d = (P, T, C)\n",
    "        self.current_state1d = self.state1d(self.current_state3d)\n",
    "        termination = False\n",
    "        self.reward_state_term = (reward, self.current_state1d, termination)\n",
    "        return self.reward_state_term\n",
    "\n",
    "\n",
    "    def env_cleanup(self):\n",
    "        \"\"\"Cleanup done after the environment ends\"\"\"\n",
    "        self.current_state3d = (0, 0, 0)\n",
    "        self.current_state1d = 0\n",
    "    \n",
    "    # helper method \n",
    "    def state1d(self, state3d):\n",
    "        P = state3d[0]\n",
    "        T = state3d[1]\n",
    "        C = state3d[2]\n",
    "        if (P == 0):\n",
    "            if (T <= self.BGRewardTime) and (C == 1): \n",
    "                raise Exception(f\"It is not possible that a reward is delivered before {self.BGRewardTime}. Hence this state does not exist. \")\n",
    "            elif (T > self.BGRewardTime) and (C == 0):\n",
    "                raise Exception(f\"It is not possible that no reward is delivered at {self.BGRewardTime}. Hence this state does not exist. \")\n",
    "            else: \n",
    "                return FindIndexOfTime(self.BGTimeArray, T)\n",
    "        \n",
    "        elif (P == 1): \n",
    "            if (C == 0): \n",
    "                return FindIndexOfTime(self.PSTimeArray, T) + len(self.BGTimeArray)\n",
    "            elif (C == 1): \n",
    "                return FindIndexOfTime(self.PSTimeArray, T) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) + len(self.PSTimeArray) + len(self.BGTimeArray)\n",
    "            else: \n",
    "                raise Exception(str(C) + \" not in the possible range of number of collected rewards [0, 1]!\")\n",
    "        \n",
    "        elif (P == 2): \n",
    "            if (C == 0): \n",
    "                return FindIndexOfTime(self.TravelTimeArray, T) + len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) + len(self.PSTimeArray) + len(self.BGTimeArray)\n",
    "            else: \n",
    "                raise Exception(f\"No reward could be delivered during travel time.\")\n",
    "                \n",
    "        elif (P == 3): \n",
    "            if (C == 0): \n",
    "                return FindIndexOfTime(self.TravelTimeArray, T) + len(self.TravelTimeArray) + len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) + len(self.PSTimeArray) + len(self.BGTimeArray)\n",
    "            else: \n",
    "                raise Exception(f\"No reward could be delivered during travel time.\")\n",
    "        \n",
    "        else: \n",
    "            raise Exception(str(P) + \" not in recognized pursuit [0: Background, 1: Pursuit, 2: travel from 0 to 1, 3: travel from 1 to 0]!\")\n",
    "    \n",
    "    def state3d(self, state1d): \n",
    "        s = state1d\n",
    "        if (s < self.BGTimeArray.shape[0]): \n",
    "            P = 0\n",
    "            T = self.BGTimeArray[s]\n",
    "            C = 0\n",
    "            if (T > self.BGRewardTime): \n",
    "                C = 1\n",
    "        elif (s >= self.BGTimeArray.shape[0]) and (s < len(self.PSTimeArray) + len(self.BGTimeArray)):\n",
    "            P = 1\n",
    "            T = self.PSTimeArray[s-self.BGTimeArray.shape[0]]\n",
    "            C = 0\n",
    "        \n",
    "        elif (s >= len(self.PSTimeArray) + len(self.BGTimeArray)) and \\\n",
    "        (s < len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) \\\n",
    "         + len(self.PSTimeArray) + len(self.BGTimeArray)):\n",
    "            P = 1\n",
    "            T = self.PSTimeArray[s-len(self.PSTimeArray)-len(self.BGTimeArray)\\\n",
    "                                 +FindIndexOfTime(self.PSTimeArray, self.PSMinDelay)]\n",
    "            C = 1\n",
    "        \n",
    "        elif (s >= len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) \\\n",
    "              + len(self.PSTimeArray) + len(self.BGTimeArray)) \\\n",
    "        and (s < len(self.TravelTimeArray) + len(self.PSTimeArray) - FindIndexOfTime(self.PSTimeArray, self.PSMinDelay) \\\n",
    "             + len(self.PSTimeArray) + len(self.BGTimeArray)): \n",
    "            P = 2\n",
    "            T = self.TravelTimeArray[s-len(self.PSTimeArray)+FindIndexOfTime(self.PSTimeArray, self.PSMinDelay)\\\n",
    "                                       -len(self.PSTimeArray)-len(self.BGTimeArray)]\n",
    "            C = 0\n",
    "        \n",
    "        else: \n",
    "            P = 3\n",
    "            T = self.TravelTimeArray[s-len(self.TravelTimeArray)\\\n",
    "                                     -len(self.PSTimeArray)+FindIndexOfTime(self.PSTimeArray, self.PSMinDelay)\\\n",
    "                                     -len(self.PSTimeArray)-len(self.BGTimeArray)]\n",
    "            C = 0\n",
    "        \n",
    "        \n",
    "        return (P, T, C)\n",
    "    \n",
    "    # function that finds the real optimal policy (that maximizes the global reward rate)\n",
    "    def PolicyMax(self): \n",
    "        PSRewardRate = np.zeros(len(self.PSTimeArray))\n",
    "        # BGLeaveTime = self.BGRewardTime\n",
    "        # csmp = self.consmp_dur\n",
    "        # transit = self.transit_dur\n",
    "        # BGRewardRate = self.BGRewardAmount / BGLeaveTime\n",
    "        GlobalRewardRate = np.zeros(len(self.PSTimeArray))\n",
    "        PS_ExpectedReward = np.zeros(len(self.PSTimeArray))\n",
    "        PS_ExpectedTime = np.zeros(len(self.PSTimeArray))\n",
    "        \n",
    "        for i in range(0, len(self.PSTimeArray)): \n",
    "            t = self.PSTimeArray[i]\n",
    "            if isInBounds(t, self.PSMinStay): \n",
    "                t_b = self.BGRewardTime + self.PSResetTime\n",
    "            else: \n",
    "                t_b = self.BGRewardTime\n",
    "            Pr_RewardedBeforeT = self.PSRewardTimeCDF[i]\n",
    "            Pr_NotRewardedBeforeT = 1 - Pr_RewardedBeforeT\n",
    "            PS_ExpectedReward = Pr_RewardedBeforeT * self.PSRewardAmount + Pr_NotRewardedBeforeT * 0\n",
    "            PS_ExpectedExitTimeIfRewardedBeforeT = np.dot(self.PSTimeArray[0:i], self.PSRewardTimePDF[0:i])\n",
    "            PS_ExpectedTime = PS_ExpectedExitTimeIfRewardedBeforeT + Pr_NotRewardedBeforeT * t\n",
    "            Total_ExpectedReward = np.mean(self.BGRewardAmount) + PS_ExpectedReward\n",
    "            Total_ExpectedTime = t_b + PS_ExpectedTime + self.transit_dur * 2\n",
    "            PSRewardRate[i] = PS_ExpectedReward / PS_ExpectedTime\n",
    "            GlobalRewardRate[i] = Total_ExpectedReward / Total_ExpectedTime\n",
    "        \n",
    "        optimal_giveup_index_rou_g = argmax(GlobalRewardRate)\n",
    "        optimal_giveup_time_rou_g = self.PSTimeArray[optimal_giveup_index_rou_g]\n",
    "        optimal_giveup_index_rou_l = argmax(PSRewardRate)\n",
    "        optimal_giveup_time_rou_l = self.PSTimeArray[optimal_giveup_index_rou_l]\n",
    "        return (GlobalRewardRate, PSRewardRate, optimal_giveup_time_rou_g, optimal_giveup_time_rou_l)\n",
    "    \n",
    "    def test_reward_function(self): \n",
    "        fig, ax = plt.subplots(1, 1)\n",
    "        ax.plot(self.PSTimeArray, self.PSRewardTimePDF)\n",
    "        fig, bx = plt.subplots(1, 1)\n",
    "        bx.plot(self.PSTimeArray, self.PSRewardTimeCDF)\n",
    "        fig, cx = plt.subplots(1, 1)\n",
    "        cx.plot(self.BGTimeArray, self.BGRewardTimePDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\n",
      "23\n",
      "[0.   0.1  0.21 0.33 0.47 0.62 0.79 0.98 1.19 1.42 1.68 1.97 2.29 2.64\n",
      " 3.03 3.47 3.96 4.5  5.1  5.76 6.5 ]\n",
      "[0.   0.1  0.21 0.33 0.47 0.62 0.79 0.98 1.19 1.42 1.68 1.97 2.29 2.64\n",
      " 3.03 3.47 3.96 4.5  5.1  5.76 6.5  7.33 8.24]\n",
      "[0.   0.1  0.21 0.33 0.47]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "env_info = {\"fundamental timestep\": 0.1, \n",
    "            \"gamma\": 0.9, \n",
    "            \"state representation structure\": \"dilating\", \n",
    "            \"pursuit port total duration\": 7, \n",
    "            \"background port total duration\": 9, \n",
    "            \"transit duration\": 0.5, \n",
    "            \"consumption duration\": 1, \n",
    "            \"first time in pursuit port that could give a reward\": 1,\n",
    "            \"required minimum wait time in pursuit port\": 1, \n",
    "            \"time to wait until the trial reset\": 1.5, \n",
    "            \"exponential distribution scale\": 4, \n",
    "            \"delivery time in the background port\": 5,\n",
    "            \"reward amount in pursuit\": 3, \n",
    "            \"reward amount in background\": [0.9, 1.5], \n",
    "            \"seed\": 0}\n",
    "\n",
    "env = GiveUpEnvironment(env_info = env_info)\n",
    "print(len(env.PSTimeArray))\n",
    "print(len(env.BGTimeArray))\n",
    "print(env.PSTimeArray)\n",
    "print(env.BGTimeArray)\n",
    "print(env.TravelTimeArray)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\shich\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:372: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "According to the global reward rate, the optimal give-up time in the pursuit port is at 3.0 second!\n",
      "According to the local reward rate, the optimal give-up time in the pursuit port is at 2.1 second!\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VOW9x/HPLxthl7BDCIRFJCyCBCKiqAUVFMFdXKl6Ra9KbbXt1dtWLV5bl2624kKRulShbrVoUYsKsskSNmU37GENi2wCmZk8949McIyBDJDJnJn5vl+vvJg585zJLwS+Ofmd5zzHnHOIiEhiSIp2ASIiUn0U+iIiCUShLyKSQBT6IiIJRKEvIpJAFPoiIgkkrNA3s4FmttLMCszsgQpev9PMvjSzRWY2w8xygtvbmNnB4PZFZvZ8VX8BIiISPqtsnr6ZJQOrgAuAQmAecJ1zblnImHrOub3Bx0OAu5xzA82sDfC+c65LZMoXEZHjEc6Rfm+gwDm3xjlXDEwAhoYOKAv8oNqArvgSEfGglDDGtAQ2hjwvBPLKDzKzu4H7gDTgByEvZZvZQmAv8Evn3PQK9h0BjACoXbt2z9NOOy3sL0BERGD+/Pk7nHONKxsXTuhbBdu+dyTvnBsNjDaz64FfAsOBLUCWc26nmfUE3jWzzuV+M8A5NwYYA5Cbm+vy8/PDKEtERMqY2fpwxoXT3ikEWoU8zwQ2H2P8BOAyAOfcYefczuDj+cBq4NRwChMRkaoXTujPAzqYWbaZpQHDgImhA8ysQ8jTS4CvgtsbB08EY2ZtgQ7AmqooXEREjl+l7R3nnN/M7gE+ApKBcc65pWY2Csh3zk0E7jGzAYAP2E1pawegHzDKzPxAALjTObcrEl+IiIhUrtIpm9VNPX0RkeNnZvOdc7mVjdMVuSIiCUShLyKSQBT6IiIJRKHvYc45JszdwJ5vfNEuRUTihELfw9bt/IYH3vmSn721GK+dcBeR2KTQ97BDvgAA/1m2jX8u3BTlakQkHij0PcwXKAGgTo0UHp64lC17Dka5IhGJdQp9D/MFSls6/zOwI/6A4+dvfaE2j4icFIW+h5Ud6bdrUof/vfg0pn+1g9fmbIhyVSISyxT6HlYW+mnJSdyQ15qz2zfiN5OWs37ngShXJiKxSqHvYf5geyc1OYmkJOPJq7qRbMb9bywmUKI2j4gcP4W+hxUHj/RTkktvadDilJr8emhn8tfv5vnPVkezNBGJUQp9Dwtt75S5vEdLLunWnD9OXsWXhXuiVZqIxCiFvoeVhX5qSOibGY9d1oVGdWpw7z8WcrA4EK3yRCQGKfQ9rGzKZll7p8wptdL4wzWns6boAI9NWhaN0kQkRin0Payi9k6Zs9o34vZzsvn77A18umJbdZcmIjFKoe9hPv/32zuhfnpRR05rVpefv/UFRfsOV2dpIhKjFPoe5g9Oy0xNqfjbVCMlmaeH9WDfIT8/fXMxJZrGKSKVUOh72JEpm0l21DEdm9Xll4Nz+GxVEeNmrq2u0kQkRin0Pczn//birGO5MS+LC3Oa8sSHKzSNU0SOSaHvYb5ACclJRvIxjvShdBrnk1d1o1GdGowcv4D9h/3VVKGIxJqwQt/MBprZSjMrMLMHKnj9TjP70swWmdkMM8sJee3B4H4rzeyiqiw+3vlKSo7Z2gl1Sq00/nRtdzbs+oaH/rUkwpWJSKyqNPTNLBkYDQwCcoDrQkM96HXnXFfnXHfgSeAPwX1zgGFAZ2Ag8Gzw/SQMPr+rcLrm0eS1bcjIH3TgnQWbeGdBYQQrE5FYFU6i9AYKnHNrnHPFwARgaOgA59zekKe1gbJpJEOBCc65w865tUBB8P0kDL5AyVFn7hzNyB+0p3ebDH757hIKtu+PUGUiEqvCSZSWwMaQ54XBbd9hZneb2WpKj/R/dJz7jjCzfDPLLyoqCrf2uOcvKSE1Obz2TpmU5CT+fF0P0lOTufu1BVqmQUS+I5zQryh1vjch3Dk32jnXDvgf4JfHue8Y51yucy63cePGYZSUGIr9jpSk4z/X3qx+On+8tjurtu/j4Ynq74vIt8JJlEKgVcjzTGDzMcZPAC47wX0lhC9QQtpxtnfKnHtqY+4+rz1v5Bfy9nz190WkVDiJMg/oYGbZZpZG6YnZiaEDzKxDyNNLgK+CjycCw8yshpllAx2AuSdfdmLwBY6/vRPqxwM6kJdd2t//atu+KqxMRGJVpaHvnPMD9wAfAcuBN5xzS81slJkNCQ67x8yWmtki4D5geHDfpcAbwDLgQ+Bu55yazGHyBU6svVOmrL9fKy2Zu15bwAHN3xdJeCnhDHLOTQImldv2UMjje4+x72PAYydaYCI7kdk75TWtl87Tw3pw07g5PPjOlzw9rDtmJ/7bg4jENl2R62G+QAlpJ9HeKXN2h0bcf8GpTFy8mZdnrTv5wkQkZin0Pcx/ku2dUHed154BnZrwf/9ezvz1u6rkPUUk9ij0Pay4Cto7ZZKSjN9f050Wp9TkrtcWaP19kQSl0PewqmrvlKlfM5XnbjyDr7/xMXL8AvzBpZtFJHEo9D2sdMpm1X6LOreoz28u78rsNbt4/IMVVfreIuJ9Yc3ekejwBxwpVRz6AFf2zGRx4deMnbGWrpn1Gdr9eytjiEic0pG+hxWf5MVZx/KrwTn0zs7g5299wZJNuvGKSKJQ6HtYaU8/Mt+i1OQknr3hDDJqp3HHq/PZuV8ndkUSgULfw0rbO5G7kKpRnRqMuSmXHfsPc/frC/DpxK5I3FPoe1hxBE7kltc1sz6/vaL0xO5j/14e0c8lItGnE7keFsn2Tqgrzshk6ea9vDhjLR2b1eW63lkR/5wiEh060vcwX8BF/Ei/zIODTuPcUxvzq3eX8PnqndXyOUWk+in0PaqkxBEoiWxPP1RKchJ/ub4HbRrV5r9fm8/6nQeq5fOKSPVS6HuUr6T0pGp1HekD1EtPZezNuQDc9nI++w75qu1zi0j1UOh7lC9QelfJ6ujph2rTqDbP3nAG63YcYOT4hQRKvnd3SxGJYQp9jypbF6e62juhzmrXiF8P7czUlUU8+v6yav/8IhI5mr3jUcWB6m/vhLohrzVriw4wdsZaWjesxS19s6NSh4hULYW+R0WrvRPqwYs7sWHXNzz6/jJaNajFgJymUatFRKqG2jse5fMHj/RTondrw+Qk40/DutOlZX1Gjl+oNXpE4oBC36P8wdk7VXXnrBNVKy2FscNzyaidxq0vzWPz1wejWo+InByFvkcV+0vbO9Hq6YdqUjedcT/sxcHiALf8bR57Dmoqp0isCitRzGygma00swIze6CC1+8zs2Vm9oWZfWJmrUNeC5jZouDHxKosPp6VLX6WFsX2TqiOzery3I09WV20nztezeewPxDtkkTkBFQa+maWDIwGBgE5wHVmllNu2EIg1znXDXgLeDLktYPOue7BjyFVVHfc80p7J9TZHRrx1NXdmL1mFz998wtKNIdfJOaEkyi9gQLn3BrnXDEwARgaOsA5N8U5903w6Wwgs2rLTDxeau+EurxHJv8z8DTeW7yZ336gVTlFYk04idIS2BjyvDC47WhuAz4IeZ5uZvlmNtvMLqtoBzMbERyTX1RUFEZJ8c9r7Z1Qd57blpv7tOav09fy4oy10S5HRI5DOPP0K0qdCn+vN7MbgVzg3JDNWc65zWbWFvjUzL50zq3+zps5NwYYA5Cbm6ueAd+GvpfaO2XMjIcv7cy2vYd49P1lNKqTpvvsisSIcBKlEGgV8jwT2Fx+kJkNAH4BDHHOHbn3nnNuc/DPNcBUoMdJ1Jswyi7O8lp7p0xykvH0sB7kZWdw/xuLmbpye7RLEpEwhJMo84AOZpZtZmnAMOA7s3DMrAfwAqWBvz1kewMzqxF83AjoC2gxlzB4ub1TJj01mb8Oz+XUpnX5778vYP763dEuSUQqUWnoO+f8wD3AR8By4A3n3FIzG2VmZbNxngLqAG+Wm5rZCcg3s8XAFOBx55xCPwy+KK+9E6566am8fGtvmtarwa0vzWPVtn3RLklEjiGstXecc5OASeW2PRTyeMBR9psFdD2ZAhOVP9jeSfF46AM0rluDV2/L48rnZnHTi3N4686zaJVRK9pliUgFvJ8oCerbVTa9294J1SqjFq/elschXwk3vjiH7XsPRbskEamAQt+jjvT0Y+BIv0zHZnV56ZZe7Nh3mBvGzmHXgeJolyQi5cROoiSYI1M2Yyj0AXpkNWDs8F5s2PUNN4+bw17dclHEU2IrURLIt1M2Y6O9E6pPu4Y8f2NPVm7dx61/m8c3xf5olyQiQQp9jzoye8eDF2eF4/zTmvD0sB4s2LCb21/J55BPC7SJeEFsJkoC8AVKSEkykpJi70i/zMVdm/PUVacza/VORrw6X8Ev4gEKfY/yB1xUbope1a7smcnjV3Rl2qoi7nptgZZkFokyhb5HFQdKPH9hVriu7ZXFY5d34dMV27nn9YUUB28FKSLVLz5SJQ75AiUxNV2zMjfktWbU0M5MXraNH41feOSchYhUr/hJlTjj88dHeyfUzX3a8NDgHD5cupWROuIXiQqFvkf5SuKnvRPq1rOz+VUw+O9+fYGCX6SaxV+qxAlfwMVVeyfUbWdn8+shpa2eu16br5O7ItUoPlMlDvj88XmkX2b4WW149LIufLx8O3dqOqdItYnfVIlx/pKSuOvpl3fTma35zeVdmbKyiNtfyedgsYJfJNIU+h5VHHBxfaRf5vq8LJ66qhszC3YwfNxc9mmtHpGIiv9UiVE+f3xN2TyWq3Nb8efrSpdsuHHsHL7+RqtzikRKYqRKDEqE9k6owd1a8MJNPVm+dR/DxsymaN/hyncSkeOm0PeoRGnvhOrfqSl/+2HpsszXvPA5hbu/iXZJInEnsVIlhsT77J2j6du+Ea/e1pud+w9z1XOf85XuuStSpRIvVWKEL1ASk2vpV4WerTP4xx19CDjH1S98zsINu6NdkkjcCCv0zWygma00swIze6CC1+8zs2Vm9oWZfWJmrUNeG25mXwU/hldl8fHMX5J47Z1QnZrX4+07z6Jeeio3jJ3D9K+Kol2SSFyoNFXMLBkYDQwCcoDrzCyn3LCFQK5zrhvwFvBkcN8M4GEgD+gNPGxmDaqu/PhVnKDtnVBZDWvx1p19yMqoxa0vzeO9xZujXZJIzAsnVXoDBc65Nc65YmACMDR0gHNuinOu7KzbbCAz+PgiYLJzbpdzbjcwGRhYNaXHN1+ghLSUxGzvhGpSL51/3NGH7q1OYeT4hYydvibaJYnEtHBCvyWwMeR5YXDb0dwGfHA8+5rZCDPLN7P8oiL9Gg+l7Z2UGL1VYlWrXzOVV2/LY2DnZvzfv5fzf+8vo6TERbsskZgUTqpUdLhZ4f84M7sRyAWeOp59nXNjnHO5zrncxo0bh1FS/EvU2TtHk56azOgbzmB4n9aMnbGWe/+xSAu1iZyAcFKlEGgV8jwT+F5z1cwGAL8AhjjnDh/PvvJ9xYESUtXe+Y7kJOORIZ15YNBpvLd4M8PHzWXPN1q2QeR4hBP684AOZpZtZmnAMGBi6AAz6wG8QGngbw956SPgQjNrEDyBe2Fwm1TCFyghVe2d7zEz7jy3HX+89nTmr9/NFc/NZOMuXcQlEq5KU8U55wfuoTSslwNvOOeWmtkoMxsSHPYUUAd408wWmdnE4L67gEcp/cExDxgV3CbHEChxlDjU3jmGy3tk8sqteRTtO8zlz87UXH6RMKWEM8g5NwmYVG7bQyGPBxxj33HAuBMtMBGV3T9W7Z1j69OuIe/c1ZdbX5rHsDGzeXpYdwZ2aR7tskQ8TYeSHlQW+omyyubJaN+kDv+86yxyWtTjv19bwPOfrcY5zewRORqligf5A6WhlZKkI/1wNKxTg/G3n8klXZvz+AcruP/NxZrZI3IUYbV3pHp9297Rz+Rwpacm85fretChSV3++PEq1u/8hhdu6kmjOjWiXZqIpyhVPKi4LPTV3jkuZsa9Azow+vozWLp5D0OfmcnyLXujXZaIpyhVPMgXbO8k6iqbJ+uSbs15846z8JeUcMWzs5j05ZZolyTiGQp9D/LrSP+kdc2sz3v3nE2n5nW567UFPPXRCgJaukFEoe9Fau9UjSb10hk/4kyG9WrF6Cmruf2VfPYc1BW8ktiUKh5U1t7RlM2TVyMlmd9e0ZVHL+vCtFVFXD56Jqt0Ny5JYEoVDypr7yTSjdEjycy46czWvH77mew95Oey0TO1Nr8kLIW+B6m9Exm9szN4f+TZdGpej5HjFzLqvWVHpseKJAqligd9O3tH356q1qx+OuNvP5MfntWGcTPXcv1fZ7N976FolyVSbZQqHuTzlx3pq70TCWkpSTwypDNPD+vOkk17ufjPM5hVsCPaZYlUC4W+B/lL1N6pDkO7t+Rf9/TllFqp3PDiHJ7++CtN65S4p1TxoGK1d6rNqU3r8q+7+3J595b88eNVDB83lx37D1e+o0iMUqp4kNo71at2jRR+f83pPHFlV+at28Wgp6czU+0eiVMKfQ9Se6f6mRnX9sri3bv7Ui89hRtfnMOTH67Q7B6JO0oVD1J7J3o6Na/HeyPP5pqerXh26mqueeFz3Y5R4opSxYPK2ju6Ijc6aqWl8MRV3fjLdT0o2Lafi/88nYm6mEvihFLFg3y6ItcTLj29BZPuPYf2Terwo/EL+fGEhew9pLV7JLYp9D3IX6L2jle0yqjFm3f04ccDOvDeF1sY9KfpzF27K9pliZwwpYoHFWv2jqekJCfx4wGn8uadfUhOMq4d8zlPfLhCt2SUmBRW6JvZQDNbaWYFZvZABa/3M7MFZuY3s6vKvRYws0XBj4lVVXg88wVKSEkyzBT6XnJGVgMm3XsOV/fM5Lmpq3VnLolJlYa+mSUDo4FBQA5wnZnllBu2Afgh8HoFb3HQOdc9+DHkJOtNCP4Sp9aOR9WpkcKTV53O2Jtz2bG/mCHPzGD0lIIjK6OKeF04ydIbKHDOrXHOFQMTgKGhA5xz65xzXwD6l18Fiv0lau143ICcpvznJ/24MKcZT320kqtf+JyC7fujXZZIpcIJ/ZbAxpDnhcFt4Uo3s3wzm21ml1U0wMxGBMfkFxUVHcdbxydfoIS0FB3pe11G7TSeub4Hf76uB2uKDnDxn6fz/GerddQvnhZOslR0yHk8q1JlOedygeuBP5lZu++9mXNjnHO5zrncxo0bH8dbx6fSnr5CPxaYGUNOb8Hk+/pxfsfGPP7BCq58bhYrt+ruXOJN4SRLIdAq5HkmEPaVKs65zcE/1wBTgR7HUV9C8gccqSlq78SSJnXTef7GnjxzfQ827j7I4L9M58+ffHVkJpaIV4QT+vOADmaWbWZpwDAgrFk4ZtbAzGoEHzcC+gLLTrTYRFEcKNGJ3BhkZgzu1oLJP+nHRZ2b8YfJqxj8l+ks2LA72qWJHFFpsjjn/MA9wEfAcuAN59xSMxtlZkMAzKyXmRUCVwMvmNnS4O6dgHwzWwxMAR53zin0K+ELlJCq9k7MalinBs9cfwYvDs9l3yE/Vz43i0cmLmX/YX+0SxMhJZxBzrlJwKRy2x4KeTyP0rZP+f1mAV1PssaEo/ZOfOjfqSl5bRvy1IcrePnzdXy0dCuPDOnMRZ2bRbs0SWA6nPQgtXfiR50aKfx6aBfeuvMs6tdM5Y5X5/NfL8+jcLdW7pToULJ4kE+hH3d6tm7AeyPP5sFBpzGzYCcX/GEaz3+2Wuv1S7VTsniQL+B0cVYcSk1O4o5z2zH5vn70bd+Qxz9YwcVPT2fWat2lS6qPQt+D/DrSj2uZDWoxdngv/npzLgd9Aa7/6xzueX0BW/YcjHZpkgCULB5UHNDaO4nggpymfHzfudzbvwP/WbaN/r//jOemrtbqnRJRShYPKu3pq72TCNJTk/nJBafy8U/O5ax2DXniwxVc+MdpTF62DeeO58J3kfAo9D1I7Z3Ek9WwtOXz8q29SU1O4vZX8rl53FxWbdNyDlK1lCwe5FN7J2Gde2pjPrj3HB6+NIfFG79m0NPT+eW7X7Jj/+FolyZxQsniQcVq7yS01OQkbumbzdSfnc8NeVmMn7uR856aynNTV3PIp36/nByFvgdpnr5A6dLNo4Z24aMf9+PMthk88eEK+v/+M/61aBMlJer3y4lRsniQX+0dCdG+SR3GDu/F6/+VR/2aqdw7YRGXPjOD6V/p3hNy/JQsHqRlGKQiZ7VvxPsjz+ZP13Znz0EfN704lxvHzmHJpj3RLk1iiJLFgzRlU44mKcm4rEdLPrn/XH41OIelm/cw+C8zuPu1Bbpdo4QlrFU2pfoEShzOoSN9OaYaKcncdnY2V+dmMnbaGsbOWMsHS7Zw5RmZ3DugA5kNakW7RPEoJYvHlC3ApdCXcNRLT+W+Czsy7efnc0vfbP61eDPn/24qv3p3CVv3HIp2eeJBShaPKT4S+mrvSPga1anBrwbn8NnPzuOqnq0YP3cD/Z6awiMTl7J9r8JfvqXQ9xifX0f6cuKa16/Jb6/oypSfnsfl3Vvy6uz1nPPkFEa9t4xtCn9Boe85/uD8a4W+nIxWGbV44qpufHr/uVx6egte/nwd5zw5hV+9u4RNX2s1z0SmZPGYYr/aO1J1Wjesze+uPp0p95/HFT1aMmHeBs57agoPvvMF63ceiHZ5EgUKfY/RiVyJhKyGtXj8ym5M/dn5DOuVxdsLNnH+76byo/ELWb5lb7TLk2oUVrKY2UAzW2lmBWb2QAWv9zOzBWbmN7Oryr023My+Cn4Mr6rC45XaOxJJLU+pyaOXdWHGz8/n9nPa8snybQx6ejq3vjSPeet2aTnnBFBpsphZMjAaGATkANeZWU65YRuAHwKvl9s3A3gYyAN6Aw+bWYOTLzt+qb0j1aFJvXQevLgTsx7oz/0XnMrCDbu5+vnPueK5WXy4ZCsBre0Tt8I5nOwNFDjn1jjnioEJwNDQAc65dc65L4Dyd3m+CJjsnNvlnNsNTAYGVkHdcUvtHalO9WulMrJ/B2Y90J9RQzuzc38xd/59PgP+8BmvzVmvVT3jUDjJ0hLYGPK8MLgtHCezb0LyBdTekepXMy2Zm/u04dP7z+WZ63tQNz2FX/xzCX1++wm//89KzfWPI+Esw1BRnyHc3/3C2tfMRgAjALKyssJ86/jk18VZEkUpyUkM7taCS7o2Z87aXbw4Yy3PTCng+c9Wc+npLbi1bzZdWtaPdplyEsIJ/UKgVcjzTGBzmO9fCJxXbt+p5Qc558YAYwByc3MTupl45IrcFB3pS/SYGWe2bciZbRuybscBXpq1jjfyN/LOgk30atOA4We14aLOzfQbaQwK5zs2D+hgZtlmlgYMAyaG+f4fAReaWYPgCdwLg9vkKI60d5L0n0m8oU2j2jwypDOfP9ifX17SiW17D3PP6ws554kpPPPpVxTt060cY0mlyeKc8wP3UBrWy4E3nHNLzWyUmQ0BMLNeZlYIXA28YGZLg/vuAh6l9AfHPGBUcJscxZH2ToraO+It9Wum8l/ntGXKT8/jxeG5dGhah9/9ZxVnPf4JI8cv1JTPGBHW0srOuUnApHLbHgp5PI/S1k1F+44Dxp1EjQmlWLN3xOOSk4z+nZrSv1NTVhft57XZG3hz/kbeW7yZ05rV5fq8LC7r0ZJ66anRLlUqoGTxGLV3JJa0a1yHhy7NYc7/9ueJK7uSmpzEQ/9aSu/HPuZnby5m4YbdOvr3GN1ExWPU3pFYVCsthWt7ZXFtryy+LNzD63M3MHHRJt6cX8hpzepyTW4rLu/Rkga106JdasLT4aTH6OIsiXVdM+vz2yu6MucXA3js8i7USEli1PvLyPvNJ9z9+gKmrSrSFb9RpCN9jylWe0fiRJ0aKdyQ15ob8lqzfMte/jFvI+8u2sS/v9hC8/rpXHFGS648I5O2jetEu9SEotD3GJ/aOxKHOjWvxyNDOvPAoNP4ePk23ppfyHNTVzN6ympyWzfgijMyuaRrc+rX0snfSFPoe4xf7R2JY+mpyQzu1oLB3Vqwbe8h/rlwE2/NL+R///klj0xcSv9OTbisR0vO79iENF2gGBEKfY8pa++kJOlIX+Jb03rp3HluO+7o15Ylm/byzsJC3lu8mQ+WbKV+zVQu7tqcIae3IC87gyT9f6gyCn2P8QVKSE02zPSPXBKDmdE1sz5dM+vzi4s7Mb1gB+8u3MS/Fm1i/NwNNKuXzuBuzbn09BZ0y6yv/xsnSaHvMf5AiVo7krBSkpM4v2MTzu/YhG+K/Xy8fDsTF23m5c/XMXbGWrIyanFJt+YM7tacnOb19APgBCj0PcYXcAp9EUrn/g85vQVDTm/Bnm98fLRsK+9/sYUx09bw3NTVZDeqzaAuzbi4a3M6t9APgHAp9D2mONjeEZFv1a+VyjW5rbgmtxW7DhTz4ZKtfLBkCy9MW8OzU1fTKqMmF3dpzkVdmtE98xSdAzgGhb7H+Pxq74gcS0btNK7Py+L6vCx2Hyhm8rJtTFqyhXEz1/LCtDU0qVuDCzs35aLOzTizbUP9fypHoe8x/hK1d0TC1aB2Gtf0asU1vVqx56CPKSu289HSrbw9fxN/n72BuukpnN+xCQNymnJex8ZaBA6FvueovSNyYurXTOWyHi25rEdLDvkCTFtVxORl2/h0xXYmLt5MarKRl92Q/p2a8IPTmtC6Ye1olxwVCn2PUXtH5OSlpyZzYedmXNi5GYESx8INu5m8fBsfL9vGr99bxq/fW0a7xrXp36n0N4Dc1hkJczGYQt9j1N4RqVrJSUZumwxy22Tw4KBOrN95gE9XbOfTFdv528y1jJm2hjo1Uji7fSPO69iY8zo2oVn99GiXHTEKfY/xqb0jElGtG9bmlr7Z3NI3m/2H/cws2MHUlUVMXbmdD5duBeDUpnXo16Ex/U5tTO/sDNJTk6NcddVR6HtMsb+EFB3pi1SLOjVSuKhzMy7q3AznHKu27eezVduZtmoHr3y+nrEz1pKemkSvNhmc3b4RZ3doRKdm9WJ6SqhC32N8gRJqpenbIlLdzIyOzerSsVldRvRrx8HiALPX7mTaqiJmfLWD336wAj6AhrXT6NOuIX3bN+Ksdg3JyqgVUxeGKV08prSnHzvJ2a9cAAAJU0lEQVT/gETiVc205CNLQgBs3XOImQU7mFGwg5kFO3j/iy0AtDylJme1a0ifdg05s21DWpxSM5plV0qh7zHFmr0j4knN6qdzZc9MruyZiXOO1UX7mbV6J7MKdvKfZdt4c34hAK0b1uLM7Ibktc2gd3YGmQ1qRbny7wor9M1sIPA0kAyMdc49Xu71GsArQE9gJ3Ctc26dmbUBlgMrg0NnO+furJrS45NPC66JeJ6Z0b5JXdo3qcvNfdpQUuJYsXUfs9fs5PM1O/lgyRb+kb8RKP1NIC87g17ZGfRqk0G7xrWj2g6qNPTNLBkYDVwAFALzzGyic25ZyLDbgN3OufZmNgx4Arg2+Npq51z3Kq47bqm9IxJ7kpKMnBb1yGlRj1vPziZQ4li5dR9z1+5k7rpdTPuqiHcWbgKgQa1Ucttk0KtNA3q2zqBLy3rUSKm+2UHhHOn3Bgqcc2sAzGwCMBQIDf2hwCPBx28Bz1gsndnwEF2cJRL7kkN+CPywbzbOOdbsOMD8dbuZt24X+et3M3nZNgDSkpPomlmfnq0bkJedQf9OTSNaWzih3xLYGPK8EMg72hjnnN/M9gANg69lm9lCYC/wS+fc9PKfwMxGACMAsrKyjusLiDfFAacpmyJxxsxo17gO7RrX4ZperQAo2neY+et3s2DDbuav381LM9exYP1uT4R+RUfsLswxW4As59xOM+sJvGtmnZ1ze78z0LkxwBiA3Nzc8u+dUHyBEtLU3hGJe43r1mBgl2YM7NIMgEO+ADsPFEf884ZzSFkItAp5nglsPtoYM0sB6gO7nHOHnXM7AZxz84HVwKknW3Q8052zRBJTemoyLathumc46TIP6GBm2WaWBgwDJpYbMxEYHnx8FfCpc86ZWePgiWDMrC3QAVhTNaXHJ5/aOyISQZW2d4I9+nuAjyidsjnOObfUzEYB+c65icCLwKtmVgDsovQHA0A/YJSZ+YEAcKdzblckvpB44JyjWO0dEYmgsObpO+cmAZPKbXso5PEh4OoK9nsbePska0wYgZLS0xlq74hIpChdPMQXCIZ+gqzrLSLVT+niIcWBEgBSYngFPxHxNoW+h/iCoZ8od/ARkeqndPEQf0A9fRGJLKWLh/jU3hGRCFPoe0ix2jsiEmFKFw9Re0dEIk3p4iFl7R2FvohEitLFQ45M2dQVuSISIQp9D/H5gz19HemLSIQoXTzEr2UYRCTClC4eovaOiESaQt9D1N4RkUhTuniI2jsiEmlKFw/5dsqm2jsiEhkKfQ8p9muevohEltLFQ3y6IldEIkzp4iH+ErV3RCSyFPoeUtbe0Y3RRSRSlC4eUtbe0ZRNEYkUpYuH+DV7R0QiLKzQN7OBZrbSzArM7IEKXq9hZv8Ivj7HzNqEvPZgcPtKM7uo6kqPP2VTNpN1ExURiZBKQ9/MkoHRwCAgB7jOzHLKDbsN2O2caw/8EXgiuG8OMAzoDAwEng2+n1SgOOBIS07CTKEvIpGREsaY3kCBc24NgJlNAIYCy0LGDAUeCT5+C3jGSpNrKDDBOXcYWGtmBcH3+7xqyv/W198Uc/XzVf621apo/2GtuyMiERVO6LcENoY8LwTyjjbGOec3sz1Aw+D22eX2bVn+E5jZCGAEQFZWVri1f0dSktGhaZ0T2tcrOjStQ5eW9aNdhojEsXBCv6JDTxfmmHD2xTk3BhgDkJub+73Xw1EvPZVnb+h5IruKiCSMcE7kFgKtQp5nApuPNsbMUoD6wK4w9xURkWoSTujPAzqYWbaZpVF6YnZiuTETgeHBx1cBnzrnXHD7sODsnmygAzC3akoXEZHjVWl7J9ijvwf4CEgGxjnnlprZKCDfOTcReBF4NXiidhelPxgIjnuD0pO+fuBu51wgQl+LiIhUwkoPyL0jNzfX5efnR7sMEZGYYmbznXO5lY3TFbkiIglEoS8ikkAU+iIiCUShLyKSQDx3ItfMioD1J/EWjYAdVVROdVC9kaV6I0v1Rtbx1NvaOde4skGeC/2TZWb54ZzB9grVG1mqN7JUb2RFol61d0REEohCX0QkgcRj6I+JdgHHSfVGluqNLNUbWVVeb9z19EVE5Oji8UhfRESOQqEvIpJA4ib0K7t5u9eY2Tgz225mS6JdS2XMrJWZTTGz5Wa21MzujXZNlTGzdDOba2aLgzX/Oto1VcbMks1soZm9H+1awmFm68zsSzNbZGaeXyXRzE4xs7fMbEXw33KfaNd0NGbWMfj3Wvax18x+XCXvHQ89/eDN1lcBF1B645Z5wHXOuWXH3DGKzKwfsB94xTnXJdr1HIuZNQeaO+cWmFldYD5wmcf/fg2o7Zzbb2apwAzgXufc7Ep2jRozuw/IBeo55wZHu57KmNk6INc5FxMXO5nZy8B059zY4L1Bajnnvo52XZUJ5tsmIM85dzIXrgLxc6R/5ObtzrlioOzm7Z7lnJtG6b0HPM85t8U5tyD4eB+wnArudewlrtT+4NPU4Idnj3DMLBO4BBgb7VrikZnVA/pReu8PnHPFsRD4Qf2B1VUR+BA/oV/Rzds9HUqxyszaAD2AOdGtpHLBdskiYDsw2Tnn5Zr/BPwcKIl2IcfBAf8xs/lmNiLaxVSiLVAE/C3YQhtrZrWjXVSYhgHjq+rN4iX0w7oBu5wcM6sDvA382Dm3N9r1VMY5F3DOdaf03sy9zcyTbTQzGwxsd87Nj3Ytx6mvc+4MYBBwd7Bl6VUpwBnAc865HsABIBbO/aUBQ4A3q+o94yX0dQP2CAv2xd8GXnPOvRPteo5H8Nf4qcDAKJdyNH2BIcEe+QTgB2b29+iWVDnn3Obgn9uBf1LaZvWqQqAw5Le9tyj9IeB1g4AFzrltVfWG8RL64dy8XU5Q8KToi8By59wfol1POMyssZmdEnxcExgArIhuVRVzzj3onMt0zrWh9N/up865G6Nc1jGZWe3gSX2CbZILAc/ORHPObQU2mlnH4Kb+lN672+uuowpbOxDGjdFjwdFu3h7lso7JzMYD5wGNzKwQeNg592J0qzqqvsBNwJfBHjnA/zrnJkWxpso0B14OznxIAt5wzsXEVMgY0RT4Z+nxACnA6865D6NbUqVGAq8FDwzXALdEuZ5jMrNalM5IvKNK3zcepmyKiEh44qW9IyIiYVDoi4gkEIW+iEgCUeiLiCQQhb6ISAJR6IuIJBCFvohIAvl/9qEWBgWsUwQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHy9JREFUeJzt3Xd8leX9//HXh4QQNoaEPRL2VEYIQ7+gFRWUgrZWwa0oqLXWVUtbq1Zra+3Paq1+2+ICEXFArSi4cNRVRpghYYWVhAAJBEIG2dfvj6R+0wjkEE64zzl5Px8PHuacc+XkLY/kzZ3rvu7rNuccIiISWhp5HUBERPxP5S4iEoJU7iIiIUjlLiISglTuIiIhSOUuIhKCVO4iIiFI5S4iEoJU7iIiISjcqy8cHR3tYmNjvfryIiJBafXq1QecczG1jfOs3GNjY0lMTPTqy4uIBCUz2+3LOE3LiIiEIJW7iEgIUrmLiIQglbuISAhSuYuIhCCVu4hICFK5i4iEIJW7iMhpUFpewerdOTy9bCspmUfq/et5dhGTiEgoc86x40ABX27N5qvUgyzfcZD84jLMoG2LJgzo1Kpev77KXUTET3KPlvJN6gG+2JbNF1sPsOfwUQC6t23G5CGd+J9e0Yzu2ZY2zSLqPYvKXUSkjpxzJGce4V9bs/l8SxZr0g5TXuFo2SScMb3actu5PRnbO4ZubZud9mwqdxGRk5BfXMZX2w7w6eb9fL4lm6y8YgAGdW7FbeN6Mq5vDEO6tqFxmLenNFXuIiK1SM8p5JNN+/lkcxbLdxyktNzRMjKcsX1iOK9vO8b2iaZdy0ivY/4XlbuISA0VFY6kPbks27Sfj1P2s3lfHgA9Yppz49lxnNe3HfGxZ3h+dH4iKncRESqXKq7YkcNHKfv4KHk/+44U0chgRGwUD1zSn/P7tycuurnXMX2mcheRBquotJyvth3g/Y37WLZpP7lHS4ls3IixvWP42cC+fK9fO85oXv8rW+qDT+VuZhOAPwNhwAvOucdrvN4NmAu0qRozyzm31M9ZRUROWVFpOZ9vyWJp0j4+3ZxFfnEZrSLDGd+/PRcN6sDY3jE0jQjzOuYpq7XczSwMeA64AMgAVpnZYudcSrVhDwBvOuf+amYDgKVAbD3kFRE5aUWl5XyxNZslSXtZlrKfgpJyzmjWmElndmTCoA6M6RlNRHjgzp/XhS9H7glAqnNuB4CZvQ5MAaqXuwP+c7lVayDTnyFFRE5WWXkFX28/yOJ1mXyUvI+84jLaNGvM5CGduGRwJ0b1iCI8gE+Inipfyr0zkF7tcQYwssaYh4GPzOwnQHNgvF/SiYicBOcca9IO8c+1mSxN2svBghJaRoYzYVAHJp3ViTE92wb0Chd/8qXc7RjPuRqPpwFznHNPmtloYJ6ZDXLOVfzXG5nNAGYAdOvWrS55RUS+Y0d2Pv9cu4e31+0hPecoTcIbMb5/eyYP6cS4PjFENg7+OfST5Uu5ZwBdqz3uwnenXaYDEwCcc/82s0ggGsiqPsg5NxuYDRAfH1/zHwgREZ8dLixh8fpMFq3Zw/r0wzQyOLtXNHed34eLBnWgRZOGvRjQl//7VUBvM4sD9gBTgatqjEkDzgfmmFl/IBLI9mdQEZGy8gr+tTWbRWsyWJaSRUl5Bf06tORXF/dn8pBOtG8VWFeJeqnWcnfOlZnZHcCHVC5zfMk5l2xmjwCJzrnFwL3A82Z2N5VTNjc453RkLiJ+sT07nzcT0/nHmj1k5xUT1TyCq0d14/LhXRjYqbXX8QKST7+3VK1ZX1rjuQerfZwCnO3faCLSkBWWlPHehr28uSqdxN2HCGtknNe3HVfEd+G8fu0azInRumrYk1IiEnBSMo+wYGUa/1y7h7ziMnrENOcXE/tx2bDOAbc5VyBTuYuI546WlPPuhkzmr0hjffphIsIbMWlwR6YmdGNE7BmYHWvRnpyIyl1EPLMjO5/5K9JYuDqD3KOl9GrXggcnDeAHwzqflrsVhTKVu4icVuUVjk83Z/HKv3fx5bYDhDcyJgzqwDWjujMyLkpH6X6icheR0yK3sJQ3EtOYt3w36TlH6dg6knsv6MOVCV01l14PVO4iUq9Ss/J5+eudLFqTQVFpBQlxUfxiYn8uHNA+pPd28ZrKXUT8zjnHV6kHePGrnXy+JZuI8EZcOqQTN4yJY0CnVrW/gZwylbuI+E1JWQXvrs/k+S93sHlfHtEtIrh7fB+uHtWN6BZNvI7XoKjcReSUHSkqZcGKNF7+ehf7jhTRp30Lnrj8TKYM6UST8Ia3aVcgULmLSJ1l5RXx4lc7mb88jfziMsb0bMvjPxzMuD4xWvXiMZW7iJy03QcL+PsXO1i4OoOy8gomDu7IbeN6Mqiz9nkJFCp3EfHZtv15PPtZKu+uzyS8USN+OLwLM8f2IDa6udfRpAaVu4jUKjkzl2c/TeWD5H00bRzGzf/Tg5vPiaOdttgNWCp3ETmujXtyeXrZNpZt2k/LJuHccV4vbjw7jqjm2hog0KncReQ7qpd6q8hw7rmgD9ePiaV108ZeRxMfqdxF5Fub9h7hqY+38lHK/5X6DWfH0ipSpR5sVO4iwo7sfJ5ato33NmTSokk4d43vzY1nx+lIPYip3EUasD2Hj/LMsm0sXJNBRFgjbhvXk5lje9K6mUo92KncRRqgQwUl/O/nqcz9925wcN3o7tx+bi9iWmqLgFChchdpQI6WlPPS1zv527+2U1Bcxg+HdeGuC/rQuU1Tr6OJn6ncRRqA8grHotUZPPnxFvYfKWZ8/3b87KJ+9O3Q0utoUk9U7iIh7stt2Ty2ZBOb9+UxpGsb/jJtGAlxUV7HknqmchcJUVv35/HYkk38a2s2XaOa8uxVQ7lkcEdt6NVAqNxFQsyhghKeWraV+SvSaB4RxgOX9Ofa0d219W4Do3IXCRGl5RXMX76bp5ZtI6+olGtGdefu8X04Q1sFNEgqd5EQ8HXqAR5enMy2rHzO6RXNrycN0MnSBk7lLhLEMg8f5bElm1iStJduUc14/rp4xvdvp3l1UbmLBKPisnJe+HInz36aisNx7wV9uGVsDyIba15dKqncRYLMN6kHeOCdjezILmDCwA48MKk/Xc5o5nUsCTAqd5EgkZ1XzGNLUvjnuky6t23GnBtHcG7fdl7HkgClchcJcBUVjvkr03jig80Ul1Zw5/d6cft5vTQFIyekchcJYNv25zHrH0ms3n2IMT3b8uilg+gZ08LrWBIEVO4iAai4rJznPtvOXz9PpXmTcJ780Vn8YFhnrYIRn6ncRQLM6t053L9wA9uzC7h0SCd+PWkAbVtoK145OSp3kQBRWFLG//twKy9/s5NOrZsy96YExvWJ8TqWBCmVu0gA+Gb7AWYtSiItp5DrR3fn/gn9aN5EP55Sd/ruEfFQQXEZv39/E68uTyO2bTPemDGKkT3aeh1LQoDKXcQjK3fmcN9b60k/VMj0c+K478K+NI3Q8kbxj0a+DDKzCWa2xcxSzWzWccZcYWYpZpZsZq/5N6ZI6CgqLeexJSlcOfvfALwxYzS/njRAxS5+VeuRu5mFAc8BFwAZwCozW+ycS6k2pjfwC+Bs59whM9NlcyLHsHFPLne9sY7UrHyuHtmNX17cX3PrUi98+a5KAFKdczsAzOx1YAqQUm3MLcBzzrlDAM65LH8HFQlm5RWO2V/s4E8fbyGqeYRWwki986XcOwPp1R5nACNrjOkDYGZfA2HAw865D2q+kZnNAGYAdOvWrS55RYJOxqFC7nlzPSt35nDx4A787rLBtGmmG2hI/fKl3I91SZw7xvv0Bs4FugBfmtkg59zh//ok52YDswHi4+NrvodIyHln3R4eeHsjDnSVqZxWvpR7BtC12uMuQOYxxix3zpUCO81sC5Vlv8ovKUWCTEFxGQ8tTmbh6gyGdz+Dp68cQtcobcsrp48v5b4K6G1mccAeYCpwVY0x/wSmAXPMLJrKaZod/gwqEiw27snlJwvWsvtgAXee35s7v9eL8DCfFqaJ+E2t5e6cKzOzO4APqZxPf8k5l2xmjwCJzrnFVa9daGYpQDnwM+fcwfoMLhJonHO8+NVO/vDBZqJbNGHBLbogSbxjznkz9R0fH+8SExM9+doi/na4sIT73trAsk37uXBAe564/EydNJV6YWarnXPxtY3TAluRU7Qm7RA/eW0tWXlFPPz9AVw/JlYnTcVzKneROvrPNMzj72+mY5tIFt46hrO6tvE6lgigchepkyNFpfzsrfV8mLyfiwa254nLz6J108ZexxL5lspd5CRt2ZfHra+uJi2nkAcu6c/0c+I0DSMBR+UuchLeWbeHWYuSaBEZzoJbRpEQF+V1JJFjUrmL+KC0vILHlmxizje7SIiN4tmrhtKuVaTXsUSOS+UuUosD+cX8eP4aVuzMYfo5ccya2I/GuihJApzKXeQEkjJymTkvkYMFJTx95RAuHdrZ60giPlG5ixzH22szmLUoiegWTVh02xgGdW7tdSQRn6ncRWoor3A88cFm/v7FDkb1iOK5q4bRtkUTr2OJnBSVu0g1eUWl3PX6Oj7ZnMW1o7rz4PcHaH5dgpLKXaRK2sFCbn5lFduzC3h0ykCuHR3rdSSROlO5iwArd+Ywc14iFQ7m3ZTAmF7RXkcSOSUqd2nw3l6bwf0LN9A1qhkvXj+CuOjmXkcSOWUqd2mwnHP8+ZNtPL1sG6N7tOVv1wyndTPtDyOhQeUuDVJxWTmzFiXx9to9XD68C7+7bDAR4TpxKqFD5S4NTm5hKTPmJbJiZw73XdiHH5/XSxt/SchRuUuDsufwUW54aSW7Dxby56lDmDJEV5xKaFK5S4ORknmEG+espLCknLk3JTC6p+5vKqFL5S4NwtepB5g5bzUtI8NZeOsY+nZo6XUkkXqlcpeQt3h9Jve+uY4e0S2Yc9MIOrZu6nUkkXqncpeQNvebXTz8bjIjYqN4/rp43QpPGgyVu4Qk5xxPLdvGM59s44IB7fnLtKFENg7zOpbIaaNyl5BTXuF4aPFGXl2exhXxlWvYw7X5lzQwKncJKSVlFdzz5jre27CXmeN6MGtCP61hlwZJ5S4ho6i0nNvnr+HTzVn8YmI/Zo7r6XUkEc+o3CUkFBSXcfPcRJbvPMhjlw3i6pHdvY4k4imVuwS93MJSbpizkg0ZufzpirO4bGgXryOJeE7lLkHtYH4x17y4ku1Z+Tx31TAmDOrgdSSRgKByl6CVnVfM1S8sJy2nkBeuj2dsnxivI4kEDJW7BKWsI0VMe345mYeLePkG7RMjUpPKXYLO3tyjXPX8CrKOFDH3pgQS4qK8jiQScFTuElT2HD7KtNnLySko4ZXpCQzvrmIXORaVuwSNzKpiP1RYwrzpCQztdobXkUQClspdgsK+3Mo59kMFJbx680jO6trG60giAU3lLgHvPydPD+ZXTsWo2EVqp92UJKBl5RUx9fnlVSdPRzBMUzEiPvGp3M1sgpltMbNUM5t1gnGXm5kzs3j/RZSG6mB+MVc/v4J9uUXMuUknT0VORq3lbmZhwHPARGAAMM3MBhxjXEvgTmCFv0NKw5N7tJRrX1xJ+qFCXrphBCNiVewiJ8OXI/cEINU5t8M5VwK8Dkw5xrhHgSeAIj/mkwYov7iMG15eybasPP5+bTyjeugCJZGT5Uu5dwbSqz3OqHruW2Y2FOjqnHvPj9mkASoqLefmuavYkJHLs1cNY5y2FBCpE1/K/Vh3OnDfvmjWCHgKuLfWNzKbYWaJZpaYnZ3te0ppEErKKrj11dWs2JnDn644i4sGahMwkbrypdwzgK7VHncBMqs9bgkMAj43s13AKGDxsU6qOudmO+finXPxMTE6IpP/U17huPuNdXy+JZvfXTaYKUM61/5JInJcvpT7KqC3mcWZWQQwFVj8nxedc7nOuWjnXKxzLhZYDkx2ziXWS2IJOc45fv3ORpYk7eVXF/dnWkI3ryOJBL1ay905VwbcAXwIbALedM4lm9kjZja5vgNK6Hvyo628tiKN287tyS1je3gdRyQk+HSFqnNuKbC0xnMPHmfsuaceSxqKF77cwbOfpTItoSv3X9TX6zgiIUNXqIpnFq3O4LdLNjFxUAd+e+lgzI517l5E6kLlLp74fEsW9y/awNm92vL01CGENVKxi/iTyl1Ouw0Zh7l9/hr6tm/J364ZTpPwMK8jiYQclbucVrsPFnDTnFVENY9gzk0jaBnZ2OtIIiFJ5S6nzYH8Yq57aSXlFY65NyXQrmWk15FEQpb2c5fTorCkjOlzVrH/SBGv3TKKnjEtvI4kEtJ05C71rrzCceeCdSTtyeXZacO0J7vIaaAjd6l3j76XwrJN+3lkykDGD2jvdRyRBkFH7lKvXv56J3O+2cX0c+K4bnSs13FEGgyVu9Sbj1P288h7KVw0sD2/vLi/13FEGhSVu9SLpIxc7lywljM7t+bpK4fqIiWR00zlLn63L7eI6XMr17I/f308TSN0kZLI6aZyF786WlLOza+soqC4jBdviNdadhGPaLWM+E1FhePet9aRnHmEF66Lp1+HVl5HEmmwdOQufvP0sq0sTdrHLyf25/z+WvIo4iWVu/jFO+v28MynqfxoeBdu/p84r+OINHgqdzllGzIOc//CDSTERvHbywZpX3aRAKByl1OSnVfMzHmriW7RhL9eM0zb94oECJ1QlTorKavg9vmrOVRYwsJbx9C2RROvI4lIFZW71Nlv3k1m1a5D/HnqEAZ1bu11HBGpRtMyUievrUhj/oo0Zo7rwZQhnb2OIyI1qNzlpK3encNDizcyrk8M91/Uz+s4InIMKnc5KVl5Rdw+fw2d2jTlmanaM0YkUKncxWel5RXc8dpaco+W8rdrhtO6me5/KhKodEJVfPbEB5tZuTOHp648i/4dtbWASCDTkbv4ZMmGvTz/5U6uH92dy4Z28TqOiNRC5S612rY/j58tXM/w7mfwq0sGeB1HRHygcpcTKigu49ZXV9MsIpz/vXoYEeH6lhEJBvpJleNyzvHLt5PYeaCAZ6YNoX0r7c0uEixU7nJcC1am8866TO4e34cxPaO9jiMiJ0HlLseUnJnLw+8mM7ZPDD8+r5fXcUTkJKnc5Tvyikr58fw1RDWL4KkrzqKRLlQSCTpa5y7/xTnHrEVJpB86yuszRmmnR5EgpSN3+S+vrkhjSdJe7ruwLyNio7yOIyJ1pHKXb23ed4RH30thXJ8YZo7t4XUcETkFKncB4GhJOXe8tpbWTRvzpObZRYKe5twFgEfeS2Z7dj7zbhpJtObZRYKejtyF9zZksmBlOreN68k5vbWeXSQU+FTuZjbBzLaYWaqZzTrG6/eYWYqZbTCzT8ysu/+jSn1IzynkF4uSGNqtDXdf0MfrOCLiJ7WWu5mFAc8BE4EBwDQzq7l71Fog3jl3JrAQeMLfQcX/ysor+Onra8HgmalDaRymX+REQoUvP80JQKpzbodzrgR4HZhSfYBz7jPnXGHVw+WA9oQNAs99tp01aYd57LLBdI1q5nUcEfEjX8q9M5Be7XFG1XPHMx14/1gvmNkMM0s0s8Ts7GzfU4rfrUk7xDOfbuOyoZ2ZfFYnr+OIiJ/5Uu7HWhPnjjnQ7BogHvjjsV53zs12zsU75+JjYmJ8Tyl+lV9cxt1vrKNDq0h+M2Wg13FEpB74shQyA+ha7XEXILPmIDMbD/wKGOecK/ZPPKkPj7ybTHpOIW/MHE2rSN0HVSQU+XLkvgrobWZxZhYBTAUWVx9gZkOBvwOTnXNZ/o8p/vLBxr28mZjB7ef20vYCIiGs1nJ3zpUBdwAfApuAN51zyWb2iJlNrhr2R6AF8JaZrTOzxcd5O/FQ1pEiZv0jiTO7tOan43t7HUdE6pFPV6g655YCS2s892C1j8f7OZf4mXOOny/aQFFpOU9dOUTLHkVCnH7CG4jXV6Xz2ZZsZk3oR8+YFl7HEZF6pnJvANJzCvnteymM6dmW60bHeh1HRE4DlXuIq6hw3PvWehqZ8ccfabdHkYZC5R7iXvp6Jyt35vDg9wfQuU1Tr+OIyGmicg9h2/bn8cSHWxjfvz2XD9eOECINico9RJWVV3Dfwg00jwjj9z8YjJmmY0QaEt2sI0S98NVO1qcf5i/ThhLTUjffEGlodOQeglKz8vnTx1u5aGB7Jp3Z0es4IuIBlXuIKa9w3L9wPc0iwnj00kGajhFpoFTuIeblr3eyJu0wD31/AO1aRnodR0Q8onIPITsPFPDHD7dwfr92XDrkRFvui0ioU7mHiIoKx88XbqBJeCN+p9UxIg2eyj1EvLYyjZW7cnjgkgG0b6XpGJGGTuUeAvblFvGH9zczpmdbfhSvi5VEROUe9Jxz/PqdjZSUV+hiJRH5lso9yH2wcR8fp+znngv60L1tc6/jiEiAULkHsdzCUh5cnMzATq2Yfk6c13FEJIBo+4Eg9vgHm8gpKOHlG0YQrjsriUg1aoQgtXzHQRasTOfmc+IY1Lm113FEJMCo3INQSVkFv3o7ia5RTblrfB+v44hIANK0TBCa/cV2tmcX8PINI2gaEeZ1HBEJQDpyDzK7Dxbwl09TuXhwB87r187rOCISoFTuQaRyTXsyjcMa8eCkgV7HEZEApnIPIkuS9vLF1mzuvbAPHVpriwEROT6Ve5A4UlTKb95NYXDn1lw3OtbrOCIS4HRCNUg8+eEWDuQX8+L18YQ10hYDInJiOnIPAsmZucxbvptrRnbnzC5tvI4jIkFA5R7gKiocD76TzBnNIrjvwr5exxGRIKFyD3D/WLuH1bsP8fMJ/WjdrLHXcUQkSKjcA1ju0VIef38TQ7u14fLh2qddRHynE6oB7KmPt3KwoIQ5NybQSCdRReQk6Mg9QKVkHuGVf+/i6pHdtDGYiJw0lXsAcs7x0OKNtG7aWCdRRaROVO4BaPH6TFbtqjyJ2qZZhNdxRCQIqdwDTGFJGb9fupnBnVtzRXxXr+OISJBSuQeYv32+nX1Hinjo+wN0ElVE6kzlHkDScwr5+xc7mHxWJ+Jjo7yOIyJBzKdyN7MJZrbFzFLNbNYxXm9iZm9Uvb7CzGL9HbQhePz9zZjBrIn9vI4iIkGu1nI3szDgOWAiMACYZmYDagybDhxyzvUCngL+4O+goW75joMsSdrLbeN60alNU6/jiEiQ8+XIPQFIdc7tcM6VAK8DU2qMmQLMrfp4IXC+mWnC2EflFY7fvJtC5zZNmTmuh9dxRCQE+HKFamcgvdrjDGDk8cY458rMLBdoCxzwR8jq3lyVzvNf7vD323qqpLyC3QcLee6qYUQ21j1RReTU+VLuxzoCd3UYg5nNAGYAdOvWzYcv/V1tmjWmd/sWdfrcQPbDYV24eHAHr2OISIjwpdwzgOoLrrsAmccZk2Fm4UBrIKfmGznnZgOzAeLj479T/r64cGAHLhyoEhQRORFf5txXAb3NLM7MIoCpwOIaYxYD11d9fDnwqXOuTuUtIiKnrtYj96o59DuAD4Ew4CXnXLKZPQIkOucWAy8C88wslcoj9qn1GVpERE7Mpy1/nXNLgaU1nnuw2sdFwI/8G01EROpKV6iKiIQglbuISAhSuYuIhCCVu4hICFK5i4iEIPNqObqZZQO76/jp0dTD1gb1KNjyQvBlVt76pbz162TydnfOxdQ2yLNyPxVmluici/c6h6+CLS8EX2blrV/KW7/qI6+mZUREQpDKXUQkBAVruc/2OsBJCra8EHyZlbd+KW/98nveoJxzFxGREwvWI3cRETmBoCv32m7WHUjM7CUzyzKzjV5n8YWZdTWzz8xsk5klm9lPvc50ImYWaWYrzWx9Vd7feJ3JF2YWZmZrzew9r7PUxsx2mVmSma0zs0Sv8/jCzNqY2UIz21z1vTza60zHY2Z9q/5u//PniJnd5Zf3DqZpmaqbdW8FLqDyBiGrgGnOuRRPgx2HmY0F8oFXnHODvM5TGzPrCHR0zq0xs5bAauDSAP77NaC5cy7fzBoDXwE/dc4t9zjaCZnZPUA80Mo5N8nrPCdiZruAeOdc0KwZN7O5wJfOuReq7kHRzDl32Otctanqtz3ASOdcXa8B+lawHbn7crPugOGc+4Jj3JEqUDnn9jrn1lR9nAdsovL+uAHJVcqveti46k9AH62YWRfgEuAFr7OEIjNrBYyl8h4TOOdKgqHYq5wPbPdHsUPwlfuxbtYdsOUTzMwsFhgKrPA2yYlVTXGsA7KAj51zAZ0XeBq4H6jwOoiPHPCRma2uugdyoOsBZAMvV019vWBmzb0O5aOpwAJ/vVmwlbtPN+KWU2NmLYBFwF3OuSNe5zkR51y5c24Ilff2TTCzgJ3+MrNJQJZzbrXXWU7C2c65YcBE4MdVU42BLBwYBvzVOTcUKAAC+twcQNX00WTgLX+9Z7CVuy8365ZTUDV3vQiY75z7h9d5fFX1q/fnwASPo5zI2cDkqnns14Hvmdmr3kY6MedcZtV/s4C3qZwaDWQZQEa13+AWUln2gW4isMY5t99fbxhs5e7LzbqljqpOUL4IbHLO/cnrPLUxsxgza1P1cVNgPLDZ21TH55z7hXOui3Mulsrv3U+dc9d4HOu4zKx51Yl1qqY2LgQCeuWXc24fkG5mfaueOh8IyAUBNUzDj1My4OM9VAPF8W7W7XGs4zKzBcC5QLSZZQAPOede9DbVCZ0NXAskVc1jA/yy6h66gagjMLdqlUEj4E3nXMAvLwwi7YG3K//NJxx4zTn3gbeRfPITYH7VAeAO4EaP85yQmTWjcgXgTL++bzAthRQREd8E27SMiIj4QOUuIhKCVO4iIiFI5S4iEoJU7iIiIUjlLiISglTuIiIhSOUuIhKC/j/xl9WpBG0PMwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFFxJREFUeJzt3X+MZWddx/HP5/5oy49C1R1BdrfsEhdlIZKSSak2QbSVtFXbxKBpE/xBCGsMBRSiKWqqqX8JRtSkoisiikCtheiGrJREajTGNp1SRNraZF2gOy6kA5T6A6H3nvP1j7n3zp07d3Yu2+c8z9w771fSdM65Z+48uZn55Nnv+T7PcUQIALBYWqUHAABIj3AHgAVEuAPAAiLcAWABEe4AsIAIdwBYQIQ7ACwgwh0AFhDhDgALqFPqB+/bty8OHTpU6scDwFx64IEHvhwRSztdVyzcDx06pJWVlVI/HgDmku0vzHIdZRkAWECEOwAsIMIdABYQ4Q4AC4hwB4AFtGO4236f7cdtf3ab1237D2yfsv0Z269IP0wAwLdilpn7+yVdc47Xr5V0ZPDfMUnvefrDAgA8HTuGe0T8o6SvnuOSGyT9Ray7V9Iltr8r1QCBefXl//mmPv7ZL5UeBvaoFDX3/ZLOjB2vDs5tYfuY7RXbK2trawl+NLB7feSBVf3CBx/QN3pV6aFgD0oR7p5ybupTtyPieEQsR8Ty0tKOq2eBufbNfq0IqVfVpYeCPShFuK9KOjh2fEDS2QTvC8y1/iDU+9XUuQ7QqBThfkLSzwy6Zq6Q9GREfDHB+wJzrV/Hpv8DOe24cZjtD0t6taR9tlcl/YakriRFxB9JOinpOkmnJH1d0uubGiwwTzbCnbIM8tsx3CPiph1eD0lvSjYiYEEMyzGUZVACK1SBhlSDGXtFWQYFEO5AQ3qUZVAQ4Q40pKq4oYpyCHegIb2aVkiUQ7gDDalohURBhDvQkI1uGWruyI9wBxoyvJHKzB0lEO5AQ4ZlGVohUQLhDjSkNyjLsHEYSiDcgYYwc0dJhDvQkOGMvUcrJAog3IGGMHNHSYQ70BC2H0BJhDvQkIoVqiiIcAcaMgx1yjIogXAHGjJcvNSjLIMCCHegIdxQRUmEO9AQWiFREuEONGRj5k5ZBvkR7kBDNrYfYOaO/Ah3oCE8QxUlEe5AQ/o8rAMFEe5AQ3hYB0oi3IGG0AqJkgh3oCHDxUvcUEUJhDvQgLoOxSDTaYVECYQ70IDxLQd6lGVQAOEONGC8zl5RlkEBhDvQgPH2R1ohUQLhDjRgfA93HtaBEmYKd9vX2H7U9inbt0x5/VLb99h+0PZnbF+XfqjA/BgPdGbuKGHHcLfdlnS7pGslHZV0k+2jE5f9uqQ7I+IySTdK+sPUAwXmyaaZO4uYUMAsM/fLJZ2KiNMR8ZSkOyTdMHFNSHrO4OvnSjqbbojA/Nl0Q5WZOwrozHDNfklnxo5XJb1y4prflPQJ22+W9CxJVycZHTCnemOzdRYxoYRZZu6ecm7yt/UmSe+PiAOSrpP0Adtb3tv2MdsrtlfW1ta+9dECc4KZO0qbJdxXJR0cOz6grWWXN0i6U5Ii4l8kXSRp3+QbRcTxiFiOiOWlpaXzGzEwBza3QlJzR36zhPv9ko7YPmz7Aq3fMD0xcc1jkq6SJNsv0Xq4MzXHnrX5hiozd+S3Y7hHRF/SzZLulvSI1rtiHrJ9m+3rB5e9XdIbbf+rpA9L+rmI4DcaexatkChtlhuqioiTkk5OnLt17OuHJV2ZdmjA/BoG+gWdFmUZFMEKVaABw1LMRZ0WZRkUQbgDDRjO1i/qtinLoAjCHWjAMNAv6rZphUQRhDvQgOE2vxdSc0chhDvQgE1lGWruKIBwBxqwUZZpUXNHEYQ70IBRt0y3za6QKIJwBxownK1f2KFbBmUQ7kADhrP1i7r0uaMMwh1oAK2QKI1wBxpQ1bRCoizCHWhAr9pohaxDqpm9IzPCHWhANdYKKbEzJPIj3IEGjGrunfbgmNIM8iLcgQaM97lLzNyRH+EONKBf17Klbnv9EcS0QyI3wh1oQL8OdVsttdvDmjtlGeRFuAMNqOpQu2V1Wx4dAzkR7kADelWtTstqtyjLoAzCHWhAVYc6bavbphUSZRDuQAN6Vajdao3N3Km5Iy/CHWhAVdfqtr3RLcPMHZkR7kAD+tX6DdV2qzU6BnIi3IEG9OtQt91SZ1iWoRUSmRHuQAOGrZCdNq2QKINwBxow2QrZoyyDzAh3oAGTrZDM3JEb4Q40oFdvboXsUXNHZoQ70ICqrtVtWd1Bt0xFWQaZEe5AA3qjVki6ZVAG4Q40oBq2QrKICYXMFO62r7H9qO1Ttm/Z5pqfsv2w7YdsfyjtMIH50h+2QrIrJArp7HSB7bak2yX9iKRVSffbPhERD49dc0TSOyRdGRFP2P7OpgYMzIN+tb79QGdQc6cVErnNMnO/XNKpiDgdEU9JukPSDRPXvFHS7RHxhCRFxONphwnMl62LmKi5I69Zwn2/pDNjx6uDc+NeLOnFtv/Z9r22r5n2RraP2V6xvbK2tnZ+IwbmwPoipo3tB5i5I7dZwt1Tzk3+pnYkHZH0akk3SXqv7Uu2fFPE8YhYjojlpaWlb3WswNwYLmLqsIgJhcwS7quSDo4dH5B0dso1fxsRvYj4nKRHtR72wJ402QrZYz93ZDZLuN8v6Yjtw7YvkHSjpBMT1/yNpB+SJNv7tF6mOZ1yoMA8qQYPyKZbBqXsGO4R0Zd0s6S7JT0i6c6IeMj2bbavH1x2t6Sv2H5Y0j2SfjkivtLUoIHdrl+H2m3T545idmyFlKSIOCnp5MS5W8e+DklvG/wH7Hn9wfYDHR7WgUJYoQo0oBp7hqpNKyTyI9yBBvTqelSS6bSsHmUZZEa4Aw2o6hjdTO20WtxQRXaEO5BYRKhXjYe7aYVEdoQ7kNhwkj5cwNRum5k7siPcgcSGe7e3x8oytEIiN8IdSGzY9tgdu6HapyyDzAh3ILHhLL096HHvtM3MHdkR7kBiw1n6+A1VFjEhN8IdSGx483TU596mFRL5Ee5AYsMFS7RCoiTCHUisqobhPmiFbNEKifwIdyCxYSvkeFmGG6rIjXAHEuvXm2funZZHgQ/kQrgDiQ07Y9p0y6Agwh1IbFSWGYY7fe4ogHAHEutPtkKy/QAKINyBxPrVlJo7rZDIjHAHEpvslqEVEiUQ7kBi1cQipi6tkCiAcAcSG5Vl2huLmCjLIDfCHUisP7n9AN0yKIBwBxIbztLpc0dJhDuQ2HCW3mX7ARREuAOJbTxmj+0HUA7hDiS20ec+1gpJWQaZEe5AYpMP66AVEiUQ7kBivXrrfu6UZZAb4Q4kVk08Q7XbohUS+RHuQGKTG4e1Wy1FiC0IkNVM4W77GtuP2j5l+5ZzXPda22F7Od0Qgfmy5WEdg5CnNIOcdgx3221Jt0u6VtJRSTfZPjrluoslvUXSfakHCcyTaYuY1s8zc0c+s8zcL5d0KiJOR8RTku6QdMOU635L0jslfSPh+IC5M7n9wDDkqbsjp1nCfb+kM2PHq4NzI7Yvk3QwIj6WcGzAXKrqUMtSa2xXyOF5IJdZwt1Tzo1+S223JL1b0tt3fCP7mO0V2ytra2uzjxKYI70qRjtCSmMzd3aGREazhPuqpINjxwcknR07vljSyyT9g+3PS7pC0olpN1Uj4nhELEfE8tLS0vmPGtjFqroelWSkjT1mKMsgp1nC/X5JR2wftn2BpBslnRi+GBFPRsS+iDgUEYck3Svp+ohYaWTEwC7Xq2JTuA/3mOGGKnLaMdwjoi/pZkl3S3pE0p0R8ZDt22xf3/QAgXlT1ZvLMqNuGVohkVFnlosi4qSkkxPnbt3m2lc//WEB86tf16M6uzTe587MHfmwQhVIrF+FuuPhTp87CiDcgcSqOtRuj4c7rZDIj3AHEuvVoW5rrBVyEPQ9au7IiHAHEqsmau5dZu4ogHAHEttuEVOPRUzIiHAHEqvqzX3uw24ZZu7IiXAHEutVE62QdMugAMIdSKyqY7TlgLTRLUOfO3Ii3IHE+nVMXcRU0S2DjAh3ILF+VY+2+ZU2yjI9yjLIiHAHEqu2zNxphUR+hDuQ2PqukNNm7pRlkA/hDiQ22Qo5nMUzc0dOhDuQWK+uN+8tM9p+gHBHPoQ7kFhVT+4KOai5U5ZBRoQ7kFi/itHTlyT2c0cZhDuQWL+uJxYxEe7Ij3AHEtvSCsmukCiAcAcS61WxzSImau7Ih3AHEpucubdals3MHXkR7kBivare1OcurT+wg+0HkBPhDiRW1THqkBlqt8zGYciKcAcSiojBrpCb/7Q6bdMtg6wIdyChYV29O1GW6bTMwzqQFeEOJDScnbcnyjKddouZO7Ii3IGE+qOZ+0RZpmX1aYVERoQ7kFA1KL20W9NuqDJzRz6EO5BQb9AR050oy3TbLXaFRFaEO5DQcHY+2S1DKyRyI9yBhIY198lFTHTLIDfCHUhoeNN0chETfe7IbaZwt32N7Udtn7J9y5TX32b7Ydufsf33tl+YfqjA7jdqhdwyc6cVEnntGO6225Jul3StpKOSbrJ9dOKyByUtR8T3SbpL0jtTDxSYB8PSy/iukBKtkMhvlpn75ZJORcTpiHhK0h2Sbhi/ICLuiYivDw7vlXQg7TCB+dAf3DSd1grJzB05zRLu+yWdGTteHZzbzhsk/d20F2wfs71ie2VtbW32UQJzYrT9wJRWSPrckdMs4e4p56b+ltp+naRlSe+a9npEHI+I5YhYXlpamn2UwJzoVdu3QlKWQU6dGa5ZlXRw7PiApLOTF9m+WtKvSfrBiPhmmuEB86XaphWyS7cMMptl5n6/pCO2D9u+QNKNkk6MX2D7Mkl/LOn6iHg8/TCB+TBqhZxWc6fPHRntGO4R0Zd0s6S7JT0i6c6IeMj2bbavH1z2LknPlvTXtj9t+8Q2bwcstNEipsk+91ZrdLMVyGGWsowi4qSkkxPnbh37+urE4wLm0jDAOzysA4WxQhVIqH+OXSEpyyAnwh1IaKMVcvOfVrdFKyTyItyBhHrbbD/QbpuaO7Ii3IGEqnp6t0yXFarIjHAHEhouYprslmm3WtTckRXhDiS0sYhpWrcMZRnkQ7gDCW27nzvdMsiMcAcSOueTmOpQBAGPPAh3IKFRWWZyP/fBMfdUkQvhDiQ0uqE6ZRHT+uvU3ZEH4Q4kVG3zsI7h/u4sZEIuhDuQ0PYz9/U/NW6qIhfCHUioqkPtlmVvvaEqiXZIZEO4Awn16nrLrF3aaI1klSpyIdyBhKoqpod7i3BHXoQ7kFC/ji1tkNLGitWKmjsyIdyBhPo7lGV61NyRCeEOJDS8oTppNHOnLINMCHcgoV4VWx7UIbGICfkR7kBC28/cWcSEvAh3IKFeVW/ZEVIaq7lzQxWZEO5AQlW9XSskNXfkRbgDCfXr2PKgDml8ERM1d+RBuAMJ9bcrywwXMVGWQSaEO5BQf7sbqm3KMsiLcAcS6leh7rSyDK2QyIxwBxLarhWyTSskMiPcgYR69fSae3e0/QDhjjwIdyCh7Voh26NWSMoyyINwBxLqV9vtCkm3DPKaKdxtX2P7UdunbN8y5fULbf/V4PX7bB9KPVBgHuy0KyT7uSOXHcPddlvS7ZKulXRU0k22j05c9gZJT0TEd0t6t6TfTj1QYB7stJ874Y5cZpm5Xy7pVEScjoinJN0h6YaJa26Q9OeDr++SdJUnHyIJ7AH9nZ7ERCskMunMcM1+SWfGjlclvXK7ayKib/tJSd8h6cspBjnuzvvP6E/+6XTqtwWSOPu1/9Plh799y/n2oCxz+z3/oQ/d91juYWGXectVR/TjL39Boz9jlnCfNgOf/LflLNfI9jFJxyTp0ksvneFHb3XJM7s68rxnn9f3Ak178fMu1k+8Yv+W8xdf2NHPv+pFOvPE1wuMCrvNc5/RbfxnzBLuq5IOjh0fkHR2m2tWbXckPVfSVyffKCKOSzouScvLy+dVfHzNS5+v17z0+efzrUAxtvWO615SehjYQ2apud8v6Yjtw7YvkHSjpBMT15yQ9LODr18r6ZMRwZ0jAChkx5n7oIZ+s6S7JbUlvS8iHrJ9m6SViDgh6U8lfcD2Ka3P2G9sctAAgHObpSyjiDgp6eTEuVvHvv6GpJ9MOzQAwPlihSoALCDCHQAWEOEOAAuIcAeABUS4A8ACcql2dNtrkr5wnt++Tw1sbTDn+Ew24/PYjM9jq3n9TF4YEUs7XVQs3J8O2ysRsVx6HLsJn8lmfB6b8XlsteifCWUZAFhAhDsALKB5DffjpQewC/GZbMbnsRmfx1YL/ZnMZc0dAHBu8zpzBwCcw9yF+04P695LbB+0fY/tR2w/ZPutpce0G9hu237Q9sdKj2U3sH2J7bts//vgd+X7S4+pJNu/NPh7+aztD9u+qPSYmjBX4T7jw7r3kr6kt0fESyRdIelNe/zzGHqrpEdKD2IX+X1JH4+I75X0cu3hz8b2fklvkbQcES/T+jbmC7lF+VyFu2Z7WPeeERFfjIhPDb7+b63/0W59xtseYvuApB+V9N7SY9kNbD9H0qu0/swFRcRTEfG1sqMqriPpGYOnxj1TW58stxDmLdynPax7T4fZkO1Dki6TdF/ZkRT3e5J+RVJdeiC7xIskrUn6s0Gp6r22n1V6UKVExH9K+h1Jj0n6oqQnI+ITZUfVjHkL95kexL3X2H62pI9I+sWI+K/S4ynF9o9JejwiHig9ll2kI+kVkt4TEZdJ+l9Je/Zele1v0/q/9g9LeoGkZ9l+XdlRNWPewn2Wh3XvKba7Wg/2D0bER0uPp7ArJV1v+/NaL9n9sO2/LDuk4lYlrUbE8F90d2k97PeqqyV9LiLWIqIn6aOSfqDwmBoxb+E+y8O69wzb1not9ZGI+N3S4yktIt4REQci4pDWfzc+GRELOSubVUR8SdIZ298zOHWVpIcLDqm0xyRdYfuZg7+fq7SgN5hneobqbrHdw7oLD6ukKyX9tKR/s/3pwblfHTzzFhh6s6QPDiZEpyW9vvB4iomI+2zfJelTWu82e1ALulKVFaoAsIDmrSwDAJgB4Q4AC4hwB4AFRLgDwAIi3AFgARHuALCACHcAWECEOwAsoP8HAhrHisAiflEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl0Ved97vHvT0cSmtA8IQkhBGIQAhuQwTY2wTZg7KZ20qStkzpNehO7Q9w2y7f33uS2q2mdrq407UrTrrpdHlv3No5rO03rJrYxnuoJbMRgG4lJDEIDkhASEkLzOe/94xyoTAQ6gMQ+w/NZ6yydfc7eWz8h9Ox93v2+7zbnHCIiEh8SvC5ARESuHoW+iEgcUeiLiMQRhb6ISBxR6IuIxBGFvohIHFHoi4jEEYW+iEgcUeiLiMSRRK8LOF9+fr6rqKjwugwRkaiyY8eOLudcwWTrRVzoV1RUUFdX53UZIiJRxcyawllPzTsiInFEoS8iEkcU+iIicUShLyISRxT6IiJxRKEvIhJHFPoiInEk4vrpi1yqgZExWnoGaT01yOmhMQaGxzgz4ufM8Bj+gCMxwUj0JZCYYCQnJpCVmkR2WhLZacnkpCVRODOF1GSf1z+GyFWh0Jeo4Q84DnScZnfzKXYd62F/+2laegY5eWbkivedl55MSXYqpdmplOelMa8gnXkFGcwvzCA7LXkKqheJDAp9iWidfUNsrm/nlYYOdjb1cGbED0BOWhJLSrLYuCSLspzUc4+s1CTSkhNJn5FIWrKPxARjLODwBxyj/gDDYwF6B0c5NTDCqYFRus+M0NE3ROupIVpPDXKw8zSv7+9kZCxwrob8jGQWz8qkuiSTmpIslpRkUpGXTkKCefXPInLZFPoScU72D/OTXa28tKedncd6cA4qC9L5/MoylpfncO3sbObkpWEWXugm+YwkH6Qk+ZgJ5GfMuOj6/oCjtWeQxhOnaezs52BHPw3H+3jynSOM+h0AM2ckcm15NsvLc1hRns3y2TlkpSVd6Y8uMu3MOed1DZ9QW1vrNPdOfGrs7OeJd47wbztbGB4LUD0rk001xdxRU0xV0Uyvy2N4zM/Bjn7q23r5sKWXnU09HOg4TSD0J7SoeCar5+ayujKP6ypyKZh58YOLyFQysx3OudpJ11Poi9d2NHXz928c4rV9nSQnJvC5FWV89aYK5hd6H/ST6R8e46PmU+xo6uGDo93UHe1hcDTYBFVVmMGa+fmsmZ/P6spcMlP0SUCmj0JfIl5n3xB//uJe/n13G7npyXzp+jl86YY5kza/RLJRf4A9rb28f6Sb9w6d5IMjJxkaDZBgcM3sbNZWFbB2QQHXzs7Gp2sCMoUU+hKxRv0BnnrvKD949SAjYwHuX1vJ79wyj7Tk2LvENDzmZ9exU7zb2MXbB7v4sOUUzkFWahI3zc9n3cIC1i0sVFOQXDGFvkSkhrY+vvGvuzjQ0c+6hQV8+xeXMDc/3euyrpqeMyO809jFWwdO8NbBE3T0DQOwrCyLWxYWcuuiQpaWZqlnkFwyhb5EnOfqmvmjf99DVmoSf/aZGjZUF4XdAycWOedoON7HG/s6eX1fJ7uag58CCmbO4LZFhaxfXMSa+fkaOCZhUehLxBga9fMnL9TzzPZmbqjM42+/sFzNGRPoPjPCfx3o5NWGTv7rwAn6h8eYkZjAzVX5bKwu5tbFhVF9vUOml0JfIkJz9wC//cMd7Gnt43fWzePBDQtI9GnKp8mMjAX44Eg3r+7tYEtDB62nBjGDleU5bFxSxO1LipmTFz/NYjI5hb54rrGzn197fBuDI36+/yvXsr66yOuSotLZZqAtDR28Ut9Bw/E+ABYWzeT2JUVsXFLMkpLMuG4qE4W+eOxAx2m++Nj7ADx932oWRMDgqljR3D3AKw0dvFLfzvaj3QQclOWkcvuSYjbVFLOiPEfdQeOQQl8809DWx71PvE9igvH0fdczvzDD65Ji1sn+YV7d28Hm+g7eOdjFiD9AfsYMNi4pYtOSYm6Yl0eSmtPigkJfPPFxSy/3PvE+ack+nr7v+rjqjum100OjvLn/BC/Xt/PGvk4GRvxkpiSyvrqIO2pmcXNVPilJ6gkUqxT6ctUdOtHPZx9+l5kpSTxz//XMzk3zuqS4NTTq5+2DXby8p51X93bQOzhKWrKPWxYVckdNMbcsLCR9RuwNhotn4Ya+fusyJXoHR7nvqToSfQkK/AiQkuRjQ3URG6qLGPUH2HroJC/taWdLQzs/++h4qCtoAXcuLea2xUVkpWpeoHihM325YmP+AP/jqTq2Hurih1+7nlVzc70uSS7AH3BsP9rNy3vaeXlPO+19QyT5jBvn5bOpppiN1UXkaSxAVFLzjlw1f/bTBh5/5wjf/aWl3LOq3OtyJEyBgOPDllO8vKedl/a0c6x7gASDVXNzuaNmFrcvKaY4K8XrMiVMCn25Kp6ra+Z/Pf8RX7mxgj+5a4nX5chlOjsW4OwBoLGzH4Dl5dncUVPMpiWzKM9Tk10kU+jLtNt1rIdffWQbq+bm8k+/cZ1G2saQxs7TvPRxOy/Xt1PfFhwMdvamNptqiqkqzNBgsAij0JdpNTji586/fZuRsQA/+72bdPPwGNbcPRD6BHCcncdOAVCZn87tNcXcvqSYa8qydACIAAp9mVbf+WkDT7xzhKfvW82N8/K9Lkeuko6+IV5p6GDznna2Hj6JP+CYlZXCxuoibq8pZlVFrj7xeWRKQ9/MNgF/A/iAx51z3z3v/d8Cvg74gX7gfudcQ+i9bwFfDb33e865zRf7Xgr9yLf9aDe/8shW7l09h+98psbrcsQjpwZGeG1vJy/Xt/PWgRMMjwXISUvitsVFbKwuYu2CAg0Gu4qmLPTNzAccADYALcB24AtnQz20TqZzri/0/C7gd5xzm8ysGvgRsAooAV4FFjjn/Bf6fgr9yHa2WWfUH2DzN9ZqgI8AMDAyxlsHTvDynnZe39dJ39AYqUk+1i4ITQu9qJCcdDUBTqepHJy1Cmh0zh0O7fgZ4G7gXOifDfyQdODskeRu4Bnn3DBwxMwaQ/vbGtZPIRHnr17Zz5GuMzx932oFvpyTlpzIpppZbKqZxag/wPuHu9lc384rDe1sru/Al2BcV5HDxupiNlQXafCeh8L5qy0FmscttwCrz1/JzL4OPAgkA7eO23bbeduWXlal4rntR7t58t0jfOn6OWrHlwtK8iVwU1U+N1Xl86d3LeHj1t7gtNAN7Tz00wYe+mkDi4pnsrG6iPXVRSwt1YXgqymc0J/ot/FzbULOuYeBh83si8AfAV8Od1szux+4H6C8XIN7ItGoP8D/+fFHlGan8s07FnldjkSJhATjmtnZXDM7mz+4fSFHu86wpaGDLXs7+Ls3Gvnb1xspzkzhtsWFrK8u4obKPF0HmGbhhH4LMHvcchnQdpH1nwH+4VK2dc49CjwKwTb9MGqSq+yZ7c0cPnGGx369Vs06ctkq8tO5b20l962tpPvMCK/v62RLQzs/2dXKD98/Rlqyj5ur8lm/uIhbFun2kNMhnL/e7UCVmc0FWoF7gC+OX8HMqpxzB0OLvwCcff4C8LSZfZ/ghdwq4IOpKFyunv7hMf7m1QOsmpvL+sWFXpcjMSI3PZnPryzj8yvLGBr1s+3wSV7d28FrezvZXN+BGVw7O5vbFhVy66IiFs+aqWagKTBp6DvnxszsAWAzwS6bTzrn6s3sIaDOOfcC8ICZrQdGgR6CTTuE1nuW4EXfMeDrF+u5I5HpsbcO09U/wmO/vkh/dDItUpJ8rFtYyLqFhXznbkd9Wx+v7+vktb0d/NUrB/irVw4wKyuFWxYVcuvCQtbMzyc1Wc1Al0ODs+SiOk8Pse4v3+SWhYU8/GsrvC5H4lDn6SHe2NfJ6/s6eedgF2dG/CQnJnBDZR63LirkloWFmhcIjciVKfKHP/mYf93ezKsPfooK3QVLPDY85mf7kR5e29fBm/tPcKTrDACVBemsW1DIuoUFrJqbG5cXgxX6csUaO/u5/Qdvce/qcv70bo28lchztOsMb+7v5I39J9h6+CQjYwFSk3zcMC+PdQsL+NSCAubkxcfJiu6cJVfsey/vIzXJx+/eVuV1KSITqshP5yv5c/nKmrkMjgQvBr+5v5M3D5zg9X2dwXXy0li7IHgAuL4yL+57n8X3Ty8XtOtYD680dPA/NyxQtzmJCqmhewDfsijYw+xI1xneOnCCtw6c4Lm6Fv55axNJPmPlnBzWLihgbVUB1bMySUiIr84Jat6RCf32v+zg3cYutn7rtrg/M5LoNzzmp+5oT/AgcLCLvceDM8fkpiezZn4+N88PjiAuyU71uNLLp+YduWzHTg6wub6d3/zUPAW+xIQZiT7WzM9nzfx8vkWwR9C7jV28daCLdxq7+M8Pg2NGKwvSuSm03vWVeTF5w3j9RcvPefLdI/gSjK/cWOF1KSLTonBmCp9dXsZnl5fhnGN/x2neORg8AJxtCkowWFqWzZp5eayZn8/KOTkx0StIoS+f0DswyrN1zfziNSUUZeqm2BL7zIxFxZksKs7kazdXMjIWYNexHt49dJJ3G7t45K3D/P2bh0j2JbBiTjY3zsvnxnl5LCvLJjkx+m4Yo9CXT/jhB00MjPj52k2VXpci4onkxARWV+axujKPBzcsoH94jO1HunnvUBfvHTrJX796gO9vgdQkH7UVOdwwL48bKvOoKc0iKQruGqbQl3NGxgI89d5RbpqfT3VJptfliESEjBmJn+gV1HNmhPePnGTroZNsPXyS7728H4D0ZB+1Fbmsrszl+so8lkboQUChL+f854dtdPQN8xefW+Z1KSIRKyc9+dwNYwC6+ofZdvgk7x/uZtu4g0Bqko+Vc3JYNTeXVXNzuXZ2dkRcE1DoCwDOOR57+zALijL41IICr8sRiRr5GTP49LISPr2sBAgeBD440s37h0/y/pFu/vrVAzgHyb4ElpVlcd3cXFZV5LKyIofMlKvfO0ihLwC823iSfe2n+d7nlmkmTZErkJ8xgzuXzuLOpcFPAqcGRqg72sMHR7v54Eg3j711mH948xBmsKg4k9o5OdRW5HBdRe5VGSeg0BcA/um9I+RnJHP38hKvSxGJKdlpyawP3RoSgjeR333sFB8c7WZHUw//trOF/7etCYBVFbk8+1s3TGs9Cn3hxOlh3th/gq/dPJcZid63OYrEsrTkRG6cn8+N84P3mR7zB9jXfpq6o91XZUoIhb7wH7tb8Qccn19R5nUpInEn0ZdATWkWNaVZV+X7RV5/Irnqnt/RwjVlWVQVzfS6FBGZZgr9OFff1su+9tN8bqXO8kXigUI/zv14RytJPuMXl+kCrkg8UOjHsVF/gP/Y3cpti4rISU/2uhwRuQoU+nHsv/af4OSZETXtiMQRhX4c+/HOFvLSk1m3UCNwReKFQj9OnRoY4bW9ndx1bUlETgolItNDf+1x6j8/bGPEH+Bz6psvElcU+nHq+Z2tLCqeyRJNoSwSVxT6ceho1xk+bD7FL60o1eRqInFGoR+HtjR0AHBHaD5wEYkfCv04tKWhg0XFM5mdm+Z1KSJylSn040z3mRHqmrrZGJrmVUTii0I/zry2t4OAgw3VxV6XIiIeUOjHmS0NHczKSqGmVL12ROKRQj+ODI36eftgF+sXF6nXjkicUujHkXcOdjE46meD2vNF4lZYoW9mm8xsv5k1mtk3J3j/QTNrMLOPzOw1M5sz7j2/me0OPV6YyuLl0mxp6GDmjESur8zzuhQR8cikt0s0Mx/wMLABaAG2m9kLzrmGcavtAmqdcwNm9tvA94BfDb036Jy7dorrlkvkDzhe29fBpxYWkJyoD3gi8Sqcv/5VQKNz7rBzbgR4Brh7/ArOuTeccwOhxW2AJnSJMLube+jqH1HTjkicCyf0S4Hmccstodcu5KvAS+OWU8yszsy2mdlnLqNGmQKvNHSQmGCsW1jodSki4qFJm3eAibp5uAlXNLsXqAU+Ne7lcudcm5lVAq+b2cfOuUPnbXc/cD9AeXl5WIXLpdnS0MHqylyyUpO8LkVEPBTOmX4LMHvcchnQdv5KZrYe+EPgLufc8NnXnXNtoa+HgTeB5edv65x71DlX65yrLSjQDT2m2qET/Rw+cYYNi9W0IxLvwgn97UCVmc01s2TgHuATvXDMbDnwCMHA7xz3eo6ZzQg9zwfWAOMvAMtV8Nre4ARr69WeLxL3Jm3ecc6NmdkDwGbABzzpnKs3s4eAOufcC8BfAhnAc6FBP8ecc3cBi4FHzCxA8ADz3fN6/chV8E7jSaoKMyjL0QRrIvEunDZ9nHMvAi+e99ofj3u+/gLbvQcsvZIC5cqMjAXYfqSbX6lVhyoR0YjcmPdRyykGR/3cME8DskREoR/z3jt0EjNYPVehLyIK/Zj33qEuqmdlkpOe7HUpIhIBFPoxbGjUz86mU9yoph0RCVHox7AdTT2M+APcOC/f61JEJEIo9GPY1kMn8SUY183N9boUEYkQCv0Y9t6hLpaVZZExI6yeuSISBxT6Map/eIwPW3rVni8in6DQj1Hbj3TjDzi154vIJyj0Y9TWwydJ9iWwck6O16WISARR6Meo9w51sWJONilJPq9LEZEIotCPQacGRqhv6+OGSjXtiMgnKfRj0LbD3TgHN87XRVwR+SSFfgzaeqiL1CQf15Rle12KiEQYhX4M2na4m+vm5pKcqF+viHySUiEGHeseYEFhhtdliEgEUujHmJGxAIOjft0AXUQmpNCPMb2DowBkpSn0ReTnKfRjzLnQ15m+iExAoR9jzoZ+pkJfRCag0I8xfTrTF5GLUOjHGDXviMjFKPRjjEJfRC5GoR9jFPoicjEK/RjTNzhKWrKPJJ9+tSLy85QMMaZ3cFRn+SJyQQr9GKPQF5GLUejHmN7BUTJTFPoiMjGFfozpHRzVwCwRuSCFfozpU/OOiFyEQj/GqE1fRC5GoR9DRv0BzoxoWmURuTCFfgz573l3Ej2uREQilUI/hmgufRGZTFihb2abzGy/mTWa2TcneP9BM2sws4/M7DUzmzPuvS+b2cHQ48tTWbx8Ut/QGKApGETkwiYNfTPzAQ8DdwDVwBfMrPq81XYBtc65ZcDzwPdC2+YC3wZWA6uAb5tZztSVL+Np3h0RmUw4Z/qrgEbn3GHn3AjwDHD3+BWcc2845wZCi9uAstDz24Etzrlu51wPsAXYNDWly/kU+iIymXBCvxRoHrfcEnrtQr4KvHQp25rZ/WZWZ2Z1J06cCKMkmYjumiUikwkn9G2C19yEK5rdC9QCf3kp2zrnHnXO1TrnagsKCsIoSSZytveOpmEQkQsJJ/RbgNnjlsuAtvNXMrP1wB8Cdznnhi9lW5kavYOjzEhMICXJ53UpIhKhwgn97UCVmc01s2TgHuCF8SuY2XLgEYKB3znurc3ARjPLCV3A3Rh6TaZB74BG44rIxU06isc5N2ZmDxAMax/wpHOu3sweAuqccy8QbM7JAJ4zM4Bjzrm7nHPdZvYdggcOgIecc93T8pOIpmAQkUmFNXTTOfci8OJ5r/3xuOfrL7Ltk8CTl1ughE+hLyKT0YjcGKLQF5HJKPRjiEJfRCaj0I8hfUO6gYqIXJxCP0b4A47TQ2M60xeRi1Lox4jTQ5qCQUQmp9CPEZqCQUTCodCPEZpsTUTCodCPEQp9EQmHQj9GKPRFJBwK/Rih0BeRcCj0Y4RCX0TCodCPEb2DoyT7EkhJ0q9URC5MCREj+gbHyExNIjTLqYjIhBT6MaJvcJSs1LAmTRWROKbQjxGabE1EwqHQjxEKfREJh0I/RvQOaoZNEZmcQj9G6ExfRMKh0I8BgYCjb0ihLyKTU+jHgNPDYzingVkiMjmFfgzo07TKIhImhX4M0BQMIhIuhX4MUOiLSLgU+jGgT6EvImFS6McAnemLSLgU+jFAoS8i4VLox4DewVESE4y0ZJ/XpYhIhFPox4CzUzBoWmURmYxCPwZoCgYRCZdCPwZosjURCZdCPwb06UxfRMKk0I8Bat4RkXAp9GNAr26VKCJhCiv0zWyTme03s0Yz++YE7681s51mNmZmnz/vPb+Z7Q49XpiqwiXIOUff0JjO9EUkLJOeHpqZD3gY2AC0ANvN7AXnXMO41Y4BXwH+YIJdDDrnrp2CWmUCZ0b8+ANOoS8iYQmnTWAV0OicOwxgZs8AdwPnQt85dzT0XmAaapSL0GhcEbkU4TTvlALN45ZbQq+FK8XM6sxsm5l95pKqk0n1Dij0RSR84ZzpTzTM013C9yh3zrWZWSXwupl97Jw79IlvYHY/cD9AeXn5JexaenUDFRG5BOGc6bcAs8ctlwFt4X4D51xb6Oth4E1g+QTrPOqcq3XO1RYUFIS7awF6BkYAnemLSHjCCf3tQJWZzTWzZOAeIKxeOGaWY2YzQs/zgTWMuxYgV66+rZfEBKMyP8PrUkQkCkwa+s65MeABYDOwF3jWOVdvZg+Z2V0AZnadmbUAvww8Ymb1oc0XA3Vm9iHwBvDd83r9yBXa0dRDdUkmqZphU0TCENaIHufci8CL5732x+OebyfY7HP+du8BS6+wRrmAMX+AD5t7+dXrZk++sogIGpEb1fa1n2Zw1M+KOTlelyIiUUKhH8V2HusBYEV5tseViEi0UOhHsZ1NPRTOnEFpdqrXpYhIlFDoR7Edx3pYOSdHd8wSkbAp9KNU5+khmrsHWVGu9nwRCZ9CP0rtbDoFoIu4InJJFPpRatexHpJ9CdSUZnpdiohEEYV+lNrR1MOS0kxmJGpQloiET6EfhUbGAnzU2stKteeLyCVS6EehhuN9jIwF1J4vIpdMoR+FdjSdHZSl0BeRS6PQj0I7j/VQmp1KcVaK16WISJRR6EehnU09LNfUCyJyGRT6UeZ47yDHe4dYqfZ8EbkMCv0oc25QltrzReQyKPSjzI6mHmYkJrB4lgZlicilU+hHmd3NPSwryyI5Ub86Ebl0So4o4g84Go73sbRUF3FF5PIo9KPI4RP9DI0GNN+OiFw2hX4U+bi1F4Ca0iyPKxGRaKXQjyJ7WvtISUqgMj/d61JEJEop9KPInrZeFs/KJNGnX5uIXB6lR5QIBBwNbX3UlKhpR0Qun0I/SjR1D9A/PMZSteeLyBVQ6EeJPaGLuEvUc0dEroBCP0rsaesl2ZdAVeFMr0sRkSgWU6HvDzivS5g2e1p7WVg8UyNxReSKJHpdwFQ5MzzG9X/+GjWlWVxXkUNtRS4r5uSQMSP6f0TnHHta+7hzabHXpYhIlIv+RAwZGvXzuZVl1DV183dvNBJwkGCweFYmtXNyWDEneCAozU71utRL1tIzSO/gqAZlicgVi5nQz8uYwZ/ctQSA/uExdh3rYfuRbnYc6+G5HS08tbUJgFlZKayYk8OK8hxWzsmhelZmxDeZ1LeFRuKqu6aIXKGYCf3xMmYkcnNVATdXFQAw5g+wr/00O5p6qGvqYWdTDz/76DgAMxITWFqaxfLybFaU57C8PCfibkO4p7UPX4KxsFgXcUXkysRk6J8v0ZdATWkWNaVZfPnGCgDae4fYeSx4ANjVfIqntjbx2NtHgOCngWtnZ597LC3LIi3Zu3+qPW29VBVmkJLk86wGEYkNcRH6EynOSuHOpbO4c+ksAIbH/Ow9fppdx3rYdewUu5tP8dKedgB8CcaCoplcOzuLa8qyuWZ2NlWFGVdlOoTgRdxe1i0snPbvJSKxL25D/3wzEn3nzux/Y03wta7+YT5sDh4Adjef4mcfHedHHzQDkJrkY0lJJsvKsrlmdhbLyrKZk5tGQoJNaV0dfcN09Y9QU6JBWSJy5cIKfTPbBPwN4AMed85997z31wI/AJYB9zjnnh/33peBPwot/plz7qmpKPxqyM+YwW2Li7htcREQPOs+enKA3c09fNTSy0ctvTz9QRNPvhsAYGZKIktLs4KPsuDX8tw0zC7/QHB2JO7SMl3EFZErN2nom5kPeBjYALQA283sBedcw7jVjgFfAf7gvG1zgW8DtYADdoS27Zma8q8uM2Nufjpz89P57PIyIHiR+EBHPx+1nOLj1l4+bu3lH989yoj/vw8ENSXBg0BNaRZLSjKZm5ce9ieCPW29WKjrqYjIlQrnTH8V0OicOwxgZs8AdwPnQt85dzT0XuC8bW8HtjjnukPvbwE2AT+64sojRKIvgeqSTKpLMrkn9NrIWIADHafPHQT2tPbyT+MOBOnJPqpLMllSkhX6mklV4cSjbfe09jGvIMPTC8kiEjvCSZJSoHnccguwOsz9T7Rt6fkrmdn9wP0A5eXlYe46ciUn/ndvoS+EXhv1Bw8E9W191Lf2sqetj3/d3szgqD+4jS+BqqIMlpRksnhWJtWzMllckkl9Wy+r5+Z698OISEwJJ/QnaocId5KbsLZ1zj0KPApQW1sbkxPoJPkSWFKSxZKSLKidDQTnCjrSdYb6tl4ajvfR0NbHq3s7ebau5RPbaiSuiEyVcEK/BZg9brkMaAtz/y3AuvO2fTPMbWOeL8GYX5jB/MIM7r42+AHIOUfn6eFzB4GWngF+YdksjysVkVgRTuhvB6rMbC7QCtwDfDHM/W8G/tzMckLLG4FvXXKVccTMKMpMoSgzhVvUN19Eptiko4ucc2PAAwQDfC/wrHOu3sweMrO7AMzsOjNrAX4ZeMTM6kPbdgPfIXjg2A48dPairoiIXH3mXGQ1odfW1rq6ujqvyxARiSpmtsM5VzvZepE9vaSIiEwphb6ISBxR6IuIxBGFvohIHFHoi4jEEYW+iEgcibgum2Z2Ami6gl3kA11TVM7VoHqnl+qdXqp3el1KvXOccwWTrRRxoX+lzKwunL6qkUL1Ti/VO71U7/SajnrVvCMiEkcU+iIicSQWQ/9Rrwu4RKp3eqne6aV6p9eU1xtzbfoiInJhsXimLyIiFxAzoW9mm8xsv5k1mtk3va5nMmb2pJl1mtker2uZjJnNNrM3zGyvmdWb2e97XdNkzCzFzD4wsw9DNf+p1zVNxsx8ZrbLzH7qdS3hMLOjZvaxme02s4ifGtfMss3seTPbF/q/fIPXNV2ImS0M/bueffSZ2TemZN+x0LxjZj7gALCB4N26tgNfcM41XHRDD5nZWqAJp7cWAAAC0UlEQVQf+GfnXI3X9VyMmc0CZjnndprZTGAH8JkI//c1IN05129mScA7wO8757Z5XNoFmdmDQC2Q6Zz7tNf1TMbMjgK1zrmo6PduZk8BbzvnHjezZCDNOXfK67omE8q3VmC1c+5KxjABsXOmvwpodM4dds6NAM8Ad3tc00U5594CouKGMs654865naHnpwneTOfnbnAfSVxQf2gxKfSI2DMcMysDfgF43OtaYpGZZQJrgScAnHMj0RD4IbcBh6Yi8CF2Qr8UaB633EKEh1K0MrMKYDnwvreVTC7UXLIb6AS2OOciueYfAP8bCHhdyCVwwCtmtsPM7ve6mElUAieAfww1oT1uZuleFxWme4AfTdXOYiX0bYLXIvasLlqZWQbwY+Abzrk+r+uZjHPO75y7FigDVplZRDajmdmngU7n3A6va7lEa5xzK4A7gK+HmiwjVSKwAvgH59xy4AwQDdf+koG7gOemap+xEvotwOxxy2VAm0e1xKRQu/iPgR865/7N63ouRehj/JvAJo9LuZA1wF2hNvJngFvN7F+8LWlyzrm20NdO4CcEm1kjVQvQMu7T3vMEDwKR7g5gp3OuY6p2GCuhvx2oMrO5oSPjPcALHtcUM0IXRZ8A9jrnvu91PeEwswIzyw49TwXWA/u8rWpizrlvOefKnHMVBP/vvu6cu9fjsi7KzNJDF/UJNZNsBCK2J5pzrh1oNrOFoZduAyK2I8I4X2AKm3Yg+JEn6jnnxszsAWAz4AOedM7Ve1zWRZnZj4B1QL6ZtQDfds494W1VF7QG+BLwcaiNHOD/Oude9LCmycwCngr1fEgAnnXORUVXyChRBPwkeD5AIvC0c+5lb0ua1O8CPwydGB4GfsPjei7KzNII9kj8zSndbyx02RQRkfDESvOOiIiEQaEvIhJHFPoiInFEoS8iEkcU+iIicUShLyISRxT6IiJxRKEvIhJH/j+uzG0G+XVuLQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ---------------\n",
    "# Debugging Cell\n",
    "# Plot the reward structures of both ports\n",
    "# Plot the global reward rate for every policy (policy is represented by its exit time)\n",
    "# ---------------\n",
    "env = GiveUpEnvironment()\n",
    "env.env_init({\"state representation structure\": \"regular\", \n",
    "              \"fundamental timestep\": 0.1, \n",
    "              \"gamma\": 0.9, \n",
    "              \"pursuit port total duration\": 7, \n",
    "              \"background port total duration\": 9, \n",
    "              \"transit duration\": 0.5, \n",
    "              \"consumption duration\": 1, \n",
    "              \"first time in pursuit port that could give a reward\": 1.2,\n",
    "              \"required minimum wait time in pursuit port\": 1.5, \n",
    "              \"time to wait until the trial reset\": 3, \n",
    "              \"exponential distribution scale\": 3, \n",
    "              \"delivery time in the background port\": 5,\n",
    "              \"reward amount in pursuit\": 10, \n",
    "              \"reward amount in background\": [0.8, 1.2], \n",
    "              \"seed\": 0})\n",
    "env.test_reward_function()\n",
    "(rou_g, rou_l, t_rou_g, t_rou_l) = env.PolicyMax()\n",
    "hazard_function = np.zeros(len(env.PSTimeArray))\n",
    "for i in range(len(env.PSTimeArray)): \n",
    "    Pr_NeverRewarded = 1 - env.PSRewardTimeCDF[i]\n",
    "    Pr_NeverRewardedNRewardedAtT = env.PSRewardTimePDF[i]\n",
    "    Pr_RewardedAtTGivenNotYetRewarded = Pr_NeverRewardedNRewardedAtT / Pr_NeverRewarded\n",
    "    hazard_function[i] = Pr_RewardedAtTGivenNotYetRewarded\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "ax.plot(env.PSTimeArray, rou_g)\n",
    "#ax.plot(env.PSTimeArray, rou_l)\n",
    "print(f\"According to the global reward rate, the optimal give-up time in the pursuit port is at {t_rou_g} second!\")\n",
    "print(f\"According to the local reward rate, the optimal give-up time in the pursuit port is at {t_rou_l} second!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 5.1, 1.0)\n",
      "18\n",
      "(2, 0, 0)\n",
      "58\n",
      "(0, 0, 0)\n",
      "0\n",
      "(2, 0, 0)\n",
      "0 58 False\n",
      "(1, 6.5, 0)\n",
      "0 43 False\n",
      "(1, 6.5, 0)\n",
      "0 43 False\n",
      "(3, 0, 0)\n",
      "0 63 False\n"
     ]
    }
   ],
   "source": [
    "# ---------------\n",
    "# Debugging Cell\n",
    "# Test if the action at a certain state leads to a certain other state and reward\n",
    "# ---------------\n",
    "def test_action():\n",
    "    env = GiveUpEnvironment()\n",
    "    env.env_init({\"state representation structure\": \"dilating\", \n",
    "              \"dt\": 0.1, \n",
    "              \"gamma\": 0.9, \n",
    "              \"pursuit port total duration\": 7, \n",
    "              \"background port total duration\": 9, \n",
    "              \"transit duration\": 0.5, \n",
    "              \"consumption duration\": 1, \n",
    "              \"first time in pursuit port that could give a reward\": 1,\n",
    "              \"required minimum wait time in pursuit port\": 1, \n",
    "              \"time to wait until the trial reset\": 1.5, \n",
    "              \"exponential distribution scale\": 4, \n",
    "              \"delivery time in the background port\": 5,\n",
    "              \"reward amount in pursuit\": 3, \n",
    "              \"reward amount in background\": [0.6, 1.8], \n",
    "              \"seed\": 0})\n",
    "    env.current_state3d = (0, 4.5, 0)\n",
    "    env.env_step(0)\n",
    "    print(env.current_state3d)\n",
    "    print(env.current_state1d)\n",
    "    # assert(env.current_state == (0, env.dt, 0))\n",
    "    \n",
    "    env.current_state3d = (0, 2.29, 0)\n",
    "    env.env_step(1)\n",
    "    print(env.current_state3d)\n",
    "    print(env.current_state1d)\n",
    "    # assert(env.current_state == (0, 2, 0))\n",
    "    \n",
    "    env.current_state3d = (0, 1.19, 0)\n",
    "    env.env_step(1)\n",
    "    print(env.current_state3d)\n",
    "    print(env.current_state1d)\n",
    "    # assert(env.current_state == (1, 0, 0))\n",
    "    \n",
    "def test_reward():\n",
    "    env = GiveUpEnvironment()\n",
    "    env.env_init({\"state representation structure\": \"dilating\", \n",
    "                  \"dt\": 0.1, \n",
    "                  \"gamma\": 0.9, \n",
    "                  \"pursuit port total duration\": 7, \n",
    "                  \"background port total duration\": 9, \n",
    "                  \"transit duration\": 0.5, \n",
    "                  \"consumption duration\": 1, \n",
    "                  \"first time in pursuit port that could give a reward\": 1,\n",
    "                  \"required minimum wait time in pursuit port\": 1, \n",
    "                  \"time to wait until the trial reset\": 1.5, \n",
    "                  \"exponential distribution scale\": 4, \n",
    "                  \"delivery time in the background port\": 5,\n",
    "                  \"reward amount in pursuit\": 3, \n",
    "                  \"reward amount in background\": 1, \n",
    "                  \"seed\": 0})\n",
    "    env.current_state3d = (0, 4.5, 0)\n",
    "    reward_state_term = env.env_step(1)\n",
    "    print(env.current_state3d)\n",
    "    print(reward_state_term[0], reward_state_term[1], reward_state_term[2])\n",
    "    # assert(reward_state_term[0] == 0 and reward_state_term[1] == (0, env.dt, 0) and reward_state_term[2] == False)\n",
    "    \n",
    "    env.current_state3d = (1, 5.76, 0)\n",
    "    reward_state_term = env.env_step(0)\n",
    "    print(env.current_state3d)\n",
    "    print(reward_state_term[0], reward_state_term[1], reward_state_term[2])\n",
    "    reward_state_term = env.env_step(0)\n",
    "    print(env.current_state3d)\n",
    "    print(reward_state_term[0], reward_state_term[1], reward_state_term[2])\n",
    "    # assert(reward_state_term[0] ==  and reward_state_term[1] ==  and reward_state_term[2] == False)\n",
    "    \n",
    "    env.current_state3d = (1, 2.29, 1)\n",
    "    reward_state_term = env.env_step(1)\n",
    "    print(env.current_state3d)\n",
    "    print(reward_state_term[0], reward_state_term[1], reward_state_term[2])\n",
    "    # assert(reward_state_term[0] == 0 and reward_state_term[1] == (0, 0, 0) and reward_state_term[2] == False)\n",
    "    \n",
    "test_action()\n",
    "test_reward()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 2. Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create BasicTDAgent class\n",
    "class BasicTDAgent(BaseAgent):\n",
    "    \n",
    "    # ---------------\n",
    "    # As we did with the environment, we first initialize the agent once when a TDAgent object is created. \n",
    "    # In this function, we create a random number generator, seeded with \n",
    "    # the seed provided in the agent_info dictionary to get reproducible results. \n",
    "    # We also set the policy, discount and step size based on the agent_info dictionary. \n",
    "    # Finally, with a convention that the policy is always specified as a mapping \n",
    "    # from states to actions and so is an array of size (# States, # Actions), \n",
    "    # we initialize a values array of shape (# States,) to zeros.\n",
    "    # ---------------\n",
    "    def __init__(self, agent_info={}):\n",
    "            \n",
    "        \"\"\"Setup for the agent called when the experiment first starts.\"\"\"\n",
    "\n",
    "        # Create a random number generator with the provided seed to seed the agent for reproducibility.\n",
    "        self.rand_generator = np.random.RandomState(agent_info.get(\"seed\"))\n",
    "\n",
    "        # Policy will be given, recall that the goal is to accurately estimate its corresponding value function. \n",
    "        self.policy = agent_info.get(\"policy\")\n",
    "        # Discount factor (gamma) to use in the updates.\n",
    "        self.discount = agent_info.get(\"discount\")\n",
    "        # The learning rate or step size parameter (alpha) to use in updates.\n",
    "        self.step_size = agent_info.get(\"step_size\")\n",
    "\n",
    "        # Initialize an array of zeros that will hold the values.\n",
    "        # Recall that the policy can be represented as a (# States, # Actions) array. With the \n",
    "        # assumption that this is the case, we can use the first dimension of the policy to\n",
    "        # initialize the array for values.\n",
    "        self.values = np.zeros((self.policy.shape[0],))\n",
    "        self.delta = 0\n",
    "        \n",
    "    def agent_init(self, agent_info={}):\n",
    "    \n",
    "        \"\"\"Setup for the agent called when the experiment first starts.\"\"\"\n",
    "\n",
    "        # Create a random number generator with the provided seed to seed the agent for reproducibility.\n",
    "        self.rand_generator = np.random.RandomState(agent_info.get(\"seed\"))\n",
    "\n",
    "        # Policy will be given, recall that the goal is to accurately estimate its corresponding value function. \n",
    "        self.policy = agent_info.get(\"policy\")\n",
    "        # Discount factor (gamma) to use in the updates.\n",
    "        self.discount = agent_info.get(\"discount\")\n",
    "        # The learning rate or step size parameter (alpha) to use in updates.\n",
    "        self.step_size = agent_info.get(\"step_size\")\n",
    "\n",
    "        # Initialize an array of zeros that will hold the values.\n",
    "        # Recall that the policy can be represented as a (# States, # Actions) array. With the \n",
    "        # assumption that this is the case, we can use the first dimension of the policy to\n",
    "        # initialize the array for values.\n",
    "        self.values = np.zeros((self.policy.shape[0],))\n",
    "        \n",
    "    # ---------------\n",
    "    # In agent_start(), we choose an action based on the initial state and policy we are evaluating. \n",
    "    # We also cache the state so that we can later update its value when we perform a Temporal Difference update. \n",
    "    # Finally, we return the action chosen so that the RL loop can continue and the environment can execute this action.\n",
    "    # ---------------\n",
    "    def agent_start(self, state):\n",
    "        \"\"\"The first method called when the episode starts, called after\n",
    "        the environment starts.\n",
    "        Args:\n",
    "            state (Numpy array): the state from the environment's env_start function.\n",
    "        Returns:\n",
    "            The first action the agent takes.\n",
    "        \"\"\"\n",
    "        # The policy can be represented as a (# States, # Actions) array. So, we can use \n",
    "        # the second dimension here when choosing an action.\n",
    "        action = self.rand_generator.choice(range(self.policy.shape[1]), p=self.policy[state])\n",
    "        self.last_state = state\n",
    "        return action\n",
    "\n",
    "    def agent_step(self, reward, state):\n",
    "        \"\"\"A step taken by the agent.\n",
    "        Args:\n",
    "            reward (float): the reward received for taking the last action taken\n",
    "            state (Numpy array): the state from the\n",
    "                environment's step after the last action, i.e., where the agent ended up after the\n",
    "                last action\n",
    "        Returns:\n",
    "            The action the agent is taking.\n",
    "        \"\"\"\n",
    "\n",
    "        # Hint: We should perform an update with the last state given that we now have the reward and\n",
    "        # next state. We break this into two steps. Recall for example that the Monte-Carlo update \n",
    "        # had the form: V[S_t] = V[S_t] + alpha * (target - V[S_t]), where the target was the return, G_t.\n",
    "\n",
    "        # your code here\n",
    "        alpha = self.step_size\n",
    "        gamma = self.discount\n",
    "        s = self.last_state\n",
    "        self.values[s] = self.values[s] + alpha * (reward + gamma * self.values[state] - self.values[s])\n",
    "        self.delta = reward + gamma * self.values[state] - self.values[s]\n",
    "\n",
    "        # Having updated the value for the last state, we now act based on the current \n",
    "        # state, and set the last state to be current one as we will next be making an \n",
    "        # update with it when agent_step is called next once the action we return from this function \n",
    "        # is executed in the environment.\n",
    "\n",
    "        action = self.rand_generator.choice(range(self.policy.shape[1]), p=self.policy[state])\n",
    "        self.last_state = state\n",
    "\n",
    "        return action\n",
    "\n",
    "    def agent_end(self, reward):\n",
    "        \"\"\"Run when the agent terminates.\n",
    "        Args:\n",
    "            reward (float): the reward the agent received for entering the terminal state.\n",
    "        \"\"\"\n",
    "\n",
    "        # Hint: Here too, we should perform an update with the last state given that we now have the \n",
    "        # reward. Note that in this case, the action led to termination. Once more, we break this into \n",
    "        # two steps, computing the target and the update itself that uses the target and the \n",
    "        # current value estimate for the state whose value we are updating.\n",
    "\n",
    "        # your code here\n",
    "        alpha = self.step_size\n",
    "        gamma = self.discount\n",
    "        s = self.last_state\n",
    "        self.values[s] = self.values[s] + alpha * (reward - self.values[s])\n",
    "\n",
    "    def agent_cleanup(self):        \n",
    "        \"\"\"Cleanup done after the agent ends.\"\"\"\n",
    "        self.last_state = None\n",
    "        \n",
    "    def agent_message(self, message):\n",
    "        \"\"\"A function used to pass information from the agent to the experiment.\n",
    "        Args:\n",
    "            message: The message passed to the agent.\n",
    "        Returns:\n",
    "            The response (or answer) to the message.\n",
    "        \"\"\"\n",
    "        if message == \"get_values\":\n",
    "            return self.values\n",
    "        elif message == \"get TD error\": \n",
    "            return self.delta\n",
    "        else:\n",
    "            raise Exception(\"TDAgent.agent_message(): Message not understood!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --------------\n",
    "# Debugging Cell\n",
    "# --------------\n",
    "# The following test checks that the TD check works for a case where the transition \n",
    "# garners reward -1 and does not lead to a terminal state. This is in a simple two state setting \n",
    "# where there is only one action. The first state's current value estimate is 0 while the second is 1.\n",
    "# Note the discount and step size if you are debugging this test.\n",
    "policy_list = np.array([[1.], [1.]])\n",
    "agent_info = {\"policy\": np.array(policy_list), \"discount\": 0.99, \"step_size\": 0.1}\n",
    "agent = BasicTDAgent(agent_info = agent_info)\n",
    "\n",
    "agent.values = np.array([0., 1.])\n",
    "agent.agent_start(0)\n",
    "\n",
    "reward = -1\n",
    "next_state = 1\n",
    "agent.agent_step(reward, next_state)\n",
    "\n",
    "assert(np.isclose(agent.values[0], -0.001) and np.isclose(agent.values[1], 1.))\n",
    "\n",
    "# The following test checks that the TD check works for a case where the transition \n",
    "# garners reward -100 and lead to a terminal state. This is in a simple one state setting \n",
    "# where there is only one action. The state's current value estimate is 0.\n",
    "# Note the discount and step size if you are debugging this test.\n",
    "policy_list = np.array([[1.]])\n",
    "agent_info = {\"policy\": np.array(policy_list), \"discount\": 0.99, \"step_size\": 0.1}\n",
    "agent = BasicTDAgent(agent_info = agent_info)\n",
    "\n",
    "agent.values = np.array([0.])\n",
    "agent.agent_start(0)\n",
    "\n",
    "reward = -100\n",
    "next_state = 0\n",
    "agent.agent_end(reward)\n",
    "\n",
    "assert(np.isclose(agent.values[0], -10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QLearningAgent(BaseAgent):\n",
    "    def __init__(self, agent_info={}):\n",
    "        \"\"\"Setup for the agent called when the experiment first starts.\n",
    "        \n",
    "        Args:\n",
    "        agent_init_info (dict), the parameters used to initialize the agent. The dictionary contains:\n",
    "        {\n",
    "            num_states (int): The number of states,\n",
    "            num_actions (int): The number of actions,\n",
    "            epsilon (float): The epsilon parameter for exploration,\n",
    "            step_size (float): The step-size,\n",
    "            discount (float): The discount factor,\n",
    "        }\n",
    "        \n",
    "        \"\"\"\n",
    "        # Create a random number generator with the provided seed to seed the agent for reproducibility.\n",
    "        \n",
    "        # Store the parameters provided in agent_init_info.\n",
    "        self.num_actions = agent_info[\"num_actions\"]\n",
    "        self.num_states = agent_info[\"num_states\"]\n",
    "        self.epsilon = agent_info[\"epsilon\"]\n",
    "        self.step_size = agent_info[\"step_size\"]\n",
    "        self.discount = agent_info[\"discount\"]\n",
    "        self.rand_generator = np.random.RandomState(agent_info[\"seed\"])\n",
    "        self.c = agent_info.get(\"degree of exploration\", 0.2)\n",
    "        self.exploration_method = agent_info.get(\"exploration method\", \"UCB\")\n",
    "        \n",
    "        # Create an array for action-value estimates and initialize it to zero.\n",
    "        self.q = 0.5 * np.ones((self.num_states, self.num_actions)) # The array of action-value estimates.\n",
    "        self.delta = 0\n",
    "        self.total_steps = 0\n",
    "        self.SAcounter = np.zeros((self.num_states, self.num_actions)) # The array of times each state-action pair is chosen.\n",
    "        self.UCB = math.inf * np.ones((self.num_states, self.num_actions)) # The array of upper-confidence-bounds for each state-action pair. \n",
    "        \n",
    "    def agent_init(self, agent_info):\n",
    "        \"\"\"Setup for the agent called when the experiment first starts.\n",
    "        \n",
    "        Args:\n",
    "        agent_init_info (dict), the parameters used to initialize the agent. The dictionary contains:\n",
    "        {\n",
    "            num_states (int): The number of states,\n",
    "            num_actions (int): The number of actions,\n",
    "            epsilon (float): The epsilon parameter for exploration,\n",
    "            step_size (float): The step-size,\n",
    "            discount (float): The discount factor,\n",
    "        }\n",
    "        \n",
    "        \"\"\"\n",
    "        # Store the parameters provided in agent_info.\n",
    "        self.num_actions = agent_info[\"num_actions\"]\n",
    "        self.num_states = agent_info[\"num_states\"]\n",
    "        self.epsilon = agent_info[\"epsilon\"]\n",
    "        self.step_size = agent_info[\"step_size\"]\n",
    "        self.discount = agent_info[\"discount\"]\n",
    "        self.rand_generator = np.random.RandomState(agent_info[\"seed\"])\n",
    "        self.c = agent_info.get(\"degree of exploration\", 0.2)\n",
    "        self.exploration_method = agent_info.get(\"exploration method\", \"UCB\")\n",
    "        \n",
    "        # Create an array for action-value estimates and initialize it to zero.\n",
    "        self.q = 0.5 * np.ones((self.num_states, self.num_actions)) # The array of action-value estimates.\n",
    "        self.delta = 0\n",
    "        self.total_steps = 0\n",
    "        self.SAcounter = np.zeros((self.num_states, self.num_actions)) # The array of times each state-action pair is chosen.\n",
    "                     \n",
    "    def agent_start(self, observation):\n",
    "        \"\"\"The first method called when the episode starts, called after\n",
    "        the environment starts.\n",
    "        Args:\n",
    "            observation (int): the state observation from the\n",
    "                environment's evn_start function.\n",
    "        Returns:\n",
    "            action (int): the first action the agent takes.\n",
    "        \"\"\"\n",
    "        \n",
    "        state = observation\n",
    "        current_q = self.q[state, :]\n",
    "        \n",
    "        if (self.exploration_method == \"epsilon-greedy\"):\n",
    "            # Choose action using epsilon greedy.\n",
    "            if self.rand_generator.rand() < self.epsilon:\n",
    "                action = self.rand_generator.randint(self.num_actions)\n",
    "            else:\n",
    "                action = self.argmax(current_q)\n",
    "        elif (self.exploration_method == \"UCB\"):\n",
    "            # Choose action using Upper-Confidence-Bound. \n",
    "            action = self.argmax(self.UCB[state])\n",
    "        else: \n",
    "            raise Exception(self.exploration_method + \" not in recognized exploration methods: 'epsilon-greedy' or 'UCB'!\")\n",
    "        \n",
    "        self.update_UCB(state)\n",
    "        self.total_steps += 1\n",
    "        self.SAcounter[state, action] += 1\n",
    "        self.prev_state = state\n",
    "        self.prev_action = action\n",
    "        return action\n",
    "    \n",
    "    def agent_step(self, reward, observation):\n",
    "        \"\"\"A step taken by the agent.\n",
    "        Args:\n",
    "            reward (float): the reward received for taking the last action taken\n",
    "            observation (int): the state observation from the\n",
    "                environment's step based on where the agent ended up after the\n",
    "                last step.\n",
    "        Returns:\n",
    "            action (int): the action the agent is taking.\n",
    "        \"\"\"\n",
    "        \n",
    "        state = observation\n",
    "        current_q = self.q[state, :]\n",
    "        \n",
    "        if (self.exploration_method == \"epsilon-greedy\"):\n",
    "            # Choose action using epsilon greedy.\n",
    "            if self.rand_generator.rand() < self.epsilon:\n",
    "                action = self.rand_generator.randint(self.num_actions)\n",
    "            else:\n",
    "                action = self.argmax(current_q)\n",
    "        elif (self.exploration_method == \"UCB\"):\n",
    "            # Choose action using Upper-Confidence-Bound. \n",
    "            action = self.argmax(self.UCB[state])\n",
    "        else: \n",
    "            raise Exception(self.exploration_method + \" not in recognized exploration methods: 'epsilon-greedy' or 'UCB'!\")\n",
    "        \n",
    "        # Perform an update\n",
    "        # --------------------------\n",
    "        # your code here\n",
    "        alpha = self.step_size\n",
    "        gamma = self.discount\n",
    "        r = reward\n",
    "        MaxQSpa = max(current_q) # meaning max(Q(S', a))\n",
    "        QSA = self.q[self.prev_state, self.prev_action] # meaning Q(S, A)\n",
    "        self.q[self.prev_state, self.prev_action] = QSA + alpha * (r + gamma * MaxQSpa - QSA)\n",
    "        self.delta = r + gamma * MaxQSpa - QSA\n",
    "        # --------------------------\n",
    "        \n",
    "        self.update_UCB(state)\n",
    "        self.total_steps += 1\n",
    "        self.SAcounter[state, action] += 1\n",
    "        self.prev_state = state\n",
    "        self.prev_action = action\n",
    "        return action\n",
    "    \n",
    "    def agent_end(self, reward):\n",
    "        \"\"\"Run when the agent terminates.\n",
    "        Args:\n",
    "            reward (float): the reward the agent received for entering the\n",
    "                terminal state.\n",
    "        \"\"\"\n",
    "        # Perform the last update in the episode\n",
    "        # --------------------------\n",
    "        # your code here\n",
    "        alpha = self.step_size\n",
    "        gamma = self.discount\n",
    "        r = reward\n",
    "        QSA = self.q[self.prev_state, self.prev_action] # meaning Q(S, A)\n",
    "        self.q[self.prev_state, self.prev_action] = QSA + alpha * (r - QSA)\n",
    "        # --------------------------\n",
    "    \n",
    "    def agent_message(self, message):\n",
    "        \"\"\"A function used to pass information from the agent to the experiment.\n",
    "        Args:\n",
    "            message: The message passed to the agent.\n",
    "        Returns:\n",
    "            The response (or answer) to the message.\n",
    "        \"\"\"\n",
    "        if message == \"get_action_values\":\n",
    "            return self.q\n",
    "        elif message == \"get TD error\": \n",
    "            return self.delta\n",
    "        else:\n",
    "            raise Exception(\"TDAgent.agent_message(): Message not understood!\")\n",
    "    \n",
    "    def update_UCB(self, state): \n",
    "        \"\"\"The method gets the current upper confidence bound of each \n",
    "        action at a certain state. \n",
    "        Args: \n",
    "            state (int): the state of which the UCBs of actions are needed.\n",
    "            \n",
    "        Returns: \n",
    "            UCB (array): the upper confidence bound of each action\n",
    "            at a certain state.\n",
    "        \"\"\"\n",
    "        for j in range(self.num_actions): \n",
    "            if (self.SAcounter[state][j] == 0):\n",
    "                self.UCB[state][j] = math.inf\n",
    "            else: \n",
    "                self.UCB[state][j] = self.q[state][j] + self.c * math.sqrt(np.log(self.total_steps) / self.SAcounter[state][j])\n",
    "                self.UCB[state][j] = np.round(self.UCB[state][j], 1)\n",
    "        \n",
    "        return self.UCB\n",
    "    \n",
    "    def argmax(self, q_values):\n",
    "        \"\"\"argmax with random tie-breaking\n",
    "        Args:\n",
    "            q_values (Numpy array): the array of action-values\n",
    "        Returns:\n",
    "            action (int): an action with the highest value\n",
    "        \"\"\"\n",
    "        top = float(\"-inf\")\n",
    "        ties = []\n",
    "\n",
    "        for i in range(len(q_values)):\n",
    "            if q_values[i] > top:\n",
    "                top = q_values[i]\n",
    "                ties = []\n",
    "\n",
    "            if q_values[i] == top:\n",
    "                ties.append(i)\n",
    "\n",
    "        return self.rand_generator.choice(ties)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 4. Simulating Learning Process with Q-learning and SARSA\n",
    "\n",
    "Finally, see the TD policy evaluation algorithm in action by looking at the estimated values, the per state value error and after the experiment is complete, the Mean Squared Value Error curve vs. episode number, summarizing how the value error changed over time.\n",
    "\n",
    "The code below runs one run of an experiment given env_info and agent_info dictionaries. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\shich\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:372: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "According to the global reward rate, the optimal give-up time in the pursuit port is at 2.0 second!\n",
      "According to the local reward rate, the optimal give-up time in the pursuit port is at 2.8 second!\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VNX9//HXZ5KQDdl3wiogoChgZBdEXMAF1IqCGyrWWrVa7fLTLtra72KrdanVCgi4VKVIRVFBpaLgwhZAZBFkUSDsiqBmQiaZnN8fmdB8aTATSObemXk/H488nLlz7+QDI++cnHPuOeacQ0REkkPA6wJERCR2FPoiIklEoS8ikkQU+iIiSUShLyKSRBT6IiJJRKEvIpJEFPoiIklEoS8ikkRSvS7gcE2aNHHt27f3ugwRkbiybNmyL51zTas6z3eh3759e/Ly8rwuQ0QkrpjZlmjOU/eOiEgSUeiLiCQRhb6ISBJR6IuIJBGFvohIEokq9M1suJmtN7ONZnZXJa/faWZrzewTM3vHzNpVeC1sZh9HvmbVZPEiIlI9VU7ZNLMU4HHgbCAfWGpms5xzayuctgLIdc4FzezHwJ+AyyOvFTrnetZw3SIichSiaen3ATY65zY750LANGBUxROcc+8654KRp4uAnJotM3kdCBbz4pKtFIdLvS5FRBJANKHfGthW4Xl+5NiRjAfmVHieYWZ5ZrbIzC6q7AIzuzFyTt7evXujKCl5vLV2F3e/vIpfzviE0lLtZywixyaaO3KtkmOVpo+ZXQXkAkMqHG7rnNthZh2BeWa2yjm36f+8mXMTgYkAubm5SrYKvjtYAsDMFdupn5nGvRd2x6yyj0REpGrRtPTzgTYVnucAOw4/yczOAn4NjHTOFZUfd87tiPx3M/Ae0OsY6k06hcVhAK7p346nP/qCR9/Z4HFFIhLPomnpLwU6m1kHYDswBrii4glm1guYAAx3zu2pcLwhEHTOFZlZE2AgZYO8EqWCohJSA8bvLjyRYCjMI//aQP3MNK4b2MHr0kQkDlUZ+s65EjO7FXgLSAGmOOfWmNl9QJ5zbhbwAFAXeCnS9bDVOTcS6AZMMLNSyn6ruP+wWT9ShWAoTFadFAIB4/5LevBNYTG/f20t9TPTuKS3xstFpHqiWmXTOTcbmH3YsXsqPD7rCNd9BPQ4lgKTXTBUQladso8pNSXAX8b24vqnl/KLGZ9QLyONs7o397hCEYknuiPX5wpCYbLSUw49z0hLYeI1uZzUqh43v7CchZu+8rA6EYk3Cn2fK4x071RUNz2Vp6/rQ9tGWfzw2TxW5R/wqDoRiTcKfZ8rKPp3905FDbPr8Nz4PtTPTGPc1CVs3POdB9WJSLxR6PtcYfF/tvTLtayfyd9v6EvA4OrJi9m2L1jpeSIi5RT6PldQVEJ2JS39ch2aZPPs9X0JhsKMnbSIHfsLY1idiMQbhb7PBSvp0z9c91b1eG58Hw4Eixk7aRG7vzkYo+pEJN4o9H0umtAHODmnAc+M78OX3xYxdtIi9n5bVOU1IpJ8FPo+FwyVkJUe1e0U9G7bkKev78PO/Qe58qlFfPWdgl9E/i+Fvo+FSkopDjuy0qpu6Zc7rX0jJl+by5avglz51GK+LgjVYoUiEm8U+j5WGCpbbC3aln65Acc3YdI1uWz+soCrpyzmQGFxbZQnInFIoe9jBaGyZZWzo+jTP9zgLk2ZcNWprN/1LddMWcK3BxX8IqLQ97VgpKWfeRShDzC0azMev6I3a7Yf4NqpSykoKqnJ8kQkDin0fSx4qKVfve6dis45sQWPje3Fx9v2c93TCn6RZKfQ97Hyln40Uza/z4geLXn48p7kfbGPcerqEUlqCn0fK2/pV3cgtzIjT2nFY2N78/G2/Vw9eYkGd0WSlELfx8pb+kczkFuZ809uyRNX9mbNjgNc9dRi9gc1nVMk2Sj0fSxYdGwDuZU558QWTLw6l/W7v2XspMW6gUskySj0faygBgZyKzO0azMmj8tl897vGDtpEXu+1Vo9IslCoe9jxzpl8/uc3rkpU687jW37ChkzcRG7Dij4RZKBQt/HgqESUgJGemrtfEwDjm/Cs+P7sOebIi6fuJDtWpZZJOEp9H2sfIVNM6u173Fa+0Y8N74P+wpCXD5hoTZiEUlwCn0fCxZFt6zyserVtiEv3NCPbw+WcNmEhdp6USSBKfR9rCD0/btm1aQeOfWZdmM/isOOyyYs1GbrIglKoe9jhaFwrQziHkm3lvWYcVN/suqkMHbSIhZu+ipm31tEYkOh72OxbOmXa98kmxk3DaBl/QzGTV3C22t2xfT7i0jtUuj7WGEoTFZ67Fr65VrUz2D6j/rTrWU9fvz8cmYsy495DSJSOxT6PlYQ5f64taFhdh1euKEv/Ts25ucvrWTyB597UoeI1CyFvo8Fi0rIinH3TkXZ6alMvjaXESe14A+vr+XPb6/HOedZPSJy7BT6PhYs9q6lXy49NYW/XtGby3Pb8Ni8jfz21dWUlir4ReKVd81IqVLZPH3vP6KUgHH/D3rQIDuNCfM3c6CwhAdHn0x6qrc/kESk+qJq6ZvZcDNbb2YbzeyuSl6/08zWmtknZvaOmbWr8No4M9sQ+RpXk8UnsuJwKaFwaY0tq3yszIy7R3TjrhFdeW3lDq6dslRr8ovEoSpD38xSgMeBEUB3YKyZdT/stBVArnPuZGAG8KfItY2Ae4G+QB/gXjNrWHPlJ67aXGztWNw05HgevvwU8rbsY/STH7FD6/WIxJVoWvp9gI3Ouc3OuRAwDRhV8QTn3LvOufJFWxYBOZHH5wJznXP7nHNfA3OB4TVTemI7tD9uDeyaVdMu7pXD09f1Yef+g1z8xId8uvMbr0sSkShFE/qtgW0VnudHjh3JeGDOUV4rETW1P25tGdipCdNv6o9hjH5yIR9s+NLrkkQkCtGEfmVLPFY6fcPMrgJygQeqc62Z3WhmeWaWt3fv3ihKSnzlu2b5YSD3SLq1rMfMWwbQukEm105dwsvLdROXiN9FE/r5QJsKz3OAHYefZGZnAb8GRjrniqpzrXNuonMu1zmX27Rp02hrT2iHNkX3aUu/XMv6mUy/qT+ntW/EndNX8vi7GzWXX8THogn9pUBnM+tgZnWAMcCsiieYWS9gAmWBv6fCS28B55hZw8gA7jmRY1IFv3fvVFQ/M42nrz+Ni3q24oG31vOrmaspCZd6XZaIVKLKvgPnXImZ3UpZWKcAU5xza8zsPiDPOTeLsu6cusBLkQ0/tjrnRjrn9pnZHyj7wQFwn3NuX638SRJMeej7cSC3MumpKTx0WU9aNsjkb+9tYteBQh67ojd146R+kWQR1b9I59xsYPZhx+6p8Pis77l2CjDlaAtMVuWbomem+b+lXy4QMP7f8K60apDJ72at4dK/fcRT43LJaZjldWkiEqFlGHwqWOTfKZtVubpfO6Zeexrb9xdy0eMfsmyLfrkT8QuFvk8Fi+OnT78yg7s0ZebNA8lOT2XsxMXMXKGZPSJ+oND3qWBRmIBBemr8fkSdmtXllZsH0qttA+74x0oeeGudFmsT8Vj8JkqCC4bCZNdJJTIwHrcaZtfhufF9uTy3DY+/u4lbXlh+aDqqiMSeQt+ngqES3627c7TqpAa4/wc9+M353XhzzS4um7CQXQcOel2WSFJS6PtUQSgcl4O4R2Jm3HB6R566JpfP9xYw8q8f8En+fq/LEkk6Cn2fKgyVxNV0zWgN69acf948gLSUAKOfXKilG0RiTKHvUwVFYbI92BQ9Frq2qMertw6kZ5sG3Dl9Jb9/bQ3FuoNXJCYU+j5VtlVi4nTvHK5J3XT+fkNfrhvYnqkffsFVTy3my++Kqr5QRI6JQt+nyjZFT8yWfrm0lAD3XngiD112Ch9v28/Ix9TPL1LbFPo+FQwldku/okt65/DPHw/AzLj0yYXMWKZ+fpHaotD3qWAo8Vv6FZ3Uuj6zbh1IbruG/Pylldz76mr184vUAoW+TxWEwmQl6EDukTSum86z1/fhhkEdeGbhFq6ctJi936qfX6QmKfR9qCRcSqiklOwk6d6pKDUlwG8u6M6jY3ryyfb9XPDY+yz5XAu2idQUhb4PxftiazVhVM/WvPzjgWSmpTB20iKenL9J6/aI1ACFvg/Fw/64sdC9VT1e+8kgzj2xOffPWceNz+WxPxjyuiyRuKbQ96F42R83Fo7LSOPxK3rzuwu7M/+zvZz/lw/4eJumdYocLYW+D8XT/rixYGZcO7AD03/UH4DRT37E0x9+rg3YRY6CQt+H4m1/3Fjp1bYhb9w2iMGdm/K719Zy6wsr+PZgsddlicQVhb4PHdofVy39/9Agqw6TrsnlrhFdeXPNLi587APW7DjgdVkicUOh70OF5S39JB/IPZJAwLhpyPG8cENfgqEwFz/xEVPV3SMSFYW+DxUUaSA3Gn07NmbO7aczqFMTfv/aWsY/k8dXWrRN5Hsp9H1IA7nRa1w3ncnjcvndhd35YOOXDH/0fd7fsNfrskR8S6HvQxrIrZ7y2T2v3jKQ+plpXD15Cf8z+1NCJVq7R+RwCn0fCoZKMIP0VH081dGtZT1eu3UQV/Zty8QFm/nB3z7i8y8LvC5LxFeUKj4UDIXJrpOKmXldStzJrJPCf1/cgyevOpVtXwc5/y/v81LeNg3yikQo9H0oGCrRdM1jNPykFsy5/XR6tK7PL2Z8wq0vruDrAi3hIKLQ96GCojDZCv1j1rJ+Ji/8sB+/OPcE3lq9i3MfWcC76/d4XZaIpxT6PhQMhcnUHP0akRIwbhnaiVduGUjDrDpcN3Upd7+86tC0WJFko9D3oWCoRC39GnZS6/q8eutAfjS4I9OWbmXEo++z9Aut0y/JR6HvQ8FQmCxN16xxGWkp3H1eN/5xY38cjssmLOR/53xKUUnY69JEYiaq0Dez4Wa23sw2mtldlbw+2MyWm1mJmV162GthM/s48jWrpgpPZMFQCVlpaunXlj4dGjHn9sGMOa0tE+ZvZuRjH2r9HkkaVYa+maUAjwMjgO7AWDPrfthpW4FrgRcqeYtC51zPyNfIY6w3KRQUJd/+uLFWNz2V/72kB1OvPY19wRAXPf4hj72zQZuxS8KLpqXfB9jonNvsnAsB04BRFU9wzn3hnPsE0L+YGlBYHNYSDDEytGsz3v7pYM49sQV/nvsZo/76Iau3q9UviSua0G8NbKvwPD9yLFoZZpZnZovM7KLKTjCzGyPn5O3dq3VTCopKtMJmDDXMrsNfr+jNhKtPZe93RYx6/EP+9OY6Dharr18STzShX9ltodW5vbGtcy4XuAJ4xMyO/483c26icy7XOZfbtGnTarx14gmXOopKSpN+f1wvnHtiC/51xxAu6dWaJ97bxPl/eZ9lWzTDRxJLNKGfD7Sp8DwH2BHtN3DO7Yj8dzPwHtCrGvUlHe2P6636WWk8MPoUnr2+DweLS7n0yYX8/rU1hz4XkXgXTegvBTqbWQczqwOMAaKahWNmDc0sPfK4CTAQWHu0xSaDQ8sqayDXU4O7NOWtOwZzdb92TP3wC859ZAEfbvzS67JEjlmVoe+cKwFuBd4CPgWmO+fWmNl9ZjYSwMxOM7N8YDQwwczWRC7vBuSZ2UrgXeB+55xC/3toLX3/qJueyn2jTmL6j/qTGghw5VOL+eWMlVrDR+JaVB3HzrnZwOzDjt1T4fFSyrp9Dr/uI6DHMdaYVP69a5b69P2ibF7/6Tzyrw1Men8z//p0D78+rxuX9G6tlVAl7uiOXJ8pLNb+uH6UkZbCXSO68sZtg2jfOIufvbSSsZMWsXHPd16XJlItCn2fKW/pa2llf+raoh4zbhrA/1zcg7U7vuG8R9/nobfXa3qnxA2Fvs/8e6tEhb5fBQLGFX3b8s7PzmBEjxb8Zd5Ghj+ygA82aKBX/E+h7zOHBnLT1L3jd02PS+fRMb14bnwfAK6avJifTlvB3m+LPK5M5MgU+j5zaJ6+Wvpx4/TOTXnzp4O57cxOvLFqJ2c++B5Pvb9Z6/iILyn0feZQ944GcuNKRloKd55zAm/+dDC92zXkv974lPMefZ+PNLdffEah7zPBohLMICNNH008Or5pXZ6+7jQmXZPLwZIwVzy1mJufX8b2/YVelyYCKPR9JxgKk5WWovnfcczMOLt7c+beMYQ7z+7CO5/uYdif3+Ov8zZolo94TqHvMwXaHzdhZKSlcNuwzrzzsyEMPaEZD779Gec8vIB3Pt3tdWmSxBT6PhMMlWi6ZoLJaZjF3646lb+P70ud1ADjn8njmilLWL/rW69LkySk0PeZYCisJRgS1KDOTZhz++n89oLufLz1a0Y8uoBfzVylKZ4SUwp9nwmGSrTYWgJLSwkwflAH5v9iKOMGtGf60m0MffA9nnhvo/r7JSYU+j5T1tJX6Ce6htl1uPfCE3n7jsH0P74xf3pzPcP+PJ9XP96Oc9XZo0ikehT6PhMsUugnk45N6zLpmlxe+GFf6memcfu0j7n4iY+0Y5fUGoW+zxSEtD9uMhpwfBNe+8kgHrj0ZHbsL+QHf1vIj/++jE17tYqn1Cyli88UhsJagiFJpQSM0bltOK9HSya9v5lJCzbz9trdXJabw+3DutCifobXJUoCUEvfZwpCJZq9k+Sy01P56VldmP/LoVzdrx0zluUz5IF3uX/OOg4Ei70uT+KcQt9HwqWOg8Wl6tMXAJrUTed3I09k3s/O4LweLZmwYBOn/2keT87fpJk+ctQU+j5SvmuWQl8qatMoi4cv78ns207n1HYNuX/OOoY88C4vLtlKiVbylGpS6PtIUPvjyvfo1rIeU6/rwz9u7EfrBpnc/fIqznpoPjNX5BMu1TRPiY5C30cObaCilr58j74dG/PPHw/gqWtyyaqTyh3/WMnZD89n1sodlCr8pQoKfR8pCKmlL9ExM87q3pzXfzKIJ6/qTVogwG0vrmD4owuYvWqnwl+OSKHvI4XaH1eqKRAwhp/Ukjm3n85jY3sRLnXc/Pxyzn/sA95es0t398p/UOj7SIG6d+QoBQLGhae04u07hvDI5T05WBzmxueWMfKvHzJ37W6Fvxyi0PcRDeTKsUoJGBf1as3cOwbz4OhTOFBYzA+fzWPEo+/z2sodGvAVhb6faCBXakpqSoBLT81h3s+G8PDlp1AcLuUnL67g7IfmM2NZvjZtT2IKfR8JaiBXalhqSoCLe+Uw944hPHFlb9LTUvj5SysZ+uB7PL94C0Uluskr2Sj0fSSogVypJYGAcV6Plsy+bRCTx+XSpG46v565msF/epfJH3x+qMEhiU+h7yPlA7kZqQp9qR1mxrBuzZl58wCev6EvHZpk84fX1zLg/nk89PZ6vvxOu3glOvUj+EhhZNesQMC8LkUSnJkxsFMTBnZqwrIt+5gwfzOPvbuRCQs2Mzo3hxsGdaR9k2yvy5RaEFVL38yGm9l6M9toZndV8vpgM1tuZiVmdulhr40zsw2Rr3E1VXgiKtCuWeKBU9s1YuI1ucy9YwgX92rN9KX5nPnn97jl+eWs3Lbf6/KkhlXZ0jezFOBx4GwgH1hqZrOcc2srnLYVuBb4+WHXNgLuBXIBByyLXPt1zZSfWIJFWlZZvNOpWV3u/8HJ3Hl2F6Z+9AV/X7SFN1btpF/HRvxoyPGc0aUpZvotNN5F09LvA2x0zm12zoWAacCoiic4575wzn0CHD4P7FxgrnNuXyTo5wLDa6DuhKT9ccUPmtXL4P8N78pHd53Jr8/rxhdfBrlu6lLOfngBzy/ecujOcYlP0YR+a2Bbhef5kWPROJZrk45CX/zkuIw0fji4Iwt+OZSHLjuFjLQAv565mv73v8Mf31zHzgOFXpcoRyGavoTKfp+L9ra+qK41sxuBGwHatm0b5VsnnmCohOx0de+Iv9RJDXBJ7xwu7tWavC1fM+WDz5kwfxMTF2zmvB4tuX5ge3q1beh1mRKlaBImH2hT4XkOsCPK988Hzjjs2vcOP8k5NxGYCJCbm5u094kHQ2Ga1E33ugyRSpkZp7VvxGntG7FtX5BnF37BtCXbeG3lDnq3bcB1Azsw/KQWpKVoJrifRfPpLAU6m1kHM6sDjAFmRfn+bwHnmFlDM2sInBM5JpUoUEtf4kSbRln8+vzuLPzVMH4/8kT2FYT4yYsrGHj/PB6e+xm7vznodYlyBFWGvnOuBLiVsrD+FJjunFtjZveZ2UgAMzvNzPKB0cAEM1sTuXYf8AfKfnAsBe6LHJNKFKpPX+JM3fRUxg1oz7yfncGUa3Pp3qoef5m3gQH3z+Pm55excNNXWuHTZ6JqVjrnZgOzDzt2T4XHSynruqns2inAlGOoMWkUFCn0JT4FAsaZXZtzZtfmbPmqgOcXb2V63jZmr9pF52Z1ubp/Oy7u1ZrjMtK8LjXpqfPNJ0pLHYXFYc3Tl7jXrnE2vzqvG4vuHsafLj2ZjLQU7nl1Df3+5x1+88oq1uw44HWJSU0J4xOFxVpWWRJLRloKl+W24bLcNny8bT/PLdzC9Lx8/r5oK6fk1GdMn7ZceEor6mocK6b0t+0Th/bH1T8ASUA92zSgZ5sG/PaCbry8fDvTlm7l7pdX8V+vr2Vkz1aM7dOWHq3r647fGFDC+MSh/XHV0pcE1iCrDtcP6sB1A9uzfOt+XlyylZkrtvPikm10b1mPsX3bMqpnK+qp77/WKPR9oqBI3TuSPMyMU9s15NR2Dbnnwu68Ggn+376ymv9+Yy0jTmrJ6FNz6NexsVadrWEKfZ8oLNauWZKc6mWkcXX/9lzVrx2rth9g2tKyG75mrthO6waZ/ODUHC7tnUPbxllel5oQlDA+oZa+JDsz4+ScBpyc04B7LujOW2t2MWNZPo/N28Bf3tlA3w6NGJ3bhhEntdBNjMdAf3M+of1xRf4tIy2FUT1bM6pna3bsL2Tmiu28lLeNn7+0kntfXc2IHi25pFdr+nZsTIq6f6pFCeMT2h9XpHKtGmRyy9BO3HzG8Szb8jUzluXz+ic7mbEsnxb1MhjZsxUX9WxNt5bHafZPFBT6PlG+P26mundEKmVm5LZvRG77Rvxu5In869PdvLJiO1M++JyJCzZzQvPjuKhXa0b1bEWrBplel+tbCn2fKIx072Sre0ekShlpKVxwcisuOLkV+wpCvLFqJ6+s2M4f31zHH99cR98OjbioV2tGnNSCBll1vC7XV5QwPlE+kJuZppa+SHU0yq7D1f3acXW/dmz5qoBXP97BKyu2c/fLq/jtK6sZ1LkJF57cirNPbK75/yj0faOwOExmWormJIscg3aNs7ltWGd+cmYnVm//htc/2cHrn+zkZy+tpM7LAYac0JQLT2nFWd2aJe2kieT8U/tQQVGJpmuK1BAzo0dOfXrk1OeuEV1ZsW0/r63cwexVO5m7djcZaQGGdW3OBSe35IwTmiXVWJpC3yeCoTBZmrkjUuPMjN5tG9K7bUN+e353ln6xj9c/2cmc1Tt5Y9VOMtICnNGlGSN6tODMrs0Sfvlnhb5PBEMlGsQVqWWBgNG3Y2P6dmzMvRd2Z8kX+5izahdvrdnFm2t2USclwKDOTRh+UgvO6d48IQeBlTI+EQyFk+pXTBGvpaYEGHB8EwYc34TfjzyR5Vu/Zs7qXby5ehfz1u3h7oDRv2Njhp/UgrO7N6d5vQyvS64RCn2fCIbCaumLeCQQ+Pc9AL85vxurth849APgN6+s5jevrOaUnPqc3b05Z3dvQZfmdeP2RjCljE8UFJXQKDvxfpUUiTcV1wD65bknsGHPd8xdu5u31+7mwbc/48G3P6Nto6zID4Dm5LZrSGpK/GxCqND3ibKWvrp3RPzEzOjS/Di6ND+OW4Z2Yvc3B/nXp7uZu3Y3zy3cwuQPPqdBVhpnntCMoV2bMbhLU+pn+nsgWKHvE2Wzd/RxiPhZ83oZXNm3HVf2bcd3RSUs+Gwvc9fuZt76Pby8YjspgbJ9AoZ1bcaZXZvRqZn/uoGUMj4RDJWQpbtxReJG3fRUzuvRkvN6tCRc6lix9WvmrdvDvHV7+N856/jfOevIaZjJmV3Lfgvo37ExGT74N67Q94HSUkdhsVr6IvEqpcJA8C+Hd2XH/kLeXb+Hd9ftYXreNp5duIWMtAB9OzRmSJemDO7SlOObZnvyW4BSxgcOloRxThuoiCSKVg0yD3UDHSwOs3DzV8xfv5cFn+3lvtfXAtC6QSaDuzRlSJcmDOjUJGbrAin0faB8sTUN5Ioknoy0FIae0IyhJzQDYNu+IPM/K/sB8NrKHby4ZCspAaN32wac070FPxzcsVbrUej7QGGofKtEfRwiia5Noyyu6teOq/q1ozhcyvItX7Ngw17mf7aXDzd9qdBPBgWHtkpUS18kmaSlBA4tC/GLc7tSHC6t9e8ZP3cUJLDyrRI1kCuS3NJicJOXQt8Hgmrpi0iMKPR9oHwgV6EvIrVNoe8DhcXaH1dEYiOq0Dez4Wa23sw2mtldlbyebmb/iLy+2MzaR463N7NCM/s48vVkzZafGNTSF5FYqbJpaWYpwOPA2UA+sNTMZjnn1lY4bTzwtXOuk5mNAf4IXB55bZNzrmcN151QCjWQKyIxEk1Lvw+w0Tm32TkXAqYBow47ZxTwTOTxDGCY+W2VIR8rn7KZ6YN1OUQksUUT+q2BbRWe50eOVXqOc64EOAA0jrzWwcxWmNl8Mzu9sm9gZjeaWZ6Z5e3du7daf4BEUBgKk5EWICWgn5MiUruiCf3KkshFec5OoK1zrhdwJ/CCmdX7jxOdm+icy3XO5TZt2jSKkhJLgfbHFZEYiSb084E2FZ7nADuOdI6ZpQL1gX3OuSLn3FcAzrllwCagy7EWnWiCRdofV0RiI5rQXwp0NrMOZlYHGAPMOuycWcC4yONLgXnOOWdmTSMDwZhZR6AzsLlmSk8c2h9XRGKlyqRxzpWY2a3AW0AKMMU5t8bM7gPynHOzgMnAc2a2EdhH2Q8GgMHAfWZWAoSBm5xz+2rjDxLPCkIlaumLSExE1bx0zs0GZh927J4Kjw8Coyu57p/AP4+xxoRXGAqTna7QF5HapztyfaAgFCYzTd07IlL7FPo+EAyVqKUvIjGh0PeBYCisDVREJCYU+j4QLCrRujsiEhMKfY+1Ag9jAAAGkUlEQVQ55wgWh7U/rojEhELfYweLS3EOMtW9IyIxoND3WPliaxrIFZFYUOh77NCyymrpi0gMKPQ9VqD9cUUkhhT6HguGtGuWiMSOQt9jwSJ174hI7Cj0PabuHRGJJYW+x8oHcrO1P66IxIBC32Nq6YtILCn0PVaogVwRiSGFvscKNJArIjGk0PdYsLiE9NQAKYHK9pYXEalZCn2PBYvCGsQVkZhR6HusIFRCZpr680UkNhT6HtP+uCISSwp9jxWEwlpWWURiRqHvscJQiTZQEZGYUeh7rKBI++OKSOwo9D0WDGl/XBGJHYW+x4IayBWRGFLoeywYCpOZpu4dEYkNhb6HnHMEQyVq6YtIzCj0PVRUUkqp07o7IhI7Cn0PFRRpWWURiS2Fvoe0P66IxFpUoW9mw81svZltNLO7Knk93cz+EXl9sZm1r/Da3ZHj683s3JorPf79O/TVvSMisVFl6JtZCvA4MALoDow1s+6HnTYe+No51wl4GPhj5NruwBjgRGA48ETk/YSyOfoAWRrIFZEYiaal3wfY6Jzb7JwLAdOAUYedMwp4JvJ4BjDMzCxyfJpzrsg59zmwMfJ+QoWWvlbZFJEYiaZfoTWwrcLzfKDvkc5xzpWY2QGgceT4osOubX3U1X6P/cEQo59cWBtvXWv+PZCr7h0RiY1o0qayLZ1clOdEcy1mdiNwI0Dbtm2jKOk/BQJG5+Z1j+paLw3JTKNLi/irW0TiUzShnw+0qfA8B9hxhHPyzSwVqA/si/JanHMTgYkAubm5//FDIRr1MtJ44spTj+ZSEZGkEU2f/lKgs5l1MLM6lA3MzjrsnFnAuMjjS4F5zjkXOT4mMrunA9AZWFIzpYuISHVV2dKP9NHfCrwFpABTnHNrzOw+IM85NwuYDDxnZhspa+GPiVy7xsymA2uBEuAW51y4lv4sIiJSBStrkPtHbm6uy8vL87oMEZG4YmbLnHO5VZ2nO3JFRJKIQl9EJIko9EVEkohCX0QkiSj0RUSSiO9m75jZXmDLMbxFE+DLGionFuKtXlDNsRJvNcdbvZBYNbdzzjWt6mLfhf6xMrO8aKYt+UW81QuqOVbireZ4qxeSs2Z174iIJBGFvohIEknE0J/odQHVFG/1gmqOlXirOd7qhSSsOeH69EVE5MgSsaUvIiJHkDChX9Xm7X5jZlPMbI+Zrfa6lmiZWRsze9fMPjWzNWZ2u9c1VcXMMsxsiZmtjNT8e69rioaZpZjZCjN73etaomFmX5jZKjP72MziYsVEM2tgZjPMbF3k/+n+Xtf0fczshMjfb/nXN2b202q/TyJ070Q2W/8MOJuyjVuWAmOdc2s9Lex7mNlg4DvgWefcSV7XEw0zawm0dM4tN7PjgGXART7/ezYg2zn3nZmlAR8AtzvnFlVxqafM7E4gF6jnnLvA63qqYmZfALnOubiZ825mzwDvO+eeiuwVkuWc2+91XdGIZN52oK9zrlr3NSVKSz+azdt9xTm3gLK9B+KGc26nc2555PG3wKfU0p7HNcWV+S7yNC3y5euWjpnlAOcDT3ldS6Iys3rAYMr2AsE5F4qXwI8YBmyqbuBD4oR+ZZu3+zqM4p2ZtQd6AYu9raRqka6Sj4E9wFznnN9rfgT4JVDqdSHV4IC3zWxZZM9rv+sI7AWmRrrRnjKzbK+LqoYxwItHc2GihH5UG7BLzTCzusA/gZ86577xup6qOOfCzrmelO3R3MfMfNudZmYXAHucc8u8rqWaBjrnegMjgFsi3Zd+lgr0Bv7mnOsFFAC+HwsEiHRFjQReOprrEyX0o9qAXY5dpF/8n8DzzrmXva6nOiK/vr8HDPe4lO8zEBgZ6SOfBpxpZn/3tqSqOed2RP67B5hJWZern+UD+RV+65tB2Q+BeDACWO6c2300FydK6Eezebsco8ig6GTgU+fcQ17XEw0za2pmDSKPM4GzgHXeVnVkzrm7nXM5zrn2lP1/PM85d5XHZX0vM8uODOwT6SI5B/D1rDTn3C5gm5mdEDk0jLK9vOPBWI6yawei2Bg9Hhxp83aPy/peZvYicAbQxMzygXudc5O9rapKA4GrgVWRPnKAXznnZntYU1VaAs9EZjsEgOnOubiYBhlHmgMzy9oEpAIvOOfe9LakqPwEeD7SUNwMXOdxPVUysyzKZin+6KjfIxGmbIqISHQSpXtHRESioNAXEUkiCn0RkSSi0BcRSSIKfRGRJKLQFxFJIgp9EZEkotAXEUki/x+eLe0EYNaOiAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl0VeW9xvHvj0CYJyEMEsIgAWUUCESxtY4Vq4IDKDhUqohYwene9tra1qG3rVbr1KItjqgoiFOxF0WtotYBSBiEMIQQgYRBwjyT6Xf/IHalMZCT5CT7nJPns5bL7HPenDwLkoed9+z9vubuiIhIbKkXdAAREQk/lbuISAxSuYuIxCCVu4hIDFK5i4jEIJW7iEgMUrmLiMQglbuISAxSuYuIxKD6QX3htm3beteuXYP68iIiUSk9PX2buydUNC6wcu/atStpaWlBfXkRkahkZutDGadpGRGRGKRyFxGJQSp3EZEYpHIXEYlBKncRkRikchcRiUEqdxGRGKRyFxGpJRu2H+CR9zNZ883eGv9agd3EJCJSF+w5VMCcrzbz+qJcFq7biRm0bd6Q5PbNa/TrqtxFRMKsqNj5dE0ebyzayNyMLRwuLKZ7QlN+dl4vLhnYieNbNa7xDCp3EZEwWfPNXl5Lz+XNxRvZuvcwLRs34PKUzlw2OJEBiS0xs1rLonIXEamGfYcL+cfSTcxMy2Hxhl3Ur2ec0SuBywYlctZJ7WhYPy6QXCp3EZFKcncW5+xi5oIc3v5qEwfyi0hu14xfXXASFw/sRNtmDYOOqHIXEQnVjv35vLEol5kLc1izdR+NG8Rx0YCOXDEkiUFJrWp12qUiKncRkWNwd77I3s70+Rt4L2MLBUXOyZ1b8YdL+3Fh/440b9Qg6IjlUrmLiJRj98ECXk/PZfr89azN20/Lxg24KrULY4Z25sQOLYKOVyGVu4hIKUtzdjF9/npmL93EoYJiTu7ciodGD+DC/h1p1CCYN0erQuUuInXewfwiZi/dyEtfbmDZxt00bhDHJQM7cVVqF/p2ahl0vCpRuYtInZWz4wAvfrmeGQs2sOdQIT3bN+O+kX24eGAnWkToXHqoQip3MxsOPAbEAU+7+/1lnn8EOLPksAnQzt1bhTOoiEg4fPsG6fOfreODld9gZpzftwM/PrUrQ7q2jqgrXqqjwnI3szhgCnAukAssNLPZ7r7i2zHufnup8ZOBgTWQVUSkyg7mF/H3JRt5/vN1rNqyl9ZNGnDTGSdw9Sld6Niy5pcDqG2hnLkPBbLcPRvAzGYAI4EVRxk/Frg7PPFERKpn466DvPjFemYs3MCuAwWc1LEFfxzVnxEDjo+qN0grK5Ry7wTklDrOBVLLG2hmXYBuwIdHeX4CMAEgKSmpUkFFRCpj+cbdPPVpNv/4ajPuznl9OjBuWFeGdjsuZqZejiWUci/vT8GPMnYM8Jq7F5X3pLtPBaYCpKSkHO01RESqpLjY+Tgzj6mfZPNF9naaNazPdad15dphXUls3SToeLUqlHLPBTqXOk4ENh1l7Bjg5uqGEhGpjMOFRfx98Sae+jSbNVv30aFFI375oxMZMzQp6q96qapQyn0hkGxm3YCNHCnwK8sOMrNeQGvgi7AmFBE5il0H8pk+fwPPf76OvL2HOaljCx694mQu6N+RBnF1e6O5Csvd3QvNbBIwlyOXQj7r7hlmdh+Q5u6zS4aOBWa4u6ZbRKRGfbPnEE9/ms30+Rs4kF/ED3omMOGK7gw7oU2dmE8PRUjXubv7HGBOmcd+U+b4nvDFEhH5rg3bD/DXT9byWlouRe5c1L8jE884ISrWeqltukNVRCLeqi17eHLeWt5euon69eoxOiWRG08/gaQ2detN0spQuYtIxFq0YSdPfLSWD1Z+Q9P4OMZ/vzvjv9eNdi0aBR0t4qncRSTiLPh6B4/9M5PPsrbTqkkDbj+nJ9cO60KrJvFBR4saKncRiRilS71ts4bc9aOTuDI1iaYNVVWVpT8xEQlc2VL/1QUncVVqFxrHx+7yADVN5S4igVGp1xyVu4jUurR1O3jkA5V6TVK5i0itydi0m4fmruaj1Xkq9RqmcheRGvf1tv08/H4mby/dRMvGDbjz/BO59tSuKvUapHIXkRqzefdBHv/nGl5Ny6Vh/XpMOrMHN5zenZaN6+ZiXrVJ5S4iYbd932GenLeWF75cDw7XnNKFm8/sQULzhkFHqzNU7iISNgfyC3nqk6+Z+slaDhYUcdmgRG49J7nOraUeCVTuIlJtRcXOrLQcHn4/k617DzO8Twf++7ye9GjXPOhodZbKXUSqzN2Zl5nH/XNWsfqbvQxKasWTVw9icJfjgo5W56ncRaRKMjbt5vdzVvJZ1na6tGnCE1cN4vy+HbSeeoRQuYtIpWzadZCH3lvNm4s30rJxA35zYW+uPqUL8fXr9s5HkUblLiIh2X+4kCfnreWpT7Nxhwnf785Pz+yhyxojVEjlbmbDgcc4ss3e0+5+fzljLgfuARxY6u7f2WdVRKKPu/P3JZv4wzsr+WbPYUYMOJ6fndeLzsfpCphIVmG5m1kcMAU4F8gFFprZbHdfUWpMMvAL4DR332lm7WoqsIjUnmW5u7nn7QzS1++kX6eWTLlyECld9WZpNAjlzH0okOXu2QBmNgMYCawoNeYGYIq77wRw963hDioitSdv72EemruaV9NzaNM0ngcu68fowZ2pV09vlkaLUMq9E5BT6jgXSC0zpieAmX3Gkambe9z93bIvZGYTgAkASUlJVckrIjUov7CYaZ+v4/F/ruFgQRHjv9eNyWcn06KR5tWjTSjlXt4/1V7O6yQDZwCJwKdm1tfdd/3HJ7lPBaYCpKSklH0NEQnQx5l53Pt2Btl5+zmjVwK/vrA3JyQ0CzqWVFEo5Z4LdC51nAhsKmfMl+5eAHxtZqs5UvYLw5JSRGrMlt2HuO8fGcxZtoVubZvy3LghnHmi3jaLdqGU+0Ig2cy6ARuBMUDZK2HeAsYCz5tZW45M02SHM6iIhFdhUTHPf76OR97PpLDY+a9zezLhB91pWF/L8MaCCsvd3QvNbBIwlyPz6c+6e4aZ3Qekufvskud+aGYrgCLgZ+6+vSaDi0jVpa3bwa/eWs6qLXs5s1cC947oS1IbXdoYS8w9mKnvlJQUT0tLC+Rri9RVO/bn88A7q5iZlkPHlo24+6I+nNenvZYMiCJmlu7uKRWN0x2qInVAcbEzKz2H+99Zxd5Dhdx4enduOTuZpg1VAbFKf7MiMS5r6z5+8cZXLFy3k6Fdj+O3F/elVwctxRvrVO4iMSq/sJi/fbyWP3+YReP4OP44qj+jBydqCqaOULmLxKAlObu48/WvWLVlLxf278jdF/XRFnd1jMpdJIYcyC/kobmZPPf517Rv3oinfpzCub3bBx1LAqByF4kRH2fmcdeby8jdeZCrT0ni58NP1LIBdZjKXSTK7dyfz2//sYI3Fm+ke0JTZk08lSFaubHOU7mLRLH3V3zDL95Yxq4D+Uw+qwc3n9mDRg10h6mo3EWi0u6DBdz7dgZvLNrIiR2aM+26IfQ5vmXQsSSCqNxFosy81Vu58/Vl5O07zOSzejD5rGTtXyrfoXIXiRL7Dhfyu/9bwSsLcujRrhl/u2YwAzq3CjqWRCiVu0gU+DxrGz977Ss27T7Ijad35/Zze2puXY5J5S4SwQ7kF3L/O6t44Yv1dGvblNcmnsrgLroSRiqmcheJUMtyd3PrjMVkb9vPuGFd+Z/hJ9I4XmfrEhqVu0iEKSp2/vbJWh5+L5O2zRry8vhUhvVoG3QsiTIqd5EIsnHXQe6YuYT5X+/ggn4d+d0lfWnVJD7oWBKFVO4iEeLtpZv45ZvLKC52HhzVn1FawVGqIaSLY81suJmtNrMsM7uznOfHmVmemS0p+W98+KOKxKa9hwq449UlTH5lMSckNGPOrd9ndEpnFbtUS4Vn7mYWB0wBzgVygYVmNtvdV5QZOtPdJ9VARpGYlb5+B7fNXMLGnQe55exkJp/VgwZxuiFJqi+UaZmhQJa7ZwOY2QxgJFC23EUkREXFzhMfZfHIB5kc36oxr954Kila7EvCKJRy7wTklDrOBVLLGXeZmZ0OZAK3u3tOOWNE6ry8vYe5beZiPsvazogBx/O/l/TV0rwSdqGUe3kTf17m+G3gFXc/bGYTgWnAWd95IbMJwASApKSkSkYViX6fZ23jlhlL2HuogPsv7ccVQzS3LjUjlMm9XKBzqeNEYFPpAe6+3d0Plxw+BQwu74Xcfaq7p7h7SkJCQlXyikSlomLn4fczueqZ+bRsXJ+/TzqNMUOTVOxSY0I5c18IJJtZN2AjMAa4svQAM+vo7ptLDkcAK8OaUiSKbd1ziFtmLObL7B1cOqgTvx3Zl6YNdRWy1KwKv8PcvdDMJgFzgTjgWXfPMLP7gDR3nw3cYmYjgEJgBzCuBjOLRI1PMvO4feYSDuQX8eCo/oxO6VzxJ4mEgbmXnT6vHSkpKZ6WlhbI1xapaYVFxTz6wRqmzMsiuV0zplw5iOT2zYOOJTHAzNLdPaWicfrdUCTMtu07zOSXF/NF9nYuT0nk3hF9teCX1DqVu0gYLd6wk59OX8SO/fmahpFAqdxFwsDdeXnBBu6dvYJ2LRry+k3D6NtJe5pKcFTuItV0qKCIX7+1nFnpuZzeM4HHrjiZ1k21kqMES+UuUg05Ow5w0/R0lm/cwy1n9eDWc3oSV0/XrkvwVO4iVfRJZh63zFhMUbHz9I9TOKd3+6Ajifybyl2kkoqLnSc/XstD762mZ7vm/PWawXRr2zToWCL/QeUuUgkH8gv5r1eX8s7yLYwYcDz3X9aPJvH6MZLIo+9KkRBt3HWQG6alsWrLHu760UmM/343rQ0jEUvlLhKC9PU7uPHFdA4XFPPMuCGc2atd0JFEjknlLlKBWWk53PXmco5v1YgZE1Lo0U7LCEjkU7mLHEVRsfOHOSt5+l9f870ebfnLlQNp1UTXr0t0ULmLlGPPoQImv7yYjzPzGDesK7+64CTqa29TiSIqd5Eyvt62n+unLWTD9gP8/pJ+XJmqXcMk+qjcRUr5fO02Jr6YTlw946XxqZzSvU3QkUSqROUuUuL19FzufOMrurZpyrPjhtD5uCZBRxKpMpW71HnuzmP/XMOjH6xh2AltePLqwbRs3CDoWCLVEtI7RGY23MxWm1mWmd15jHGjzMzNrMJdQkQiQX5hMf81aymPfrCGUYMTef4nQ1XsEhMqPHM3szhgCnAukAssNLPZ7r6izLjmwC3A/JoIKhJuuw8WMPHFdL7I3s4d5/Zk8lk9dMepxIxQztyHAlnunu3u+cAMYGQ5434L/BE4FMZ8IjUiZ8cBLnvyc9LW7+CRKwZwy9nJKnaJKaGUeycgp9Rxbslj/2ZmA4HO7v6PMGYTqRFf5e7ikic+Z+ueQ7xwXSqXDEwMOpJI2IXyhmp5pzP+7yfN6gGPAOMqfCGzCcAEgKQkXTsste+9jC3cOmMJbZrFM2NCqpYSkJgVypl7LlB6l99EYFOp4+ZAX2Cema0DTgFml/emqrtPdfcUd09JSEioemqRKnjpy/Xc+FI6PTs0582fnqZil5gWypn7QiDZzLoBG4ExwJXfPunuu4G23x6b2Tzgv909LbxRRaqm9KWOZ5/Yjr9cOYjG8XFBxxKpURWWu7sXmtkkYC4QBzzr7hlmdh+Q5u6zazqkSFUVFTv3zM7gxS/Xc9mgRO6/rB8NtEaM1AEh3cTk7nOAOWUe+81Rxp5R/Vgi1Xe4sIg7Zi7l/5Zt5sYfdOfO4SfqihipM3SHqsSkfYcLufHFND7L2s5dPzqJG07vHnQkkVqlcpeYs23fYcY9t4CVm/fyp9EDuGywLnWUukflLjElZ8cBrnlmPlv2HOLpH6dw5onaDk/qJpW7xIyVm/dw7bMLOFxYzPTxpzC4S+ugI4kERuUuMSF9/U7GPbeApvH1mTXxVHq21zXsUrep3CXqfb52G+OnpdG+RSNeGp9Kp1aNg44kEjiVu0S1j1ZvZeKL6XRp04SXxqfSrnmjoCOJRASVu0Std5dvZvIri+nVoTkvXJfKcU3jg44kEjFU7hKV/r5kI3e8upQBiS15ThtsiHyHyl2izowFG/jFm8s4pVsbnr42haYN9W0sUpZ+KiSqPPfZ19z79grO6JXAX68eTKMGWgBMpDwqd4kaT8zL4o/vrua8Pu15fOxAGtZXsYscjcpdIp678/D7mfz5wyxGnnw8fxo9gPpa2VHkmFTuEtHcnQfnruaJeWu5IqUzv7+0H3H1tLKjSEVU7hKx3J0/vZfJE/PWMnZoEr+7uC/1VOwiIdHvthKRvp2K+ctHWYwd2lnFLlJJKneJSI9+sIY/f5jFFSmd+d3F/VTsIpWkcpeI8+gHmTz2zzWMHpzIHy5VsYtURUjlbmbDzWy1mWWZ2Z3lPD/RzJaZ2RIz+5eZ9Q5/VKkLHi/ZyHrU4EQeuKy/il2kiiosdzOLA6YA5wO9gbHllPfL7t7P3U8G/gg8HPakEvP+8uEaHn4/k0sHdVKxi1RTKGfuQ4Esd89293xgBjCy9AB331PqsCng4YsodcGUj7J46L1MLhnYiQdHDdDljiLVFMqlkJ2AnFLHuUBq2UFmdjNwBxAPnFXeC5nZBGACQFJSUmWzSox6ct5aHpy7mpEnH89Do1XsIuEQypl7eT9p3zkzd/cp7n4C8D/Ar8p7IXef6u4p7p6SkJBQuaQSk5777GseeHcVIwYcufNUxS4SHqGUey7QudRxIrDpGONnABdXJ5TUDa+m5XDv2ys4r097Hr5cSwqIhFMoP00LgWQz62Zm8cAYYHbpAWaWXOrwAmBN+CJKLPq/rzZz5+tf8f3ktjw+dqCKXSTMKpxzd/dCM5sEzAXigGfdPcPM7gPS3H02MMnMzgEKgJ3AtTUZWqLbR6u3ctvMxQxKas3frhms1R1FakBIa8u4+xxgTpnHflPq41vDnEti1JfZ25n4Yjq9OjTn2Z8MoUm8ljcSqQn6XVhqzdKcXYyflkbn45ow7SdDadFIW+OJ1BSVu9SK1Vv2cu1zC2jdtAEvXZ9Km2YNg44kEtNU7lLj1m3bz9XPzKdh/XpMv/4UOrRsFHQkkZincpcatXn3Qa56ej5Fxc5L16eS1KZJ0JFE6gSVu9SYHfvzuerp+ew5WMAL1w0luX3zoCOJ1Bm6VEFqxIH8Qq57fiEbdx7kxetT6dupZdCRROoUnblL2BUUFXPz9EV8lbuLx8cOZGi344KOJFLn6Mxdwsrd+eUby/hodR6/u6Qv5/XpEHQkkTpJZ+4SVn96L5NZ6bncenYyV6V2CTqOSJ2lcpeweeGLdf/e0Pq2c5IrHC8iNUflLmExZ9lm7p6dwTkntee3I/tipqV7RYKkcpdq+zJ7O7fNWMKgpNb8WSs8ikQE/RRKtazasocbXkgjqU0Tnrk2hcbxWuFRJBKo3KXKNu46yLXPLqBJfBzTrhtKqybxQUcSkRK6FFKqZPeBAq59dgEH8ouYNfFUOrVqHHQkESlFZ+5SafmFxdw0PZ312/cz9ZoUTuzQIuhIIlKGztylUtydX7+1nM/XbudPowdw6gltgo4kIuUI6czdzIab2WozyzKzO8t5/g4zW2FmX5nZP81Md6/EqL9+nM3MtBwmn9WDywYnBh1HRI6iwnI3szhgCnA+0BsYa2a9ywxbDKS4e3/gNeCP4Q4qwZuzbDMPvLuKiwYczx3n9gw6jogcQyhn7kOBLHfPdvd8YAYwsvQAd//I3Q+UHH4J6JQuxizesJPbZy5hcJfWPDiqv25SEolwoZR7JyCn1HFuyWNHcz3wTnlPmNkEM0szs7S8vLzQU0qgcnYc4IYX0mjXoiFTrxlMowa6ll0k0oVS7uWdonm5A82uBlKAB8t73t2nunuKu6ckJCSEnlICs+dQAddPW8jhwmKeGzdEe5+KRIlQrpbJBTqXOk4ENpUdZGbnAHcBP3D3w+GJJ0H6dl327Lz9vHDdUHq0005KItEilDP3hUCymXUzs3hgDDC79AAzGwj8DRjh7lvDH1Nqm7tz9+wMPl2zjd9f0o9hPdoGHUlEKqHCcnf3QmASMBdYCbzq7hlmdp+ZjSgZ9iDQDJhlZkvMbPZRXk6ixNOffs3L8zdw0xkncPmQzhV/gohElJBuYnL3OcCcMo/9ptTH54Q5lwToo1Vb+f07K/lRvw787Ie9go4jIlWg5QfkP6zN28ctryymd8cW/Gn0ydSrp0seRaKRyl3+bffBAm6YlkZ8/XpM/bGW7xWJZlpbRgAoKnZum7GYDTsOMH18qlZ5FIlyKncB4KH3VvPR6jz+9+K+pHbXYmAi0U7TMsLfl2zkyXlruTI1iatP0ZpvIrFA5V7HLd+4m/95/SuGdG3NPRf1CTqOiISJyr0O27bvMBNeSOO4JvE8cdVg4uvr20EkVmjOvY7KLyzmpy8tYvv+fF6/aRgJzbVmjEgsUbnXUfe+ncGCdTt4bMzJ9O3UMug4IhJm+j28Dpo+fz3T529g4g9OYOTJx1q9WUSilcq9jlm0YSf3zM7gjF4J/Ow8LS0gEqtU7nXI9n2HuXn6Itq3aMRjVwwkTksLiMQszbnXEUXFzq0zlrB9fz5v3DSMlk0aBB1JRGqQztzriEc/yORfWdv47cg+egNVpA5QudcBH676hj9/mMXlKYlcMSQp6DgiUgtU7jEuZ8cBbpuxhN4dW3DfyL5BxxGRWqJyj2GHCoq4aXo6AH+9ejCNGmgJX5G6IqRyN7PhZrbazLLM7M5ynj/dzBaZWaGZjQp/TKmKe9/OYPnGPTx8+ckktWkSdBwRqUUVlruZxQFTgPOB3sBYM+tdZtgGYBzwcrgDStW8mpbDKwtyuPnMEzind/ug44hILQvlUsihQJa7ZwOY2QxgJLDi2wHuvq7kueIayCiVlLFpN79+azmn9WjDHefqRiWRuiiUaZlOQE6p49ySxyQC7T5YwE0vLaJ1k3geG6MblUTqqlDKvbx28Kp8MTObYGZpZpaWl5dXlZeQY3B3/nvWUjbtOsiUqwbRtplWehSpq0Ip91ygc6njRGBTVb6Yu0919xR3T0lISKjKS8gxPPfZOt5f8Q2/+NFJDO7SOug4IhKgUMp9IZBsZt3MLB4YA8yu2VhSWctyd/OHd1ZyzkntuO60rkHHEZGAVVju7l4ITALmAiuBV909w8zuM7MRAGY2xMxygdHA38wsoyZDy3/ad7iQya8sok3Thjw4agBmmmcXqetCWjjM3ecAc8o89ptSHy/kyHSN1DJ351dvLmPDjgO8csMptG4aH3QkEYkAukM1yr2+aCNvLdnErWf3JLV7m6DjiEiEULlHsayt+/j1W8s5pftxTDqrR9BxRCSCqNyj1KGCIia9vIjG8XG6nl1EvkObdUSp389Zyaote3l2XArtWzQKOo6IRBiduUehd5dv4YUv1jP+e90460StGyMi36VyjzK5Ow/w89eW0j+xJT8ffmLQcUQkQqnco0hBUTG3vLKYYoc/jx1IfH399YlI+TTnHkUe/SCTRRt28fjYgXRp0zToOCISwXTqFyUWfL2DJ+at5fKUREYMOD7oOCIS4VTuUWDvoQJun7mEpOOacPdFfYKOIyJRQNMyUeCe2SvYvPsgsyYOo2lD/ZWJSMV05h7h5izbzOuLcpl0Zg8t4ysiIVO5R7Bv9hzil28uY0BiSyafnRx0HBGJIir3CFVcfGRXpUMFRTx8xck0iNNflYiETo0RoV74Yh2frtnGXRf05oSEZkHHEZEoo3KPQGu+2csf3lnFmb0SuDo1Keg4IhKFVO4RJr+wmNtmLqFpw/o8MKq/dlUSkSoJqdzNbLiZrTazLDO7s5znG5rZzJLn55tZ13AHrSse/SCTjE17uP/SfrRrrtUeRaRqKix3M4sDpgDnA72BsWbWu8yw64Gd7t4DeAR4INxB64KF63bw14/XckVKZ37Yp0PQcUQkioVy5j4UyHL3bHfPB2YAI8uMGQlMK/n4NeBs03xCpXx7F2pi6yb8+qKy/3aKiFROKLc7dgJySh3nAqlHG+PuhWa2G2gDbAtHyNJeXZjDU59mh/tlA7f3UCFb9x5i1sRhNNNdqCJSTaG0SHln4F6FMZjZBGACQFJS1a4CadWkAcntY/PSwHN7t9ddqCISFqGUey7QudRxIrDpKGNyzaw+0BLYUfaF3H0qMBUgJSXlO+Ufih/26aD5aBGRCoQy574QSDazbmYWD4wBZpcZMxu4tuTjUcCH7l6l8hYRkeqr8My9ZA59EjAXiAOedfcMM7sPSHP32cAzwItmlsWRM/YxNRlaRESOLaR37tx9DjCnzGO/KfXxIWB0eKOJiEhV6Q5VEZEYpHIXEYlBKncRkRikchcRiUEqdxGRGGRBXY5uZnnA+ip+eltqYGmDGqbMtSPaMkdbXlDm2nK0zF3cPaGiTw6s3KvDzNLcPSXoHJWhzLUj2jJHW15Q5tpS3cyalhERiUEqdxGRGBSt5T416ABVoMy1I9oyR1teUObaUq3MUTnnLiIixxatZ+4iInIMUVfuFW3WHWnM7Fkz22pmy4POEgoz62xmH5nZSjPLMLNbg85UETNrZGYLzGxpSeZ7g84UKjOLM7PFZvaPoLOEwszWmdkyM1tiZmlB56mImbUys9fMbFXJ9/SpQWc6FjPrVfJn++1/e8zstiq9VjRNy5Rs1p0JnMuRDUIWAmPdfUWgwY7BzE4H9gEvuHvfoPNUxMw6Ah3dfZGZNQfSgYsj/M/YgKbuvs/MGgD/Am519y8DjlYhM7sDSAFauPuFQeepiJmtA1LcPSquGTezacCn7v50yX4UTdx9V9C5QlHSdxuBVHev9D1B0XbmHspm3RHF3T+hnF2pIpW7b3b3RSUf7wVWcmSP3IjlR+wrOWxQ8l/En7WYWSJwAfB00FlikZm1AE7nyH4TuHt+tBR7ibOBtVUpdoi+ci9vs+6ILp5oZmZdgYHA/GCTVKxkemMJsBV4390jPjPwKPBzoDjoIJXgwHtmll6yJ3Ik6w7kAc+VTH09bWZNgw5VCWOAV6r6ydFW7iFtxC3VZ2bNgNeB29x9T9B5KuLuRe5+Mkf2+B1qZhE9BWZmFwJb3T096CxA1oZWAAABdUlEQVSVdJq7DwLOB24umXaMVPWBQcCT7j4Q2A9E/Pt0ACVTSCOAWVV9jWgr91A265ZqKpm3fh2Y7u5vBJ2nMkp+7Z4HDA84SkVOA0aUzGHPAM4ys5eCjVQxd99U8v+twJscmSqNVLlAbqnf4l7jSNlHg/OBRe7+TVVfINrKPZTNuqUaSt6cfAZY6e4PB50nFGaWYGatSj5uDJwDrAo21bG5+y/cPdHdu3Lk+/hDd7864FjHZGZNS95kp2R644dAxF4F5u5bgBwz61Xy0NlAxF4YUMZYqjElAyHuoRopjrZZd8CxjsnMXgHOANqaWS5wt7s/E2yqYzoNuAZYVjKHDfDLkn10I1VHYFrJ1QX1gFfdPSouLYwy7YE3j/z7T33gZXd/N9hIFZoMTC85GcwGfhJwngqZWROOXBF4Y7VeJ5ouhRQRkdBE27SMiIiEQOUuIhKDVO4iIjFI5S4iEoNU7iIiMUjlLiISg1TuIiIxSOUuIhKD/h/xKaTXQSzhKgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF4tJREFUeJzt3X+M5Pdd3/Hna+fHemedYLW3jYLP4Yw4UC1EcbQ4IEs04AB2oDZ/0Mon0QKKcP+I+aEgkEORoe4fqCCVUsmlNUkg/IprAmlP6FpTQSJohYPXSUiwjctxJNxiwEuShmRmvd+dmTd/zHzn5vZmZ747O3ez3+/n9ZBOnh/fnX379vS6z30+n+/nrYjAzMyqZWXZBZiZ2eI53M3MKsjhbmZWQQ53M7MKcribmVWQw93MrIIc7mZmFeRwNzOrIIe7mVkF1Zf1jU+dOhVnzpxZ1rc3Myul55577m8jYmPWdUsL9zNnzrC1tbWsb29mVkqSPlXkOk/LmJlVkMPdzKyCHO5mZhXkcDczqyCHu5lZBc0Md0nvkfSKpD8+5H1J+k+SLkr6uKQ3Lr5MMzM7iiIj918E7p3y/n3A2eGvh4CfO35ZZmZ2HDPDPSJ+D/jMlEseAH4pBp4BbpH0+kUVaEfz4Uuf5k//5vPLLsPMlmwRc+63ApfHnm8PX7uGpIckbUna2tnZWcC3toPe+YFP8LO/86fLLsPMlmwR4a4Jr03suh0RT0TEZkRsbmzMvHvW5vD5V7t8/tXussswsyVbRLhvA7eNPT8NvLyAz7U57GY9drPessswsyVbRLifB/7VcNfM1wKfi4i/WsDn2hFFBO2sSzvzyN0sdTMPDpP0PuDNwClJ28CPAw2AiPgvwAXgrcBFoAN8z/Uq1qbb6/aJwCN3M5sd7hFxbsb7Abx9YRXZ3Np7gxG7R+5m5jtUK6QzHLF39jxyN0udw71CRuG+32PwDyozS5XDvULy6ZheP9jr9pdcjZktk8O9QsYXUr2oapY2h3uF5Auq4EVVs9Q53CukMzZa73jkbpY0h3uFONzNLOdwr5DO2FRMZ8/TMmYpc7hXiEfuZpZzuFfI+CKqF1TN0uZwr5DxO1M9cjdLm8O9QjpZj9fcVB89NrN0OdwrpJN1OXXz6uCxF1TNkuZwr5BO1uO1aw0aNdHZ98jdLGUO9wrpZF3WmzVazbpH7maJc7hXSHuvR6tZZ71Zo+05d7OkOdwrZHe/R6tZY61Z88FhZolzuFdIe6/L+mqN9dW697mbJc7hXiG7WY+1Rp21Rs1bIc0S53CviIignV0ZuXc8cjdLmsO9Iva6ffoBa80arWbNfVTNEudwr4h8Gma9WR+Eu6dlzJLmcK+IvAtTa7jP3QuqZmlzuFfE7vCO1NZw5L6b9YiIJVdlZsvicK+I0ch9uKDa7QdZr7/kqsxsWRzuFZHPsbcaNdYatcFrXlQ1S5bDvSJGC6qrddZXh+Huw8PMkuVwr4h8X3u+oAo+9tcsZQ73ihhNywwXVMdfM7P0ONwrYnxBNR+5ezukWboKhbukeyW9JOmipEcmvP8GSR+U9FFJH5f01sWXatOML6iORu5eUDVL1sxwl1QDHgfuA+4Azkm648BlPwY8FRF3Ag8C/3nRhdp0naxHs75CvbbiBVUzKzRyvwu4GBGXIiIDngQeOHBNAK8dPv4i4OXFlWhF5F2YAC+omhn1AtfcClwee74NvOnANT8B/Lak7wPWgbcspDorrJP1RqHuBVUzKzJy14TXDt7Xfg74xYg4DbwV+GVJ13y2pIckbUna2tnZOXq1dqhO1h2F+mjk7gVVs2QVCfdt4Lax56e5dtrlbcBTABHxB8BNwKmDHxQRT0TEZkRsbmxszFexTTTonzoI92Z9hfqK3EfVLGFFwv1Z4Kyk2yU1GSyYnj9wzV8A9wBI+scMwt1D8xtod2xaBhgdHmZmaZoZ7hHRBR4GngZeZLAr5nlJj0m6f3jZDwHfK+mPgPcB3x0+kvCGao9Ny8DgGIK2F1TNklVkQZWIuABcOPDao2OPXwDuXmxpdhS7WY/W6pUf51qz5q2QZgnzHaoV0R7bCgmDjkzeCmmWLod7RXSyHmtj4b7mVntmSXO4V0BE0Ml6rI8tqK473M2S5nCvgL1un14/rhq5t1bdR9UsZQ73Csi3PI7Pubca3gppljKHewXkI/Tx3TLeCmmWNod7BeyOGnVcvaC6662QZslyuFdAezQtc/WC6n4vyLr9ZZVlZkvkcK+AfD/7VQuqPjzMLGkO9wroTBi5+9hfs7Q53CvgyoLq1VshwSN3s1Q53Ctg0oJqq+GRu1nKHO4V0B6F+9i0zHAU33aTbLMkOdwrIF9QbR04OAw8LWOWKod7BXT2ezRrKzRqV36cXlA1S5vDvQI6e92rtkGCF1TNUudwr4DBiZAHwt0LqmZJc7hXQOdAFya4sqDqcDdLk8O9Ag72TwVo1laorciHh5klyuFeAZ2sd024S6Llhh1myXK4V0An6161xz233qx7QdUsUQ73Cpg0cgc8cjdLmMO9Ajp7V/dPzbVWHe5mqXK4V0A7u3afO0Cr4W5MZqlyuJdcRLCb9VhfnRDuq+7GZJYqh3vJZb0+3X4cuqDqkbtZmhzuJTfpuN/cWrM2et/M0uJwL7lJ/VNz683a6H0zS4vDveQm9U/NrXmfu1myHO4lN+qfOmFBdb1ZY78XZN3+jS7LzJbM4V5yef/Utcakfe6D1zzvbpaeQuEu6V5JL0m6KOmRQ675F5JekPS8pF9bbJl2mN0pI/dRw459T82Ypeba4d4BkmrA48A3AdvAs5LOR8QLY9ecBd4J3B0Rn5X0j65XwXa19pTdMvlr7qNqlp4iI/e7gIsRcSkiMuBJ4IED13wv8HhEfBYgIl5ZbJl2mCv9UydMy7iPqlmyioT7rcDlsefbw9fGfTnw5ZL+r6RnJN076YMkPSRpS9LWzs7OfBXbVToztkKOX2Nm6SgS7prwWhx4XgfOAm8GzgHvknTLNV8U8UREbEbE5sbGxlFrtQnyUfnEs2XcR9UsWUXCfRu4bez5aeDlCdf8j4jYj4g/B15iEPZ2nXWyHo2aaNav/VG2PHI3S1aRcH8WOCvpdklN4EHg/IFr/jvwDQCSTjGYprm0yEJtsk7WY61x7agdxsLdC6pmyZkZ7hHRBR4GngZeBJ6KiOclPSbp/uFlTwOflvQC8EHghyPi09eraLuivddlfXXypqd8QbXtaRmz5MzcCgkQEReACwdee3TscQDvGP6yG6izP7kLE3haxixlvkO15Dp7k/unAqzWV6ityAuqZglyuJfcYf1TASTRarjVnlmKHO4lNy3cYdhH1QuqZslxuJdcO+uO9rNP0mrWvaBqliCHe8ntZr3RnaiTtNyNySxJDveSa09ZUIVhH1WP3M2S43Avud0pWyHBfVTNUuVwL7Gs22e/F1PDfX3VfVTNUuRwL7F8//q0aZm1Rn10LLCZpcPhXmKdKY06cuurNTr7HrmbpcbhXmKjkfuMrZDe526WHod7iV1p1DF9K2TW67Pf69+osszsBHC4l1jeG3VSo46cDw8zS5PDvcTyaZlJLfZy7qNqliaHe4kVXVAdv9bM0uBwL7EiC6p5lyYvqpqlxeFeYkUWVNfdJNssSQ73EsvD3QuqZnaQw73EOlmX+opo1g7/MV5ZUHW4m6XE4V5i7b0ea80akg69Jh+5+2RIs7Q43Eusk3WnboOEsWkZny9jlhSHe4l1sh6t1cPn22FsQdXny5glxeFeYrP6pwKs1ldYkbdCmqXG4V5inWx6FyYASYPDw7ygapYUh3uJFRm5w2De3fvczdLicC+x9t7sBVUYhLu7MZmlxeFeYrtZb+oNTLlWs86uR+5mSXG4l1g76009eiC3vlobHQ9sZmlwuJfYbtabemhYbq1Z91ZIs8Q43Esq6/bJen1ajQIj92bNNzGZJaZQuEu6V9JLki5KemTKdd8hKSRtLq5Em2Q3P8u90Mi95q2QZomZGe6SasDjwH3AHcA5SXdMuO41wPcDH150kXatzv7wLPcic+7NurdCmiWmyMj9LuBiRFyKiAx4EnhgwnX/Dvgp4NUF1meHyBdIC+1zX/VWSLPUFAn3W4HLY8+3h6+NSLoTuC0ifmuBtdkUu6NGHQX2uTfqZN0+3V7/epdlZidEkXCfdJ5sjN6UVoCfAX5o5gdJD0nakrS1s7NTvEq7Rn6Eb6FpmbyPqnfMmCWjSLhvA7eNPT8NvDz2/DXAVwIfkvRJ4GuB85MWVSPiiYjYjIjNjY2N+au2Qv1Tc2tN91E1S02RcH8WOCvpdklN4EHgfP5mRHwuIk5FxJmIOAM8A9wfEVvXpWIDrnRWKrqgOvgaL6qapWJmuEdEF3gYeBp4EXgqIp6X9Jik+693gTZZ5ygLqu6japac2f+mByLiAnDhwGuPHnLtm49fls2Sj8KLHRzmPqpmqfEdqiWVb20sdHDYqvuomqXG4V5SnaxLbUWs1mf/CFteUDVLjsO9pDpZj1ajhjRpp+rVvKBqlh6He0l19mY3x855QdUsPQ73kurs92b2T815QdUsPQ73kursdQttgwS4qbGC5GkZs5Q43EuqnRXrnwogiVbD3ZjMUuJwL6mi/VNzrdU6u/seuZulwuFeUu2sNzoQrIhW0yN3s5Q43EtqN+ux1ig2LQODRVUvqJqlw+FeUu2se6SR+3qz5gVVs4Q43Euqs1d8KyQMjilwNyazdDjcS2i/1yfr9QtvhYTBXaq7HrmbJcPhXkJHOcs95wVVs7Q43EtodxTuR1hQXa2x6zZ7ZslwuJdQfnTv0RZU67T3PC1jlgqHewld6cJ0tAXVvW6fXj9mX2xmpedwL6FRc+wjLqiOf62ZVZvDvYTmWVBd87G/ZklxuJdQZ44F1Xx+3uFulgaHewm155iWyf8i8KKqWRoc7iXU2Zsn3D1yN0uJw72EOsP96uurRzs4DLygapYKh3sJdfZ6rAhW68V/fB65m6XF4V5CnWxwaJikwl+z7j6qZklxuJdQJyvePzXXGu2W8bSMWQoc7iXUznpHD/fh9T48zCwNDvcS2s26R9rjDnBTvYaEj/01S4TDvYTae0frnwqwsiLWGm7YYZYKh3sJdfZ7rB1x5A7uo2qWEod7CXX2uqwfcc4dBkcQeEHVLA2Fwl3SvZJeknRR0iMT3n+HpBckfVzS70j6ksWXarlO1hsdBHYUaw13YzJLxcxwl1QDHgfuA+4Azkm648BlHwU2I+KrgPcDP7XoQu2KTtYd7Vs/ivXVOrv7HrmbpaDIyP0u4GJEXIqIDHgSeGD8goj4YER0hk+fAU4vtkwb1856o33rR+E+qmbpKBLutwKXx55vD187zNuA/znpDUkPSdqStLWzs1O8Shvp9vpk3T6txjwLqrVR/1Uzq7Yi4T7pHveJvdokfSewCfz0pPcj4omI2IyIzY2NjeJV2siVQ8PmWFBt1kfHBZtZtRUZ/m0Dt409Pw28fPAiSW8B/g3wTyNibzHl2UH5yHuuBVWP3M2SUWTk/ixwVtLtkprAg8D58Qsk3Qn8V+D+iHhl8WVaLm+2Me+CqkfuZmmYGe4R0QUeBp4GXgSeiojnJT0m6f7hZT8N3Az8uqSPSTp/yMfZMXWOM3Jv1Hh1v0+vP3FWzcwqpNDwLyIuABcOvPbo2OO3LLguO0Qe7vON3Ad/Iezu97j5CI0+zKx8fIdqyYz6p86xoJofWdBxH1WzynO4l0y+IHrUI3+B0ZEFPl/GrPoc7iVznAXV/JhgL6qaVZ/DvWSOs6DqPqpm6XC4l8wiFlQd7mbV53AvmU7WRYKbGkf/0a01vKBqlgqHe8l0sh6tRg1p0qkQ03nkbpYOh3vJdLIurTn3qOcLqm7YYVZ9DveSae/15toGCVcWVN1H1az6HO4l08l6oxH4Ua01PC1jlgqHe8kMujDNN3JfWRFrjZoXVM0S4HAvmXn7p+bWV2ujM+HNrLoc7iUzb//UXKtZ98jdLAEO95I5zoIqDPuoes7drPIc7iWzuz9fc+yc+6iapcHhXjLtve7cu2VgMC3jg8PMqs/hXiK9frDX7R97WsYjd7Pqc7iXSH5n6XEWVN1H1SwNDvcSOc5xv7m1Zo3OnkfuZlXncC+R0XG/x1hQXW/WfIeqWQIc7iWSd2HKj+6dx1qzzu5+j34/FlWWmZ1ADvcS2d1fzMh9/LPMrJoc7iWSj9yPtRVy1X1UzVLgcC+RfK78WFsh85MhvahqVmkO9xI5Tv/UnLsxmaXB4V4i+T73422FdDcmsxQ43EtkUVshxz/LzKrJ4V4inb0uEtxUP87xAx65m6XA4V4i7azHWqPGyorm/oxRH1UvqJpVmsO9RI7TPzWXHxfsbkxm1VYo3CXdK+klSRclPTLh/VVJ/234/oclnVl0oTaYSjnONkgYm5ZxNyazSpsZ7pJqwOPAfcAdwDlJdxy47G3AZyPiy4CfAf79ogu1fOR+vHBfa3hB1SwFRUbudwEXI+JSRGTAk8ADB655AHjv8PH7gXskzT8xbBN1si7rq8eblqmtiJsaK15QNau4IklxK3B57Pk28KbDromIrqTPAf8Q+NtFFDnuqWcv8/O/f2nRH1sKlz/b4WvO/INjf856s85TW9t86KWdBVRlZkf1/fec5Z/9ky++rt+jSLhPGoEfPFKwyDVIegh4COANb3hDgW99rVtaDc6+7ua5vrbszr7uZh746luP/Tlv/4YvY+tTn1lARWY2jy9aa1z376GI6Ue/Svo64Cci4luGz98JEBE/OXbN08Nr/kBSHfhrYCOmfPjm5mZsbW0t4H/BzCwdkp6LiM1Z1xWZc38WOCvpdklN4EHg/IFrzgPfNXz8HcDvTgt2MzO7vmZOywzn0B8GngZqwHsi4nlJjwFbEXEeeDfwy5IuAp9h8BeAmZktSaGtFxFxAbhw4LVHxx6/CvzzxZZmZmbz8h2qZmYV5HA3M6sgh7uZWQU53M3MKsjhbmZWQTNvYrpu31jaAT4155ef4jocbXCdueYbo2w1l61ecM03ymE1f0lEbMz64qWF+3FI2ipyh9ZJ4ppvjLLVXLZ6wTXfKMet2dMyZmYV5HA3M6ugsob7E8suYA6u+cYoW81lqxdc841yrJpLOeduZmbTlXXkbmZmU5Qu3Gc16z5pJL1H0iuS/njZtRQh6TZJH5T0oqTnJf3AsmuaRdJNkv5Q0h8Na/63y66pKEk1SR+V9FvLrqUISZ+U9AlJH5N04hsySLpF0vsl/cnwz/TXLbumaSR9xfD3Nv/1d5J+cK7PKtO0zLBZ9/8DvolBu79ngXMR8cJSC5tC0tcDXwB+KSK+ctn1zCLp9cDrI+Ijkl4DPAd8+wn/PRawHhFfkNQA/g/wAxHxzJJLm0nSO4BN4LUR8W3LrmcWSZ8ENiOiFHvGJb0X+P2IeNewH0UrIv7/susqYph3fwm8KSKOfE9Q2UbuRZp1nygR8XsMzrgvhYj4q4j4yPDx54EXGfTIPbFi4AvDp43hrxM/apF0GvhW4F3LrqWKJL0W+HoG/SaIiKwswT50D/Bn8wQ7lC/cJzXrPtHBU2aSzgB3Ah9ebiWzDac3Pga8AvzviDjxNQP/EfgRoL/sQo4ggN+W9NywJ/JJ9qXADvALw6mvd0laX3ZRR/Ag8L55v7hs4V6oEbcdn6Sbgd8AfjAi/m7Z9cwSEb2I+GrgNHCXpBM9BSbp24BXIuK5ZddyRHdHxBuB+4C3D6cdT6o68Ebg5yLiTqANnPh1OoDhFNL9wK/P+xllC/dt4Lax56eBl5dUS2UN561/A/jViPjNZddzFMN/dn8IuHfJpcxyN3D/cA77SeAbJf3KckuaLSJeHv73FeADDKZKT6ptYHvsX3HvZxD2ZXAf8JGI+Jt5P6Bs4V6kWbcdw3Bx8t3AixHxH5ZdTxGSNiTdMny8BrwF+JPlVjVdRLwzIk5HxBkGf45/NyK+c8llTSVpfbjIznB645uBE7sLLCL+Grgs6SuGL90DnNiNAQec4xhTMlCwh+pJcViz7iWXNZWk9wFvBk5J2gZ+PCLevdyqprob+JfAJ4Zz2AA/Ouyje1K9HnjvcHfBCvBURJRia2HJvA74wODvf+rAr0XE/1puSTN9H/Crw8HgJeB7llzPTJJaDHYE/utjfU6ZtkKamVkxZZuWMTOzAhzuZmYV5HA3M6sgh7uZWQU53M3MKsjhbmZWQQ53M7MKcribmVXQ3wNtf30tak9odAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VdW9//H3NxOQEIaMQGamQBgqEOZ5sKCoWC0Wx9rWcuvUWlutrb23/uy91/bWodY6UYeqdQa1OIETs0xhhoQhJEBCBhIiJBAynvX7Yx0FEcnJuM85+b6eZz9n2tn5gjwf11lr7bXEGINSSin/EuB0AUoppVqehrtSSvkhDXellPJDGu5KKeWHNNyVUsoPabgrpZQf0nBXSik/pOGulFJ+SMNdKaX8UJBTvzgqKsokJyc79euVUsonbdq0qdQYE93QeY6Fe3JyMhkZGU79eqWU8kkictCT87RbRiml/JCGu1JK+SENd6WU8kMa7kop5Yc8CncRmSUie0QkW0TuOcfnN4pIiYhsdR83tXypSimlPNXgbBkRCQQeBy4E8oGNIrLYGJN51qmvG2Nua4UalVJKNZInLfdRQLYxJscYUwO8Bsxp3bKUUko1hyfz3OOAvDNe5wOjz3HelSIyCdgL/NIYk3f2CSIyH5gPkJiY2PhqVbPU1LnYX3KCvcUVHD9VS73LUO8yuIzBZbDPXYZ69+vIsBASI0NJigglvnsoIUE6RKOUr/Ak3OUc75298eq7wKvGmGoR+RnwAjDtGz9kzAJgAUB6erpu3tpKjDEcPnaKPUUV7HYfe4rKySk5SZ2raX/tAQI9u3YiKTKUpMhQEiPCSI4M5TsJ3ejVrVML/wmUUs3lSbjnAwlnvI4HCs48wRhz9IyX/wD+3PzSVGNUVNWyZGcRi7cVsPXQMSqq6776LL57Jwb0COfCtFhSe3RhQI9wIsNCCBAhIEAIDBACRRDha89LKqo5WFbJwaOVHDp68qvnS3cVU3ay5qvrp0SFMb5vJOP7RDG2TyTdQkOc+CtQSp3Bk3DfCPQTkRTgMDAPuObME0SkpzGm0P3yMiCrRatU51RdV8/yPSX8e+thPsk6Qk2di8SIUOYM68XAnjbE+8eGE94xuEnXj+nSkZguHRmZHPGNzyqqajlQWsmGA2WsyS7l7c2H+de6Q4jA4F5dGdc3kgl9o0hPiqBTSGBz/6hKqUYSYxr+mi4iFwN/BQKB54wx/yMi9wMZxpjFIvIANtTrgDLgZmPM7vNdMz093ejaMo3nchnW55axeNth3t9eSHlVHZFhIVwytCdzhsUxLKEbIufqSWtdtfUutuUdY032UdbsL2XLoS+orTeEBAYwJTWaK4bHM3VANB2CNOiVag4R2WSMSW/wPE/CvTVouDdORVUtz6zK5Y2MPAqPVxEaEsjMQT2Yc0EvJvSNIijQuwY7K2vq2JBbxsq9pby7vYCSimq6hQZz6dBeXDE8jgsc+p+QUr5Ow91PVNfV89Lagzy+LJsvKmuZmhrN5cPiuDAtltAQxxb1bJS6ehers0t5a/Nhlu4qorrORe+oMK4YHsflw+KI7x7qdIlK+QwNdx9X7zK8veUwj3y8l8PHTjGxXxR3zxzAkPiuTpfWLBVVtXy4o4hFm/NZn1sGwOiUCK4bk8RFg3t43TcQpbyNhruPMsbw2e4j/N+SPewprmBIXFd+M2sAE/pFOV1ai8srq+SdLYdZuDmfg0crievWiR+NT+YHIxOaPAislL/TcPdBmw6W8acPd7PxwBckR4by65mpXDy4JwEB/t03Xe8yfJpVzDOrctlwoIzwDkHMG5XAjeNTiNM59Ep9jYa7j3n447387dN9RId34BfT+/GDkQkEt8Muim15x3hmdS4f7LAza2cP6clNE1MYGt/N4cqU8g4a7j5m/J8+IykylGd+mO4zA6WtKf+LSl74/ACvbsjjRHUdo1IiuG1qXyb2i9JZNqpd8zTc21/T0Asdr6x1D5pGa7C7xXcP5d7Zaaz97TR+P3sg+WWV3PDcBuY+tZbPs0txqlGilK/QcPcCmYXlAAzsGe5wJd4nvGMwN03szbK7pvDHyweT/8UprnlmPfMWrGNdztGGL6BUO6Xh7gWy3OGe1quLw5V4rw5BgVw/Jonld03hvkvTyCk9ybwF67jmH+vIOFDmdHlKeR0Ndy+QWVhOVOcOxIR3dLoUr9cxOJAbx6ew6u6p/H72QPYWV/D9p9Zy/bPr2XzoC6fLU8praLh7gcyCcm21N1LH4EBumtiblXdP5XcXD2BXQTlXPPE5P30xg/0lJ5wuTynHabg7rKbOxb4jFaT11HBvitCQIOZP6sOqu6fyqwv7s3b/Ub77yEp+/84OSiqqnS5PKcdouDss+8gJauuNttybKaxDELdP78fyu6Zw7ehEXtuQx5S/LOPRT/ZRWVPX8AWU8jMa7g77cqZMms6UaRFRnTtw/5zBfPTLSUzqH80jn+xl8l+W8+qGQ9TVu5wuT6k2o+HusKzCcjoGB5AS1dnpUvxK7+jOPHndCBbdPJbEiFB++9YOLnp0FZ9mFescedUuaLg7LLOgnNQeXQj08/VjnDIiKYKFPxvLU9eNoM5l+MkLGVz37Hp2F5U7XZpSrUrD3UHGGDILy3UwtZWJCLMG9+CjX07ivkvT2FVQzsWPruLet3dw9IQOuir/5HPhXu8yZB+pcLqMFlFwvIrjp2p1MLWNBAcGcOP4FJb/ego3jE3mtY15THlwOc+syqGmTvvjlX/xuXB/7LN9XPrYGpbtOeJ0Kc2WWaCDqU7oFhrCfZcNYukdExmR1J3/fj+LmX9dySeZ2h+v/IfPhfu1o5PoHR3GTS9ksGhTvtPlNEtWYTkikNpDW+5O6BsTzj9/NIrnfzSSAIGbXszghuc2sKfIP74ZqvbN58I9OrwDr80fw5jeEfzqzW08vWK/0yU1WWZBOcmRYXTuoCtBOmlqagxL7rD98dvzj3PRoyv5w793cqyyxunSlGoynwt3sCsFPnfjSGYP7ckDH+7mv9/LxOXyva/TOpjqPc7sj79uTBIvrTvI1AeX89K6g9T74L8tpXwy3MGuEvjYvGH8cGwSz6zO5c43tvrUoFh5VS2Hyip1MNXLdA8L4f45g3n/5xNJ7RHOf76zk0seW816XV5Y+RifDXeAgADhvssGcdfMVN7ZWsBNL2Zwsto3bjXfXWj7dbXl7p0G9uzCqz8dwxPXDqf8VC0/WLCO217ZTMGxU06XppRHfDrcwc5hvnVqX/585RBW7yvhmn+s84m5y1lfbdCh4e6tRISLh/Tkkzsnc8eMfnycWcy0h5bz6Cf7qKqtd7o8pc7L58P9Sz8YmcjT16ezu6iCuU+tJa+s0umSziuzoJyIsBBiu3RwuhTVgE4hgdwxoz+f/moy0wfE8sgne5n+0Ao+3FGoUyeV1/KbcAe4MC2Wl28aTemJaq548nO25h1zuqRv9eVgqm727Dviu4fy+LXDefWnYwjvGMTNL2/m2mfW69RJ5ZX8KtwB0pMjWHjzODoEBXDV02t5Z8thp0v6htp6F3uKK3Qw1UeN7RPJe7dP4P45g+xSBn9bxX2Ld3G8stbp0pT6it+FO0D/2HAW3zaBYQnduOP1rfx5yW6vmiqZU3KSmjqXDqb6sKDAAG4Ym8zyX0/hmlGJvLj2AFMeXMbL63XqpPIOfhnuABFhIbz0k9FcMzqRJ5fvZ/5LGZzwkpk0uiG2/+geFsIfLx/Me7dPpF9sOPe+vZNLH1vNRt20WznMo3AXkVkiskdEskXknvOc930RMSKS3nIlNl1IUAD/c/lg7p8ziGV7SrjiiTUcOur8QGtmYTkhQQH0jgpzuhTVQtJ6deH1+WP4+zXDOFZZw9yn1nL7q1t06qRyTIPhLiKBwOPARUAacLWIpJ3jvHDg58D6li6yOUSEG8Ym8+KPR1FcXs2cx1ezdr+zN6RkFpSTGhtOUKDffnFql0SES4b24tNfTeHn0/uxdFfRV1MnT9Xo1EnVtjxJl1FAtjEmxxhTA7wGzDnHeX8E/g+oasH6Wsz4vlG8c+t4IsJCuP7Z9by8/qAjdega7v6vU0ggd17Yn0/vPHPq5HLe3VagUydVm/Ek3OOAvDNe57vf+4qIDAMSjDHvtWBtLS4lKoy3bx3PhH5R3Pv2Tv7znZ1U17Vti6q4vJqykzXa394OJETYqZOvzx9Dt9AQbn91C1c9vZYd+cedLk21A56E+7kmYn/V/BCRAOAR4FcNXkhkvohkiEhGSUmJ51W2oC4dg3n2hyOZP6k3L607yFVtfMOTDqa2P6N7R/Lu7RN44Ioh5JSc5LLHV/ObhdspqfD+O6mV7/Ik3POBhDNexwMFZ7wOBwYDy0XkADAGWHyuQVVjzAJjTLoxJj06OrrpVTdTYIDwu4sH8tR1w8kpPcnsv63io11FbfK7M93hPqCHbtDRngQGCFePSmTZXVO4aUIKb23JZ+qDy3lqxf42//ao2gdPwn0j0E9EUkQkBJgHLP7yQ2PMcWNMlDEm2RiTDKwDLjPGZLRKxS1o1uCevH/7RBIjQ5n/0ib++71Mautbd2XJzIJyEiNCCe8Y3Kq/R3mnLh2DuXd2GkvvmMTolAj+9OFuZjy8gg90KQPVwhoMd2NMHXAbsBTIAt4wxuwSkftF5LLWLrC1JUaGsujmcdzgXjr4qqfXcrgVp6/pYKoC6B3dmWdvHMmLPx5FaHAQt7y8maueXss2L14yQ/kWcaq1kJ6ebjIyvKtx/972Au5ZtIOgQOHhq77DtAGxLXr9k9V1DL5vKb+c0Z+fT+/XotdWvqveZXgjI4+HPtpD6YkavjcsjrtmptKrWyenS1NeSEQ2GWMavJdIJ1qf4ZKhvXj39gn06tqJH/8zgz99uJu6Fuym2V1UgTG6hrv6ui/745ffNZVbp/bh/R2FTH1wOQ99tMdn9idQ3kfD/SwpUWG8dcs4rhmdyFMr9jNvwboWu6s1U2fKqPPo3CGIu2YO4LNfTWbW4B489lk2Ux5czusbD+l6NarRNNzPoWNwIP/7vSE8Ou8C9hRVcNGjK3l946FmD3hlFpTTtVMwPbt2bKFKlT+K7x7Ko/OG8dYt40jo3onfLNrBxY+u4rPdxTroqjym4X4ecy6IY8kvJzEkviu/WbSDn764idJm7PKka7irxhie2J1FN4/jyWuHU11Xz4//mcHV/1ing67KIxruDYjr1olXbhrD72cPZOW+EmY+spKPM4sbfZ16l2FPUbl2yahGEREuGtKTj++czP1zBrGv+ARzHl/Dra9s5uDRk06Xp7yYhrsHAgKEmyb25t3bJhDbpSM/fTGDuxdua9QSwrmlJ6mq1TXcVdMEu9ePX3H3VH4+rS+fZR1hxsMruG/xLp/YM1i1PQ33RkjtEc47t47nlil9WLgpn4seXenxut06mKpaQucOQdz53VRW3DWF749I4KV1B5n8l+X8/bN9OrNGfY2GeyOFBAVw96wBvPEfYxGEq55eywMfZFFVe/5byDMLygkOFPpEd26jSpU/i+nSkQeuGMLSOyYytk8kD360l8l/Wcbza3J1OQMFaLg3WXpyBB/8YiLzRibw9MocZv51JZ9nl37r+ZmF5fSLCSckSP/KVcvpGxPOP25IZ9HN4+gb05n/924m0x5cwRsb81r0Hg3lezRpmqFzhyAeuGIor9w0GoBrnlnP3Qu3cayy5hvnZhboYKpqPSOSuvPqT8fwr5+MJqpzCHcv2s53/7qS97cXetX+wartaLi3gHF9o1h6xyR+NrkPizYfZsbDK3hv++mNGY5UVFF6oloHU1WrEhEm9LOb0jx9/QgCRbj1lc1c+vfVLNtzROfItzMa7i2kY3Ag91w0gMW3jadn107c9soWbnohg4Jjp8gqrAB0MFW1DRFh5qAeLLljEg9f9R3Kq2r50fMbmfvUWtZkl2rItxO6cFgrqKt38c/PD/DQR3sJEBgS35V1OWVs+8N36dpJl/pVbaumzsXrGXk8/lk2ReVVjEqO4I4Z/RjbJ1JvqPNBni4cpuHeivLKKvnd2ztYta+UuG6dWHPPNKdLUu1YVW09b2Tk8fiybIrLqzXkfZSGu5cwxvDhziJCQwKZkhrjdDlKUVVbz+sb83hiuTvkU2zIj+sT5XRpygMa7kqp8zo75EenRPCLGf0Y21tb8t5Mw10p5ZGq2npe23CIJ5bv50hFNcMTu3Hr1L5MGxCjIe+FNNyVUo1SVVvPm5vyeWr5fg4fO8WAHuHcMrUvs4f0JDBAQ95baLgrpZqktt7F4q0FPLE8m/0lJ0mODOXmKX343rB4vcPaC2i4K6WaxeUyfJRZxN+XZbPzcDk9u3Zk/qTezBuZSKeQQKfLa7c03JVSLcIYw8p9pTy+LJsNuWV0Dw3m+rHJ3DA2iajOHZwur93RcFdKtbiNB8p4ekUOn2QV0yEogCtHxPOTCSm62mkb8jTcg9qiGKWUfxiZHMHI5Aiyj5zg2dW5LNyUz6sbDjFjYCzzJ/UmPam7zrDxEtpyV0o1WemJal5ce5CX1h7gi8paLkjoxvxJvZk5qIfOsGkl2i2jlGozp2rqWbg5n2dX5XDgaCXx3Tvxw7HJXJWeQNdQXU+pJWm4K6XaXL3L8HFmMc+vyWV9bhmdggP53vA4bhyXTP/YcKfL8wsa7kopR2UWlPPC5wd4Z+thqutcjO8byY3jUpg2IEa7bJpBw10p5RXKTtbw2sZDvLT2IIXHq0iIsF02c0dol01TaLgrpbxKXb2LjzKL+eeaA2w4UEbH4AAuHdqLa8ck8Z34rjrLxkMa7kopr7Wr4Dgvrz/EO1sOU1lTz6BeXbh2dBKXXdCLzh10hvb5aLgrpbxeRVUt/95awMvrD5FVWE5YSCCXD4vj2tFJui3lt2jRcBeRWcCjQCDwjDHmT2d9/jPgVqAeOAHMN8Zknu+aGu5KqS8ZY9iSd4xX1h/i3W0FVNe5GJbYjatHJTJ7SE/CtDX/lRYLdxEJBPYCFwL5wEbg6jPDW0S6GGPK3c8vA24xxsw633U13JVS53K8spZFm/N5ef1B9pecJDQkkEuG9uSq9ARG6B2wLbr8wCgg2xiT477wa8Ac4Ktw/zLY3cIA3V5dKdUkXUOD+fGEFH40PpnNh77gjY35vLe9gDcy8ukdFcbc9ASuHB5HTJeOTpfq1TwJ9zgg74zX+cDos08SkVuBO4EQQHeCVko1i4gwIimCEUkR/NelaXywo5A3M/L585LdPPjRHqb0j2ZuegLTBsToOvPn4Em4n+s70Dda5saYx4HHReQa4PfAD79xIZH5wHyAxMTExlWqlGq3wjoEMTc9gbnpCeSUnGDhpnwWbsrn091H6B4azCVDe/G94XEMS+jW7rttvuRJn/tY4D5jzEz3698CGGMe+JbzA4AvjDFdz3dd7XNXSjVHXb2LVftKWbQ5n48zi6muc5EcGcrlw+K4/II4kqPCnC6xVbTkgGoQdkB1OnAYO6B6jTFm1xnn9DPG7HM/vxT4Q0O/XMNdKdVSKqpqWbKziLe3HGZtzlGMgWGJ3bhiWByzh/YiIizE6RJbTEtPhbwY+Ct2KuRzxpj/EZH7gQxjzGIReRSYAdQCXwC3nRn+56LhrpRqDYXHT7F4awFvbznM7qIKggKESf2juWRoTy5MiyW8o28veaA3MSml2r2swnLe2XKY97YXcvjYKUKCApjSP5pLvtOLGQNjCA3xvfnzGu5KKeXmctmbpN7dVsAHOwo5UlFNx+AApg+I5ZKhPZk6IIaOwb6x6beGu1JKnUO9y7DxQBnvbS/gwx1FHD1ZQ1hIINMHxjJrcA8m94/26jtiNdyVUqoBdfUu1uXYoP8os5iykzV0CApgUv9oZg3qwfSBMXQL9a7BWA13pZRqhLp6FxkHv2DJziKW7iqi8HgVQQHC2D6RzBzUg++mxXrFXbEa7kop1UTGGLbnH2fJriKW7Cwit/QkInBBQjdmDIxl+sAYUmPDHblhSsNd+Yb6WijcDhWFcKoMTn0Ble7HU2Vw6ph9XXUcQsIgNAI6RUBod/djxOnH0CiI6g9hkU7/qZQfMcaw78gJluws4pOsYrbnHwcgvnsnpg+IYUZaLKNTIttsCQQNd+W9jh2C7E9h/6eQswKqy7/+eUDw6dDu1N0+79AFak6cEf5l9rG++pvX7xwLMWkQO8j9mAbRAyC4U9v8+ZRfKy6v4rPdR/g0q5hV+0qprnPRuUMQk/pHMX1ALFNSo4ns3KHVfr+Gu/IetafgwBob5tmfQOle+36XeOg7HfpMg4iU02EeEgaefN01BmorT4f9iRIo2Q1HMqF4J5Tsgboqe64EQERvG/gJYyBpHPQYAgG+Mf1NeadTNfWsyS7l093FfJp1hCMV1YjA0LiuTO4fzeTUGC5I6NaiG4JruCvnFW6DZQ9AzjIbskEdIWm8DfS+M2wXSmv2WbrqoSwHine5A3+X7QI6fsh+3qELJLqDPmkC9LoAAn377kXlHJfLsLPgOMv3lLB8zxG25h3DZaBrp2Am9otiSmoMk/tHEx3evFa9hrtyTnkhfPZH2PqK7VIZ+gMb6EnjvaNr5Hg+HFwLB1fDwc9Pf5MIDoX4kZA8AfpMh17DIECXklVNc6yyhlX7Slm+p4QVe0soPWG7EAfHdeGO6f2ZkRbbpOtquKu2V3MSPn8M1jwKrjoY/TOY+Cvo1M3pys7vRAkc+tx2HR383HbpYCA00oZ8vwvtow7UqiZyuQyZheWs2FvCij0l3DylD1MHxDTpWhruqu24XLD9Nfj0fjvrJe1ymHGf7Uf3RSePwv7PIPtjO0ZQeRQQiBthg77vhdqqV47RcFdtI3cVfHSv7V+PGwEz/9f2Y/sLlwsKt8C+j+1xeBO2VR8F/WdC6sV2QDgk1OlKVTuh4a5aV81JeOcWyHzHznqZcR8MvtL/W7Nftur3LYW9H0H1cQjqBH2m2qDvPws6RztdpfJjLblBtlJfd+oYvDwXDmfA1Hth3O3eMVDaFsIiYehce9TXwsE1sPsD2P0+7PkAEEgYDQNm2yOyj9MVq3ZKW+6qcU6UwEvfs/PJv/8spM1xuiLvYAwUbbdBv+d9KNph349Jg4GX2iN2cOtO/VTtgnbLqJZ3PB9enAPHD8O8f9m56urcjh2yrfms9+xMHOOC7snuoL8M4tL9vwtLtQoNd9Wyju63wV51HK59078GTVvbiRLbZZP1LuQsB1ctdO4BAy+xYZ80Xm+eUh7TcFctp2in7Yox9XD929DzO05X5LuqjtuB2KzFdpplbaVdciH1Ytui7z0Fgp1fVlZ5Lx1QVS0jbyO8fCUEh8EN70N0f6cr8m0du54ekK2ptOvtZL1ru2+2vgwh4dD/u7ZF3/dC6NDZ6YqVj9JwV98uZzm8eg10joEb/g3dk5yuyL+EhJ4ebK2rgdyVtkW/+33YuciuxdNnuv08dZZt4SvlIe2WUee2+wN484cQ2dd2xYT3cLqi9sNVD4fWQuZi26qvKICAILvmzcBLIXU2dOnpdJXKIdrnrpquOBMWTLHL4163yC7+pZzhckHBFtj9rg36o9n2/fiRNugHXKJz6dsZDXfVNHXV8I/pcKIIbl6rd1t6E2PsGvVfBn3hNvt+TJoN+QGz7WC3zqX3azqgqppm2f9C8Q64+jUNdm8jAjED7DHprq/PpV/1IKz8P7sUROpFMOBiu0Z9UIjTVSuHaMtdnXZgDfxzNgy/AS77m9PVqMY4eRT2LrHz6bM/hbpTdjOSfhfaaZb9LrQzdZTP024Z1ThV5fDkeLvt3M9W6xQ8X1Z7ys502v0+7PkQKkvtvrTJE+zCZv1n+u5yzEq7ZVQjLbkHyvPhx0s12H1dcCfbNZN6kZ15k7/RBv3eJbDkN/aIHmBDvv8siB8FgRoF/kZb7spOuXvjetuPO+33TlejWtPR/bB3qQ36g2vsjlmdutsbpvrPtNsh6nx6r6bdMsozFUXwxFjolgg3faJrnLQnVcdh/zIb9Ps+sjtOSaCdZtlvhg38HkN1gTMvo+GuGmaMXZf9wGr4j5W6tEB75qqH/Awb8tmfQOFW+35YjF39s98M6D1V73nwAi3a5y4is4BHgUDgGWPMn876/E7gJqAOKAF+bIw52OiqVdvKeNbuE3rxgxrs7V1AICSOtsf0/4QTR+ysm+yPYe+HsO0VkADbqu/rDvpew7Sv3os12HIXkUBgL3AhkA9sBK42xmSecc5UYL0xplJEbgamGGN+cL7rasvdYaXZ8NQESBpn70LVG1/Ut3HV271j931sw75gK2Ds1MqUSTbo+0zTGThtpCVb7qOAbGNMjvvCrwFzgK/C3Riz7Izz1wHXNa5c1abqa+Gtn9qlZec8rsGuzi8gEBJG2WPavXZOfe5y21+/f5m9WxbsZiS9p9r9ZFMm6cCswzwJ9zgg74zX+cDo85z/E+DD5hSlWtmqh6BgM8x9QRegUo0XFmk3Qx98pR23OZrtDvrPYMebsOl5QOxSCCmTIGWy3dxFp9i2KU/C/VzNunP25YjIdUA6MPlbPp8PzAdITEz0sETVosoLbLgPmQuDLne6GuXrRCCqnz1Gz7ffCvM32uWLc1fCuifh87/ZVS3jRrjDfpKdW6+bkrQqT8I9H0g443U8UHD2SSIyA7gXmGyMqT7XhYwxC4AFYPvcG12tar7PH7N9qDqfXbWGwGA7jpM0DqbcYzckyVt/OuxXPQQr/wKBHWw3T9J4e278SLu+vWoxnoT7RqCfiKQAh4F5wDVnniAiw4CngVnGmCMtXqVqGSdKION5GPoD2z+qVGsLCbV98H2m2tdV5Xat+pwVcHC1XezMuOzyCL2GQfJ4G/gJo3QtnGZqMNyNMXUichuwFDsV8jljzC4RuR/IMMYsBv4CdAbeFDs4d8gYc1kr1q2aYt0TUFcFE+90uhLVXnXs4l72YKZ9XXUc8jbYu2UPrLHfLFc/Yqdd9hgCieNs0CeOgS69nK3dx+hNTO3FqS/gkSF2dcC5zztdjVLnVlNp++wPfm4DPz/DrnAJ0DXxdNAnjLabyQQEOluvA3ThMPV16xdATQVM/JXTlSj17UJCofdke4AdoC3aDofWQ946ezf1zoXucztDfLoN+rh0+1zvoP2Khnt7UF1hu2SDSF5fAAAL80lEQVRSL4Yeg52uRinPBQbbWTZxI2DsLXbq5bFDdpA2b70N/ZV/sf32ABG93UE/EuJHQOyQdrthiYZ7e7DxWag6BhN/7XQlSjWPCHRPssfQq+x71SfsPrOHM2w3Tu5K2PGG/Sywg51vHzfCDtjGDYeIPu1iMTQNd39XewrW/t3eHh4/wulqlGp5HTpDykR7gG3dlx+2QZ+/0S6dsPkFWP+k/TwkHHpd4D6G2aN7it/dqa3h7u82vwgnS+xa7Uq1ByLQNd4eX96oV18HpXttC79gi71De/3TUF9jP+/YDXoOtUsc9/yOPSL7+vSArYa7P6urhjWPnr5RRKn2KjAIYtPsMexa+15dDZRkubt0NtuB2w3/gHr3PZjBoXZGTo+hp4M/Js1n7qzVcPdn2161X0/n/N3pSpTyPkEhp1vpI26079XXQskeG/SF2+3jjjft8thg599H9rOhHzvIzsWPHQRd4ryuW0fD3V/V18Gqh+1AUu+pTlejlG8IDLYzynoMhgvcN+K7XHDsgA374l1QvNMO3u566/TPdewGsYNt0McMtC38mAGO3mWr4e6vdi6EYwdh1p+8rkWhlE8JCLBTLCN6f32xvarjcCQLina4Q38XbH0Zak6cPqdLnN2MPGagPaIHQnRqm6yQqeHuj1wuu0BT7GC7u71SquV17Grvlk0cc/o9lwuO50HJbjiSCUfcjxvX2KU/vjT7IRh5U6uWp+Huj7IW25kB33++XcznVcprBAScnof/5fo5YFdi/eKAbemXZNmbrFqZhru/MQZWPmgHfdLmOF2NUgrslMrIPvYYeEnb/Mo2+S2q7exdCsU77BoyPjxHVynVPBru/mb1I9AtCYZ83+lKlFIO0nD3J0f325XzRv7ETulSSrVbGu7+ZKd73u3gK52tQynlOA13f2GMvZMucZxdU0Mp1a5puPuL4p1Qukf72pVSgIa7/9jxJgQEQdrlDZ+rlPJ7Gu7+wOWy/e19pkFYpNPVKKW8gIa7P8jfYG95HqxdMkopS8PdH+x4E4I6wYCLna5EKeUlNNx9XX0t7HoHUmdBh3Cnq1FKeQkNd1+XswIqS2HIXKcrUUp5EQ13X7dzoV16tO8MpytRSnkRDXdfVnsKst6FgZdCUAenq1FKeRENd1+2d6nd9UW7ZJRSZ9Fw92U73oTOsZA80elKlFJeRsPdV506Bvs+hkFX6LrtSqlv0HD3Vbvfg/pqXUtGKXVOGu6+asdC6J4McSOcrkQp5YU8CncRmSUie0QkW0TuOcfnk0Rks4jUiYg2JVtbRTHkrrDLDYg4XY1Sygs1GO4iEgg8DlwEpAFXi0jaWacdAm4EXmnpAtU5ZL4DxqWzZJRS3yrIg3NGAdnGmBwAEXkNmANkfnmCMeaA+zNXK9SozrbjTYgdDDEDnK5EKeWlPOmWiQPyznid736v0URkvohkiEhGSUlJUy6hynIhf6MOpCqlzsuTcD9Xp65pyi8zxiwwxqQbY9Kjo6Obcgm1c5F91H1SlVLn4Um45wMJZ7yOBwpapxzVoJ2LIGEMdEt0uhKllBfzJNw3Av1EJEVEQoB5wOLWLUudU/EuOJKpXTJKqQY1GO7GmDrgNmApkAW8YYzZJSL3i8hlACIyUkTygbnA0yKyqzWLbrd2LAQJ1H1SlVIN8mS2DMaYD4APznrvv854vhHbXaNaU9a7kDIJOut4hVLq/PQOVV9RXgBH90Hf6U5XopTyARruviJ3pX1MmexsHUopn6Dh7ityV0KnCHvzklJKNUDD3RcYY/dKTZkIAfqfTCnVME0KX1CWA+X5djBVKaU8oOHuC3JX2Eftb1dKeUjD3RfkroTwXhDZ1+lKlFI+QsPd27lckLvKdsno2u1KKQ9puHu7I5lQWQq9tUtGKeU5DXdv99X8dh1MVUp5TsPd2+WugIg+0FVXd1BKeU7D3ZvV18GBNdpqV0o1moa7NyvcCjUVGu5KqUbTcPdmOcvto4a7UqqRNNy9We5Ku5ZMWJTTlSilfIyGu7eqrYK89XpXqlKqSTTcvVX+Bqir0i4ZpVSTaLh7q9yVdku9pHFOV6KU8kEa7t4qZwXEDYeOXZyuRCnlgzTcvVF1BRzepF0ySqkm03D3Rgc/B1Ovg6lKqSbTcPdGuSshsAMkjHK6EqWUj9Jw90a5KyBxNAR3croSpZSP0nD3NiePQtEO7W9XSjWLhru3ObDKPmp/u1KqGTTcvU3uCggJh17Dna5EKeXDNNy9Te5Ke+NSYJDTlSilfJiGuzc5fhiOZuuWekqpZtNw9ya6pZ5SqoVouHuT3JUQGgkxg5yuRCnl4zwKdxGZJSJ7RCRbRO45x+cdROR19+frRSS5pQv1e8bYwdTkiRCg/89VSjVPgykiIoHA48BFQBpwtYiknXXaT4AvjDF9gUeAP7d0oX6vLAfKD2uXjFKqRXjSRBwFZBtjcowxNcBrwJyzzpkDvOB+vhCYLiLScmW2A7kr7GPvKU5WoZTyE57Mt4sD8s54nQ+M/rZzjDF1InIciARKW6LIRlu/ADKedeRXN9mJYugSBxG9na5EKeUHPAn3c7XATRPOQUTmA/MBEhMTPfjVTRQWCdGprXf91hCdCqmzQb/wKKVagCfhng8knPE6Hij4lnPyRSQI6AqUnX0hY8wCYAFAenr6N8K/xQy+0h5KKdVOedLnvhHoJyIpIhICzAMWn3XOYuCH7uffBz4zxrReeCullDqvBlvu7j7024ClQCDwnDFml4jcD2QYYxYDzwIviUg2tsU+rzWLVkopdX4eLWBijPkA+OCs9/7rjOdVwNyWLU0ppVRT6d0ySinlhzTclVLKD2m4K6WUH9JwV0opP6ThrpRSfkicmo4uIiXAwSb+eBROLW3QdFpz2/C1mn2tXtCa28q31ZxkjIlu6IcdC/fmEJEMY0y603U0htbcNnytZl+rF7TmttLcmrVbRiml/JCGu1JK+SFfDfcFThfQBFpz2/C1mn2tXtCa20qzavbJPnellFLn56std6WUUufhc+He0Gbd3kZEnhORIyKy0+laPCEiCSKyTESyRGSXiPzC6ZoaIiIdRWSDiGxz1/z/nK7JUyISKCJbROQ9p2vxhIgcEJEdIrJVRDKcrqchItJNRBaKyG73v+mxTtd0PiKS6v67/fIoF5E7mnQtX+qWcW/WvRe4ELtByEbgamNMpqOFnYeITAJOAC8aYwY7XU9DRKQn0NMYs1lEwoFNwOVe/ncsQJgx5oSIBAOrgV8YY9Y5XFqDROROIB3oYoy5xOl6GiIiB4B0Y4xPzBkXkReAVcaYZ9z7UYQaY445XZcn3Hl3GBhtjGn0PUG+1nL3ZLNur2KMWck5dqXyVsaYQmPMZvfzCiALu0eu1zLWCffLYPfh9a0WEYkHZgPPOF2LPxKRLsAk7H4TGGNqfCXY3aYD+5sS7OB74X6uzbq9Onh8mYgkA8OA9c5W0jB398ZW4AjwsTHG62sG/grcDbicLqQRDPCRiGxy74nszXoDJcDz7q6vZ0QkzOmiGmEe8GpTf9jXwt2jjbhV84lIZ2ARcIcxptzpehpijKk3xlyA3eN3lIh4dReYiFwCHDHGbHK6lkYab4wZDlwE3OrudvRWQcBw4EljzDDgJOD143QA7i6ky4A3m3oNXwt3TzbrVs3k7rdeBLxsjHnL6Xoaw/21ezkwy+FSGjIeuMzdh/0aME1E/uVsSQ0zxhS4H48Ab2O7Sr1VPpB/xre4hdiw9wUXAZuNMcVNvYCvhbsnm3WrZnAPTj4LZBljHna6Hk+ISLSIdHM/7wTMAHY7W9X5GWN+a4yJN8YkY/8df2aMuc7hss5LRMLcg+y4uze+C3jtLDBjTBGQJyKp7remA147MeAsV9OMLhnwcA9Vb/Ftm3U7XNZ5icirwBQgSkTygT8YY551tqrzGg9cD+xw92ED/M69j6636gm84J5dEAC8YYzxiamFPiYWeNv+/58g4BVjzBJnS2rQ7cDL7sZgDvAjh+tpkIiEYmcE/kezruNLUyGVUkp5xte6ZZRSSnlAw10ppfyQhrtSSvkhDXellPJDGu5KKeWHNNyVUsoPabgrpZQf0nBXSik/9P8BDGL2JgBo/aMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env_info = {\"state representation structure\": \"regular\", \n",
    "            \"fundamental timestep\": 0.2, \n",
    "            \"gamma\": 0.9, \n",
    "            \"pursuit port total duration\": 7, \n",
    "            \"background port total duration\": 7, \n",
    "            \"transit duration\": 0.5, \n",
    "            \"consumption duration\": 1, \n",
    "            \"first time in pursuit port that could give a reward\": 1.2,\n",
    "            \"required minimum wait time in pursuit port\": 1.2, \n",
    "            \"required minimum wait time in background port\": 1, \n",
    "            \"time to wait until the trial reset\": 1.5, \n",
    "            \"exponential distribution scale\": 4, \n",
    "            \"delivery time in the background port\": 1.6,\n",
    "            \"reward amount in pursuit\": 2.4, \n",
    "            \"reward amount in background\": [1.2, 2.4]}\n",
    "env = GiveUpEnvironment(env_info=env_info)\n",
    "env.test_reward_function()\n",
    "(rou_g, rou_l, t_rou_g, t_rou_l) = env.PolicyMax()\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "ax.plot(env.PSTimeArray, rou_g)\n",
    "ax.plot(env.PSTimeArray, rou_l)\n",
    "print(f\"According to the global reward rate, the optimal give-up time in the pursuit port is at {t_rou_g} second!\")\n",
    "print(f\"According to the local reward rate, the optimal give-up time in the pursuit port is at {t_rou_l} second!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████| 500000/500000 [00:35<00:00, 14086.11it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████| 500000/500000 [00:34<00:00, 14446.27it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████| 500000/500000 [00:34<00:00, 14287.95it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████| 500000/500000 [00:34<00:00, 14502.49it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████| 500000/500000 [00:35<00:00, 14104.03it/s]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJztnXl83GW1/99nluxrk7RNmy7pQkuhe6TsUGRVoSIiICCKWr0sF1zuT714FeWiXHD3olxkXwSUzaJFQaGAQJG2ULpRaNMtTdqszb7NzPP745lJJ+kkmSSzJXPer9e8vpnn+3y/c775Jp853/Oc5zxijEFRFEVJHhzxNkBRFEWJLSr8iqIoSYYKv6IoSpKhwq8oipJkqPAriqIkGSr8iqIoSYYKv6IoSpKhwq8oipJkqPAriqIkGa54GxCKwsJCM3369HiboSiKMmpYv359rTGmKJy+CSn806dPZ926dfE2Q1EUZdQgInvC7auhHkVRlCRDhV9RFCXJUOFXFEVJMhIyxq8oSnLS3d1NRUUFHR0d8TYlYUlLS6OkpAS32z3sc6jwK4qSMFRUVJCdnc306dMRkXibk3AYY6irq6OiooLS0tJhn0dDPYqiJAwdHR0UFBSo6PeDiFBQUDDiJyIVfkVREgoV/YGJxO9n0FCPiEwBHgImAj7gbmPML/v0uRz4lv9tC/BvxpiN/n27gWbAC3iMMWUjtlpJboyBDQ9CWz2kZkN6Psz9OLjT422ZoowKwonxe4BvGGM2iEg2sF5EXjTGbA3qsws4zRjTICLnAXcDy4L2LzfG1EbObCWp2fkPeO6G3m1zPg6XPAKOQR5iO5pgz+sw57ze7U2VULcTSk+JrK2KkoAMGuoxxlQZYzb4f24GtgGT+/R5wxjT4H+7FiiJtKGKAlhv/x+3QO5U+PZe+OYOOOsW2P4XWPOjwY9/4bvw2KWw71+92//yTXj4QvsUoShh8PnPf54nn3wy3mYMiyHF+EVkOrAYeGuAbl8Eng96b4AXRGS9iKwcqoGK0ottz0HVu3D6tyEtF7KK4MTrYfEV8OodsPkp69V/8AK8eSd0tR4+tuYDeOdh+/O6+w+3N+6HD54HXzdsXx3b61GUOBB2OqeIZAFPATcaY5r66bMcK/wnBzWfZIypFJHxwIsi8r4x5tUQx64EVgJMnTp1CJegJA0+L7z031B4FCy45HC7CHz8Z1D7ITy9EozPvgD2roWLH7QhoH/8ANyZMOsM2PI0nPsjOz7wzsO2f0YhbHnWfokocecHz21ha2VIqRk28ybl8P3zjxmwz6233spDDz3ElClTKCoqYunSpXzzm98c8Jj169fz9a9/nZaWFgoLC3nggQcoLi7md7/7HXfffTddXV3MmjWLhx9+mO7ubhYuXEh5eTkOh4O2tjbmzJlDeXk5e/fu5dprr6WmpoaMjAx+97vfMXfu3Ej+CoAwPX4RcWNF/1FjzNP99FkA3AOsMMbUBdqNMZX+bTXwDHBcqOONMXcbY8qMMWVFRWEVmFOSjff+ALXbYflN4Ozjs7hSbYz/mE/Bqf8Bn/sTfPT7sG0VrPmxDe28/2c46QY45Rvg6YCNT4DXAxsegplnwKLPQvkaaG8I+fHK2Gf9+vU8/vjjvPPOOzz99NO8/fbbgx7T3d3N9ddfz5NPPsn69eu5+uqruemmmwD41Kc+xdtvv83GjRs5+uijuffee8nNzWXhwoW88sorADz33HOcc845uN1uVq5cya9//WvWr1/PT37yE6655pqoXGc4WT0C3AtsM8b8rJ8+U4GngSuNMR8EtWcCDmNMs//ns4EfRsRyJbkwBl75HyheCEdfELpP1ni46HeH35eeZgdsX70dNj4GmePhhGsgJRMmL4V190H+NGjaD+feBrmT4Y1fwfurYfHl9hzNB+yXRP70qF+i0pvBPPNo8Nprr3HhhReSkZEBwAUX9PO3FsT27dvZvHkzZ511FgBer5fi4mIANm/ezHe/+10OHTpES0sL55xzDgCXXHIJTzzxBMuXL+fxxx/nmmuuoaWlhTfeeIOLL76459ydnZ2RvkQgvFDPScCVwCYRedff9p/AVABjzF3A94AC4Df+HNNA2uYE4Bl/mwv4vTHmrxG9AiU5aKuHhl1w9n8PnrkTQAQ+8TOo3wl737ThoJRMu2/pF2DVdfD8tyBrgs3ycbggbypsecYKf/NB+L/TwOGEGzaCc/hT5JXRw1Dz5I0xHHPMMbz55ptH7Pv85z/Ps88+y8KFC3nggQdYs2YNYL9QvvOd71BfX8/69es544wzaG1tJS8vj3ffffeI80SacLJ6/mmMEWPMAmPMIv9rtTHmLr/oY4z5kjEmP2h/mb+93Biz0P86xhhza7QvSBmj1O+024JZQzvOlQqX/h5W/AaWXHW4/dhPQWoOHNoDi6+0oi4C8z5pwz0tNfDHq6C12j4RbP1TxC5FSVxOPfVUnnnmGdrb22lubua5554b9Jg5c+ZQU1PTI/zd3d1s2bIFgObmZoqLi+nu7ubRRx/tOSYrK4vjjjuOG264gU984hM4nU5ycnIoLS3lj3/8I2C/UDZu3BiFq9SZu8pooc4v/ONmDv3YjHHWgw8eF0jJhIWXgThhadAXwjGftNk9D55vnxIuvNt+2az97cjsV0YFS5Ys4ZJLLmHRokVcdNFFnHLK4PM6UlJSePLJJ/nWt77FwoULWbRoEW+88QYAt9xyC8uWLeOss846YpD2kksu4ZFHHuGSSw4nKjz66KPce++9LFy4kGOOOYY//Sk6DocYY6Jy4pFQVlZmdAUupRcv/Te89lO46SC4UiJzzs4WqNsBkxYdbjMGfrkADu2F46+1mT//+h2s/iZ88e8w5SPg89k5A+Pn2ScHJWJs27aNo48+Ot5m9HDzzTeTlZU1aFZPrAn1exKR9eFWRlCPXxkd1O208fdIiT5AalZv0Qcb7jntW7DocjjrB7Zt4WWQmgtv/dZ+Mfzl63bOwHM3agaQMipR4VdGB/U7hxfmGQ6Lr4BP/ubwYG5qFiy50sb5V10P6++H+RdDZ6OdJKaMWW6++WZ27drFokWLer3uv//+wQ9OYLQev5L4GGM9/inLBu8bLY5bCWt/Yyd7feTL8LE7wOexsf9lX4XMwvjZpkSVO+8ce1/u6vEriU9LNXS1xM7jD0X+NDj5a3DCdXDe7TYkdPp3oLsNXv9F/OxSlGGgHr+S+Aw3lTPSfPR7vd8XzYH5n7GDvydcB9kT42OXogwR9fiVxCeQylkwI752hOL0bx0O+SjKKEGFX0l86nfaWbW5CVi8b9wMmHqCXSNASXrWrFnTk8OfyKjwK4lP3U5bK6dvYbZEofRUOLBZa/krKvyKEjHqYpjKORxKTwUM7P5nvC1RIsCtt97KnDlzOPPMM7nsssv4yU9+ErLfr371K+bNm8eCBQu49NJL2b17N3fddRc///nPWbRoEa+99hrPPfccy5YtY/HixZx55pkcPHgQn8/H7NmzqampAcDn8zFr1ixqa2O3SGGCulCK4sfng/pymHF6vC3pn0lLwJ0Bu1+DeYNXc1TC5Plvw4FNkT3nxPlw3m397g4uy+zxeFiyZAlLly4N2fe2225j165dpKamcujQIfLy8vjqV7/aa6ZvQ0MDa9euRUS45557uP322/npT3/KFVdcwaOPPsqNN97I3//+dxYuXEhhYexSgtXjVxKb5irwtCfmwG4AVwpMPR52vRZvS5QRElyWOScnZ8CyzAsWLODyyy/nkUceweUK7UNXVFRwzjnnMH/+fO64446e4m1XX301Dz30EAD33XcfX/jCFyJ/MQOgHr+S2NSPoDhbLCk9Ff5+s51zkDU+3taMDQbwzKNJuGWZ//KXv/Dqq6+yatUqbrnllh5RD+b666/n61//OhdccAFr1qzh5ptvBmDKlClMmDCBl156ibfeeqtX5c5YoB6/ktj0pHImuPBPP9Vud6vXP5oJtyyzz+dj3759LF++nNtvv71noZXs7Gyam5t7+jU2NjJ58mQAHnzwwV7n+NKXvsQVV1zBZz7zGZxOZ/QuKgSDCr+ITBGRl0Vkm4hsEZEbQvQREfmViOwQkfdEZEnQvqtE5EP/66q+xyrKgNTvBGcq5JTE25KBKV5o6/vvOmI5aWUUEW5ZZq/XyxVXXMH8+fNZvHgxX/va18jLy+P888/nmWee6Rncvfnmm7n44os55ZRTjojhX3DBBbS0tMQ8zAPhhXo8wDeMMRtEJBtYLyIvGmO2BvU5D5jtfy0DfgssE5FxwPeBMsD4j11ljBl9JQ2bD9gSvtNPHryvEjnqdtpc+XBX3YoXThdMO1Hj/GOAm266qWfN3EBopi9ut5t//vPILK6jjjqK9957r1fbihUrQp5j48aNLFy4MCqLqQ9GOCtwVRljNvh/bga2AZP7dFsBPGQsa4E8ESkGzgFeNMbU+8X+ReDciF5BrHj9V/DYZ+NtRfJRtzPxwzwBpp9in1Aa98fbEiXBue2227jooov48Y9/HJfPH5IbJSLTgcXAW312TQb2Bb2v8Lf11z76aNwH3a3xtiK58HntOrvjEjijJ5hSf5x/1yvxtUOJGNEqy/ztb3+bPXv2cPLJ8YkghJ3VIyJZwFPAjcaYpr67QxxiBmgPdf6VwEqAqVMTcGp+8wFbk8UYW5lRiT6N+8DbBYWz421JeEw41n5JvfxjmPMxSM+Lt0VKBEjasswi4saK/qPGmKdDdKkApgS9LwEqB2g/AmPM3caYMmNMWVFRUThmxZbmKrv1eeNrRzJRu8Nu412VM1wcDvjU76C5Ep67wToJypBJxOVgE4lI/H7CyeoR4F5gmzHmZ/10WwV8zp/dczzQaIypAv4GnC0i+SKSD5ztbxtd+HzW4we7ELcSG+pGmfADlJTB8ptg67N20RZlSKSlpVFXV6fi3w/GGOrq6khLSxvRecIJ9ZwEXAlsEpF3/W3/CUz1G3IXsBr4GLADaAO+4N9XLyK3AG/7j/uhMWb0VbJqrz8s+D5PfG1JJup22BTJzAR8AhyIk26E8jXw/Leg8Cg7q1cJi5KSEioqKnrq2ChHkpaWRknJyNKbBxV+Y8w/CR2rD+5jgGv72XcfcN+wrEsUAmEeAK96/DGjbof19kfbmIrDARf+H9xzJtx3Diy6As78vs7oDQO3201paWm8zRjzJHhydILQFCT86vHHjoDwj0ZyiuHatXDSDfDeE/DrMji4dfDjFCUGqPCHg3r8sae73Wb1jFbhB0jNhrN+CF/9J3Q2wft/jrdFigKo8IdHYGAX1OOPFfXldjtaJm8NxPi59gus8p14W6IogAp/eDQHZaCq8MeGQEbPaMnhH4xJi6Dy3cH7KUoMUOEPh2CPX0M9saH2Q7tN9HLM4VK8yDoQLdXxtkRRVPjDolkHd2NO3U7ILobUrHhbEhkmLbJb9fqVBECFPxyaDxzOJdcJXLFhNGf0hGLiAkCgSoVfiT8q/IPh7baP57n+yhNe9fhjQt2OsTGwGyAtxz/Aq8KvxB8V/sFoqQYM5PmFXz3+6NNWb2dLF4yRgd0Akxapx68kBCr8gxEY2A14/Brjjz6jsUZPOBQvgqb90KLlCJT4osI/GIGB3Tx/qWjN6ok+Y1X4AwO8Aa+/tQ5euR08XfGzSUlKVPgHIyD86vHHjrodIE7InxZvSyLLxAV2G5jI9ecb4eVbYf/6+NmkJCUq/IPRXGVFKHuifa/CH33qdkD+dHC6421JZAke4N3yDGxbZds7GuNrl5J0hL0CV9LSfMCKvivVvtdQT/SpHWOpnMFMWmxLNu9bCzkl0FShwq/EHPX4B6O5ygq/w/8dqR5/dPF5rcdfdFS8LYkOxYugtQY6muDC39q2jkPxtUlJOsJZges+EakWkc397P8PEXnX/9osIl4RGefft1tENvn3rYu08TGhqcrOIA0Iv3r80aVhN3g7oXBOvC2JDiUfsdvTvgVT/Au0qMevxJhwPP4HgHP722mMucMYs8gYswj4DvBKn1W2lvv3l43M1DjR7Bf+QLxZPf7oUrPdbovGqPBPOQ6++CKc8g1wpYA7Q4VfiTmDCr8x5lUg3OUSLwMeG5FFiUR3u30Mz54IjoDwq8cfVWr9wl84RkM9Ilb8Hf5/vbRcDfUoMSdiMX4RycA+GTwV1GyAF0RkvYisjNRnxYzA5K1eoR71+KNKzQeQNRHS8+JtSWxIy1WPX4k5kczqOR94vU+Y5yRjTKWIjAdeFJH3/U8QR+D/YlgJMHXq1AiaNQICOfzZE8Gpg7sxoXb72B3YDUVangq/EnMimdVzKX3CPMaYSv+2GngGOK6/g40xdxtjyowxZUVFRRE0awT0CH+xhnpigTHW4x+rA7uhSMuFdg31KLElIsIvIrnAacCfgtoyRSQ78DNwNhAyMyhhaQr2+P3Cr1k90aOpErqax+7Abig01KPEgUFDPSLyGHA6UCgiFcD3ATeAMeYuf7cLgReMMa1Bh04AnhGRwOf83hjz18iZHgPqd9p/zPR8MD7b5vPG16axTO0Yz+gJhQq/EgcGFX5jzGVh9HkAm/YZ3FYOLByuYQlBIOwgYss2IBrqiSY1H9htsoV6OpvA5zuc6aMoUUb/0gaidntv79Pp1lBPNKndboUwa3y8LYkd6Xn2abKrJd6WKEmECn9/tNXbqfXBwu9waVZPNAl+wkoW0nLtVsM9SgxR4e+PwAzS4LCDw63CH01q3k+uVE4IEn7N7FFihwp/f4QaaHS6NNQTLdrqoa0WiubG25LYoh6/EgdU+PujZrutoxJYgAX8Hr8Kf1QI9YSVDKjwK3FAhb8/arbbmvDBmRYOl6ZzRoueJ6xkC/X4S1Oo8CsxRIW/P2o/ODKfXEM90aPmA3ClQ26ClOuIFerxK3FAhT8UnS3QuO9I4ddQT/So3Q6Fs5Ivlz01x261bIMSQ5LsvyxMavuZSORQjz9qHNwC44+JtxWxx+mClGz1+JWYosIfioDwhwr1aIw/8rTW2YJ4E5JQ+EHLNigxR4U/FDXbrXc/bkbvdg31RIfqLXY78dj42hEvVPiVGKPCH4raD2DczMMVOQNoyYbocMBftHVCkgp/utbkV2KLCn8o+ptBqumc0eHgFsgsSq4aPcGox6/EGBX+vni6oH5X6IlEDpeGeqLBwc3J6+2DrrurxBwV/r7U7wTjDV06QEM9kcfrgeptyTuwC+rxKzFHhb8vNQPMIFWPP/LU7wRvp3r8nU0aRlRixqDCLyL3iUi1iIRcNlFETheRRhF51//6XtC+c0Vku4jsEJFvR9LwqFG9FcQBhRrjjwkH/X9WyZrRA4fLNnQ2xdcOJWkIx+N/ADh3kD6vGWMW+V8/BBARJ3AncB4wD7hMROaNxNiYcGATFMwGd/qR+zTUE3kObLZfqKG+aJMFLdugxJhBhd8Y8ypQP4xzHwfsMMaUG2O6gMeBFcM4T2w5sBkmzg+9T/P4I8/BLVb0XanxtiR+qPArMSZSMf4TRGSjiDwvIoFRusnAvqA+Ff62kIjIShFZJyLrampqImTWEGlvgMa9/YcdNNQTeQ5uSe74PhwWfq3Xo8SISAj/BmCaMWYh8GvgWX97qPXzTH8nMcbcbYwpM8aUFRUVRcCsYXAwMIO0H49fq3NGlrZ6aKpI7oweUI9fiTkjFn5jTJMxpsX/82rALSKFWA8/aBUTSoDKkX5eVDmwyW4naKgnJlRvtVv1+O1WhV+JESMWfhGZKGJXxxaR4/znrAPeBmaLSKmIpACXAqtG+nlR5cBmyBwP2RNC73e4bN65EhkOJnmNngDpuhiLEltcg3UQkceA04FCEakAvg+4AYwxdwGfBv5NRDxAO3CpMcYAHhG5Dvgb4ATuM8ZsicpVRIoD7w0sQk5dbD2iHNgEGQWQ1c8XbbKQkg2ICr8SMwYVfmPMZYPs/1/gf/vZtxpYPTzTYoy329bomflv/ffRCVyRpWqjHU+RUMNBSYTDAWk5WrZBiRk6czdA7Qfg7eo/vg+axx9JujtsjH/Sknhbkhho2QYlhqjwBwiUBu4vowesx48Bny8mJo1pDmyyYbPJKvyACr8SU1T4Axx4D5ypUDCr/z4Of2RMwz0jp/Idu1WP35KmNfmV2KHCH+DgZpgwz+bq90dgYRYN94ycyg02gypnUrwtSQzU41diiAo/gDE29DBYPrl6/JFj/wYb5kn2gd0A6vErMUSFH6D5ALTVwcQFA/dz+D1+LdswMjqb7WC6hnkOk5arJRuUmKHCD4dn7A42kSgQBtJQz8io2ggYmLQ43pYkDmm50N2qf1tKTFDhB/9Aowyc0QNBHr/+c46InoFdFf4eembvak1+Jfqo8IMdaCyaA6nZA/frifHr7N0RsX8D5E6BrDgV40tEAouxtA+nArqiDA0VfmOsEIUTb+7J6lHhHxGVG9Tb70sgu6mxIr52KEmBCn9jBbRWhzeRSLN6Rk5bPTTs1olbfcnzF7Jt3DdwP0WJACr8lRvsdkgevwr/sNH4fmhyJgOiHr8SE1T492+wg7bhlAbu8fg1nXPYBIS/eFF87Ug0nG7ILoZD6vEr0UeFv3KDXQEqnDVfNdQzcirW2cXsA1ksymHypmioR4kJyS38Ph9UvguTl4bXX0M9I8Png31rYeqyeFuSmOSWqPArMWFQ4ReR+0SkWkQ297P/chF5z/96Q0QWBu3bLSKbRORdEVkXScMjQv1O6GwKf6BR0zlHRt2HdkH7qSfE25LEJHcKNO7X6q9K1AnH438AOHeA/buA04wxC4BbgLv77F9ujFlkjCkbnolRZP96uw23dEDPBC4V/mGx9027nXJ8fO1IVPKm2DBiy8F4W6KMcQYVfmPMq0C/s0qMMW8YYxr8b9diF1UfHezfAO5MO3krHLRkw8jY+xZkFELBzHhbkpjkBlI6NbNHiS6RjvF/EXg+6L0BXhCR9SKyMsKfNXIqN0DxQnA4w+uvHv/I2PsmTD1eK3L2R4/w742vHcqYJ2LCLyLLscL/raDmk4wxS4DzgGtF5NQBjl8pIutEZF1NTU2kzOofb7ctzjaUiUSa1TN8mg9Cwy4r/Epocv0Py5rSqUSZiAi/iCwA7gFWGGPqAu3GmEr/thp4Bjiuv3MYY+42xpQZY8qKimJQw6V6K3g6hjaRSEs2DJ99a+1W4/v9k5Zjq3RqqEeJMiMWfhGZCjwNXGmM+SCoPVNEsgM/A2cDITOD4sLet+y25CPhH6Me//DZuxZcaTa0pvRPrubyK9FngHUGLSLyGHA6UCgiFcD3ATeAMeYu4HtAAfAbsbFbjz+DZwLwjL/NBfzeGPPXKFzD8Nj7pp0mnzc1/GM0nXP47F1r50u4UuJtSWKjwq/EgEGF3xhz2SD7vwR8KUR7OZCY7p0xVvinnTS0gUadwDU8ulrt4isn3xhvSxKfvCmw5414W6GMcZJz5u6hPdBcNfSBRvX4h8f+9WC8Gt8Ph9wS6GzU9XeVqJKcwr/HP5Fo2olDO86p6ZzDYt+/7HbKEMZTkhXN5VdiQHIK/943bfZE0dFDO86hE7iGRWstpOZAen68LUl8VPiVGJC8wj/leHAM8fJ1Atfw6GqBlMx4WzE6CCzIckgncSnRI/mEv7UWaj8Y3kSiwAxf9fiHRncbuDPibcXoIHM8OFPU41eiSvIJ/17/RKKhxvfBZgA5XOrxD5WuVvX4w8XhsGnGmtKpRJEkFP43wZk6/KX/HG6dwDVUVPiHRt4U9fiVqJKcwj95aXgrboXC6daSDUOlq1VDPUMhd4rW61GiSnIJf2Ai0UgKhWmoZ+h0t6nHPxRyp9h5Jp6ueFuijFGSS/j3rrWiPe2k4Z/D4dJQz1DpUuEfEhPmAcaWDVeUKJBcwl/+ss2YmDaCpf801DN0NJ1zaJSeCgjsfDnelihjlOQS/p0vw5RlIxMh9fiHjqZzDo30fJt8UL4m3pYoY5TkEf6Waji4GWaeMbLzaIx/aPi8dt2DlKx4WzK6mLkcKt6GjqZ4W6JEgL11bXz7qffo9vribQqQTMIf8J5mLh/ZeZxuncA1FLpa7TZFPf4hMWO5LWy3+5/xtkSJAM9vruLxt/exp64t3qYAyST8O1+C9HEwcYSVoh1u9fiHQkD4NdQzNKYcZ39n5RrnHwvsrrP/By2diaEdYQm/iNwnItUiEnIFLbH8SkR2iMh7IrIkaN9VIvKh/3VVpAwfEsbY+P6M04den6cvTg31DIluv4ejoZ6h4Uq12Wc6wDsm2F1r/w+aOxIjWhCuCj4AnDvA/vOA2f7XSuC3ACIyDrti1zLservfF5HYl2is3gYtB0Ye5gEb49dQT/hoqGf4zFwOdR/qLN4xQI/H35EYTmNYwm+MeRWoH6DLCuAhY1kL5IlIMXAO8KIxpt4Y0wC8yMBfINEh8Lg8IxLCr6GeIdEj/JrOOWQCf6/q9Y9q2ru8VDV2ANA8moQ/DCYDwXPMK/xt/bXHlp0vQcHswyVvR4JTPf4h0R2I8avwD5nxR0PWRI3zj3L21h8e0G0eTTH+MAi1cK0ZoP3IE4isFJF1IrKupqYmQmYBnk7Y/frI0zgDaDrn0NBQz/ARseNS5a/YcSplVLKrtrXn59EW4x+MCiDYnS4BKgdoPwJjzN3GmDJjTFlRUVGEzAJ2vQaedpj10cicT6tzDo2uwOCuevzDongBtNVCx6F4W6IMk0B83+WQ0RXjD4NVwOf82T3HA43GmCrgb8DZIpLvH9Q9298WO7avtmlxpadF5nxasmFodLXYrYZ6hkduid3qAO+oZU9dK+MyUyjISkmYGL8rnE4i8hhwOlAoIhXYTB03gDHmLmA18DFgB9AGfMG/r15EbgHe9p/qh8aYgQaJI4sxsP15G+Zxp0XmnBrqGRrd6vGPiGDhnzg/vrYow2JXbSvTCzJo6vAkTB5/WMJvjLlskP0GuLafffcB9w3dtAhQ9S40V8Lc/4rcObVWz9AIhHp0Atfw0MXXRz27a9s4cVYB5TWtNI2xGH9i8v5qEAfMPidy59RQz9DoarGiP9KJc8lKRqFdMU6XYhyVtHd5OdDUwfSCTLLTXAkT6hnb/43bV8OU4yGzIHLnVI9/aGhlzpHhcEDuZPX4Ryl76u3A7vRCK/yJEuoZu8LfsMdW45z7scie16kTuIbSC5pQAAAgAElEQVREV6umco6UHBX+0cpufypnaUEm2anuMZfOmXhsf95u50RY+LVkw9DoatU6PSMlVxdfH63s9lfjnFaYQVaaa8ylcyYe21dD4RwomBnZ82rJhqGhC62PnNwSuwavji2NOnbXtlKQmUJOmpvsNBetXV68vvhPxhubwt9WD3tej3yYB7Q651DRhdZHTm4JGJ8Vf2VUsau2lemF9u8/K9UmUSZCnH9sCv+2VVac530y8ufWUM/Q0IXWR45O4hq17K5rZXqB/fvPSXMDiVG2YWwK/+anoGAWFI9w0ZVQONx2ZSStnRIeutD6yNFc/lFJW5eHg02dTC+woc6sNPX4o0fzAVuf59iLbJGrSOP0z3lTrz88NJ1z5OT6C9pqLv+oIrDMYiDUk+0X/kTI5R97wr/lWcDAMZ+Kzvkd9nFN4/xh0tWqHv9IScmE9Hz1+EcZPamcPcJvtSMRMnvGnvBvfgomHAvj50bn/A6/x6+TuAbHGBX+SJFbosI/yijvI/yBwd1EKNswtoS/YQ9U/MuGeaKF0+/xa2rd4HS3A0ZDPZFAc/lHHTtrWpiYk0amX/BzNNQTJbY8bbfHRinMA0Eef/xvXsKjC61HDvX4Rx3lNa3MKDr8tKuDu9Fi81MwuQzyp0fvMzTUEz66+lbkyC2BzkboaIq3JUoYGGMor2npJfzpbidOh2g6Z0TpaoWUbJh/cXQ/pyfUE/+bl/AEhF9DPSMnkMvftD++dihhUdfaRVOHhxmFh592RYSs1MQo2xDuQiznAr8EnMA9xpjb+uz/ObDc/zYDGG+MyfPv8wKb/Pv2GmMuiIThR5CSCVc/H/38en9Wj8fTHd4vL5nRUE/kCM7lH390fG1RBqW8xjo9wR4/kDClmQf1+EXECdwJnAfMAy4TkXnBfYwxXzPGLDLGLAJ+DTwdtLs9sC9qot/b4Oie35/H/7XH1kX3c8YCgWUXNdQzcnI0l380UV5j//ZnFvV2erJSXTSPkhj/ccAOY0y5MaYLeBxYMUD/y4DHImFcQuKP8VfWaax1UHSh9ciRPRHEqQO8o4Ty2lZSXA4m5aX3as9JS4zSzOEI/2Qg2M2o8LcdgYhMA0qBl4Ka00RknYisFZEoFM+JMf5QT3d3F50eb5yNSXB6Yvwq/CPG4dS6/KOI8poWSgsycTp6RyCyEmQxlnCEP1TspL9A+qXAk8aYYEWcaowpAz4L/EJEQtZJFpGV/i+IdTU1NWGYFSf8oR4XXg61xf+bO6HpDmT1qPBHBE3pHDX0TeUMMGpi/FgPf0rQ+xKgsp++l9InzGOMqfRvy4E1wOJQBxpj7jbGlBljyoqKisIwKz60eez3oAsv9a1dcbYmwekJ9WiMPyLklsAhjfEnOt1eH3vr20IK/0BZPY1t3XR0xyaKEI7wvw3MFpFSEUnBivuqvp1EZA6QD7wZ1JYvIqn+nwuBk4CtkTA8XtS1+wBwiZeGNhX+AdFQT2SZcAw07rWFCJWEZW99Gx6f6ZXKGSA7zd2vx/+TF7az7Ef/iLZ5QBjCb4zxANcBfwO2AX8wxmwRkR+KSHCWzmXA48b0yqc8GlgnIhuBl4HbjDGjWvhr2/zCj5eGVg31DEh3qx0TcaXE25KxwYzT7bb8lXhaoQxCf6mcYEM9XV5fSM9+X0MbJfnpR7RHg7BS0Y0xq4HVfdq+1+f9zSGOewOYPwL7Eo5g4a9Xj39gdKH1yDJxga3SWb4GFl4Sb2uUfgikcs4oCuXxHy7bkOZ29tq3r76N2eOzo28gY2nmboyobrWPaW68HNIY/8B0tenkrUjicEDpaVb4dSGghKW8ppXCrBRy091H7OuvJr/PZ6hoaGfKuNh4/Cr8Q+Rgq/X4s9xGPf7B6GrRcg2RZuZyaK6E2g/jbYnSD+W1LSHj+wBZqaFr8te0dNLp8TF1XGz+X1T4h8iBFhuby0kVTeccjO42DfVEmhmn2235y/G0QhmA/lI5Idjj760d++ptBlyJCn9icqDF3rDcFDSdczA01BN58qfbV/maOBuihKKxrZu61q5+hT+wGEvfsg37GqzwT8lX4U9IKpvtDctJQdM5B0NDPdFhxul2XWldDCjh2FHTQipd/YZ6cvzLL/aN8e+rbweIWVaPCv8QaOn00Nhhf85OERX+wehu01m70WDGcuhqhsoN8bZE6cP+7evYnPpFFnk3hdzfsxhLiFDP+OzUIzJ9ooUK/xA40NhON/bGZLnRPP7B0PV2o0PpqYBouCcBMeWv4BYvBVsfCLm/J9TTx+PfW98Ws4FdUOEfElWNHXgDwu/y0dLpocvji7NVCYwKf3TIGAfFC2Hrnw7PjlYSguy69wCQ91eHnGGd4nKQ6nIcUajNpnKq8CckVY0dPR5/pj9F95CGe/qnu01j/NHixOuheis8tALa6uNtjQJ0dHuZ2fU+1RmzwXjhnUdC9stOc9MU5PF3e31UNbYzJUbxfVDhHxIHGjvw9BF+zeXvB283eLvU448W8z8Nn3kIqt6D+87R4m0JwAe79jBNDtIw4wKYfgpseBB8R0YEsvuUZq481I7PxC6VE1T4h0RVYzt5mfZbOcNpb6jG+fuhS0syR52jz4crn4Hmg/DUl+JtTdJT8/4bAIybcyKUfQEO7YWdLx3Rz5ZmPqwbgYwejfEnKFWNHUzMSwdxkh4QfvX4Q6MLrceG6SfByTfCvrXq9ccZ3751+BAKZx8Hcz8BGQWw/v4j+mWl9q7Jv9c/eUtj/AnKgcYOinPTwekm3Z91pZO4+kEXWo8d8/wroW57Lr52JDl5De+x3z0NScsBVyosuhy2Pw8t1b36Zaf1rsm/r6ENt1OYmJMWM1tV+IdA5aF2inPTwOEmze/x6+BuP+hC67GjYCZMONZm+ShxoaPLw8zu7TTkLTjcOPtsO8h7cEuvvlmp7j6hnjYm5aUfsUxjNFHhD5PWTg9NHR4m5qaBw4nTeMhKdVGvMf7Q6ELrsWXeCtj3FjRVxduSQfH6DO1dXjshsq2bQ21dPa9AirQZZdVHd324hXHSgqNk6eHGXP/S5E37e/XNTnP1Ktmwr6E9pvF9CLMevwIHmuyU3eLcNHC6wdtNXoZbPf7+CIR6dPWt2DBvBbx8K7z/Zzjuy0M+vKXTw69f+pCKhnaa2rtp7vBgsAtuOwTcTgdupwMR6wS1dHpoD1pMxOeDTo+PTo8Xp0MoyEyhIMvORO3s9tLl9dHc4aG+tYuGtq5Bq0qLQGFWKsW5aRRlpdLp8XGovYumdg8up5DmcuJ2Oejyf2a314dTBKdDcDkcOB2C2ymkpziZnJfBlHHplBZmMq84hxlFWRH3ruu2vw5A4dyTDjfmBIS/90q1Of6sHp/P4HAI++rbOOeYiRG1ZzDCEn4RORf4JeAE7jHG3NZn/+eBO4DAV9v/GmPu8e+7Cviuv/2/jTEPRsDumFN1KCD86XZVKZ+HcZkpms7ZH0kY6un2i5vH68NgS+bnprtJTzlyGr7PZ9h/qJ09dW20dnl6VmQqyc+gtDCT/Aw3xkCHx4tDZPCp/EVzoHCODfcMUfg7PV6+8vA63txZx/TCTHLS3GSnuRARjDEYY6+trcuDz9jByfHZaaSnOOmRT4E0t5NUlwOP11Df2kVtSyeN7d2kuhxkpbqYmJPGuMwUCjJTyEh14RTB4RACGmyMfRro9Hjp6PZR29JJVWMHlY0dpLsdjM9OY2aRC4/P0Nnto8vrI8XpINXtIMXpwGcMHp/B4/Xh9YHH56Olw8PrO2o52NzR82WT5nYwa3wW08ZlMmVcBvMm5XDq7ELyMoa/UpzsX087qUyYtehwoysVMougsaJX33GZKRgDa8vrWDglj/rWrpjV4e8xbbAOIuIE7gTOwi68/raIrAqxhOITxpjr+hw7Dvg+UAYYYL3/2IaIWB9DqhptypWN8bvA5yE/I4UGHdwNzRgM9azZXs2WyiZcDsHldFDd3MGOgy18WN1CTXNnLw84mMwUJwVZqaS4HLgcgtdn2FvfRucAs75T/N4sWO93ekEmRxdnM684h2Mn5zJ/ci4FWam9D5q3Al77CbTUQFZRWNfk9Rm+9sS7vL6jjp9evJCLlpaE98sYZXR6vOyqbWVrZRNbKpvYUd3C1qomXth6gG6vwSGweGo+ZdPzmVmYRWlRJtMKMijKSkVk8KeDcYc2sSd1NnOdfRZfyZl0RKjnoqUlPLx2D9c/9g4/+cxCIHZVOQOE4/EfB+wwxpQDiMjjwArCWzT9HOBFY0y9/9gXgXOBx4Znbvyo8ldnm5CTBk4XeLvJz3CzqzZCU+Y9XTY84u3CdLUirbXQXAVttWB8NHV42H6gGd8oiX2Or19PKYyJUE9bl4cfrNrKE+t6p0u6ncKMwiwWlORSnJtGtt9TdjsdOEQwGBrbu6lt7qKutZNurw+vz96/0+cUMbMoi+mFmWSlukhzOzHGfiHsqm2lurmTNLeTjBQnHd1e3q9qZktlE6s3HS4DEPCex2WmkJ+RwlHM5evGx5uP3cr+3CW4TBduDC6XkOIQHKYbPB2IpxOnGESEXQ3dvLZnDt/9+NIxK/oAqS4ncyfmMHdiDp9acrjd4/WxaX8jL2+vYc32au7/5266vIe/kNPcDqbkZzAuM4Vx7m6myUEKpYkcWsiWNlIdBrdTOM5TzrvjP3PkB+eUQMOuXk3ZaW7+78oyPnnn69z12FNc43yHKeNOOvLYKBKO8E8Ggv/iK4BlIfpdJCKnAh8AXzPG7Ovn2MmhPkREVgIrAaZOnRqGWbHDGMM/th1k1vgs+8jtD/XkZw7D42/cD1UboXoLVL8PDbuhcR+0HOzpEsq/yAE+MpKLiAMNJgvjS2dcvA0ZJsYY3iyv47vPbGZXXSvXLp/JdctnYzB0eXxkplqRjySzJwy85mpTRzdb9jexeX8ju+taqW/toq6li/LaFt5ty+WTZhIn7L8f9h+ZPx6KE4EzCuZRfNwLEbB+9OFyOlg8NZ/FU/P5+llH4Wlvomb3Fur3bqOzegfSuJe0lgqKWvZR6Kvt/0QCGXPOOLI9dzLs/ucRzbPGZ/HTzyyk4bHfcKl7DfVZ/xPBqxqccIQ/lA71dTufAx4zxnSKyFeBB4EzwjzWNhpzN3A3QFlZWUK5te/sO8TGikZ+uOIY2+D0x/gzUmj2ZyGkuAYQgKYq2PwUbHkG9q/raW5Ln0S1ezKVjiV8kJLHvlYn4k7jqMlFbGxw805DGnUmB4fLxfkLJnHJR6aEXMczEVm3u54bn93Jow1djMuJtzVD41BbF4++tZc/rtvH7ro2Juak8fsvHc8JMwt6+owgHDwictLcnDCzoJctvTj0AtTtAFcaxpWKBycerx0AxZmKIyUdlzsVnzjw+gyuircoXvVl+P2lcMWT4B4k1tzdDhXrYM/rdvlHV5o9xpliUxd9HkBsm9s/HmZ8dh8CDqd9dTTZImbNVeDpBHHYlysFXOngTrPH+Tzg89r/OVea/RyfF7yddptZCFkTIWu8nTCVnm+3WeMhLc+uUxyKrlY4sAkq37U1j+rLoW4HruYqioHiQL/M8VA4DcadCQWzoHCW/bz0PEjLxYuTDo8Xr8PN/LwQ4bWcSdDZCJ3NkNr7S/2cYyZyML8GmiG/+wCQP/DvPoKEI/wVwJSg9yVAr2FqY0xd0NvfAYGvrwrg9D7HrhmqkfHm/td3k53q4qIl/kdhh9Nm9WTa//5D7V2Mzw4x+cLbDW/8Gl75H/B0QPFCuk7/L1Y3z+QXG53sbnDidgpT8jOYWpzBecdO5IKFk0lPcfIZYPuBZt7Z28AZR48Pff4EZq5rHB1UsLOmlaXTIuPzN3fYbBOXU0h1OaPyJdjl8XHlvf9i0/5GlpWO4/ozZnPe/IlkpIySBLi8KfaF9brc/le/cl5UAi5jSz48cQV85MuHxTejwAorwPbVsOVZ2PWKrcGEQN5UK8zdbfZv3eEEcVrB9nTYV3843JA90b7c6XZk1/jsF4Kn2n7BiMMKvjjB123bvF32WKfb7m+rhfZ+hgwdLkgfZxMM3JnWvq4WK/qtNfbzwF5nwSy7zkHBDCiYDYWzIb900OQEJzBgMDPHrxlNlXYAPhifjwkduwGQQ3th/NEDflYkCeev+W1gtoiUYrN2LgU+G9xBRIqNMYEE4guAbf6f/wb8SEQCX2VnA98ZsdUx5EBjB89vquKqE6eT6a+lbUM93Yzzu30Nrd1HCvP+9bDq3+HgZjj6Avjo9/nQO4Er7n2Lg02dnDizgFsvncXxMwr6TS2bMzGbORMHfvRPVEry03E7JWJjIB3dXpb/ZA21LYdDayfNKuCmj81j3qTIPVL84u8fsGl/I7+9fAnnzS8e/ICxwPxPW0F87gbY8ff+++VOtV8MpafC1OOt1zsQxtgvBnHaEWqwXrqvG5yp/XvjQ6W7A1qrbZXS9gZoq7PC3lIN7fU20aCr1T51pGTa2eTZE2HSYiheBDlRvM+BXP7GiiOF/9Bu6Pb/fxzaGz0bQjCo8BtjPCJyHVbEncB9xpgtIvJDYJ0xZhXw7yJyAeAB6oHP+4+tF5FbsF8eAD8MDPSOFh5ZuwevMVx1wvTDjc5AjN96nL3q9Xg6Yc2P4fVf2kfCS38Pcz/O+weauPx3a3E4hCe/egJl00dr5Ds8XE4HU8dlsKsmMsL/0vvV1LZ0cf0Zsxifk0ZdSycPvLGbj//6NT6zdArXLJ/JtIKRDSS/VV7Hb1/ZySVlU5JH9AMs/bz1eNtqDycatNVbAe1ug5lnWKEMI8OlBxH7vxKM02VfkcSdZp8+8hJrbBCwoR44IrMHgOpth39u2B0TcwKEdQeMMauB1X3avhf083fox5M3xtwH3DcCG+NGR7eX3/9rLx+dO4GpBUGPfA4XeG06J3B4gLfyXXjmK1DzPiz5HJx9K6TlsKWykSvueYtUl5Pff3kZM4qSo35NaWEW5bUtETnXM+/sZ0JOKjeeeVTPE9IXTizlf1/+kAfe2M0T6/ZxyuxCLl82jbPnTcAxxAk6je3dfP0PG5k6LoPvnT8vIjaPOvKn2ZcSObInAXLEJC4ADvoTI3Mmx9zj15INA/DUhgrqW7u4+qTpvXc4XDbU44/x17d12fKr959nY5SXPwUX/BrScqhqbOdz9/6LdLeTJ75yfNKIPsDMokx217X1pDAOl4bWLtZsr2bFosm9wmK5GW5u+vg8/vmtM/jamUfx4cEWvvrIei6/5y32H2oPea7ymhZ+9sJ21u9p8E9OMqzeVMWFd77OgaYOfnHJosMhPUUZKa4UO9DcZxIXYDP78qfb2H6ihXqSlZrmTm7/63bKpuUfmUHhD/XkZdjH2Py9f4MXboLCo2x99KzxgB0ovObRDXR0e3niKyeMOBQx2igtzKTL46Py0MiWlfvLpiq6vYYViyaF3D8hJ40bzpzNtctn8od1Fdz6l62c+/NX+d758/j00pKeCThbKhv53L3/oq61i1+9tIPJeenkprvZWtXE7PFZ3HtVGYunxi6zQkkScib17/GPPwayJ9gxwRiiwt8PNz+3hfYuL7ddtODImXsON3g9pLqcXJLyBudu+Q2ULIXL/2jTyfz8aPU23tl7iN9cvoRZ45PH0w9QWmi/6MprW0ck/M++s5+jJmQxr3jgQVyX08Fnl03l5FmFfPOPG/mPJ9/jvtd3c93yWUzMTeUL979NVqqLVdedxIcHW3juvUqqDnVw+6cXcNGSkphWR1SSiJzJNsU2GE+nbTv6fJvm2d5gowVpscl9VuEPwYtbD/KX96r4xllHhRZshz+9bPNT/MjxG3ZkLOKoK5+FVNu3vcvLUxsqeOCN3Vx9UikfS7aBQj+BsNaumhZOOyq8EgJ92VvXxro9Dfy/c+eENXUeYGpBBo+tPJ5n3tnPb17ewbW/3wDAtIIMHv3SMkryM1hQkjemZ6oqCURuCZS/0rut9gObZTRhHj3TnQ7thYnHxsQkFf4+NHV0891nNzF3YjZfOW1m6E5Ot71JT69km+to7sj5Hp8rb2Xdngre3lXPxopDdHsNZdPy+c7H5sb2AhKIwqwUslNdlI8gpfNP79psiBWLQk747henQ/j00hIuXDyZ1ZuqeO3DGr559hzGx3CxC0UBbKinq7m3Rx8Y2B1/TO+UThX+2FPd3MGXHlxHTXMnd19Z1v9sXIfbprhNLuPX8l+8sqOVV3avw+UQjpmcyxdPnsHxM8Zx/IyCiE/pH02ICKVFmcPO5e/o9vKH9ftYVjqOyXnDq17odAjnL5zE+QtDjw8oStTJCarLHxD+6i12FnLBTOhotG2H9sTMJBV+Px8cbOYL979NfWsXd19ZxsIpA0xOmbzU3sRLHmZltWHRrAYWT81jYUleyBK8ycyMwkze3j28Yqw/f/ED9tW386ML50fYKkWJIbn+kGLj/sOzcw9utWW0nW47c9idEdPMHhV+YEd1Mxf95g3SUpz84SsnML8kd+ADlq20L2DpNCJWkmAsUlqYxZ82VtLR7R28pnwQ6/fUc/dr5Xx22VROmT288QFFSQhyQqzEVb0VpvkrcopA3rSYCn/yxiGCeOjNPXR5fTxzzYmDi74yJEqLMjEGdteFH+5p7/LyzT++x6TcdP7zY7GrX6IoUSF7InYSl1/42xvszxOCJgrmTYWG2IV6kl74Oz1eVm2s5OxjJlIS48UQkoEZgZTOMEs3+HyGH/55C7tqW7nj4gVk6WQqZbTj9Beka/QLf/X7djv+mMN98mPr8Sf9f9XL71dzqK2bi5YMLWtECY9ALn84A7yNbd3c+MQ7vLy9hq+cOoMTZxZG2zxFiQ05kw97/IFCeH09/s5G+zSQHv1JhEkv/E+u38/47FSNI0eJTP9aq4N5/Fsrm/jqI+upamznlk8eyxXLErDglqIMl5xJtobX5qfs8pjHXHh40BdsjB+s1x8D4U/qUE9dSydrtldz4eLJOmszipQWZg5YrG31piou+u0bdHq8PL7yBK48flrYk7UUZVSQWwL1u+CZr8LUE+GTd/XeH6gsGqM4f1IL/5/ercTjMzqDM8qUFmXyflUzD6/dQ1uXp6fd5zP87MUPuObRDcwtzua5609m6TStlaOMQXIm29n++aVw6aO2lHQwAeGPUZw/qUM9T22oYP7kXI4aZJ1TZWRcfdJ03qs4xH89u5k7/vo+x5UWUN/aSeWhDg40dfDppSXceuGxpLp0DoQyRik91b5W3AkZIdK/0/MhNSexhF9EzgV+iV2I5R5jzG199n8d+BJ2IZYa4GpjzB7/Pi+wyd91rzHmggjZPiJe3l7Nlsombk7W2usxZNb4bJ677mQ27G3g/td3s/1AM+NzUjlxZgHHzyzg4qAKmooyJileAFc91/9+8S9lGaPZu4MKv4g4gTuBs7Br6L4tIquMMVuDur0DlBlj2kTk34DbgUv8+9qNMYsibPeIePn9ar7yyHrmTszWME+MEBGWThunk90UpT/ypkHDrph8VDgx/uOAHcaYcmNMF/A4sCK4gzHmZWNMm//tWuyi6nHBGMM/P6zlO0+/x5/e3Y8xvRcBeWHLAVY+vI6jJmTx2JePJzst8gt2K4qiDJn8aQMvUB9Bwgn1TAb2Bb2vAJYN0P+LwPNB79NEZB02DHSbMebZIVsZBh3dXh7/114eXruHnTWtuJ3CY//ax7Pv7OeWTx7L3vo2Hn5zD3/bcoD5JXk8dPVx5Kar6CuKkiCc8yM498cx+ahwhD9U8DXkWnoicgVQBpwW1DzVGFMpIjOAl0RkkzFmZ4hjVwIrAaZOHXoOt0OE/315JyX56fzsMws579hiHn97L3f8bTun3P4yxkBehpsvnzKD686YpZ6+oiiJRQzHucIR/gpgStD7EuCIdcRE5EzgJuA0Y0xnoN0YU+nflovIGmAxcITwG2PuBu4GKCsrG/IirSkuB8/fcApF2ak9bV84qZSz5k3g/td3c3RxDp9YUDykQmGKoihjkXCE/21gtoiUAvuBS4HPBncQkcXA/wHnGmOqg9rzgTZjTKeIFAInYQd+o0Kw6Acoyc/gvz6hmTuKoigBBhV+Y4xHRK4D/oZN57zPGLNFRH4IrDPGrALuALKAP/rT8gJpm0cD/yciPuxA8m19soEURVGUGCN9s14SgbKyMrNu3bp4m6EoijJqEJH1xpiycPomdckGRVGUZESFX1EUJclQ4VcURUkyVPgVRVGSDBV+RVGUJEOFX1EUJclIyHROEakBhluftBCojaA5iUqyXCckz7Umy3VC8lxrLK9zmjEmrDVkE1L4R4KIrAs3l3U0kyzXCclzrclynZA815qo16mhHkVRlCRDhV9RFCXJGIvCf3e8DYgRyXKdkDzXmizXCclzrQl5nWMuxq8oiqIMzFj0+BVFUZQBGDPCLyLnish2EdkhIt+Otz2RRESmiMjLIrJNRLaIyA3+9nEi8qKIfOjf5sfb1kggIk4ReUdE/ux/Xyoib/mv8wkRSYm3jZFARPJE5EkRed9/b08Yi/dURL7m/7vdLCKPiUjaWLmnInKfiFSLyOagtpD3UCy/8mvUeyKyJF52jwnhFxEncCdwHjAPuExExtLqKx7gG8aYo4HjgWv91/dt4B/GmNnAP/zvxwI3ANuC3v8P8HP/dTZg13UeC/wS+KsxZi6wEHvNY+qeishk4N+BMmPMsdg1PS5l7NzTB4Bz+7T1dw/PA2b7XyuB38bIxiMYE8IPHAfsMMaUG2O6gMeBFXG2KWIYY6qMMRv8PzdjBWIy9hof9Hd7EPhkfCyMHCJSAnwcuMf/XoAzgCf9XcbKdeYApwL3AhhjuowxhxiD9xS74FO6iLiADKCKMXJPjTGvAvV9mvu7hyuAh4xlLZAnIsWxsbQ3Y0X4JwP7gt5X+NvGHCIyHbtu8VvABGNMFdgvB2B8/CyLGL8A/h/g878vAA4ZYzz+92Pl3s4AaoD7/WGte0QkkzF2T40x+4GfAHuxgt8IrGds3tMA/d3DhNGpsSL8oZanH3PpSoiwmjIAAAHOSURBVCKSBTwF3GiMaYq3PZFGRD4BVBtj1gc3h+g6Fu6tC1gC/NYYsxhoZZSHdULhj2+vAEqBSUAmNuTRl7FwTwcjYf6Wx4rwVwBTgt6XAJVxsiUqiIgbK/qPGmOe9jcfDDwq+rfV/R0/SjgJuEBEdmPDdWdgnwDy/GECGDv3tgKoMMa85X//JPaLYKzd0zOBXcaYGmNMN/A0cCJj854G6O8eJoxOjRXhfxuY7c8USMEOHq2Ks00Rwx/nvhfYZoz5WdCuVcBV/p+vAv4Ua9siiTHmO8aYEmPMdOw9fMkYcznwMvBpf7dRf50AxpgDwD4RmeNv+iiwlTF2T7EhnuNFJMP/dxy4zjF3T4Po7x6uAj7nz+45HmgMhIRijjFmTLyAjwEfADuBm+JtT4Sv7WTsI+F7wLv+18ew8e9/AB/6t+PibWsEr/l04M/+n2cA/wJ2AH8EUuNtX4SucRGwzn9fnwXyx+I9BX4AvA9sBh4GUsfKPQUew45ddGM9+i/2dw+xoZ47/Rq1CZvpFBe7deauoihKkjFWQj2KoihKmKjwK4qiJBkq/IqiKEmGCr+iKEqSocKvKIqSZKjwK4qiJBkq/IqiKEmGCr+iKEqS8f8BeLV2c/4Nmx8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# get number of states\n",
    "env_info = {\"state representation structure\": \"regular\", \n",
    "            \"fundamental timestep\": 0.2, \n",
    "            \"gamma\": 0.9, \n",
    "            \"pursuit port total duration\": 7, \n",
    "            \"background port total duration\": 7, \n",
    "            \"transit duration\": 0.5, \n",
    "            \"consumption duration\": 1, \n",
    "            \"first time in pursuit port that could give a reward\": 1.2,\n",
    "            \"required minimum wait time in pursuit port\": 1.2, \n",
    "            \"required minimum wait time in background port\": 1, \n",
    "            \"time to wait until the trial reset\": 1.5, \n",
    "            \"exponential distribution scale\": 4, \n",
    "            \"delivery time in the background port\": 1.6,\n",
    "            \"reward amount in pursuit\": 2.4, \n",
    "            \"reward amount in background\": [0.6, 1.8]}\n",
    "\n",
    "\n",
    "env = GiveUpEnvironment(env_info=env_info)\n",
    "num_states = env.num_states\n",
    "\n",
    "\n",
    "# run experiment\n",
    "MaxRuns = 5\n",
    "leave_values = np.zeros([MaxRuns, num_states])\n",
    "stay_values = np.zeros([MaxRuns, num_states])\n",
    "for j in range(MaxRuns):\n",
    "    \n",
    "    seed = j+10\n",
    "    env_info[\"seed\"] = seed\n",
    "\n",
    "    env = GiveUpEnvironment(env_info=env_info)\n",
    "    num_states = env.num_states\n",
    "\n",
    "    agent_info = {\"num_actions\": 2, \n",
    "                  \"num_states\": num_states, \n",
    "                  \"epsilon\": 0.05, \n",
    "                  \"discount\": 0.9, \n",
    "                  \"step_size\": 0.1, \n",
    "                  \"seed\": seed + 5000, \n",
    "                  \"degree of exploration\": 0.5, \n",
    "                  \"exploration method\": \"UCB\"}\n",
    "    agent = QLearningAgent(agent_info=agent_info)\n",
    "    #agent.agent_init(agent_info)\n",
    "    #print(agent.policy)\n",
    "    # agent_info.update({\"policy\": policy})\n",
    "\n",
    "    rl_glue = RLGlue(env, agent)\n",
    "    rl_glue.rl_init(env_info, agent_info)\n",
    "    rl_glue.rl_start(env_info, agent_info)\n",
    "    values = rl_glue.agent.agent_message(\"get_action_values\")\n",
    "    MaxSteps = 500000\n",
    "    TD_error = np.zeros(MaxSteps)\n",
    "    prev_state = np.zeros(MaxSteps)\n",
    "    current_state = np.zeros(MaxSteps)\n",
    "    prev_action = np.zeros(MaxSteps)\n",
    "    rewarded_bool = np.zeros(MaxSteps)\n",
    "\n",
    "    for i in tqdm(range(MaxSteps)):\n",
    "        prev_state[i] = rl_glue.agent.prev_state\n",
    "        prev_action[i] = rl_glue.agent.prev_action\n",
    "        roat = rl_glue.rl_step()\n",
    "        rewarded_bool[i] = (roat[0] != 0)\n",
    "        TD_error[i] = rl_glue.agent.agent_message(\"get TD error\")\n",
    "        current_state[i] = rl_glue.agent.prev_state\n",
    "        \n",
    "\n",
    "    values = rl_glue.agent.agent_message(\"get_action_values\")\n",
    "    leave_values[j] = values[:, 1]\n",
    "    stay_values[j] = values[:, 0]\n",
    "    \n",
    "    prev_state = prev_state.astype(int)\n",
    "    current_state = current_state.astype(int)\n",
    "# fig, ax = plt.subplots(1, 1)\n",
    "plot1 = plt.plot(np.mean(leave_values, axis=0), label = \"q_leave\")\n",
    "plot2 = plt.plot(np.mean(stay_values, axis=0), label = \"q_stay\")\n",
    "plt.legend(loc=\"upper right\")\n",
    "plt.show()\n",
    "\n",
    "# FindExitIndex = np.where(values[80:120, 0] < values[80:120, 1])\n",
    "# IndexArray = FindExitIndex[0] + 80\n",
    "# BestExitIndex = min(IndexArray)\n",
    "# print(BestExitIndex)\n",
    "# print(env.state3d(BestExitIndex))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66\n",
      "(1, 6.2, 0)\n"
     ]
    }
   ],
   "source": [
    "FindExitIndex = np.where(np.mean(stay_values[:, 60:120], axis=0) <= np.mean(leave_values[:, 60:120], axis=0))\n",
    "IndexArray = FindExitIndex[0] + 60\n",
    "BestExitIndex = min(IndexArray)\n",
    "print(BestExitIndex)\n",
    "print(env.state3d(BestExitIndex))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot heatmap of TD errors aligned by reward time\n",
    "first_PS_state = rl_glue.environment.state1d((1, 0, 0))\n",
    "last_PS_time = rl_glue.environment.PSTimeArray[-1]\n",
    "last_unrewarded_PS_state = rl_glue.environment.state1d((1, last_PS_time, 0))\n",
    "first_rewarded_PS_state = last_unrewarded_PS_state + 1\n",
    "last_rewarded_PS_state = rl_glue.environment.state1d((1, last_PS_time, 1))\n",
    "\n",
    "rewarded_steps = np.nonzero(rewarded_bool)\n",
    "rewarded_steps = rewarded_steps[0]\n",
    "PS_rewarded_steps = []\n",
    "\n",
    "for step in rewarded_steps: \n",
    "    if (prev_state[step] <= last_unrewarded_PS_state) and (prev_state[step] >= first_PS_state):\n",
    "        PS_rewarded_steps.append(step)\n",
    "\n",
    "PS_num_rewards = len(PS_rewarded_steps)\n",
    "# data = np.zeros((PS_num_rewards, 50))\n",
    "# data_states = np.zeros((PS_num_rewards, 50))\n",
    "\n",
    "# for trial in range(PS_num_rewards): \n",
    "#     plot_range = np.arange(rewarded_steps[trial] - 25, rewarded_steps[trial] + 25)\n",
    "#     data[trial, :] = TD_error[plot_range]\n",
    "#     data_states[trial, :] = current_state[plot_range]\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PS_rewarded_time = np.zeros(PS_num_rewards)\n",
    "PS_rewarded_TDerror = np.zeros(PS_num_rewards)\n",
    "for trial in range(PS_num_rewards): \n",
    "    rewarded_state_1d = current_state[PS_rewarded_steps[trial]]\n",
    "    rewarded_state_3d = rl_glue.environment.state3d(rewarded_state_1d)\n",
    "    PS_rewarded_time[trial] = rewarded_state_3d[1]\n",
    "    PS_rewarded_TDerror[trial] = TD_error[PS_rewarded_steps[trial]]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.cm as cm\n",
    "import matplotlib.colors as colors\n",
    "\n",
    "# colormap = ['#0091ad', '#d6f6eb', '#fdf1d2', '#faaaae', '#ff57bb'] # pink\n",
    "# colormap = ['#0f3c4c', '#134b5f', '#165a72', '#1a6985', '#1e7898', '#2187ab', '#2596be'] #tinted blue\n",
    "# colormap = ['#be8a25', '#ab7c21', '#986e1e', '#85611a', '#725316', '#5f4513', '#4c370f', '#39290b', '#261c07', '#130e04'] #tinted gold\n",
    "colormap = ['#be8a25', '#d6f6eb', '#faaaae']\n",
    "\n",
    "num_categories = len(colormap)\n",
    "categories = np.ones(PS_num_rewards) * (num_categories - 1)\n",
    "category_size = math.ceil((MaxSteps - 8000) / (num_categories - 1))\n",
    "\n",
    "for trial in range(PS_num_rewards): \n",
    "    for c in range(num_categories - 1): \n",
    "        if (PS_rewarded_steps[trial] < category_size * (c+1)) and (PS_rewarded_steps[trial] >= category_size * c): \n",
    "            categories[trial] = c\n",
    "            \n",
    "\n",
    "categories = categories.astype(int)\n",
    "category_color = []\n",
    "for trial in range(PS_num_rewards): \n",
    "    category_color.append(colormap[categories[trial]])\n",
    "\n",
    "fig, ax = plt.subplots(1,1)\n",
    "ax.scatter(PS_rewarded_time, PS_rewarded_TDerror, c = category_color, alpha = 0.5)\n",
    "ax.set_xlabel(\"Time to Reward at the Pursuit Port (sec)\")\n",
    "ax.set_ylabel(\"TD error\")\n",
    "ax.set_title(\"TD-dilating\")\n",
    "ax.set_xlim(1, 7)\n",
    "# plt.savefig(\"TD errors from TD-dilating (3)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "ax = sns.heatmap(data, cmap=\"YlGnBu\")\n",
    "bx = sns.heatmap(data_states)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_PS_state = rl_glue.environment.state1d((1, 0, 0))\n",
    "last_PS_time = rl_glue.environment.PSTimeArray[-1]\n",
    "last_unrewarded_PS_state = rl_glue.environment.state1d((1, last_PS_time, 0))\n",
    "first_rewarded_PS_state = last_unrewarded_PS_state + 1\n",
    "last_rewarded_PS_state = rl_glue.environment.state1d((1, last_PS_time, 1))\n",
    "PS_unrewarded_exit_bool = np.zeros(MaxSteps)\n",
    "\n",
    "for i in range(MaxSteps): \n",
    "    if not (i >= MaxSteps - 1):\n",
    "        if (prev_state[i] <= last_unrewarded_PS_state) and (prev_state[i] >= first_PS_state) and ((prev_state[i + 1] == 0) or (prev_state[i + 1] == first_PS_state)): \n",
    "            PS_unrewarded_exit_bool[i] = 1\n",
    "\n",
    "        else: \n",
    "            pass\n",
    "\n",
    "PS_unrewarded_exit = np.nonzero(PS_unrewarded_exit_bool) # indices of steps where the agent exits pursuit port unrewarded\n",
    "exit_state_time = np.zeros(len(PS_unrewarded_exit[0]))\n",
    "for i in range(len(PS_unrewarded_exit[0])): \n",
    "    exit_step = PS_unrewarded_exit[0][i]\n",
    "    # print(exit_step)\n",
    "    exit_state1d = prev_state[exit_step]\n",
    "    exit_state1d = int(exit_state1d)\n",
    "    # print(exit_state1d)\n",
    "    exit_state3d = rl_glue.environment.state3d(exit_state1d)\n",
    "    # print(exit_state3d)\n",
    "    exit_state_time[i] = exit_state3d[1]\n",
    "\n",
    "plt.plot(exit_state_time)\n",
    "# len(PS_unrewarded_exit)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot action values in background port. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "BGlen = len(rl_glue.environment.BGTimeArray)\n",
    "xticklabel = np.zeros(BGlen)\n",
    "print(BGlen)\n",
    "average_leave_values = np.mean(leave_values, axis = 0)\n",
    "average_stay_values = np.mean(stay_values, axis = 0)\n",
    "for i in range(BGlen): \n",
    "    X = rl_glue.environment.state3d(i)\n",
    "    xticklabel[i] = X[1]\n",
    "\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "y0 = average_stay_values\n",
    "y1 = average_leave_values\n",
    "ax.plot(xticklabel, y0[0:BGlen], label = \"Q_stay\")\n",
    "ax.plot(xticklabel, y1[0:BGlen], label = \"Q_leave\")\n",
    "ax.legend(loc = \"upper right\")\n",
    "ax.set_xticks(xticklabel)\n",
    "ax.set_xticklabels(ax.get_xticks(), rotation = 60)\n",
    "ax.set_xlabel('Time in Background port (sec)')\n",
    "ax.set_title('Q(state-action pair)')\n",
    "# ax.set_xticks(rl_glue.environment.BGTimeArray)\n",
    "# ax.set_xticklabels(xticklabel)\n",
    "ax.set_ylabel('Value')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot action values in pursuit port when the agent has not been rewarded. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PSlen = len(rl_glue.environment.PSTimeArray)\n",
    "xticklabel = np.zeros(PSlen)\n",
    "\n",
    "average_leave_values = np.mean(leave_values, axis = 0)\n",
    "average_stay_values = np.mean(stay_values, axis = 0)\n",
    "for i in range(PSlen): \n",
    "    X = rl_glue.environment.state3d(i + BGlen)\n",
    "    xticklabel[i] = X[1]\n",
    "\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "y0 = average_stay_values\n",
    "y1 = average_leave_values\n",
    "ax.plot(xticklabel, y0[BGlen:BGlen + PSlen], label = \"Q_stay\")\n",
    "ax.plot(xticklabel, y1[BGlen:BGlen + PSlen], label = \"Q_leave\")\n",
    "ax.legend(loc = \"upper right\")\n",
    "ax.set_xticks(xticklabel)\n",
    "ax.set_xticklabels(ax.get_xticks(), rotation = 60)\n",
    "ax.set_xlabel('Time in Pursuit port (sec)')\n",
    "ax.set_title('Q(state-action pair)')\n",
    "# ax.set_xticks(rl_glue.environment.BGTimeArray)\n",
    "# ax.set_xticklabels(xticklabel)\n",
    "ax.set_ylabel('Value')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "length = len(average_stay_values)\n",
    "merged = np.zeros((2, length))\n",
    "merged[0] = average_stay_values\n",
    "merged[1] = average_leave_values\n",
    "merged[:, 0:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import colors\n",
    "\n",
    "BGlen = len(rl_glue.environment.BGTimeArray)\n",
    "\n",
    "plotpart = range(0, BGlen)\n",
    "xticklabel = np.zeros(BGlen)\n",
    "\n",
    "average_leave_values = np.mean(leave_values, axis = 0)\n",
    "average_stay_values = np.mean(stay_values, axis = 0)\n",
    "\n",
    "action_values = np.zeros((2, length))\n",
    "action_values[0] = average_stay_values\n",
    "action_values[1] = average_leave_values\n",
    "\n",
    "data = action_values[:, plotpart]\n",
    "\n",
    "# create discrete colormap\n",
    "cmap = colors.Colormap('Blues')\n",
    "# cmap = colors.ListedColormap(['red', 'blue'])\n",
    "# bounds = [0,10,20]\n",
    "# norm = colors.BoundaryNorm(bounds, cmap.N)\n",
    "\n",
    "plt.figure(figsize=((1,0.1)))\n",
    "plt.imshow(data, cmap=cmap)\n",
    "\n",
    "# draw gridlines\n",
    "# plt.grid(which='major', axis='both', linestyle='-', color='k', linewidth=2)\n",
    "# ax.set_xticks(np.arange(-.5, 10, 1));\n",
    "# ax.set_yticks(np.arange(-.5, 10, 1));\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = 499600\n",
    "duration = 100\n",
    "fig, bx = plt.subplots(1,1)\n",
    "bx.plot(TD_error)\n",
    "plt.xlim([start, start + duration])\n",
    "\n",
    "fig, cx = plt.subplots(1,1)\n",
    "cx.plot(prev_state)\n",
    "plt.xlim([start, start + duration])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 3. Policy Evaluation with Basic TD\n",
    "Finally, see the TD policy evaluation algorithm in action by looking at the estimated values, the per state value error and after the experiment is complete, the Mean Squared Value Error curve vs. episode number, summarizing how the value error changed over time.\n",
    "\n",
    "The code below runs one run of an experiment given env_info and agent_info dictionaries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_info = {\"state representation structure\": \"regular\", \n",
    "            \"fundamental timestep\": 0.1, \n",
    "            \"gamma\": 0.95, \n",
    "            \"pursuit port total duration\": 7, \n",
    "            \"background port total duration\": 7, \n",
    "            \"transit duration\": 0.5, \n",
    "            \"consumption duration\": 1, \n",
    "            \"first time in pursuit port that could give a reward\": 1.2,\n",
    "            \"required minimum wait time in pursuit port\": 1.2, \n",
    "            \"required minimum wait time in background port\": 1, \n",
    "            \"time to wait until the trial reset\": 1.5, \n",
    "            \"exponential distribution scale\": 4, \n",
    "            \"delivery time in the background port\": 1.6,\n",
    "            \"reward amount in pursuit\": 3, \n",
    "            \"reward amount in background\": 0.3}\n",
    "env = GiveUpEnvironment(env_info=env_info)\n",
    "env.test_reward_function()\n",
    "(rou_g, rou_l, t_rou_g, t_rou_l) = env.PolicyMax()\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "ax.plot(env.PSTimeArray, rou_g)\n",
    "ax.plot(env.PSTimeArray, rou_l)\n",
    "print(f\"According to the global reward rate, the optimal give-up time in the pursuit port is at {t_rou_g} second!\")\n",
    "print(f\"According to the local reward rate, the optimal give-up time in the pursuit port is at {t_rou_l} second!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(env.PSTimeArray)\n",
    "print(env.TravelTimeArray)\n",
    "print(env.PSMinDelay)\n",
    "FindStateStart(env.PSTimeArray, env.PSMinDelay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_policy(ps_exit_time, env): \n",
    "    policy = np.ones(shape=(env.num_states, 2)) * 0.5\n",
    "    for i in range(env.state1d((0, env.BGRewardTime, 0)) + 1):\n",
    "        policy[i] = [1, 0]\n",
    "    policy[env.state1d((0, env.BGRewardTime, 0)) + 1] = [0, 1]\n",
    "    \n",
    "    exit_state_time = FindStateStart(env.PSTimeArray, ps_exit_time)\n",
    "    for i in range(env.state1d((1, 0, 0)), env.state1d((1, exit_state_time, 0))):\n",
    "        policy[i] = [1, 0]\n",
    "    policy[env.state1d((1, exit_state_time, 0))] = [0, 1]\n",
    "    \n",
    "    min_delay_state = FindStateStart(env.PSTimeArray, env.PSMinDelay)\n",
    "    for i in range(env.state1d((1, min_delay_state, 1)), env.state1d((1, env.PSTimeArray[-1], 1)) + 1):\n",
    "        policy[i] = [0, 1]\n",
    "    policy[env.state1d((2, env.TravelTimeArray[-1], 0))] = [0, 1]\n",
    "    policy[env.state1d((3, env.TravelTimeArray[-1], 0))] = [0, 1]\n",
    "    return policy\n",
    "\n",
    "# make_policy(1.6, env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_policy_basic_TD(PS_exit_time, max_steps, state_rep_struct, seed): \n",
    "    from rl_glue import RLGlue\n",
    "    from agent import BaseAgent\n",
    "    from environment import BaseEnvironment  \n",
    "    \n",
    "    env_info = {\"state representation structure\": state_rep_struct, \n",
    "                \"fundamental timestep\": 0.1, \n",
    "                \"gamma\": 0.95, \n",
    "                \"pursuit port total duration\": 7, \n",
    "                \"background port total duration\": 7, \n",
    "                \"transit duration\": 0.5, \n",
    "                \"consumption duration\": 1, \n",
    "                \"first time in pursuit port that could give a reward\": 1.2,\n",
    "                \"required minimum wait time in pursuit port\": 1.2, \n",
    "                \"required minimum wait time in background port\": 1, \n",
    "                \"time to wait until the trial reset\": 1.5, \n",
    "                \"exponential distribution scale\": 4, \n",
    "                \"delivery time in the background port\": 1.6,\n",
    "                \"reward amount in pursuit\": 3, \n",
    "                \"reward amount in background\": 0.3}\n",
    "    \n",
    "    env = GiveUpEnvironment(env_info=env_info)\n",
    "    \n",
    "    MaxSteps = max_steps\n",
    "    num_sessions = 5\n",
    "    value_all_sessions = np.zeros((num_sessions, env.num_states))\n",
    "    \n",
    "\n",
    "    for j in tqdm(range(num_sessions)): \n",
    "        \n",
    "        env_info[\"seed\"] = seed + 7*j\n",
    "        env = GiveUpEnvironment(env_info=env_info)\n",
    "\n",
    "        # define policy\n",
    "        policy = make_policy(PS_exit_time, env)\n",
    "\n",
    "\n",
    "        agent_info = {\"policy\": policy, \"discount\": 0.95, \"step_size\": 0.1, \"seed\": seed + 500 + 6*j}\n",
    "        agent = BasicTDAgent(agent_info=agent_info)\n",
    "        #agent.agent_init(agent_info)\n",
    "        #print(agent.policy)\n",
    "        # agent_info.update({\"policy\": policy})\n",
    "        rl_glue = RLGlue(env, agent)\n",
    "        rl_glue.rl_init(env_info, agent_info)\n",
    "        rl_glue.rl_start(env_info, agent_info)\n",
    "    \n",
    "        TD_error = np.zeros(MaxSteps)\n",
    "        state_of_last_step = np.zeros(MaxSteps)\n",
    "\n",
    "\n",
    "        for i in range(MaxSteps):\n",
    "            roat = rl_glue.rl_step()\n",
    "    #         TD_error[i] = rl_glue.agent.agent_message(\"get TD error\")\n",
    "    #         state_of_last_step[i] = rl_glue.agent.last_state\n",
    "        values = rl_glue.agent.agent_message(\"get_values\")\n",
    "        value_all_sessions[j,:] = values\n",
    "    \n",
    "    mean_value_across_session = np.mean(value_all_sessions, axis = 0)\n",
    "    \n",
    "    return mean_value_across_session\n",
    "\n",
    "v_16 = evaluate_policy_basic_TD(1.6, 50000, \"dilating\", 5000)\n",
    "v_24 = evaluate_policy_basic_TD(2.4, 50000, \"dilating\", 5001)\n",
    "v_32 = evaluate_policy_basic_TD(3.2, 50000, \"dilating\", 5002)\n",
    "v_40 = evaluate_policy_basic_TD(4.0, 50000, \"dilating\", 5003)\n",
    "v_48 = evaluate_policy_basic_TD(4.8, 50000, \"dilating\", 5004)\n",
    "v_56 = evaluate_policy_basic_TD(5.6, 50000, \"dilating\", 5005)\n",
    "v_64 = evaluate_policy_basic_TD(6.4, 50000, \"dilating\", 5006)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot average state values under different policies\n",
    "vm_16 = np.mean(v_16)\n",
    "vm_24 = np.mean(v_24)\n",
    "vm_32 = np.mean(v_32)\n",
    "vm_40 = np.mean(v_40)\n",
    "vm_48 = np.mean(v_48)\n",
    "vm_56 = np.mean(v_56)\n",
    "vm_64 = np.mean(v_64)\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "policy_list = [1.6, 2.4, 3.2, 4.0, 4.8, 5.6, 6.4]\n",
    "mean_state_value_dilating = [vm_16, vm_24, vm_32, vm_40, vm_48, vm_56, vm_64]\n",
    "ax.scatter(policy_list, mean_state_value_regular, label = \"regular\")\n",
    "ax.scatter(policy_list, mean_state_value_dilating, label = \"dilating\")\n",
    "ax.set_xticks(policy_list)\n",
    "ax.set_xticklabels(ax.get_xticks())\n",
    "ax.legend(loc = \"upper left\")\n",
    "ax.set_xlabel('Policy indicated by Exit Time (sec)')\n",
    "ax.set_ylabel('Average State Values')\n",
    "ax.set_ylim(0.19, 1.45)\n",
    "plt.savefig(\"Average State Value Under Each Policy\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(mean_state_value_regular)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot all state values under different policies\n",
    "BGlen = len(rl_glue.environment.BGTimeArray)\n",
    "PSlen = len(rl_glue.environment.PSTimeArray)\n",
    "xticklabel = np.zeros(PSlen)\n",
    "\n",
    "for i in range(PSlen): \n",
    "    X = rl_glue.environment.state3d(i + BGlen)\n",
    "    xticklabel[i] = X[1]\n",
    "\n",
    "\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "\n",
    "ax.plot(xticklabel, v_16[BGlen:BGlen + PSlen], label = \"T_exit = 1.6 s\")\n",
    "ax.plot(xticklabel, v_24[BGlen:BGlen + PSlen], label = \"T_exit = 2.4 s\")\n",
    "ax.plot(xticklabel, v_32[BGlen:BGlen + PSlen], label = \"T_exit = 3.2 s\")\n",
    "ax.plot(xticklabel, v_40[BGlen:BGlen + PSlen], label = \"T_exit = 4.0 s\")\n",
    "ax.plot(xticklabel, v_48[BGlen:BGlen + PSlen], label = \"T_exit = 4.8 s\")\n",
    "ax.plot(xticklabel, v_56[BGlen:BGlen + PSlen], label = \"T_exit = 5.6 s\")\n",
    "ax.plot(xticklabel, v_64[BGlen:BGlen + PSlen], label = \"T_exit = 6.4 s\")\n",
    "ax.legend(loc = \"upper right\")\n",
    "ax.set_xticks(xticklabel)\n",
    "ax.set_xticklabels(ax.get_xticks(), rotation = 60)\n",
    "ax.set_xlabel('Time in Pursuit port (sec)')\n",
    "ax.set_title('V(state)')\n",
    "# ax.set_xticks(rl_glue.environment.BGTimeArray)\n",
    "# ax.set_xticklabels(xticklabel)\n",
    "ax.set_ylabel('Value')\n",
    "ax.set_ylim(-0.2, 4.2)\n",
    "plt.savefig(\"State Values under Different Policies - dilating\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_info = {\"state representation structure\": \"dilating\", \n",
    "            \"fundamental timestep\": 0.1, \n",
    "            \"gamma\": 0.95, \n",
    "            \"pursuit port total duration\": 7, \n",
    "            \"background port total duration\": 7, \n",
    "            \"transit duration\": 0.5, \n",
    "            \"consumption duration\": 1, \n",
    "            \"first time in pursuit port that could give a reward\": 1.2,\n",
    "            \"required minimum wait time in pursuit port\": 1.2, \n",
    "            \"required minimum wait time in background port\": 1, \n",
    "            \"time to wait until the trial reset\": 1.5, \n",
    "            \"exponential distribution scale\": 4, \n",
    "            \"delivery time in the background port\": 1.6,\n",
    "            \"reward amount in pursuit\": 3, \n",
    "            \"reward amount in background\": 1.5}\n",
    "\n",
    "env = GiveUpEnvironment(env_info=env_info)\n",
    "#env.env_init(env_info)\n",
    "\n",
    "# The Optimal Policy that maximizes the global reward rate\n",
    "num_states = env.num_states\n",
    "(rou_g, rou_l, t_rou_g, t_rou_l) = env.PolicyMax()\n",
    "OptimalExit = t_rou_g\n",
    "\n",
    "policy = make_policy(OptimalExit, env)\n",
    "\n",
    "\n",
    "agent_info = {\"policy\": policy, \"discount\": 0.95, \"step_size\": 0.1, \"seed\": 0}\n",
    "agent = BasicTDAgent(agent_info=agent_info)\n",
    "#agent.agent_init(agent_info)\n",
    "#print(agent.policy)\n",
    "# agent_info.update({\"policy\": policy})\n",
    "rl_glue = RLGlue(env, agent)\n",
    "rl_glue.rl_init(env_info, agent_info)\n",
    "rl_glue.rl_start(env_info, agent_info)\n",
    "values = rl_glue.agent.agent_message(\"get_values\")\n",
    "\n",
    "# MaxSteps = 500000\n",
    "# TD_error = np.zeros(MaxSteps)\n",
    "# state_of_last_step = np.zeros(MaxSteps)\n",
    "\n",
    "# for i in tqdm(range(MaxSteps)):\n",
    "#     roat = rl_glue.rl_step()\n",
    "#     TD_error[i] = rl_glue.agent.agent_message(\"get TD error\")\n",
    "#     state_of_last_step[i] = rl_glue.agent.last_state\n",
    "# #     print(roat)\n",
    "# #     print(TD_error[i])\n",
    "# #     if (i > MaxSteps * 0.75):\n",
    "# #         values = rl_glue.agent.agent_message(\"get_values\")\n",
    "# #         ax.plot(values, str(0.9 * (1 - i / MaxSteps)))\n",
    "# values = rl_glue.agent.agent_message(\"get_values\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = 45\n",
    "duration = 25\n",
    "fig, bx = plt.subplots(1,1)\n",
    "bx.plot(TD_error)\n",
    "plt.xlim([start, start + duration])\n",
    "\n",
    "fig, cx = plt.subplots(1,1)\n",
    "cx.plot(state_of_last_step)\n",
    "plt.xlim([start, start + duration])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = agent_info[\"policy\"]\n",
    "x.shape\n",
    "y = np.array([[1.], [1.]])\n",
    "z = y[1]\n",
    "z[0]\n",
    "agent_info.get(\"policy\")\n",
    "agent = BasicTDAgent()\n",
    "agent.agent_init(agent_info)\n",
    "print(agent.policy)\n",
    "print(agent.values)\n",
    "print(agent.policy.shape[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "\n",
    "# ---------------\n",
    "# Discussion Cell\n",
    "# ---------------\n",
    "\n",
    "def run_experiment(env_info, agent_info,num_episodes=5000, experiment_name=None, plot_freq=100, true_values_file=None, value_error_threshold=1e-8):\n",
    "    env = CliffWalkEnvironment\n",
    "    agent = TDAgent\n",
    "    rl_glue = RLGlue(env, agent)\n",
    "\n",
    "    rl_glue.rl_init(agent_info, env_info)\n",
    "\n",
    "    manager = Manager(env_info, agent_info, true_values_file=true_values_file, experiment_name=experiment_name)\n",
    "    for episode in range(1, num_episodes + 1):\n",
    "        rl_glue.rl_episode(0) # no step limit\n",
    "        if episode % plot_freq == 0:\n",
    "            values = rl_glue.agent.agent_message(\"get_values\")\n",
    "            manager.visualize(values, episode)\n",
    "\n",
    "    values = rl_glue.agent.agent_message(\"get_values\")\n",
    "    return values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def measure_convergence(ExplorationStrategy, ExplorationConstant, StepSize): \n",
    "    # get number of states\n",
    "    env_info = {\"state representation structure\": \"dilating\", \n",
    "                \"fundamental timestep\": 0.2, \n",
    "                \"gamma\": 0.95, \n",
    "                \"pursuit port total duration\": 7, \n",
    "                \"background port total duration\": 9, \n",
    "                \"transit duration\": 0.5, \n",
    "                \"consumption duration\": 1, \n",
    "                \"first time in pursuit port that could give a reward\": 1.2,\n",
    "                \"required minimum wait time in pursuit port\": 1.2, \n",
    "                \"time to wait until the trial reset\": 1.5, \n",
    "                \"exponential distribution scale\": 4, \n",
    "                \"delivery time in the background port\": 5,\n",
    "                \"required minimum wait time in background port\": 5,\n",
    "                \"reward amount in pursuit\": 3, \n",
    "                \"reward amount in background\": 0.6}\n",
    "    env = GiveUpEnvironment(env_info=env_info)\n",
    "    num_states = env.num_states\n",
    "\n",
    "\n",
    "    # run experiment\n",
    "    MaxRuns = 1\n",
    "    leave_values = np.zeros([MaxRuns, num_states])\n",
    "    stay_values = np.zeros([MaxRuns, num_states])\n",
    "    for j in range(MaxRuns):\n",
    "        \n",
    "        env_info[\"seed\"] = j\n",
    "\n",
    "        env = GiveUpEnvironment(env_info=env_info)\n",
    "        num_states = env.num_states\n",
    "\n",
    "        agent_info = {\"num_actions\": 2, \n",
    "                      \"num_states\": num_states, \n",
    "                      \"epsilon\": ExplorationConstant, \n",
    "                      \"discount\": 0.95, \n",
    "                      \"step_size\": StepSize, \n",
    "                      \"seed\": j, \n",
    "                      \"degree of exploration\": ExplorationConstant, \n",
    "                      \"exploration method\": ExplorationStrategy}\n",
    "        agent = QLearningAgent(agent_info=agent_info)\n",
    "        #agent.agent_init(agent_info)\n",
    "        #print(agent.policy)\n",
    "        # agent_info.update({\"policy\": policy})\n",
    "\n",
    "        rl_glue = RLGlue(env, agent)\n",
    "        rl_glue.rl_init(env_info, agent_info)\n",
    "        rl_glue.rl_start(env_info, agent_info)\n",
    "        values = rl_glue.agent.agent_message(\"get_action_values\")\n",
    "        MaxSteps = 500000\n",
    "        TD_error = np.zeros(MaxSteps)\n",
    "        prev_state = np.zeros(MaxSteps)\n",
    "        prev_action = np.zeros(MaxSteps)\n",
    "        average_reward = np.zeros(MaxSteps)\n",
    "\n",
    "        for i in tqdm(range(MaxSteps)):\n",
    "            roat = rl_glue.rl_step()\n",
    "#             TD_error[i] = rl_glue.agent.agent_message(\"get TD error\")\n",
    "#             prev_state[i] = int(rl_glue.agent.prev_state)\n",
    "#             prev_action[i] = rl_glue.agent.prev_action\n",
    "            if (i != 0):\n",
    "                average_reward[i] = rl_glue.total_reward / i\n",
    "        \n",
    "        return average_reward\n",
    "\n",
    "plt.plot(measure_convergence(\"UCB\", 0.3, 0.1), label = \"UCB, C=0.3, alpha=0.1\")\n",
    "plt.plot(measure_convergence(\"UCB\", 0.5, 0.1), label = \"UCB, C=0.5, alpha=0.1\")        \n",
    "plt.plot(measure_convergence(\"UCB\", 0.8, 0.1), label = \"UCB, C=0.8, alpha=0.1\")\n",
    "plt.plot(measure_convergence(\"UCB\", 0.3, 0.5), label = \"UCB, C=0.3, alpha=0.5\")\n",
    "\n",
    "\n",
    "\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.xlabel(\"Steps\")\n",
    "plt.ylabel(\"Average Reward\")\n",
    "plt.title(\"Performance of UCB on different parameters\")\n",
    "plt.savefig('narrowed_UCB_parameter_study.png')\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}