transformer_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9562809d",
   "metadata": {},
   "source": [
    "### Acknowledgement\n",
    "\n",
    "Upon building the final model we use for the result of the competition, we consulted [TensorFlow tutorials](https://www.tensorflow.org/text/tutorials/transformer) and PyTorch tutorials with practical examples such as [Language Modeling](https://pytorch.org/tutorials/beginner/transformer_tutorial.html) and [Language Translation](https://pytorch.org/tutorials/beginner/translation_transformer.html.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8a3d7322",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import os, os.path \n",
    "import numpy \n",
    "import pickle\n",
    "from glob import glob\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "from tqdm.notebook import tqdm\n",
    "import math\n",
    "\n",
    "\"\"\"\n",
    "    number of trajectories in each city\n",
    "    # austin --  train: 43041 test: 6325 \n",
    "    # miami -- train: 55029 test:7971\n",
    "    # pittsburgh -- train: 43544 test: 6361\n",
    "    # dearborn -- train: 24465 test: 3671\n",
    "    # washington-dc -- train: 25744 test: 3829\n",
    "    # palo-alto -- train:  11993 test:1686\n",
    "\n",
    "    trajectories sampled at 10HZ rate, input 5 seconds, output 6 seconds\n",
    "\"\"\"\n",
    "None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "066e3292",
   "metadata": {},
   "source": [
    "## Create a Torch.Dataset class for the training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e6db7f9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "ROOT_PATH = \"./\"\n",
    "\n",
    "cities = [\"austin\", \"miami\", \"pittsburgh\", \"dearborn\", \"washington-dc\", \"palo-alto\"]\n",
    "splits = [\"train\", \"test\"]\n",
    "\n",
    "def transform_data(np_data, bch_id):\n",
    "    df = pd.DataFrame(np_data[bch_id], columns = ['x','y'])\n",
    "    df['x_vel'] = np.gradient(df.x)\n",
    "    df['y_vel'] = np.gradient(df.y)\n",
    "    df['vel'] = np.sqrt(df.x_vel**2 + df.y_vel**2)\n",
    "    df['x_acc'] = np.gradient(df.x_vel)\n",
    "    df['y_acc'] = np.gradient(df.y_vel)\n",
    "    df['acc'] = np.gradient(df.vel)\n",
    "    tangent = np.array([1/df.vel]*2).T * np.array([df.x_vel, df.y_vel]).T\n",
    "    df['curvature'] = np.abs(df.x_acc * df.y_vel - df.x_vel * df.y_acc) / (df.vel)**3\n",
    "    out = df[['x', 'y', 'curvature']]\n",
    "    return out.to_numpy()\n",
    "\n",
    "\n",
    "def rotate(X, startpoint, endpoint, default_angle):\n",
    "    \n",
    "    # Find the slope of the path\n",
    "    dx = X[:, endpoint, 0] - X[:, startpoint, 0]\n",
    "    dy = X[:, endpoint, 1] - X[:, startpoint, 1]\n",
    "    \n",
    "    # Convert theta to degree in the range(0, 360)\n",
    "    theta = np.arctan2(dy, dx)\n",
    "    angle = np.degrees(theta)\n",
    "    angle[angle < 0] += 360\n",
    "    \n",
    "    # Generate the degree we want to rotate by and convert back to theta\n",
    "    rotate_degree = -1 * (angle - default_angle)\n",
    "    rotate_theta = np.deg2rad(rotate_degree)\n",
    "    \n",
    "    # Reshape the array from [4, batchsize] to [batchsize, 2, 2]\n",
    "    rot = np.array([np.cos(rotate_theta), -np.sin(rotate_theta),\n",
    "                np.sin(rotate_theta), np.cos(rotate_theta)])\n",
    "    rot = rot.T.reshape(-1, 2, 2)\n",
    "    \n",
    "    return rot\n",
    "\n",
    "\n",
    "def get_city_trajectories(city=\"palo-alto\", split=\"train\", normalized=False):\n",
    "    f_in = ROOT_PATH + split + \"/\" + city + \"_inputs\"\n",
    "    inputs = pickle.load(open(f_in, \"rb\"))\n",
    "    inputs = np.asarray(inputs)\n",
    "    \n",
    "    outputs = None\n",
    "    \n",
    "    if split==\"train\":\n",
    "        f_out = ROOT_PATH + split + \"/\" + city + \"_outputs\"\n",
    "        outputs = pickle.load(open(f_out, \"rb\"))\n",
    "        outputs = np.asarray(outputs)\n",
    "\n",
    "    return inputs, outputs\n",
    "\n",
    "class ArgoverseDataset(Dataset):\n",
    "    \"\"\"Dataset class for Argoverse\"\"\"\n",
    "    def __init__(self, city: str, split:str, transform=None, normalized=False):\n",
    "        super(ArgoverseDataset, self).__init__()\n",
    "        self.transform = transform\n",
    "        self.normalized = normalized\n",
    "        self.split = split\n",
    "\n",
    "        self.inputs, self.outputs = self.get_city_trajectories(city=city, split=split)\n",
    "        \n",
    "        # add shifted outputs\n",
    "        if self.outputs is not None:\n",
    "            self.shifted = np.roll(self.outputs, shift=1, axis=1)\n",
    "            self.shifted[:, 0, :] = self.inputs[:, -1, :]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.inputs)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        if self.split == 'train':\n",
    "        \n",
    "            data = ((self.inputs[idx], self.shifted[idx]), self.outputs[idx])\n",
    "\n",
    "#             if self.transform:\n",
    "#                 data = self.transform(data)\n",
    "\n",
    "            return data\n",
    "        \n",
    "        return self.inputs[idx]\n",
    "    \n",
    "    def get_city_trajectories(self, city=\"palo-alto\", split=\"train\"):\n",
    "        assert city in cities and split in splits\n",
    "\n",
    "        # get input\n",
    "        f_in = ROOT_PATH + split + \"/\" + city + \"_inputs\"\n",
    "        inputs = pickle.load(open(f_in, \"rb\"))\n",
    "        inputs = np.asarray(inputs)\n",
    "\n",
    "        # store input starting positions and rotation matrix\n",
    "        start_pos = inputs[:, 0, :].copy()\n",
    "        rotate_factor = rotate(inputs, 0, 29, 30)\n",
    "        max_factor = inputs.max(axis=1)\n",
    "\n",
    "        # normalize inputs (translation + rotation)\n",
    "        if self.normalized:\n",
    "            for i in range(len(inputs)):\n",
    "                inputs[i] -= start_pos[i, :]\n",
    "                \n",
    "            for i in range(len(inputs)):\n",
    "                inputs[i] = inputs[i] @ rotate_factor[i].T\n",
    "            \n",
    "            max_factor = inputs.max(axis=1)\n",
    "            \n",
    "#             for i in range(len(inputs)):\n",
    "#                 inputs[i] = inputs[i] / max_factor[i]\n",
    "\n",
    "        # get output\n",
    "        outputs = None\n",
    "        if split == \"train\":  # get and normalize outputs\n",
    "            f_out = ROOT_PATH + split + \"/\" + city + \"_outputs\"\n",
    "            outputs = pickle.load(open(f_out, \"rb\"))\n",
    "            outputs = np.asarray(outputs)\n",
    "            if self.normalized:\n",
    "                for i in range(len(inputs)):\n",
    "                    outputs[i] -= start_pos[i, :]\n",
    "                    \n",
    "                for i in range(len(inputs)):\n",
    "                    outputs[i] = outputs[i] @ rotate_factor[i].T\n",
    "                \n",
    "#                 for i in range(len(inputs)):\n",
    "#                     outputs[i] = outputs[i] / max_factor[i]\n",
    "        \n",
    "            print(inputs.shape)\n",
    "            print(outputs.shape)\n",
    "        \n",
    "            # Adding curvature as features\n",
    "            if self.transform:\n",
    "#                 print(inputs.shape)\n",
    "#                 print(outputs.shape)\n",
    "                inputs = np.array([transform_data(inputs, i) for i in range(len(inputs))])\n",
    "                print(inputs.shape)\n",
    "\n",
    "        self.start_pos = start_pos\n",
    "        self.rotate_matrix = rotate_factor # np.linalg.inv(rot[i].T) to reverse back\n",
    "        \n",
    "        if self.normalized:\n",
    "            self.n_max = max_factor\n",
    "\n",
    "        return inputs, outputs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b30b1b28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11993, 50, 2)\n",
      "(11993, 60, 2)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "11993"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# intialize a dataset\n",
    "city = 'palo-alto' \n",
    "split = 'train' #'test' #'train'\n",
    "train_dataset = ArgoverseDataset(city = city, split = split, transform=False, normalized=True)\n",
    "\n",
    "len(train_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7877ed6",
   "metadata": {},
   "source": [
    "## Create a DataLoader class for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "97632d1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2998"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_sz = 4  # batch size \n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_sz, drop_last=True)\n",
    "len(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "25fd4df4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 50, 2])\n",
      "torch.Size([4, 60, 2])\n",
      "torch.Size([4, 60, 2])\n",
      "50\n",
      "60\n"
     ]
    }
   ],
   "source": [
    "for (inp, shift), out in train_loader:\n",
    "    print(inp.shape)\n",
    "    print(shift.shape)\n",
    "    print(out.shape)\n",
    "    for bs in range(len(inp)):\n",
    "        print(len(inp[bs]))\n",
    "        print(len(out[bs]))\n",
    "        x = inp.detach().numpy()\n",
    "        y = out.detach().numpy()\n",
    "        break\n",
    "    break\n",
    "\n",
    "# # FOR TEST SET\n",
    "# for inp in train_loader:\n",
    "#     for bs in range(len(inp)):\n",
    "#         print(inp.shape)\n",
    "#         print(len(inp[bs]))\n",
    "#         print(inp[0][:5])\n",
    "#         break\n",
    "#     break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddbfef58",
   "metadata": {},
   "source": [
    "## Sample a batch of data and visualize "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9ea7e317",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "\n",
    "\n",
    "def show_sample_batch(sample_batch):\n",
    "    \"\"\"visualize the trajectory for a batch of samples\"\"\"\n",
    "    inp, out = sample_batch\n",
    "    batch_sz = inp.size(0)\n",
    "    agent_sz = inp.size(1)\n",
    "    \n",
    "    \n",
    "    fig, axs = plt.subplots(1,batch_sz, figsize=(15, 3), facecolor='w', edgecolor='k')\n",
    "    fig.subplots_adjust(hspace = .2, wspace=.5)\n",
    "    axs = axs.ravel()   \n",
    "    for i in range(4):\n",
    "        axs[i].xaxis.set_ticks([inp[i,0,0],\n",
    "                                   inp[i,-1,0], out[i,-1,0]])\n",
    "        axs[i].yaxis.set_ticks([inp[i,0,1],\n",
    "                                   inp[i,-1,1], out[i,-1,1]])\n",
    "\n",
    "        # first two feature dimensions are (x,y) positions\n",
    "        axs[i].scatter(inp[i,:,0], inp[i,:,1])\n",
    "        axs[i].scatter(out[i,:,0], out[i,:,1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8b1a2362",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA34AAADCCAYAAAAWyRCHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA6M0lEQVR4nO3df1TUdb4/8OcIUlip600MRCNFgQaGUUel8+X084DpGgWmaXTrJsbWdW9tptb31mZ5dNW8LbhXjy1t7rXyK2snAzIgU/ZYx3IJZfzZIdbFGwxuUYmaqPx6f/9gZ1ZkZj7vgc9nZj6feT7O6ZQz88bPTPPi83l93q/3620SQggQERERERGRYQ0K9AEQERERERGRtpj4ERERERERGRwTPyIiIiIiIoNj4kdERERERGRwTPyIiIiIiIgMjokfERERERGRwYUH+gB8ceONNyIuLi7Qh0EGderUKXz//feBPgzdYDySlhiPvmE8ktYYk75hTJKW+huPukr84uLiUFNTE+jDIIOy2WyBPgRdYTySlhiPvmE8ktYYk75hTJKW+huPLPUkIiIiIiIyOCZ+REREREREBieV+MXFxSElJQVWq9U1tfjrX/8aFosFVqsVmZmZaG5u7jOurq4OVqvV9c/QoUNRWFgIAPjxxx+RkZGBCRMmICMjA2fOnFHvXRFd7cgOoCAZeGV4z7+P7Aj0EfWbu3g8fPgwbrvtNqSkpOC+++7DuXPn3I4tKCiA2WxGcnIyFixYgEuXLgEA7HY70tLSXD+zurrab++HQpCB4pHIL1wxM6wnbl4Z1vufdbcwjv7B3TlS9pqzsrISCQkJiI+Px9q1a12PL1u2DImJibBYLMjOzkZra6s/3gqFKg3PkdIzfn/+859ht9td9crLli3DkSNHYLfbMXv2bKxcubLPmISEBNjtdtjtdhw8eBBDhgxBdnY2AGDt2rW45557UF9fj3vuuadXgBGp6sgOoHQxcLYRgOj5d+liXZ8kr47HRYsWYe3atTh69Ciys7Oxfv36PmMcDgd+97vfoaamBseOHUNXVxeKi4sBAMuXL8eKFStgt9uxcuVKLF++3K/vh0LIriXAzicMFY9Emjiyoyehe2XYFTEDAKLvay/+2POa38QwltD3HClzzdnV1YXFixejoqICJ06cwPbt23HixAkAQEZGBo4dO4YjR45g4sSJWLNmjV/fD4WQXUuAnfm9z5EfPq1aXPe71HPo0KGu/75w4QJMJpPX1+/duxfjx4/HzTffDAAoLS3FY489BgB47LHHUFJS0t9DIfJu16+Arvbej3W1AxXPB+RwtFBXV4fbb78dQM8J6v3333f7us7OTly8eBGdnZ1oa2tDTEwMAMBkMrlmCc+ePet6nEg1R3YAq2OAmrf6PmeweCQaMOcNkos/+jau/QITQDdkrjmrq6sRHx+PcePGISIiAvPnz0dpaSkAIDMzE+HhPf0Q09LS0NTU5LdjpxCya8k/zpFX3dzpuAjs7TvB1h9SiZ/JZEJmZiamTJmCoqIi1+MvvvgixowZg23btrmd8btScXExFixY4Przt99+i+joaABAdHQ0vvvuO7fjioqKYLPZYLPZ0NLSInO4RP+0a0nPidAdX0+oQcJdPCYnJ6OsrAwA8N5776GxsbHPuNGjR2Pp0qUYO3YsoqOjMWzYMGRmZgIACgsLsWzZMowZMwZLly71eDeT8Ug+c85a7HwC6PAQi4Bu45FIVd5ukPjCmQCGYAmou3OkzDWnw+HAmDFjXH+OjY2Fw+Ho87otW7Zg5syZbv9uniOp31xJnwdn1bnZIJX47d+/H4cOHUJFRQU2bdqETz/9FACwevVqNDY2Ijc3Fxs3bvQ4vr29HWVlZZg7d67PB5ifn4+amhrU1NRg5MiRPo+nEOW82BzoyTMIuYvHLVu2YNOmTZgyZQrOnz+PiIiIPuPOnDmD0tJSNDQ0oLm5GRcuXMC7774LANi8eTMKCgrQ2NiIgoIC5OXluf27GY/kE2fJCpM6ImXOWT5vN0h8dfHHnhjctUS9nxnkPF2zKhGibwnt1dVsq1evRnh4OHJzc93+DJ4jqV+O7ABqtnh/zbBYVf4qqcTPWfYVFRWF7OzsPo0fHn74YY+lZQBQUVGByZMnY9SoUa7HRo0ahdOnTwMATp8+jaioKJ8Pnsgt2YvNyBH+OR6VuYvHxMRE7N69GwcPHsSCBQswfvz4PuP27NmDW265BSNHjsTgwYORk5ODzz//HACwdetW5OTkAADmzp3L5i40ML1uvLhZj+SOTuORaMA0v1Epen52iCR/7s6RMtecsbGxvaplmpqaei172Lp1K3bt2oVt27YpLm8iknZkB/DBk/B+rjQB97ysyl+nmPhduHAB58+fd/337t27kZycjPr6etdrysrKkJiY6PFnbN++vVeZJwBkZWVh69atAHqC6f777+/XGyDqxXXXROJic+Y6zQ9HbZ7i0Vm20t3djVWrVuHJJ5/sM3bs2LE4cOAA2traIITA3r17kZSUBKDnRLlv3z4AQFVVFSZMmOCnd0SG059ZPlOYLuORaMD8OSseAsmfp3OkzDXn1KlTUV9fj4aGBrS3t6O4uBhZWVkAerp9rlu3DmVlZRgyZIj/3hAZmzP+RZf319kWApZ5qvyV4Uov+Pbbb12dODs7O/Hwww/j3nvvxZw5c1BXV4dBgwbh5ptvxhtvvAEAaG5uxqJFi1BeXg4AaGtrwyeffILf//73vX7uCy+8gHnz5uGtt97C2LFj8d5776nyhijEVTwPqaTPlqdaEPmTp3jcsGEDNm3aBADIycnB448/DqB3PE6fPh0PPvggJk+ejPDwcEyaNAn5+fkAgDfffBPPPPMMOjs7ce211/Zay0sk5ciOnvjz9QI24jpgdqEu45FoQJTW9LgTOaLnJokzXo7sAD78lXx5aM0WYGyaYePN0zly6tSpbq85rzxHhoeHY+PGjZgxYwa6urqwcOFCmM1mAMAvf/lLXL58GRkZGQB6Grw4r3uJ+kV2osKWB8z+rWp/rUm4K2oOUjabzdWal6gX6YtOU8+dEzdBxO+Xb/h5kcuuJfIz7U5XX8Behd8v3/Dz0pkjO3ru9MvGjNLFny83XiJHAM83yP29Vx4Cv2M+4edFHjnLO73O9Hm+XgX6//1SnPEjCnpHdvTscdJx0fvrFC40ichHPs/yeT+REYUM2eoU2Rlxy7x/vkZpJvHijz2vYRwS+Z/zmtVb0mcKA7Lf0OR6lYkf6ZvUXROoPlVOFPJ8neXjjReiHruWyDUf62+8OM913pI/g5d8EgWtiucVJipMmiV9ABM/0jOZuyZAzwmUSR+ROjjLR9R/Muv61LhRqZj8iZ44ZuJH5D9HdiicO02qNnJxR2o7B6KgpHjXBMDgSHYLJFKLrx0II0cAOUVM+ogAub261KxOmf1b79ukXPwx5DZ3JwoY17YNHpjC/HK+ZOJH+uPc80imVOa+3/GOJtFA+bwvn6nnAvb5BsYfkZPSuj4tqlNmrgPgZc+5vSvV/fuIqC+ZCjUNyzuvxFJP0heZRi4aLoolCjlcy0c0cIrr+kzaVKdY5gHfHPBc8nm20f3jRKQOmV4UkSP8ds7kjB/pi0x5J5M+ooHjLB+ROqRKPDVc1zP7t4DJy+Ueyz2JtCGzQbuflyQx8SP9UFwUC7/eNSEyLK7lI1KPUomnP7pOi27Pz1U8r+3fTRSKZDZoN4X5fUkSSz1JH5QWxQJs5EI0UOzYSaQupRuW/uo6PWyM57JOZ5MX3jQlUofrmtVL0jc4MiB9KDjjR8FPZqqcjVyIBoazfETq8zqbptG6PnfueRlem7xw1o9IHbIbtAfompUzfhTcZKbKI0f0rCsiov6RiTMXzvIRSVFq6KLxfl29KDV54awfkToCvEG7Es74UfCSnSpneSdR/8nEmRNn+YjkKDV08VeJ55WU9vXjrB9R/0ltNab9Bu1KmPhRcJIp7wzgVDmRIcjEGQB27CTygczNlEDdsPT293JDd6L+cZZ3ekv6/LRBuxImfhR8pMrOAjtVTqRrvmzVwFk+Inmya9IDde6yzOOsH5HadLTVGBM/Cj5Kra+DYKqcSLekm7hwlo/IJ7I3LQO9PIGzfkTq0dlWY0z8KHjI1EcHyVQ5kS7tWiI3y8c4I/KdXm5aKs367V3pv2Mh0jMdbjXGxI+Cg0x9NMs7ifpPqdmEC+OMyGdKd/2D7WaKtwtRT/v9EdE/yWzbEIRbjTHxo+AgUx8dDHdKifRIunNnkMxIEOmN0n59wXYzxTIPMHm5BGS5J5F3Stetzq3GginuwcSPgoFsfXSw3Ckl0hPZzp1s4kLUP8G0X58vRLfn51juSeSezLKkICvvvBITPwosHdZHE+mCL5072cSlXy5duoRp06YhNTUVZrMZK1asAAC88sorGD16NKxWK6xWK8rLy92Or6ysREJCAuLj47F27VrX4++99x7MZjMGDRqEmpoav7wX6qdg3K9P1rAxnp9juSdRX7LbNgRZeeeVwgN9ABTCdi1R7oAWOaIn6QvSACIKSjKxBcBV2hmsF6ZB7pprrkFVVRWuv/56dHR0ID09HTNnzgQAPPvss1i6dKnHsV1dXVi8eDE++eQTxMbGYurUqcjKysKtt96K5ORk7Ny5E7/4xS/89VaoP4J5vz4Z97wM7Hwi0EdBpA/OeFeqngm2su6rcMaPAkOm7XWQ1kcTBTWplvIIvmYTOmQymXD99dcDADo6OtDR0QGTySQ1trq6GvHx8Rg3bhwiIiIwf/58lJaWAgCSkpKQkJCg2XGTCmQbOwTz+Uvp2LjOj6iHTLwDwR/zYOJHgSBzl5TlnUS+86WJS5DfldSLrq4uWK1WREVFISMjA9OnTwcAbNy4ERaLBQsXLsSZM2f6jHM4HBgz5p+ldrGxsXA4HH47bhogxYZkQbBf30BxnR9RD5kGhDq5bmXiR/4l02giyOujiYKSbBMXdu5UVVhYGOx2O5qamlBdXY1jx47hqaeewsmTJ2G32xEdHY3nnnuuzzgh+ibnsrOFTkVFRbDZbLDZbGhpaen3eyAfKTYk01GMedvP72yT/46DKFjJNiDUyXUrEz/yH6kSNM5EEPlMdmN2du7UzPDhw3HnnXeisrISo0aNQlhYGAYNGoQnnngC1dXVfV4fGxuLxsZ/NtBoampCTEyMT39nfn4+ampqUFNTg5EjRw74PZAkb1s36K2E2tsMxbBY/x0HUTBSakBoCgNy3tTVsiQmfuQfUiVoOrpLShQspDZmN7FzpwZaWlrQ2toKALh48SL27NmDxMREnD592vWaDz74AMnJyX3GTp06FfX19WhoaEB7ezuKi4uRlZXlr0On/pBp4663G5eWeT2/G3DVbPPgyJ7mL0ShyBnrO5/wXkWjt3gHu3qSP8h0GDSF6TKAiAJK5oYKY0szp0+fxmOPPYauri50d3dj3rx5mD17Nv71X/8VdrsdJpMJcXFx+P3vfw8AaG5uxqJFi1BeXo7w8HBs3LgRM2bMQFdXFxYuXAiz2QygJ1n8j//4D7S0tODnP/85rFYrPv7440C+VXI2d1DasFmPcTb7t8DYtJ41fWebemb67nlZn++FaKBkYh3Qbbwz8SNtsbyTSBtSWzYwtrRksVhQW1vb5/F33nnH7etjYmJ67ek3a9YszJo1q8/rsrOzkZ2drd6B0sDJNHfQQWMHjyzz+HuCCDBUIxd3WOpJ2mF5J5E2ZLdsYGwRDZxscwfGGpG+ycS6zhsQcsaPtMHyTiLtVDwPqRsqemkwQRTMvDVzAXR995+I/kGpkQvQE+s6TvoAJn6kBZZ3EmnjyI6ei1BvdyR5Q4VIPUozAJEjepI+xhuRfsls0G6QWGfiR+qTnY3QefAQ+RXX9BH5l9IMQOSInk65RKRvSuv6DBTrXONH6pFpda23PY6IggHX9BH5l8wMAMs7ifRN5rrVYKXcnPEjdUi1v+VsBFG/KM6io+eOJG+oEKlDZgaA5zIi/ZK5btV5Ixd3OONH6pBpf8vZCCLfyXQZM9gdSaKA2rUkpGYAiEKSzHWrAScrmPjRwMhMkwOcjSDqD5kuY5EjDHdHkihgXGXVHhhwBoAo5ITwFi0s9aT+kyrvBO+OEvWHTDMXWx5vqBCpRWbvWQPOABCFFNltGwx63coZP+o/mWlyzkYQ+U6mmQtn0QOusbERd911F5KSkmA2m7FhwwYAwOHDh3HbbbchJSUF9913H86dO9dnbF1dHaxWq+ufoUOHorCwEACwbNkyJCYmwmKxIDs7G62trX58VyFKtp07z2VE+rVrCbAzXznODXzdysSP+kd2mvz5BsMGD5FmlJq5GPhupJ6Eh4fj9ddfx1dffYUDBw5g06ZNOHHiBBYtWoS1a9fi6NGjyM7Oxvr16/uMTUhIgN1uh91ux8GDBzFkyBBkZ2cDADIyMnDs2DEcOXIEEydOxJo1a/z91kJOW8XLys3JGHNE+iV7Q9Xg161M/Mh3IT5NTqQZ2S1RDHw3Uk+io6MxefJkAMANN9yApKQkOBwO1NXV4fbbbwfQk8S9//77Xn/O3r17MX78eNx8880AgMzMTISH96zESEtLQ1NTk4bvgkpqHbi27e9eXsG9Z4l0TaaMO0SuW5n4kW84TU6kDWepmdeZdG6JEqxOnTqF2tpaTJ8+HcnJySgrKwMAvPfee2hsbPQ6tri4GAsWLHD73JYtWzBz5kzVj5f+6dUPj6NZ/Iv7J7n3LJF+OW+m7nzC+3VrCN1QZeJH8jhNTqQdbomiWz/99BPmzJmDwsJCDB06FFu2bMGmTZswZcoUnD9/HhERER7Htre3o6ysDHPnzu3z3OrVqxEeHo7c3Fy3Y4uKimCz2WCz2dDS0qLa+wklJbUOnGnrwGud89Amev9/ahMRvNFCpFdSN1OBULuhyq6eJI/rjoi0obRvGMBmLkGqo6MDc+bMQW5uLnJycgAAiYmJ2L17NwDg66+/xkcffeRxfEVFBSZPnoxRo0b1enzr1q3YtWsX9u7dC5PJ5HZsfn4+8vPzAQA2m02NtxNSSmodeG7HYQBAWXc60AEsD9+BGNMPaBb/gj9EPIJXQuRikMhwZG6mhmAZNxM/UnZkR08Acd0RkfqU9g0DeFMlSAkhkJeXh6SkJCxZssT1+HfffYeoqCh0d3dj1apVePJJz2uit2/f3qfMs7KyEuvWrcO+ffswZMgQzY4/lL1UchTbDnzT61ZmWXc6ytrTXX8uzLb6/biIaIBkrlmBnuvWEJrpc2KpJ3nHdUdE2pFZcM41s0Fr//79eOedd1BVVeXalqG8vBzbt2/HxIkTkZiYiJiYGDz++OMAgObmZsyaNcs1vq2tDZ988olrptDpl7/8Jc6fP4+MjAxYrVaviSP5rqTW0Sfpu9rwyMF4YNJovx0TEalAtrxzcGTIXrdyxo+847ojIm3IbNDuXDNLQSk9PR1CuP//98wzz/R5LCYmBuXl5a4/DxkyBD/88EOf1/31r39V7yCpF2d5p7ekL3JwGF7JMvvtmIhIJbL7S89cF7LXrUz8yD3ZqXKuOyLynUyjJO4bRqSqkloH/u/Oo+jykKwDQJjJhDU5KZztI9IbX/aXDmFM/Kgv51S50l0Trjsi6h+lRkkhuOCcSGuvfngcFzs8t3Q3AXh9XiqTPiK94f7S0pj4UV+cKifShmyjpBBde0CkFee2DZ6YAOSmjWXSR6Q3sssmeM0KgIkfXU22rXyIT5UT+UxqJp2NkojUduW2De6EmUyc6SPSI1/2lyYATPzoSmwrT6QdNkoi8jt32zZcjUkfkQ7JdMXmNWsf3M6BerCtPJF2uEE7kd9x2wYig3JW0AjPa3a5v7R7nPEjtpUn0hJn0on8jts2EBmYYgUNl014whm/UMe28kTaUurgyZl0IlVx2wYigzqyA1h3i0IFDbtie8MZv1DHtvJE2lHaV4gz6USq47YNRAYk0yCNXbEVMfELVWwrT6QtxX2FOJNOpDZu20BkUDIN0njNqoiJXyiSWdPH+mii/pOJMc6kE6mK2zYQGZRS9QzQU0HDc6oirvELNVJr+sCLUqL+kt1XiB08iVQjs66PSR+RDilWz4AN0nzAGb9Qo7imD7woJRoIpRjjCYpIdUrr+rhtA5EOyXadn7mOkxWSmPiFCpk1fQAvSon6S3bdLDt4EqmmpNaBV8qOo/Wi53V93LaBSIdkq2fYIM0nTPxCgUwnJIB3TYj6SyrGuG6WSE3O8k5vM33ctoFIh1zlnayeURsTv1Ag0wnJlsfyTqL+kooxrpslUpNSeSfAdX1EuiNT3snqmX5jcxej27VErhMSkz6i/mGMEfmd0rYNANf1EemOVANCVs8MBBM/I3MFkBecKifqP8YYkd8pbdsAcF0fke7IlHfCxOqZAWKpp1HJBBDX9BENjFIHT8YYkapktm342ZDBWHGfmbN9RHrhXCcvvJRum8I406cCJn5GJNv+lp2QiPpHpoMnY4xIdTLbNtS+nOnHIyKiAVNcJ8/yTrWw1NNodi0Bat6C4lQ5S890qbGxEXfddReSkpJgNpuxYcMGAMDhw4dx2223ISUlBffddx/OnTsnPVZ2PP2D886k13V9jDEitSmt62N5J5HOHNkBrLtF+XzK8k7VMPEzEpn1RgwgXQsPD8frr7+Or776CgcOHMCmTZtw4sQJLFq0CGvXrsXRo0eRnZ2N9evXS48FIDWeerRVvMwOnkQB8OqHxz0+x20biHRG5iaqKQzIKWJzNBUx8TMKmTV9DCDdi46OxuTJkwEAN9xwA5KSkuBwOFBXV4fbb78dAJCRkYH3339feiwAqfEEvFRyFNe2nfb+InbwJFKd0mwft20g0hmZbZBY3qk6Jn5GsGsJsDPf+6JY1kcbzqlTp1BbW4vp06cjOTkZZWVlAID33nsPjY2N0mMB+Dw+FJXUOrDtwDdoFjd6fhE7eBKpTqmLJ7dtINIRqfJO9NxE5TWr6pj46Z3Umj6w9MxgfvrpJ8yZMweFhYUYOnQotmzZgk2bNmHKlCk4f/48IiIipMcCkB5fVFQEm80Gm82GlpYWTd5bsHr1w+MQAF7rnIc20fvzEQI9JyluKEukKpkunlzXR6QTUmvkwZuoGmJXTz3zZU0fS88Mo6OjA3PmzEFubi5ycnIAAImJidi9ezcA4Ouvv8ZHH30kPdaX8fn5+cjPzwcA2Gw21d5TsLuyzKysOx3oAJaH70CM6Qc0i3/BHyIewSvPvxrgoyQyHpkunpztI9IJmfJOboOkKSZ+eqa0hxj3PDEcIQTy8vKQlJSEJUuWuB7/7rvvEBUVhe7ubqxatQpPPvmk9FjZ8aHKXZlZWXc6ytrTAQAmAAXZVv8fGJHBsYsnkUHIbIEEcBskP2Cppx7Jtr9l0mc4+/fvxzvvvIOqqipYrVZYrVaUl5dj+/btmDhxIhITExETE4PHH38cANDc3IxZs2Z5HQvA4/hQ91LJUTz7J7vXMrPctLGccSBSmdK6PnbxJNIJlncGFc746Y3M5uwA1/QZVHp6OoSHJOSZZ57p81hMTIwruVMa6258KHM2c/EWacMjB2PVAyl+OyaiUCCzro9dPIl0guWdQYWJn5641vQpJX15XNNHNEDOZi6esMyMSBtc10dkEEd2sLwzyLDUU0+U1vQB3EOMSAUvlRz1uraIZWZE2uC6PiIDqXje+/Ms7/Q7Jn56sWsJ66OJ/MBZ4umJCSwzI9IC1/URGYRMLwpugRQQiolfY2Mj7rrrLiQlJcFsNmPDhg0AgGXLliExMREWiwXZ2dlobW11O76yshIJCQmIj4/H2rVrXY/b7XakpaXBarXCZrOhurpanXdkQF+W/R7dNW95fxEDiGjAnBee3ubV2cyFSH1c10dkEDLNXJzlnbxm9TvFxC88PByvv/46vvrqKxw4cACbNm3CiRMnkJGRgWPHjuHIkSOYOHEi1qxZ02dsV1cXFi9ejIqKCpw4cQLbt2/HiRMnAADLly/HihUrYLfbsXLlSixfvlz9d2cAJbUOxBx8zfv/KFseA4hogGQ6eLKZC5E2uK6PyCBkmrmwOi1gFBO/6OhoTJ48GQBwww03ICkpCQ6HA5mZmQgP7+kNk5aWhqampj5jq6urER8fj3HjxiEiIgLz589HaWkpAMBkMuHcuXMAgLNnzyImJka1N2UUztmHaHzv+UVc00c0YDIdPE0A1xYRaYDr+ogMQGqrMfRct3KiImB86up56tQp1NbWYvr06b0e37JlCx566KE+r3c4HBgzZozrz7GxsfjLX/4CACgsLMSMGTOwdOlSdHd34/PPP3f7dxYVFaGoqAgA0NLS4svh6tpLJUddF6LN4kbEmtwlfybeNSFSgVIHTxNY4kmkBa7rIzIAZ3mn0kwfe1EEnHRzl59++glz5sxBYWEhhg4d6np89erVCA8PR25ubp8x7vYMM5lMAIDNmzejoKAAjY2NKCgoQF5entu/Nz8/HzU1NaipqcHIkSNlD1fXXio5inevmH14rXMe2kREr9d0A9yrj0gFSrMNYSYTCh6yssSTSGVc10dkELJ79bEXRcBJJX4dHR2YM2cOcnNzkZOT43p869at2LVrF7Zt2+ZK6K4UGxuLxsZG15+bmppcJZ1bt251/ay5c+eyucs/uOsoWNadjhc6FqGp+0Z0CxMc4kYcnPwaSzyJVPDqh8c9PscOnkTacM70cV0fkc7JdJ1nM5egoZj4CSGQl5eHpKQkLFmyxPV4ZWUl1q1bh7KyMgwZMsTt2KlTp6K+vh4NDQ1ob29HcXExsrKyAAAxMTHYt28fAKCqqgoTJkxQ4/3onqeSs7LudKS3/w4T2v8fvnzgU0zN+oXfj43IaJT262N5J5H6ZGb6uK6PSAeO7ABqtnh/Dcs7g4riGr/9+/fjnXfeQUpKCqxWKwDgN7/5DZ5++mlcvnwZGRkZAHoavLzxxhtobm7GokWLUF5ejvDwcGzcuBEzZsxAV1cXFi5cCLO55xf5m2++iWeeeQadnZ249tprXev4QpnSRShnH4jUo7RfHzt4EmlDqYMn1/UR6cCRHcAHTwLeVshHjuhJ+jjTFzQUE7/09HS3a/VmzZrl9vUxMTEoLy/v9Tp3r01PT8fBgwd9OVbDKql14JWy42i96DnpAzj7QKQWmf36ONtApD6ZDp5M+oiC3K4l/5jpU0j6nm/w2yGRHOnmLqQN595hSknfI2ljOftApALZ/fp44UmkvvUf13l8jjN9NFALFy5EVFQUkpOTXY899NBDsFqtsFqtiIuLc1WvXenSpUuYNm0aUlNTYTabsWLFCtdzdrsdaWlpsFqtsNls7EnhKu9U6IXN8s6gxMQvgGT2DgNYckakFu7XRxQ4JbUOOFo9d/7jUgYaqH/7t39DZWVlr8f+9Kc/wW63w263Y86cOb2aFDpdc801qKqqwuHDh2G321FZWYkDBw4AAJYvX44VK1bAbrdj5cqVWL58uV/eSzAqqXXg7zv/E4pJH7vOBy2f9vEjdSntHQbwIpRITdyvjygwnA1dPOEsO6nh9ttvx6lTp9w+J4TAjh07UFVV1ec5k8mE66+/HkBPJ/uOjg5Xt3qTyYRz584BAM6ePevqTh9qnPtLn7ympedk6Y4pDMh+g0lfEGPiFyBKjVwAXoQSqUkp5sJMJs44EGnEW0MXdvAkf/jss88watQoj13ku7q6MGXKFPz1r3/F4sWLMX36dABAYWEhZsyYgaVLl6K7uxuff/65Pw87KFxZLdMsbkSs6Xs3rzIx6dMBlnoGgHODdm9+NmQwN40mUolSB092zCXSjlJDF67rI3/Yvn07FixY4PH5sLAw2O12NDU1obq6GseOHQMAbN68GQUFBWhsbERBQQHy8vI8/oyioiLYbDbYbDa0tLSo/h4C5cpqmdc656FNRFz1CpZ36gUTPz+TSfoeSRuL2pczeSIkUolSiSdn1om04eyg68no4ZGMPdJcZ2cndu7ciYceekjxtcOHD8edd97pWiu4detW17rAuXPnem3ukp+fj5qaGtTU1GDkyJHqHHwAldQ6YH11d68bN2Xd6XihYxGaum9EtzDh7xgJ5BQBs38bwCMlWUz8/Ehp1gFgIxciNbk7aV2NMUekDZmN2pfNSPDjEVGo2rNnDxITExEbG+v2+ZaWFrS2tgIALl686Ho90LNN2b59+wAAVVVVHktFjcYZv+66zpd1pyO9/XcYf3kbDty/jzN9OsLEz49kGktwnQOROrydtJwYc0TaUdqonQ1dSG0LFizAbbfdhrq6OsTGxuKtt94CABQXF/cp82xubnbtM3369GncddddsFgsmDp1KjIyMjB79mwAwJtvvonnnnsOqamp+M///E8UFRX5900FiFL8AqyW0SM2d/EDbtBO5H88aREFjsxG7bzpQmrbvn2728f/53/+p89jMTExKC8vBwBYLBbU1ta6HZueno6DBw+qdox6INOAkNUy+sTET2PO9rdK2zZwg3Yi9fCkRRRY3KidSJ9kliXxxo1+MfHTkEwjF4BJH5GaeNIiCixu1E6kX0rLkn42ZDBW3GdmDOsUEz+NyCZ9nHUgUhdPWkSBw43aifRJZlnS8MjBqH05049HRWpj4qcB2aSPsw5E6lIq8eRJi0hb3KidSH+cN2y8rYtnMzRjYOKnMpkyM4CzDkRqk9mknSctIu1wo3YifWIztNDBxE9lSmVmANf0EanNuUk0N2knCgxu1E6kT2yGFlqY+KlEdssGJn1E6pLpnMuTFpF2uFE7kT7JLE1iibaxMPFTAbdsIAoMZ3mnt9hjiSeRttZ/XMeN2ol0RmZpEpclGQ8TvwHilg1EgaNUWm0CSzyJtKS0dQNnC4iCj8zyCDZDMyYmfgMg28iFZWZE6lNqJBFmMnG/MCINKW3dwI3aiYKPTJUaK2WMi4nfAMg0cuHdTiJtvPrhcY/PmcBNoom0prR1A5M+ouAiW6XGShnjYuLXTzJdkFgbTaQNpfjjSYtIW9y6gUhfZJI+5/IIVqkZFxO/fpAJHq7pI9KGUok1S6uJtMWtG4j0Q7brPJdHhAYmfj7glg1EgSWzIJ2l1UTa4dYNRPrhjFelzdm5PCJ0MPGTJLtlA2cbiLQhu18fT1xE2uHWDUT64W0d7pW4PCJ0DAr0AeiBzF5hALsgEWmF+/URBQdu3UCkDzK9KABWqYUaJn4SZLp3ArxjQqQV7tdHFHgltQ6YPDzHrRuIgodsIxcmfaGHpZ4KZO6YsAsSkXaUYpAL0om05219LdcHEQWHnjV9R3Cxo9vr69h1PnQx8fNC5o4Jg4dIO0odPHnBSaQ9pYYuAmAMEgWY7B59wyMHo/blTD8cEQUjJn5usHsnUXBQKvFkeSeR9pQauoweHunHoyGiq8kmfVwLT0z8riLb+pbdO4m0pVTiyRgk0l5JrUOxoQu3byAKnJJah1TSB/BmKbG5Sx8yrW95x4RIWzIlnoxBIm05b4R6woYuRIH34geeY/RKrFIjgIlfL7Ktb3nHhEhbLPEkCjxvN0IjB4dxfS1RgL1UchQX2pX36WPSR04s9fwH2da37N5JpC2WeBIFXkmtw2sccqaPKLBkrluviwjD6mzGKv0TEz8ol5UB7N5J5A8s8SQKPOfWDZ6MHh7JcyFRAMkkfZGDB+H4ynv9dESkF0z8oFxWxta3RNrztk+YE0s8ibSltHUDADZzIQog2Q6ea3Isfjga0puQTvxktm3gDAOR9l4qOYptB75RvAHDEk8ibSlt3TA8cjBvvhAFiGzS9whvkpIHIZv4yVxoApxhINKas7zTWyzyBgyR9mS2bmAcEgWGL0kfb5KSJyHZ1VPmQhNg8BD5g1KptbOpEm/AEGmHWzcQBS/ZpC9y8CBet5JXITnjp3ShCbCsjMgflDp4hplMbBlP5AdKWzcw6SMKDNmkb5CJ6/pIWcjN+Mns1cdyFiLtKZ3MTACTPiI/4NYNRMFJNum7LiIMv51nZZySopCa8ZMJIG7bQKQ9mS1UWN5J5B/rP67z+By3biAKDK7pIy2ETOInc6HJ4CHyD5ktVBiLRNpTaujCrRuI/K+k1sGkjzQREqWeMvuD8UKTSHsltQ5YX93ttayMHTyJ/EOpoQu3biAKjBc/8ByXTkz6qD8MP+Mns20DLzSJtOe8yPS2RxjAEk8if3DeEPW0UTvXuhMFxkslR3Gh3ft5kkkf9ZehZ/xkt23ghSaR9rx1DXTiyYxIe86bMJ6SPoANXYgCQWZdH8+TNBCGnvGT3R+MAUSkLZluuiy3JvKP9R/Xeb0Jw4YuRP4nk/Rxnz4aKEMmfiW1DrxSdhytF7k/GFGgyTRWYlkZkX8oNXOJHBzGhi5EfibbwZP79NFAGS7xk13Tx6SPyD+UZt65hQqRfyg1cwkzmVjiSeRnvmzbwNikgTJU4sc1fUTBRanEc3jkYNS+nOnHIyIKXd7W2UYODmPSR+RnskkfSzxJLYZq7qI0swBwHRGRvyid0NhNl8h/SmodXm/CMOkj8i/ZpG+QiSWepB7DJH4yzSO4jojIP2TW9XHmnch/Xv3wuMfn2MyFyL9kk77rIsLw23lWxiepxhClnjIBxHVERP6jNPvOmXci/5BpdsZmLkT+48uaPp4nSW0DmvGrrKxEQkIC4uPjsXbt2j7PCyHw9NNPIz4+HhaLBYcOHZIeKyv3zS+k9jypfTmTSR8ZWjDEY0mtA0m/rvA6+84STwoVgY7Jl0qO4tk/2b0mfcMjB/PcSCEh0PEIyF2zAkz6SDv9Tvy6urqwePFiVFRU4MSJE9i+fTtOnDjR6zUVFRWor69HfX09ioqK8NRTT0mPlfFSyVHsP/mj19dwZoFCQTDEY0mtA8veO4yLHd1eX8cSTwoFgY5J2WZnvAlDoSDQ8QjIXbMCTPpIW/1O/KqrqxEfH49x48YhIiIC8+fPR2lpaa/XlJaW4tFHH4XJZEJaWhpaW1tx+vRpqbEytv+l0evznFmgUBEM8bj+4zp0dHu/zOQJjUJFoGNy/cd1Us3OeBOGQkGg4xFQvmYFeI4k7fU78XM4HBgzZozrz7GxsXA4HFKvkRnrVFRUBJvNBpvNhpaWll7PdQnvpzXOLFCoCIZ4bPayKTTA2XcKLf6IyYHEI5udUSgJhnOk0jUrkz7yh34nfsLNF9hkMkm9RmasU35+PmpqalBTU4ORI0f2ei7MwxiAAUShJRjiMWZ4pMfj4+w7hRp/xGR/4/FnQwZz+wYKKcFwjuQ1KwWDfid+sbGxaGz857R1U1MTYmJipF4jM1bGgulj3D7+f8aPYABRSAmGeFw2IwGDB7k/sXH2nUJNoGNy2YwERA4O6/WYCWx2RqEp0PEI8JqVgkO/E7+pU6eivr4eDQ0NaG9vR3FxMbKysnq9JisrC2+//TaEEDhw4ACGDRuG6OhoqbEyVj2QgkfSxrruooSZTHgkbSy2PXFbf98WkS4FQzw+MGk01s9NxfDIwa7HfjZkMAofsvKkRiEn0DH5wKTRWJOTgtHDI2FCz159BYxFClGBjkeA16wUHPq9j194eDg2btyIGTNmoKurCwsXLoTZbMYbb7wBAHjyyScxa9YslJeXIz4+HkOGDMEf//hHr2P7Y9UDKTyRUcgLlnh8YNJoziQQIThikvFI1CMY4hHgNSsFnkm4K14OUjabDTU1NYE+DDIofr98w8+LtMTvl2/4eZHW+B3zDT8v0lJ/v18D2sCdiIiIiIiIgh8TPyIiIiIiIoPTVannjTfeiLi4OLfPtbS09GmdazSh8B615u0zPHXqFL7//ns/H5F+MR6N/x61xnhUT6jHo1pC/bNiTKrHyDGp9+PXCy3iUVeJnzehUEsdCu9Ra/wM/SMUPudQeI9a42foH/yc5YX6ZxXq799f9P456/349UKLz5mlnkRERERERAbHxI+IiIiIiMjgDJP45efnB/oQNBcK71Fr/Az9IxQ+51B4j1rjZ+gf/JzlhfpnFerv31/0/jnr/fj1QovP2TBr/IiIiIiIiMg9w8z4ERERERERkXtBn/hVVlYiISEB8fHxWLt2bZ/nhRB4+umnER8fD4vFgkOHDkmPDZRLly5h2rRpSE1NhdlsxooVKwAAdrsdaWlpsFqtsNlsqK6udju+tbUVDz74IBITE5GUlIQvvvjCp/FGsWHDBiQnJ8NsNqOwsLDXc//1X/8Fk8nktdVtV1cXJk2ahNmzZ7se+/Wvfw2LxQKr1YrMzEw0NzdrdfiGMJD4DGZxcXFISUlxxRIA/Pjjj8jIyMCECROQkZGBM2fO9BlXV1cHq9Xq+mfo0KF9vptGtXDhQkRFRSE5Odn12LJly5CYmAiLxYLs7Gy0trYCAKqrq12fUWpqKj744AO3P5Px6BujxqNa3MW1k9I5w933O9i5O2Zvv8fWrFmD+Ph4JCQk4OOPP/b6s2XOsaFIj9esnq5JZc55nsYCnn//h6qCggKYzWYkJydjwYIFuHTpEgDgv//7v5GQkACz2Yzly5e7Hevpu/HKK69g9OjRrvNpeXm58oGIINbZ2SnGjRsnTp48KS5fviwsFos4fvx4r9d89NFH4t577xXd3d3iiy++ENOmTZMeGyjd3d3i/PnzQggh2tvbxbRp08QXX3whMjIyRHl5uRCi533dcccdbsc/+uij4s033xRCCHH58mVx5swZIYSQHm8ER48eFWazWVy4cEF0dHSIe+65R3z99ddCCCG++eYbkZmZKcaOHStaWlo8/ozXX39dLFiwQPz85z93PXb27FnXf2/YsEH84he/0O5N6NxA4jPY3XzzzX2+O8uWLRNr1qwRQgixZs0asXz5cq8/o7OzU4waNUqcOnVKs+MMJvv27RMHDx4UZrPZ9djHH38sOjo6hBBCLF++3PWZOeNWCCGam5vFyJEjXX++EuNRnpHjUS3u4loIuXOGu+93sHN3zJ5+jx0/flxYLBZx6dIl8be//U2MGzdOdHZ2uv25sufYUKPXa1ZP16Qy5zxPY4Xw/Ps/FDU1NYm4uDjR1tYmhBBi7ty54o9//KOoqqoS99xzj7h06ZIQQohvv/22z1hv340VK1aI9evX+3QsQT3jV11djfj4eIwbNw4RERGYP38+SktLe72mtLQUjz76KEwmE9LS0tDa2orTp09LjQ0Uk8mE66+/HgDQ0dGBjo4OmEwmmEwmnDt3DgBw9uxZxMTE9Bl77tw5fPrpp8jLywMAREREYPjw4a6fqzTeKL766iukpaVhyJAhCA8Pxx133OGaNXj22Wfx2muvwWQyeRzf1NSEjz76CIsWLer1+NChQ13/feHCBa8/I9QNJD71qLS0FI899hgA4LHHHkNJSYnX1+/duxfjx4/HzTff7IejC7zbb78dI0aM6PVYZmYmwsPDAQBpaWloamoCAFfcAj13jD3FGeNRXqjFo5pkzhnuvt/Bzt0xe/o9Vlpaivnz5+Oaa67BLbfcgvj4eI9VQzKfVyjS6zWrp2tSmXOep7GA59//oaqzsxMXL15EZ2cn2traEBMTg82bN+OFF17ANddcAwCIiorqM07t70ZQJ34OhwNjxoxx/Tk2NhYOh0PqNTJjA6mrqwtWqxVRUVHIyMjA9OnTUVhYiGXLlmHMmDFYunQp1qxZ02fc3/72N4wcORKPP/44Jk2ahEWLFuHChQsAIDXeKJKTk/Hpp5/ihx9+QFtbG8rLy9HY2IiysjKMHj0aqampXsf/6le/wmuvvYZBg/qGwIsvvogxY8Zg27ZtWLlypVZvQfcGEp/BzmQyITMzE1OmTEFRUREA4Ntvv0V0dDQAIDo6Gt99953Xn1FcXIwFCxZofqx6sWXLFsycOdP157/85S8wm81ISUnBG2+84bpAuBrjUY6R41Et7uJa9pxhFJ5+j8l+N0Lt8/KFnq9Z3V2Typ7z3I292tW//0PN6NGjsXTpUowdOxbR0dEYNmwYMjMz8fXXX+Ozzz7D9OnTcccdd+DLL7/sM1bpu7Fx40ZYLBYsXLjQbTnu1YI68RNuGo5efYfJ02tkxgZSWFgY7HY7mpqaUF1djWPHjmHz5s0oKChAY2MjCgoKXLN6V+rs7MShQ4fw1FNPoba2Ftddd52r3ldmvFEkJSXh+eefR0ZGBu69916kpqYiPDwcq1evVrw43LVrF6KiojBlyhS3z69evRqNjY3Izc3Fxo0btTh8QxhIfAa7/fv349ChQ6ioqMCmTZvw6aef+jS+vb0dZWVlmDt3rkZHqC+rV69GeHg4cnNzXY9Nnz4dx48fx5dffok1a9a41ju4G8t4VGbkeFSLu7iWOWeEApnvRltbGz8vL/R8zerumlStse5+/4eaM2fOoLS0FA0NDWhubsaFCxfw7rvvorOzE2fOnMGBAwewfv16zJs3r893wdt346mnnsLJkydht9sRHR2N5557TvFYgjrxi42NRWNjo+vPTU1NfcoXPb1GZmwwGD58OO68805UVlZi69atyMnJAQDMnTvXbZlFbGwsYmNjXXdUHnzwQdfiYJnxRpKXl4dDhw7h008/xYgRIxAXF4eGhgakpqYiLi4OTU1NmDx5Mv7+97/3Grd//36UlZUhLi4O8+fPR1VVFR555JE+P//hhx/G+++/76+3ozsDic9g5zzGqKgoZGdno7q6GqNGjXKVxZ0+fdptSYZTRUUFJk+ejFGjRvnleIPZ1q1bsWvXLmzbts3thUxSUhKuu+46xQsNxqN3Ro5HtVwd1/v27ZM6ZxiJp99jMt+NkydPhtzn5QsjXLNeeU3qyznv6rFOSr//Q8WePXtwyy23YOTIkRg8eDBycnLw+eefIzY2Fjk5OTCZTJg2bRoGDRrUp2GSt+/GqFGjEBYWhkGDBuGJJ56Quu4P6sRv6tSpqK+vR0NDA9rb21FcXIysrKxer8nKysLbb78NIQQOHDiAYcOGITo6WmpsoLS0tLi6G128eBF79uxBYmIiYmJisG/fPgBAVVUVJkyY0GfsTTfdhDFjxqCurg5AzzqiW2+9FQCkxhuJs+zgm2++wc6dO/Hoo4/iu+++w6lTp3Dq1CnExsbi0KFDuOmmm3qNW7NmDZqamnDq1CkUFxfj7rvvxrvvvgsAqK+vd72urKwMiYmJ/ntDOjOQ+AxmFy5cwPnz513/vXv3biQnJyMrKwtbt24F0HMyu//++z3+jO3bt7PMEz2dyNatW4eysjIMGTLE9XhDQwM6OzsBAP/7v/+Luro6xMXF9RnPeJRn1HhUi7u4njp1qtQ5w0g8/R7LyspCcXExLl++jIaGBtTX12PatGm9xqakpITc5+ULvV6zeromlTnneRoLeP79H4rGjh2LAwcOoK2tDUII7N27F0lJSXjggQdQVVUFAPj666/R3t6OG2+8sddYb9+NK9dof/DBB3Jdh31qBRMAH330kZgwYYIYN26cWLVqlRBCiM2bN4vNmzcLIXo6Cv37v/+7GDdunEhOThZffvml17HB4PDhw8JqtYqUlBRhNpvFq6++KoQQ4rPPPhOTJ08WFotFTJs2TdTU1AghhHA4HGLmzJmu8bW1tWLKlCkiJSVF3H///eLHH3/0Ot6o0tPTRVJSkrBYLGLPnj19nr+yg9vVn6HTn//8515dPXNycoTZbBYpKSli9uzZoqmpSbs3YAADic9gdfLkSWGxWITFYhG33nqr6319//334u677xbx8fHi7rvvFj/88IMQou9368KFC2LEiBGitbU1IMcfKPPnzxc33XSTCA8PF6NHjxZ/+MMfxPjx40VsbKxITU0Vqamprq6cb7/9trj11ltFamqqmDRpkvjggw9cPycvL8/1PWE8+saI8agWT3F9JW/nDHff72Dn7pg9/R4TQohVq1aJcePGiYkTJ7o6hAvROyav5KlLaijT4zWrp2tSmXOep7FCCI+//0PVyy+/LBISEoTZbBaPPPKIuHTpkrh8+bLIzc0VZrNZTJo0Sezdu1cI0ff3j6fvxiOPPCKSk5NFSkqKuO+++0Rzc7PicZiEcFM8SkRERERERIYR1KWeRERERERENHBM/IiIiIiIiAyOiR8REREREZHBMfEjIiIiIiIyOCZ+REREREREBsfEj4iIiIiIyOCY+BERERERERkcEz8iIiIiIiKD+//Xq/Etg6nr2AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x216 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i_batch, ((inp, _), out) in enumerate(train_loader):\n",
    "    show_sample_batch((inp, out))\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "604ee1e4",
   "metadata": {},
   "source": [
    "## Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5ea3c0bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "print(DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aa130635",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerTrajectory(nn.Module):\n",
    "    def __init__(\n",
    "        self,\n",
    "        batch_size,\n",
    "        model_dim, \n",
    "        n_heads=8, \n",
    "        linear_dim=1024,\n",
    "        dropout_rate=0.1, \n",
    "        encoder_layers=1, \n",
    "        decoder_layers=1,\n",
    "        input_dim=2,\n",
    "        teacher_ratio=0.3\n",
    "    ):\n",
    "        super(TransformerTrajectory, self).__init__()\n",
    "        self.batch_size = batch_size\n",
    "        self.model_dim = model_dim\n",
    "        self.n_heads = n_heads\n",
    "        self.linear_dim = linear_dim\n",
    "        self.dropout_rate = dropout_rate\n",
    "        self.encoder_layers = encoder_layers\n",
    "        self.decoder_layers = decoder_layers\n",
    "        self.input_dim = input_dim\n",
    "        self.input_length = 50\n",
    "        self.output_length = 60\n",
    "        self.teacher_ratio = teacher_ratio\n",
    "        \n",
    "        # input linear embedding\n",
    "        self.input_projection = nn.Linear(input_dim, model_dim)\n",
    "        \n",
    "        # encoder\n",
    "        encoder = nn.TransformerEncoderLayer(\n",
    "            d_model=model_dim, \n",
    "            nhead=n_heads, \n",
    "            dim_feedforward=linear_dim, \n",
    "            dropout=dropout_rate, \n",
    "            batch_first=True,\n",
    "            device=DEVICE\n",
    "        )\n",
    "        self.encoder = nn.TransformerEncoder(encoder, num_layers=encoder_layers, norm=None)\n",
    "        \n",
    "        # bridge\n",
    "        self.cell_bridge = nn.Sequential(\n",
    "            nn.Linear(self.input_length * model_dim, linear_dim),\n",
    "            nn.LeakyReLU(),\n",
    "            nn.Linear(linear_dim, linear_dim),\n",
    "            nn.LeakyReLU(),\n",
    "            nn.Linear(linear_dim, model_dim)\n",
    "        )    \n",
    "        self.hidden_bridge = nn.Sequential(\n",
    "            nn.Linear(self.input_length * model_dim, linear_dim),\n",
    "            nn.LeakyReLU(),\n",
    "            nn.Linear(linear_dim, linear_dim),\n",
    "            nn.LeakyReLU(),\n",
    "            nn.Linear(linear_dim, model_dim)\n",
    "        )\n",
    "        \n",
    "        # decoder\n",
    "        self.decoder = nn.LSTM(\n",
    "            input_size=model_dim, \n",
    "            hidden_size=model_dim,\n",
    "            batch_first=True,\n",
    "            num_layers=decoder_layers\n",
    "        )\n",
    "        \n",
    "        # final linear layer\n",
    "        self.output_projection = nn.Linear(model_dim, input_dim)\n",
    "    \n",
    "    def get_angles(self, pos, i, D):\n",
    "        angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(D))\n",
    "        return pos * angle_rates\n",
    "\n",
    "\n",
    "    def positional_encoding(self, D, position=110, dim=3, device=DEVICE):\n",
    "        angle_rads = self.get_angles(np.arange(position)[:, np.newaxis],\n",
    "                                np.arange(D)[np.newaxis, :],\n",
    "                                D)\n",
    "        # apply sin to even indices in the array; 2i\n",
    "        angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
    "        # apply cos to odd indices in the array; 2i+1\n",
    "        angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
    "        if dim == 3:\n",
    "            pos_encoding = angle_rads[np.newaxis, ...]\n",
    "        elif dim == 4:\n",
    "            pos_encoding = angle_rads[np.newaxis,np.newaxis,  ...]\n",
    "        return torch.tensor(pos_encoding, device=device)\n",
    "    \n",
    "    def generate_square_subsequent_mask(self, size):\n",
    "        return torch.triu(torch.ones(size, size) * float('-inf'), diagonal=1)\n",
    "    \n",
    "    def forward(self, inputs, ground_truth=None, device=DEVICE):\n",
    "        # transformer encoder\n",
    "        proj_inputs = self.input_projection(inputs)\n",
    "        proj_inputs += self.positional_encoding(self.model_dim)[:, :self.input_length, :]\n",
    "        \n",
    "        memory = self.encoder(proj_inputs)\n",
    "        \n",
    "        # bridge\n",
    "        memory = memory.reshape(self.batch_size, -1)\n",
    "        lstm_cell = self.cell_bridge(memory).unsqueeze(0)\n",
    "        lstm_hidden = self.hidden_bridge(memory).unsqueeze(0)\n",
    "        \n",
    "        # LSTM decoder\n",
    "        if ground_truth is not None:\n",
    "            ground_truth = self.input_projection(ground_truth)\n",
    "            \n",
    "        outputs = torch.zeros(self.batch_size, self.output_length, 2).to(device)\n",
    "        lstm_out = self.input_projection(inputs[:, -1:, :])\n",
    "        \n",
    "        for t in range(self.output_length):    \n",
    "            lstm_out, (lstm_hidden, lstm_cell) = self.decoder(lstm_out, (lstm_hidden, lstm_cell))\n",
    "            outputs[:, t] = self.output_projection(lstm_out).squeeze(1)\n",
    "            \n",
    "            # DECIDER: TEACHER FORCING GROUND TRUTH\n",
    "            if (ground_truth is not None) and random.random() < self.teacher_ratio:\n",
    "                lstm_out = ground_truth[:, t:t+1, :]\n",
    "        \n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57610ecc",
   "metadata": {},
   "source": [
    "## Train the Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d8a2d0cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(\n",
    "    batch_size, \n",
    "    city, \n",
    "    model_dim,\n",
    "    n_heads,\n",
    "    linear_dim,\n",
    "    dropout_rate,\n",
    "    encoder_layers,\n",
    "    decoder_layers,\n",
    "    num_iters,\n",
    "    learning_rate,\n",
    "    factor,\n",
    "    patience,\n",
    "    teacher_ratio,\n",
    "    device=DEVICE\n",
    "):\n",
    "    # Create the training/validation set\n",
    "    train_dataset = ArgoverseDataset(city=city, split='train', transform=False, normalized=True)\n",
    "    train_sz = int(len(train_dataset) * 0.9)\n",
    "    val_sz = len(train_dataset) - train_sz\n",
    "    train_loader, val_loader = torch.utils.data.random_split(train_dataset, [train_sz, val_sz])\n",
    "    train_loader = DataLoader(train_loader, batch_size=batch_size, drop_last=True, shuffle=True)\n",
    "    val_loader = DataLoader(val_loader, batch_size=batch_size, drop_last=True, shuffle=False)\n",
    "    \n",
    "    # Initialize the transformer/optimizer/loss function\n",
    "    transformer = TransformerTrajectory(\n",
    "        batch_size=batch_size,\n",
    "        model_dim=model_dim, \n",
    "        n_heads=n_heads, \n",
    "        linear_dim=linear_dim, \n",
    "        dropout_rate=dropout_rate, \n",
    "        encoder_layers=encoder_layers, \n",
    "        decoder_layers=decoder_layers,\n",
    "        teacher_ratio=teacher_ratio\n",
    "    ).to(device)\n",
    "\n",
    "    optimizer = torch.optim.Adam(transformer.parameters(), lr=learning_rate) \n",
    "    loss_function = nn.MSELoss()\n",
    "    scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=factor, patience=patience, verbose=True) \n",
    "    \n",
    "    # Print out how many parameters to train\n",
    "    param_sizes = [p.numel() for p in transformer.parameters()]\n",
    "    print(f\"number of weight/biases matrices: {len(param_sizes)} \"\n",
    "          f\"for a total of {np.sum(param_sizes)} parameters \")\n",
    "    \n",
    "    avg_train_loss, avg_val_loss_teachers, avg_val_loss_auto = [], [], []\n",
    "    train_time, elapsed_time = [], []\n",
    "    best_val_score = float('inf')\n",
    "    \n",
    "    # Start training\n",
    "    for epoch in tqdm(list(range(num_iters))):\n",
    "        print(f'Epoch {epoch+1}')\n",
    "        print('Training & Validating ', end='')\n",
    "        \n",
    "        start_time = time.time()\n",
    "        train_loss, val_loss_teachers, val_loss_auto = [], [], []\n",
    "        \n",
    "        # Training set\n",
    "        for batches, ((X_enc, X_dec), y) in enumerate(train_loader):\n",
    "            X_enc = X_enc.to(device).float()\n",
    "            X_dec = X_dec.to(device).float()\n",
    "            y = y.to(device).float()\n",
    "            \n",
    "            # Track progress\n",
    "            if (batches + 1) % 20 == 0:\n",
    "                print('-', end='')\n",
    "            \n",
    "            # Forward pass\n",
    "            optimizer.zero_grad()\n",
    "            out = transformer(X_enc, ground_truth=y)\n",
    "            \n",
    "            # Backpropagation\n",
    "            loss = loss_function(out, y)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            train_loss.append(loss.item())\n",
    "            \n",
    "        print()\n",
    "        avg_train = np.mean(train_loss)\n",
    "        avg_train_loss.append(avg_train)\n",
    "        \n",
    "        # End the time\n",
    "        end_train_time = time.time()\n",
    "        train_time.append(end_train_time - start_time)\n",
    "        \n",
    "        # Evaluate on val set\n",
    "        with torch.no_grad():\n",
    "            for batches, ((X_enc, X_dec), y) in enumerate(val_loader):\n",
    "                X_enc = X_enc.to(device).float()\n",
    "                X_dec = X_dec.to(device).float()\n",
    "                y = y.to(device).float()\n",
    "                \n",
    "                out_teachers = transformer(X_enc, ground_truth=y)               \n",
    "                loss_teachers = loss_function(out_teachers, y)\n",
    "                val_loss_teachers.append(loss_teachers.item())\n",
    "                \n",
    "                out_auto = transformer(X_enc)\n",
    "                loss_auto = loss_function(out_auto, y)\n",
    "                val_loss_auto.append(loss_auto.item())\n",
    "                \n",
    "            avg_val_teachers = np.mean(val_loss_teachers)\n",
    "            avg_val_loss_teachers.append(avg_val_teachers)\n",
    "            avg_val_auto = np.mean(val_loss_auto)\n",
    "            avg_val_loss_auto.append(avg_val_auto)\n",
    "        \n",
    "        end_time = time.time()\n",
    "        elapsed_time.append(end_time - start_time)\n",
    "\n",
    "        print(f'- Training Loss: {avg_train}')\n",
    "        print(f'- Teachers Validation Loss: {avg_val_teachers}')\n",
    "        print(f'- Auto Validation Loss: {avg_val_auto}')\n",
    "        print(f'- Train Time: {sum(train_time)}\\n- Elapsed Time: {sum(elapsed_time)}\\n')\n",
    "        \n",
    "        scheduler.step(avg_val_auto)\n",
    "        \n",
    "        # save better model\n",
    "        if avg_val_auto < best_val_score:\n",
    "            best_val_score = avg_val_auto\n",
    "            torch.save(transformer, f'best_{city}_auto.pt')\n",
    "        \n",
    "    return transformer, (avg_train_loss, avg_val_loss_auto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3f29b3a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_loss(start_from, city, split, losses):\n",
    "\n",
    "    avg_train_loss, avg_val_loss_teachers, avg_val_loss_auto = losses\n",
    "    plt.plot(np.sqrt(avg_train_loss[start_from:]), label='train_loss')\n",
    "    plt.plot(np.sqrt(avg_val_loss_teachers[start_from:]), label='validation_loss_teachers')\n",
    "    plt.plot(np.sqrt(avg_val_loss_auto[start_from:]), label='validation_loss_auto')\n",
    "    plt.title(f'{city} RMSE {split} Loss vs. Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "59a82219",
   "metadata": {},
   "outputs": [],
   "source": [
    "def vis_results(city, split, batch_size, model, idx, device=DEVICE):\n",
    "    train_dataset = ArgoverseDataset(city = city, split = split, transform=False, normalized=True)\n",
    "    train_loader = DataLoader(train_dataset, batch_size=batch_size)\n",
    "#     model.cpu()\n",
    "#     model.pos_encoding.to('cpu')\n",
    "#     print(model.pos_encoding.get_device())\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for (X_enc, X_dec), y in train_loader:\n",
    "            X_enc = X_enc.to(device).float()\n",
    "            X_dec = X_dec.to(device).float()\n",
    "            y = y.to(device).float()\n",
    "\n",
    "            out_auto = model(X_enc)\n",
    "            out_teachers = model(X_enc, ground_truth=y)\n",
    "            break\n",
    "#     print(X_enc[idx])\n",
    "#     print(out_auto[idx])\n",
    "    \n",
    "#     print(X_enc.shape, X_dec.shape, y.shape, out_auto.shape)\n",
    "    X = X_enc.cpu()\n",
    "    y = y.cpu()\n",
    "    out_auto = out_auto.cpu()\n",
    "    out_teachers = out_teachers.cpu()\n",
    "    \n",
    "    x_jump = train_dataset.start_pos[idx, 0]\n",
    "    y_jump = train_dataset.start_pos[idx, 0]\n",
    "    rot = train_dataset.rotate_matrix[idx].T\n",
    "    X = X[idx] @ np.linalg.inv(rot) + train_dataset.start_pos[idx]\n",
    "    y = y[idx] @ np.linalg.inv(rot) + train_dataset.start_pos[idx]\n",
    "    out_auto = out_auto[idx] @ np.linalg.inv(rot) + train_dataset.start_pos[idx]\n",
    "    out_teachers = out_teachers[idx] @ np.linalg.inv(rot) + train_dataset.start_pos[idx]\n",
    "    \n",
    "    plt.scatter(X[:, 0], X[:, 1], label='seed')\n",
    "    plt.scatter(y[:, 0], y[:, 1], label='ground truth')\n",
    "    plt.scatter(out_auto[:, 0], out_auto[:, 1], label='prediction auto')\n",
    "    plt.scatter(out_teachers[:, 0], out_teachers[:, 1], label='prediction teachers')\n",
    "    plt.title(f'Random Sample From {city}_{split} Projectile Visualization')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "66b3efc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def vis_test(city, split, batch_sz, model, idx, norm_viz=True):\n",
    "    '''\n",
    "    This is the last batch, which is usually not complete and need to fill with 0s.\n",
    "    Check if I convert the prediction back correctly or not\n",
    "    '''\n",
    "    \n",
    "    # Create the dataset for testing\n",
    "    test_dataset = ArgoverseDataset(city = city, split = split,\n",
    "                                    transform=False, normalized=True)\n",
    "        \n",
    "    test_loader = DataLoader(test_dataset, batch_size=batch_sz)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for X in test_loader:\n",
    "            if len(X) == batch_sz:\n",
    "                continue\n",
    "            print(len(X))\n",
    "            to_fill = np.zeros([batch_sz-len(X), 50, 2])\n",
    "            X = torch.from_numpy(np.append(X, to_fill, axis=0)).float()\n",
    "            X = X.to(device).float()\n",
    "            S = X.shape[1]\n",
    "            mask = create_look_ahead_mask(S)\n",
    "\n",
    "            output = model(X, mask)[0].reshape(batch_size, -1, 2)\n",
    "\n",
    "            X = X.cpu()\n",
    "            output = output.cpu()\n",
    "\n",
    "            if norm_viz:\n",
    "                plt.scatter(X[idx, :, 0], X[idx, :, 1], label='input')\n",
    "                plt.scatter(output[idx, :, 0], output[idx, :, 1], label='pred')\n",
    "                plt.legend()\n",
    "                plt.show()\n",
    "            else:\n",
    "                rotation =  test_dataset.rotate_matrix[-1].T\n",
    "                X[idx] = X[idx] @ np.linalg.inv(rotation)\n",
    "                X[idx] = X[idx] + test_dataset.start_pos[-1, : ]\n",
    "                output[idx] = output[idx] @ np.linalg.inv(rotation)\n",
    "                output[idx] = output[idx] + test_dataset.start_pos[-1, : ]\n",
    "\n",
    "                plt.scatter(X[idx, :, 0], X[idx, :, 1], label='input')\n",
    "                plt.scatter(output[idx, :, 0], output[idx, :, 1], label='pred')\n",
    "                plt.legend()\n",
    "                plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "6b019b96",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_pred(test_loader, batch_sz, model):\n",
    "    '''\n",
    "    Remember to use test_dataset stats, NOT train_dataset\n",
    "    '''\n",
    "    count_row = 0\n",
    "    out = []\n",
    "\n",
    "    for X in test_loader:\n",
    "        if len(X) != batch_sz:\n",
    "            print(len(X))\n",
    "            to_fill = np.zeros([batch_sz-len(X), 50, 2])\n",
    "            X = torch.from_numpy(np.append(X, to_fill, axis=0))\n",
    "            \n",
    "#             a = test_dataset.rotate_matrix[-1].T\n",
    "#             temp = X[20]@np.linalg.inv(a) + test_dataset.start_pos[-1]\n",
    "#             plt.scatter(temp[:, 0], temp[:, 1], label='input')\n",
    "\n",
    "        X = X.to(device).float()\n",
    "    \n",
    "        S = X.shape[1]\n",
    "        mask = create_look_ahead_mask(S)\n",
    "\n",
    "        pred = model(X, mask)[0].reshape(batch_size, -1, 2).cpu().detach().numpy()\n",
    "\n",
    "        for i in range(batch_sz):\n",
    "            if count_row >= len(test_dataset):\n",
    "                break\n",
    "                \n",
    "#             if count_row == (len(test_dataset) - 1):\n",
    "#                 plt.scatter(X[i, :, 0], X[i, :, 1], label='input')\n",
    "#                 plt.scatter(pred[i, :, 0], pred[i, :, 1], label='pred')\n",
    "#                 plt.legend()\n",
    "#                 plt.show()\n",
    "\n",
    "            rotation =  test_dataset.rotate_matrix[count_row].T\n",
    "            pred[i] = pred[i] @ np.linalg.inv(rotation)\n",
    "            pred[i] = pred[i] + test_dataset.start_pos[count_row, : ]\n",
    "\n",
    "    #         print(pred[i, 0, :])\n",
    "                \n",
    "            out.append(pred[i])\n",
    "            count_row += 1 \n",
    "            \n",
    "    #         print(count_row)\n",
    "    #         print(pred[0, :5, :])\n",
    "\n",
    "    out = np.array(out).reshape(len(test_dataset), -1)\n",
    "\n",
    "    return out\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e6c9874",
   "metadata": {},
   "source": [
    "### Pittsburgh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2febcd04",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(43544, 50, 2)\n",
      "(43544, 60, 2)\n",
      "number of weight/biases matrices: 68 for a total of 560834 parameters \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65256d3aad5142c79b190317475cb1b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/60 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1\n",
      "Training & Validating ------------------------------------------------------------------------------------------------------------- Training Loss: 103.6516999699711\n",
      "- Teachers Validation Loss: 95.42486920076259\n",
      "- Auto Validation Loss: 96.46355174569523\n",
      "- Train Time: 384.1564929485321\n",
      "- Elapsed Time: 399.8114449977875\n",
      "\n",
      "Epoch 4\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 98.39263149647931\n",
      "- Teachers Validation Loss: 95.87858887279735\n",
      "- Auto Validation Loss: 97.43501438814052\n",
      "- Train Time: 515.0228810310364\n",
      "- Elapsed Time: 535.5945045948029\n",
      "\n",
      "Epoch 5\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 86.08113981539907\n",
      "- Teachers Validation Loss: 83.52708670672249\n",
      "- Auto Validation Loss: 96.7853531556971\n",
      "- Train Time: 644.2324688434601\n",
      "- Elapsed Time: 669.8426465988159\n",
      "\n",
      "Epoch 6\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 73.39972886540531\n",
      "- Teachers Validation Loss: 76.33859850378597\n",
      "- Auto Validation Loss: 87.01075983047485\n",
      "- Train Time: 780.5732634067535\n",
      "- Elapsed Time: 811.2412109375\n",
      "\n",
      "Epoch 7\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 65.98706985766592\n",
      "- Teachers Validation Loss: 53.19441817788517\n",
      "- Auto Validation Loss: 64.38735075557933\n",
      "- Train Time: 909.6455323696136\n",
      "- Elapsed Time: 944.9945869445801\n",
      "\n",
      "Epoch 8\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 60.62932502522188\n",
      "- Teachers Validation Loss: 55.14921139268314\n",
      "- Auto Validation Loss: 65.5214359479792\n",
      "- Train Time: 1035.9456872940063\n",
      "- Elapsed Time: 1076.040028333664\n",
      "\n",
      "Epoch 9\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 55.98977097031338\n",
      "- Teachers Validation Loss: 55.48326862559599\n",
      "- Auto Validation Loss: 71.5978966881247\n",
      "- Train Time: 1165.173180103302\n",
      "- Elapsed Time: 1210.426971912384\n",
      "\n",
      "Epoch 10\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 53.290344242956124\n",
      "- Teachers Validation Loss: 56.694367478875556\n",
      "- Auto Validation Loss: 79.17173127567067\n",
      "- Train Time: 1296.6390166282654\n",
      "- Elapsed Time: 1346.7138013839722\n",
      "\n",
      "Epoch 11\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 49.020068777152915\n",
      "- Teachers Validation Loss: 47.6082596358131\n",
      "- Auto Validation Loss: 65.27975500331206\n",
      "- Train Time: 1426.1776840686798\n",
      "- Elapsed Time: 1481.2823491096497\n",
      "\n",
      "Epoch    11: reducing learning rate of group 0 to 1.0000e-03.\n",
      "Epoch 12\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 40.57064786923477\n",
      "- Teachers Validation Loss: 38.21905342270346\n",
      "- Auto Validation Loss: 57.81365314651938\n",
      "- Train Time: 1556.5095119476318\n",
      "- Elapsed Time: 1616.5194494724274\n",
      "\n",
      "Epoch 13\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 39.35632085566427\n",
      "- Teachers Validation Loss: 37.33652940217186\n",
      "- Auto Validation Loss: 58.19269183102776\n",
      "- Train Time: 1684.4786128997803\n",
      "- Elapsed Time: 1749.3942983150482\n",
      "\n",
      "Epoch 14\n",
      "Training & Validating -------------------------------------------------------------------------------\n",
      "- Training Loss: 38.47312640442568\n",
      "- Teachers Validation Loss: 39.934633900137506\n",
      "- Auto Validation Loss: 54.105240260853485\n",
      "- Train Time: 1940.4199750423431\n",
      "- Elapsed Time: 2015.090651512146\n",
      "\n",
      "Epoch 16\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 36.61728622477039\n",
      "- Teachers Validation Loss: 51.69463924800648\n",
      "- Auto Validation Loss: 64.5533010819379\n",
      "- Train Time: 2071.5694782733917\n",
      "- Elapsed Time: 2151.132552623749\n",
      "\n",
      "Epoch 17\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 36.91058127747642\n",
      "- Teachers Validation Loss: 35.21840178265291\n",
      "- Auto Validation Loss: 55.25037379825817\n",
      "- Train Time: 2200.4946839809418\n",
      "- Elapsed Time: 2284.7847142219543\n",
      "\n",
      "Epoch 18\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 35.34453274923212\n",
      "- Teachers Validation Loss: 33.77520936376908\n",
      "- Auto Validation Loss: 55.381089953815234\n",
      "- Train Time: 2329.111624956131\n",
      "- Elapsed Time: 2418.0546367168427\n",
      "\n",
      "Epoch 19\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 36.760894062472325\n",
      "- Teachers Validation Loss: 37.236428919960474\n",
      "- Auto Validation Loss: 53.692757704678705\n",
      "- Train Time: 2462.0860612392426\n",
      "- Elapsed Time: 2556.115427017212\n",
      "\n",
      "Epoch 20\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 36.29670932635762\n",
      "- Teachers Validation Loss: 32.866261068512415\n",
      "- Auto Validation Loss: 56.17549096836763\n",
      "- Train Time: 2588.2886340618134\n",
      "- Elapsed Time: 2687.4223852157593\n",
      "\n",
      "Epoch 21\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 35.591036910325094\n",
      "- Teachers Validation Loss: 33.92316437468809\n",
      "- Auto Validation Loss: 53.46947533944074\n",
      "- Train Time: 2718.4933042526245\n",
      "- Elapsed Time: 2822.2234723567963\n",
      "\n",
      "Epoch 22\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 35.82231566991681\n",
      "- Teachers Validation Loss: 39.611764928873846\n",
      "- Auto Validation Loss: 55.58807983117945\n",
      "- Train Time: 2847.6286568641663\n",
      "- Elapsed Time: 2956.025490999222\n",
      "\n",
      "Epoch 23\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 35.39964888220519\n",
      "- Teachers Validation Loss: 37.91964039381813\n",
      "- Auto Validation Loss: 58.249700349919934\n",
      "- Train Time: 2978.8055670261383\n",
      "- Elapsed Time: 3091.9226722717285\n",
      "\n",
      "Epoch 24\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 33.49954144432654\n",
      "- Teachers Validation Loss: 34.714710670359\n",
      "- Auto Validation Loss: 52.8200103114633\n",
      "- Train Time: 3103.8101358413696\n",
      "- Elapsed Time: 3221.6044561862946\n",
      "\n",
      "Epoch 25\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 35.99801273867975\n",
      "- Teachers Validation Loss: 37.61221367471359\n",
      "- Auto Validation Loss: 59.84525070470922\n",
      "- Train Time: 3232.2725241184235\n",
      "- Elapsed Time: 3354.7956550121307\n",
      "\n",
      "Epoch 26\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 35.53910509667365\n",
      "- Teachers Validation Loss: 34.44250712675207\n",
      "- Auto Validation Loss: 54.87299119724947\n",
      "- Train Time: 3359.708266019821\n",
      "- Elapsed Time: 3487.478050470352\n",
      "\n",
      "Epoch 27\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 34.422977543344686\n",
      "- Teachers Validation Loss: 36.4830841106527\n",
      "- Auto Validation Loss: 58.008704634273755\n",
      "- Train Time: 3488.4046216011047\n",
      "- Elapsed Time: 3620.8393461704254\n",
      "\n",
      "Epoch 28\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 33.49196652064916\n",
      "- Teachers Validation Loss: 39.50130684936748\n",
      "- Auto Validation Loss: 61.91019431282492\n",
      "- Train Time: 3619.127914905548\n",
      "- Elapsed Time: 3756.7474851608276\n",
      "\n",
      "Epoch    28: reducing learning rate of group 0 to 5.0000e-04.\n",
      "Epoch 29\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 30.873478981794097\n",
      "- Teachers Validation Loss: 30.39106712621801\n",
      "- Auto Validation Loss: 52.53675114407259\n",
      "- Train Time: 3750.9295744895935\n",
      "- Elapsed Time: 3893.327353000641\n",
      "\n",
      "Epoch 30\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 30.921184978843513\n",
      "- Teachers Validation Loss: 30.520008546464584\n",
      "- Auto Validation Loss: 52.000842949923346\n",
      "- Train Time: 3881.893231868744\n",
      "- Elapsed Time: 4028.9117255210876\n",
      "\n",
      "Epoch 31\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 31.139626449619243\n",
      "- Teachers Validation Loss: 32.78020617541145\n",
      "- Auto Validation Loss: 52.009789270513195\n",
      "- Train Time: 4012.285727739334\n",
      "- Elapsed Time: 4164.279017686844\n",
      "\n",
      "Epoch 32\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 30.29975902722552\n",
      "- Teachers Validation Loss: 28.06387572779375\n",
      "- Auto Validation Loss: 50.91125312973471\n",
      "- Train Time: 4140.722851991653\n",
      "- Elapsed Time: 4297.4441838264465\n",
      "\n",
      "Epoch 33\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 31.243356693803875\n",
      "- Teachers Validation Loss: 32.889491852592016\n",
      "- Auto Validation Loss: 53.00621729738572\n",
      "- Train Time: 4267.740208864212\n",
      "- Elapsed Time: 4429.466768741608\n",
      "\n",
      "Epoch 34\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 30.12724309731153\n",
      "- Teachers Validation Loss: 31.739185946829178\n",
      "- Auto Validation Loss: 50.682844400405884\n",
      "- Train Time: 4395.939040660858\n",
      "- Elapsed Time: 4562.609919309616\n",
      "\n",
      "Epoch 35\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 29.662798417159934\n",
      "- Teachers Validation Loss: 27.294016010621014\n",
      "- Auto Validation Loss: 51.78661935469683\n",
      "- Train Time: 4527.442181110382\n",
      "- Elapsed Time: 4698.908215045929\n",
      "\n",
      "Epoch 36\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 28.702842319323345\n",
      "- Teachers Validation Loss: 33.51482442371986\n",
      "- Auto Validation Loss: 52.1771202508141\n",
      "- Train Time: 4657.857722759247\n",
      "- Elapsed Time: 4834.0042843818665\n",
      "\n",
      "Epoch 37\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 29.24141742279327\n",
      "- Teachers Validation Loss: 34.12032035519095\n",
      "- Auto Validation Loss: 52.69155890801374\n",
      "- Train Time: 4791.778576374054\n",
      "- Elapsed Time: 4972.686131238937\n",
      "\n",
      "Epoch 38\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 29.15377604104335\n",
      "- Teachers Validation Loss: 31.315883166649762\n",
      "- Auto Validation Loss: 51.596859861822686\n",
      "- Train Time: 4922.854242324829\n",
      "- Elapsed Time: 5108.608115434647\n",
      "\n",
      "Epoch    38: reducing learning rate of group 0 to 2.5000e-04.\n",
      "Epoch 39\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 28.18724845028391\n",
      "- Teachers Validation Loss: 27.478851167594684\n",
      "- Auto Validation Loss: 50.543657288831824\n",
      "- Train Time: 5051.803881645203\n",
      "- Elapsed Time: 5242.178629159927\n",
      "\n",
      "Epoch 40\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 27.627813983197306\n",
      "- Teachers Validation Loss: 27.35620998985627\n",
      "- Auto Validation Loss: 50.88056592380299\n",
      "- Train Time: 5178.100423336029\n",
      "- Elapsed Time: 5373.474324464798\n",
      "\n",
      "Epoch 41\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 27.69276416087462\n",
      "- Teachers Validation Loss: 28.645454315578235\n",
      "- Auto Validation Loss: 50.02892342735739\n",
      "- Train Time: 5306.502374172211\n",
      "- Elapsed Time: 5506.852672815323\n",
      "\n",
      "Epoch 42\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 27.505638304099538\n",
      "- Teachers Validation Loss: 26.87101132028243\n",
      "- Auto Validation Loss: 50.56294345855713\n",
      "- Train Time: 5436.64866733551\n",
      "- Elapsed Time: 5641.631251335144\n",
      "\n",
      "Epoch 43\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 27.778453212158354\n",
      "- Teachers Validation Loss: 28.147425577921027\n",
      "- Auto Validation Loss: 50.91151586700888\n",
      "- Train Time: 5561.669951438904\n",
      "- Elapsed Time: 5771.633337259293\n",
      "\n",
      "Epoch 44\n",
      "Training & Validating ------------------------------------------------------------------------------\n",
      "- Training Loss: 28.07196970157374\n",
      "- Teachers Validation Loss: 25.56208529016551\n",
      "- Auto Validation Loss: 50.094372777377856\n",
      "- Train Time: 5815.846421480179\n",
      "- Elapsed Time: 6036.367101430893\n",
      "\n",
      "Epoch    45: reducing learning rate of group 0 to 1.2500e-04.\n",
      "Epoch 46\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.153423334843193\n",
      "- Teachers Validation Loss: 27.430818019544375\n",
      "- Auto Validation Loss: 49.97801995277405\n",
      "- Train Time: 5940.676589488983\n",
      "- Elapsed Time: 6166.225352525711\n",
      "\n",
      "Epoch 47\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.52630338131213\n",
      "- Teachers Validation Loss: 23.55997285246849\n",
      "- Auto Validation Loss: 49.92833094035878\n",
      "- Train Time: 6068.910736322403\n",
      "- Elapsed Time: 6299.097553491592\n",
      "\n",
      "Epoch 48\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.521736769699583\n",
      "- Teachers Validation Loss: 25.023188576978797\n",
      "- Auto Validation Loss: 49.82666516304016\n",
      "- Train Time: 6199.297742605209\n",
      "- Elapsed Time: 6434.030227899551\n",
      "\n",
      "Epoch 49\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 25.531932412214527\n",
      "- Teachers Validation Loss: 28.75609289197361\n",
      "- Auto Validation Loss: 49.658701952765966\n",
      "- Train Time: 6328.945638418198\n",
      "- Elapsed Time: 6568.352303981781\n",
      "\n",
      "Epoch 50\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.2361937428222\n",
      "- Teachers Validation Loss: 24.860862525070416\n",
      "- Auto Validation Loss: 50.042021358714386\n",
      "- Train Time: 6459.910414218903\n",
      "- Elapsed Time: 6704.254035949707\n",
      "\n",
      "Epoch 51\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.43219904416527\n",
      "- Teachers Validation Loss: 28.15622597406892\n",
      "- Auto Validation Loss: 50.202536035986505\n",
      "- Train Time: 6588.026022434235\n",
      "- Elapsed Time: 6837.208369731903\n",
      "\n",
      "Epoch 52\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.24812223124348\n",
      "- Teachers Validation Loss: 27.168505963157205\n",
      "- Auto Validation Loss: 49.81781489708845\n",
      "- Train Time: 6714.207359313965\n",
      "- Elapsed Time: 6968.243698358536\n",
      "\n",
      "Epoch 53\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.65052102204242\n",
      "- Teachers Validation Loss: 26.72929216658368\n",
      "- Auto Validation Loss: 49.67331042009241\n",
      "- Train Time: 6841.737630367279\n",
      "- Elapsed Time: 7100.551590919495\n",
      "\n",
      "Epoch    53: reducing learning rate of group 0 to 6.2500e-05.\n",
      "Epoch 54\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.560233211205677\n",
      "- Teachers Validation Loss: 28.095492026385138\n",
      "- Auto Validation Loss: 49.99542607980616\n",
      "- Train Time: 6966.903398513794\n",
      "- Elapsed Time: 7230.559933185577\n",
      "\n",
      "Epoch 55\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.252536360344855\n",
      "- Teachers Validation Loss: 27.060055259396048\n",
      "- Auto Validation Loss: 49.621035744162164\n",
      "- Train Time: 7098.032380819321\n",
      "- Elapsed Time: 7366.339066505432\n",
      "\n",
      "Epoch 56\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.665586175092685\n",
      "- Teachers Validation Loss: 26.031034850022372\n",
      "- Auto Validation Loss: 49.384802748175225\n",
      "- Train Time: 7228.643755912781\n",
      "- Elapsed Time: 7502.130629777908\n",
      "\n",
      "Epoch 57\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 25.35647735311315\n",
      "- Teachers Validation Loss: 25.440530577126673\n",
      "- Auto Validation Loss: 49.753740072250366\n",
      "- Train Time: 7354.018292188644\n",
      "- Elapsed Time: 7632.625529527664\n",
      "\n",
      "Epoch 58\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 26.64919750480091\n",
      "- Teachers Validation Loss: 24.98757312928929\n",
      "- Auto Validation Loss: 49.79470457750208\n",
      "- Train Time: 7475.975942373276\n",
      "- Elapsed Time: 7759.268485784531\n",
      "\n",
      "Epoch 59\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 25.526500514909333\n",
      "- Teachers Validation Loss: 25.025283280540915\n",
      "- Auto Validation Loss: 49.783507767845606\n",
      "- Train Time: 7601.9751307964325\n",
      "- Elapsed Time: 7890.817732334137\n",
      "\n",
      "Epoch 60\n",
      "Training & Validating -------------------------------------------------------------\n",
      "- Training Loss: 25.69166101447118\n",
      "- Teachers Validation Loss: 25.630162987638922\n",
      "- Auto Validation Loss: 49.58581880962147\n",
      "- Train Time: 7728.023066282272\n",
      "- Elapsed Time: 8022.41695857048\n",
      "\n",
      "Epoch    60: reducing learning rate of group 0 to 3.1250e-05.\n"
     ]
    }
   ],
   "source": [
    "# Hyperparameter\n",
    "batch_size = 32\n",
    "city = 'pittsburgh'\n",
    "model_dim = 64\n",
    "n_heads = 8\n",
    "linear_dim = 64 # [32, 128]\n",
    "dropout_rate = 0 # [0, 0.5]\n",
    "encoder_layers = 4\n",
    "decoder_layers = 1\n",
    "num_iters = 60 # [50, 100]\n",
    "learning_rate = 0.002 # [0.001, 0.005]\n",
    "factor = 0.5 # 0.1 ~ 0.99\n",
    "patience = 3\n",
    "teacher_ratio = 0.02\n",
    "\n",
    "pitts_net, pitts_loss = train(\n",
    "    batch_size, \n",
    "    city, \n",
    "    model_dim,\n",
    "    n_heads,\n",
    "    linear_dim,\n",
    "    dropout_rate,\n",
    "    encoder_layers,\n",
    "    decoder_layers,\n",
    "    num_iters,\n",
    "    learning_rate,\n",
    "    factor,\n",
    "    patience,\n",
    "    teacher_ratio,\n",
    "    device=DEVICE\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "08d0813b",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(43544, 50, 2)\n",
      "(43544, 60, 2)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEICAYAAACJalkVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABAKUlEQVR4nO3deXxU9bn48c9DEgiLJMpSk4KCFUEJEHYVhCjiUgR3qNUqaqXauuCt1u2KaZSWq/wqcLW2/Mp16VUUlyoRa/2poAJ1AUQExAVEwYAgkECAQJbn98c5E06GM8lMZiYzSZ736xWY+Z7tmTNn5pnvcs4RVcUYY4yJpxaJDsAYY0zTZ8nGGGNM3FmyMcYYE3eWbIwxxsSdJRtjjDFxZ8nGGGNM3DXLZCMi+SLyv4mOoyGJyEYROTPRccSDiBwjIqUikhKj9TXYvhKRPBHZ3BDb8tn25SLyRiK2HRTHX0Tk3niuN577OV7xB21jkYj80n0cl/dNRO4Wkb/Fer0BSZNs3A/4fvdLY6uIPCEi7RIdV7TcN/Br93VtFpHnEh1TJNwPaZUbf+CvMNFxeanqt6raTlUroeYHM0BEVESOT0yEsed+Ph6IZh2q+rSqnlXP7eeLSLl7PBSLyFIROaWecVyvqvfXZ1lPPBNFZHEc1nuXiLzrU95RRA6KSE4sthOJaN63AL/kq6p/UNVfhlomWkmTbFxjVbUdkAv0B+5KbDjREZGrgF8AZ7qvaxDwVmKjqpci98s88Dc2eAYRSU1EYMkuVrWtemy3Id6P59zjuhOwGHhJRMQnloTsgxj5O3CqiHQPKv8Z8Kmqrk5ATI1SsiUbAFR1K/AvnKQDgIjcKSLrRWSPiKwVkQs90yaKyGIRmS4iu9yaxLme6d1F5B132f8HdPRuT0TGicga9xfaIhE50TNto4jcLiKrRGSviMwRkR+JyD/d9b0pIkeGeCmDgX+p6vrA61LV2Z51Xy0in7nr2SAiv/JMy3NrQr8TkW0iskVELhCRn4rIFyKyU0Tu9syfLyIviMhz7vpWiEg/v6BEpIVnf+4QkXkiclRd70vQOiaKyBIReVhEdgL5IpIhIk+JyHYR+UZE/lNEWvjMX+y+3lPd8k3ua7yqlu0tEpE/isiHIlIiIq8EYhaRbm7NJVVEpgKnAY+4v7of8fwy/cQtm+D+Mn3VjWWniLwXiDXw3rnH2S4ReVxE0j2vY3FQbNW1JnFqHI+JyGsishc4XUQGiMjH7vvyvPsePRC0jt963uer69j3k4DLgd+Jp6bpHqt3iMgqYK+7P+r83AS9jutF5Ev3dT8qcnjyCKaq5cCTwNFAhxD74ET3PSwW57M2zrPdGrU0ETlPRFbKoRpTX8+0riLyknuM7XDf3xOBvwCnuPuj2G+9QfswW0RedNfztYjcHOK1bQbexvnR6HWl+5prbKe240qCatdByx3pLrfd3fevikiXELFXv2/ifD94Wx3KReQJd9rV4vP9IiJtgX8C2Z7lsiWoe0Hq/l68TZzvxRL3mE73i9e7M5PiD9iIUwMA6AJ8Csz0TL8UyMZJkBOAvUCWO20iUA5cB6QANwBFgLjT/w38CWgFjAD2AP/rTjvBXddoIA34HfAV0NIT1/vAj4AfA9uAFTg1r1Y4B+J9IV7TFcBO4HacWk1K0PQxwE8AAUYC+4AB7rQ8oAKY4sZ1HbAdeAY4AugNlAHHufPnu/vgEnf+24CvgTSf/TvZfU1d3NfwV2BuiNeQB2z2KZ/oxncTkAq0Bp4CXnHj6wZ8AVwbNP/V7nv0APAt8Kgbw1nu+9IuRByLgO+AHKAt8KLnPewGKJDqmfeXQcsrcLzn+R9xvqDS3L/TOHS8bARWA12Bo4AlwAOe17E41LqBJ4ASYBjOsdoe+Aa4xd3ORcBBz/oC73OBO/2nOMfBkXV8Xp4IrCPoM7TSjbt1mJ+bxUGv41UgEzgG53g7J8T28z37vxXwELApxD44AuczdTfQEjjDfa97Br8WYADOZ2woznFylfu6WrnPPwEedo+BdGB4Le+Ld715uMexG9NynM9WS+A4YANwdojXejnwped5T/c97OSzndqOq+Bj0LtcB+BioI27v54HXg46/n8Z6rW65V1xvvd+Gub3y+ag5b3vaTjfix/iHFtHAZ8B19d6zEabJGL15wZf6h6EitPclFnL/CuB8z07/yvPtDbuOo7G+dBUAG0905/x7NR7gXmeaS1wvtTyPHFd7pn+IvCY5/lN3oMixIH6pvvG7QDurGXel4FbPAfDftwE5R6ACgz1zL8cuMBzoLwf9Dq2AKd5Xkcg2XwGjPLMm4WTqFJ9YsoDqoBiz994d59/65kvBTgAnOQp+xWwyPMeeT+wfdzX8yNP2Q4gN8S+WQRM8zw/CecDn0L9kk0BTmI83mdbG/F8cHASwHrP66gr2TzlmTbCPZ7EU7aYml+C+737HufL9uQ6Pi9P4J9srqljuZXU/NwEJ5vhnufzCHG84hxvB93jYRvOj66BIfbBacBWoIWnbC6QH/xagMeA+4O29TnOl+UpOAnQ7zj1e1+8683jULIZiufYdcvuAh4P8VrbALuBU93nU4FXQmyntuMqZLLxmTcX2BV0/IdMNjg/9pYDd9Ty3r9Mze+X2pJNON+LV3imPwj8pbZjL9ma0S5Q1SNwdkQvPM1dInKlp2pdjPML19sctjXwQFX3uQ/b4WTeXaq61zPvN57H2d7nqloFbMKpxQR873m83+d5yIEM6nTmnYnza/F6oEBEznZf07ki8r5b3S7G+VLzvqYd6nZ6u9vxi8W77U1Br2Oz+/qCHQv8w7MvPwMqcWpvfopUNdPzNy94e27cLam5b7+h9v2Iqoa9L4O29w3OL66OIeaty0M4v9TecJsY7qxjW377MRTvstnAd+p+In2mg/M+V3ie76P2/RDutsP53ATb6nlcVxzz3OOhs6qeoarLQ8SRjVPrqfKUBR8bAccCvw3E68bc1V1HV+CboH1VH8fiNCF5t3E3IY5/9/vkeeBKt1nxctwmNB91HVe+RKSNiPxVnObn3cC7QKaE3981B/hcVf/Ls866vl9qE873YiTHStIlGwBU9R2crD8dQESOBf4vcCPQQVUzcZo56mxPxvl1f6TbThlwjOdxEc7Bh7stwTmov6v/Kzicqpar6vPAKiBHRFrh1JKm4/y6zwReI7zXFErXwAO3nbgLzusLtgk4NyiBpKtqpK/Z+wX6A07t6FhP2THEdj929Tw+xt3eD3XE5UtV96jqb1X1OGAs8B8iMqqWbQX2416cX7oAiMjRdWx/C/Bj97jyW3d9hXqN1eVRfm6i5Y2vCOgqNfvEQh0bm4CpQcdmG1Wd6047RvwHP9T5ngdt4+ugbRyhqj+tZZkncWr0o3FaGV71m6mO42ofnmMHp+Ul4Lc4zXNDVbU9To0Ywniv3ITWE7jWU1bX90td+yvm34tJmWxcM4DRIpKL0z6rOFVoxOlAzQlnJar6DbAM+L2ItBSR4TgHQcA8YIyIjBKRNJw3/QCwNNoX4HbkjRGRI8TplD8Xp6/lA5xaQCv3NVW406IazggMFJGL3A/jZJzX8b7PfH8BprpfRohIJxE5P5oNuzWwee56j3DX/R9ALM9nukJEThKRNjjNFS94an5e3+O0w4csE6cT+nj3Q7Qbp2bnXddvRKSLOIMQ7gYCQ9Y/AXqLSK7bIZpfR8z/dtd7ozgd9ucDQ8J5sXXwe43B6v25ibEPcJL070QkTUTycD6Dz/rM+3+B60VkqDjaBj5DOH0EW4Bpbnm6iAxzl/se6CIiLcOI50NgtziDKVqLSIqI5IjI4FqWeQ+nyXA28KyqHvSbqY7jaiXwc3d75+A0DQYcgVOzL3aPufvCeB243xs347QK7fdMquv75XucwRwZIVYd8+/FpE02qrodp8P5XlVdC/wfnA/u9zjt/UsiWN3Pcdppd+K8iU95tvM5Tkf+f+P8Sh6LMwTb92CK0G6cL6pvcQ7UB4EbVHWxqu7BOUjmAbvcGOdHub1XcDqBd+GMnrlInZFCwWa623pDRPbgJKShUW4bnP6rvTidrYtx+sb+JwbrDfg7To13K07nsO8IIpzXd4k4o3pmuWX5wJNus8l4oAdOX1opznH1Z1Vd5FnHM8Ab7mvZgDOgAVX9AifRvQl8ifM6Q3KPo4twfnUW4xxrr+J8cKMxBzjJfT0vh9h2tJ+bmHD3wTjgXJzP2J+BK1V1nc+8y3AGwzyCcxx/hdNHEfhBMxY4HucztRnneAenz2gNsFVE/Gq73m0E1pOLM4jmB+BvQKgvXtxm0Kdwfu0/FWo+aj+ubnG3W4zTFPeyZ7kZOP0uP+B8Hl+v7TV4TMAZev6ZHBpZ9pe6vl/cfT8X2OAeQzWaiePxvRgYJWEaORHJx+l8vCLRscSDiCzC6byM2xnODUVEPsDpTH080bEkAxF5CmeAT0GiYzHxk7Q1G2OaChEZKSJHu81oVwF9Cf+Xa5PmNvn2xKlhmCbMko0x8dcTp6+nBKft+xJV3VLXQuKcUFfq83d5vANuQFtxmpVeTHAcJs6sGc0YY0zcWc3GGGNM3DWpiyd27NhRu3XrlugwjDGmUVm+fPkPqtopnttoUsmmW7duLFu2LNFhGGNMoyIi39Q9V3SsGc0YY0zcWbIxxhgTd5ZsjDHGxJ0lG2OMMXFnycYYY0zcWbIxxhiPksJCvjxjFJ+deBJfnjGKksLCRIfUJDSpoc/GGBONksJCttw7BS0rA6CiqIgt904BIGPs2NoWNXWwmo0xxri2PTyjOtEEaFkZ2x6ekZiAmhBLNsYY46rY4n991FDlJnyWbIwxxpWalRVRuQmfJRtjjHF1vnUykp5eo0zS0+l86+TEBNSE2AABY4xxBQYBbHt4BhVbtpCalVWdaL48Y1SNMhswEBlLNsYY45ExdmyNRGIj1GIjqmY0EblfRFaJyEoReUNEst3yDiKy0L2r4CNBy0wVkU0iUlrLetNE5EkR+VREPhORu6KJ0xhj6stGqMVGtH02D6lqX1XNBV4FprjlZcC9wG0+yxQCQ+pY76VAK1XtAwwEfiUi3aKM1RhjImYj1GIjqmSjqrs9T9sC6pbvVdXFOEkneJn3w7j/ugJtRSQVaA0cBHbXvogxxsSejVCLjahHowWaxYDLOVSzidYLwF5gC/AtMF1Vd4bY/iQRWSYiy7Zv3x6jzRtjjMNvhBpAu5EjEhBN41VnshGRN0Vktc/f+QCqeo+qdgWeBm6MUVxDgEogG+gO/FZEjvObUVVnq+ogVR3UqVNc72pqjGmGMsaOJePCC0CkRnnJP15usOumNYXrtdU5Gk1VzwxzXc8AC4D7oorI8XPgdVUtB7aJyBJgELAhBus2xpiIlL7zLqjWKAsMEoj3iLSmMhou2tFoPTxPxwHrogun2rfAGeJoC5wcw3UbY0xEEjlIoKmMhou2z2aa26S2CjgLuCUwQUQ2An8CJorIZhE5yS1/UEQ2A23c8ny3fJyIFLiLPwq0A1YDHwGPq+qqKGM1xph6SeQggaYyGi6qkzpV9eJapnULUf474Hc+5fOB+e7jUpzhz8YYk3Cdb51coykLGu4yNqlZWVQUFfmWNyZ2bTRjjKlDxtixZN1fQGp2NoiQmp1N1v0FDdJn0lSu12aXqzHGmDAEX8amIbcLh1+vrTENDgBLNsYYU62ksDApv9QTlehiyZrRjDEGJ9EU3XW30z+iSkVREUV33R3WOS1N4TyYeLNkY4wxwJapf4CKipqFFRVOeS0C58F4k9SWe6dYwgliycYYYwAtLo6oPKCpnAcTb5ZsjDEmCk3lPJh4s2RjjDFASmZmROUBdlXo8FiyMcYY4Ef33I2kpdUok7Q0fnTP3bUu11TOg4k3G/psjDHU/3yWpnIeTLyJBl3JtDEbNGiQLlu2LNFhGGNMoyIiy1V1UDy3Yc1oxhhj4s6SjTHGmLizZGOMMQnSnK48YAMEAFbNg7cKoGQzZHSBUVOg7/hER2WMacKayh04w2U1m1XzoPBmKNkEqPN/4c1OuTHGxElzu/KAJZu3CqB8f82y8v1OuTHGxElzu/KAJZuSzSHKN1ntxhgTN83tygOWbDK6hJ5mzWnGmDhpblcesGQzagqktfafZs1pxph6WjSngCVDc1jT60SWDM1h0Zya3yWJvNV0IthotMCos5eu858eqpnNGGNCWDSngMwZc2lV7jw/qqSSAzPmsgjIu3ZK9XxN4Q6c4bKaDTgJJ6NriIkKD+dYc5oxJmxps+dVJ5qAVuVOeXNlySagtuY0Gw5tjIlAZkllROXNgSWbgL7jYeys0DUc678xxoSpOCMlovLmIKpkIyL3i8gqEVkpIm+ISLZb3kFEFopIqYg84pm/jYgsEJF1IrJGRKbVsu67ROQrEflcRM6OJs6w9R0Pt64GxH96ySanSS0/05rWjDEhlU8az4Gat8bhQJpT3lxFW7N5SFX7qmou8CoQ6PkqA+4FbvNZZrqq9gL6A8NE5NzgGUTkJOBnQG/gHODPItJwPwlCDocWu9KAMaZOeddOoXjyZezMSKEK2JmRQvHky2oMDmhuoko2qrrb87QtoG75XlVdjJN0vPPvU9WF7uODwArA75v9fOBZVT2gql8DXwFDook1Ir79N4L78g6xpjVjmoR4XBAz79opDPtgNb3XfcawD1Y360QDMeizEZGpIrIJuJxDNZtwlssExgJv+Uz+MbDJ83yzW9YwavTfiPt/iJvM2dBoYxq1wAUxK4qKQLX6gphN+QrMiVBnshGRN0Vktc/f+QCqeo+qdgWeBm4MZ6MikgrMBWap6ga/WXzKfL/tRWSSiCwTkWXbt28PZ/PhCfTf5Bc7/4caOFDbFQiMMUmvuV0QM1HqTDaqeqaq5vj8vRI06zPAxWFudzbwparOCDF9M+D9du8CFIWIb7aqDlLVQZ06dQpz8/Xg17SW1topXzXPBg4Y00g1twtiJkq0o9F6eJ6OA9aFscwDQAYwuZbZ5gM/E5FWItId6AF8GEWo0fNrWhs7y5lmtygwptFqbhfETJRoL1czTUR6AlXAN8D1gQkishFoD7QUkQuAs4DdwD04SWmFiAA8oqp/E5FxwCBVnaKqa0RkHrAWqAB+o6qJPxuq7/jDb6r2cE7oWxTYDdiMSXqdb51c4yZmkPgLYpYUFrLt4RlUbNlCalYWnW+d3OgvaxNVslHVkM1mqtotxCTfk1hUdT5OjSbwfCowNZr4GkTIWxTYwAFjGoPAl3iyfLk31Tt42oU4o5XRxW1C8ykPsNtOG5PUkumCmLUNWEiWGOvDLlcTrdoGDoDddtoYE5GmOmDBkk20Qg0cCNRc7LbTxpgINNUBC9aMFgt+AwcCwu3TsaY2YwzJOWAhFizZxFu4fTqFNx+qAQWa2sASjjHNTLINWIgVSzbxNmpKzUQCNft0oPamNks2xkSlMQ4jTqYBC7FifTbxVlefDtjwaWPipClc9yweFwlNBKvZNITa+nQgvKY2L+vfMSYsjX0YcVM658ZqNsmgruHTXjaU2piwNfZhxE3pIqGWbJJBOE1tATaU2piwNdZhxIGms4oi3+sPN5pk6WXNaMmirqa2AOvfMSZsjXEYcXDTmZ9kT5Z+LNk0NpH273hZX49pZhrjMGK/pjOvZE+WoViyaWzCGUrtx87lMc1UYxtGXFsTWWp2dtIny1Csz6axiaR/x8v6eoxpFEL2M2Vn0+PttxplogGr2TRO4fbveFlfjzGNQmPsZwqHJZvmIpq+HmNMg2mM/UzhsGTTXNS3ryfABheYJNYYL0lTm8bWzxQOSzbNhfeWB5EmDBtcYJJYUzrLvikTVU10DDEzaNAgXbZsWaLDaHoezgnRBNcVbl3d8PEY4xHq5MdAh7qpm4gsV9VB8dyGjUYzdYvF4IJV85yklZ/p/G+X1zEx0tgvSdNcWLIxdQs1iCDcwQV2PTcTR8l2SZpFcwpYMjSHNb1OZMnQHBbNsdMLwJKNCUckFwr1Y+f4mDjqfOtkJD29RlmihgovmlNA5oy5HFVSSQvgqJJKMmfMtYSDJRsTjvqeSBpg5/iYOMoYO5as+wtIzc4GEVKzs8m6vyAhgwPSZs+jVXnNslblTnlzZ6PRTHjqcyJpQKzO8bHh1yaEZBkqnFlSGVF5c2I1GxN/0TbDgfX7mEahOCMlovLmJKpkIyL3i8gqEVkpIm+ISLZb3kFEFopIqYg84pm/jYgsEJF1IrJGRKaFWO9oEVkuIp+6/58RTZwmwaJthgPr9zEJF87tmcsnjedAWs2yA2lOeXMX1Xk2ItJeVXe7j28GTlLV60WkLdAfyAFyVPVGd542wFBVXSgiLYG3gD+o6j+D1tsf+F5Vi0QkB/iXqv64rnjsPJsmLD8T8DtWBfKLGzYW0+z43WNG0tN9+4YWzSkgbfY8MksqKc5IoXzSePKujaAWnwANcZ5NVH02gUTjaov7baCqe4HFInJ80Pz7gIXu44MisgI4rOFeVT/2PF0DpItIK1U9EE28phGza7uZBKrt9szBySbv2imQ5MklEaLusxGRqSKyCbgcCHsPi0gmMBandlObi4GPQyUaEZkkIstEZNn27dvD3bxpbKLt97GTSk0U7MTR6NWZbETkTRFZ7fN3PoCq3qOqXYGngRvD2aiIpAJzgVmquqGW+XoD/wX8KtQ8qjpbVQep6qBOnTqFs3nTGEXT72ODC0yUku3E0caozmY0VT0zzHU9AywA7gtj3tnAl6o6I9QMItIF+AdwpaquDzMG05TVd/h1bYMLbOi0CUNTvcdMQ4qqz0ZEeqjql+7TccC6MJZ5AMgAflnLPJk4iesuVV0STYzGxPSkUjvXp1lqqveYaUjRntQ5TUR6AlXAN8D1gQkishFoD7QUkQuAs4DdwD04SWmFiAA8oqp/E5FxwCBVnYLTHHc8cK+I3Ouu8ixV3RZlvKY5iuVJpXarhWYrWU4cbazsFgOm6QtOEuAMLoj0XB+71YJpouwWA8bEQixOKgW7xpsxUbBro5nmIZpruwXEojnO+nxMM2U1G2PCFYtzfWwIdrNh97WpyZKNMeGKtjkuFtd3s5NTGwW7r83hrBnNmEhE0xwXbZ+PjYZrNGq9r00zvZSN1WyMaSjR3l472pqR1YoaTEPe1yacq1EnA0s2xjSUaPt8oqkZWX9Rg2qo+9oErkZdUVQEqlQUFbHl3ilJmXAs2RjTUKLt84mmZpTg+wE1xc7y2moUDXVfm9quRp1srM/GmIYUTZ/PqCn+J6eGUzOqT60oRsO0A53lgT6Mo0oqOTBjLosg6e/zEkrw/W0CNQpwrjSQd+0UFkHc72vTmK5GbVcQMKYxqW8CiPTqB7G66gKwZGgOR/n0VezMSGHYB8lz5YVIbnr25RmjnKarIKnZ2fR4u667psROrOKwKwgYY2rqO95JDvnFzv/hfvFH2l8UabNbLYMPGrKzvL4iHaocquZQXlTUoE2EnW+djKSn1yhL1qtRW7IxpjmItL8okma3OgYfNFRneTRqHarsI9R9bAQa9HyajLFjybq/gNTsbBAhNTvb91bVycCSjTHNRSS1okgGI9RRC/LtLE+F8qFVSTMaLtLal1+NIqC2JBUPGWPH0uPttzjxs7X0ePutpEw0YAMEjDF+IhmMUEct6FBn+XNkllRR3B7K++8l76gS/5NS43n9OO+6Wx/plO3fRXH7LI7affjsxe1xmgWDYgh8oX93++8Qn80kUxNhsrABAsY0AyWFhdU3/krJyKAK0JKSQ4+LiyElBSorScnMdKcXk9ISqqoq0fIUUo5oTVVKK7SkhNSsLNqNHEHpO+9SUVRESstKqhS0vAWpbSppl1VG6fftqNgrh2409tVddQ9SCGdgQiTJKDi5HNgDVeWHzbZoSwaZS9rSquJQ2YFUKB62l7ysErdEAHXidbfZWAY/1KUhBghYsjGmiahOKEVFNRPHgQPo/v11ryBCCr6/6v2mqfvvrvbChz+BIevhyN1KWiAx7T/BuQNmW6Vzzi4yugXFG0hIdSUjn+Sy6Ls2pH3clszdHKpVVSeQQxZtyQhrPgCkBVz4VxZ9tI6j/s9c0qoOTSpvATt/e1mjGtbdEMnGmtGMaeRKCgv5fuofqCwuPlRY6fzarlEWY6ESjd80cf89ajec/fGh5xX7Utm1vh2CM3y3ohQ2LjuS/cWt6fBtKhX7UkhtU0nnvj+QAXWPkvMmov07D6uxHLUbDixpy6JhHJZI8rJKIFRyCaZV8MqN9M+dw+YW86DqUO0mpUUK/Tv3D289zYjVbIxpRA5rDotTrSXRgmtGVS0gpX0mWrzLTT57ANi26ohDCenkNDI6f1djPUtey/bti9nZHob99PDzUyL15buDkuJ8m2hZzcYYA/jXXuJZa0m04JpRiyq3X8mtDX374ZFUKbR0fytX7EtlyzuV7OvWntIt6dUJKHOf//ozfRJQfVQUfecTbXKewZ9olmyMSWK+TWRJIpI+m1hLrTq8TCtbuE1yjop9qSFj2JcOXxYe7QxgCNSKfnoOfPnGof6eigNQvtd3+ZKNrdm26ojQ8YU4D6c5s2RjTJIKvv5WLCiwPw0qUqBdGZS2dgqPKIMqAdFDZd7pwY93tIdlx8Ogr6DD7tDTOu52ttkiKIbgwQO1PY+EX1+R3/rbHBAq1ImqYl8qW5a2gnNPh+NPr26mTM06js4XnwwfzanRXNcuq4ySjW3QSv/TFCVFk/IM/kSzPhtjklSo617VRQmM/jo8CTyTJyzpXfeZ+1ltsxjRZQTvbn6XrXu3ktEqA1Vl98HdNR4f3fboOufr8/EurnhHOLKkkl0ZKXz4k0oGr2/BkSVVhyWtwPPTV0F6RZ1hxlRKZiZVZWU1krukpaEVB0BrS5eHylPbVNKp7x4yn9ga73BjyvpsjGnGIm33V2BPOjx+VngJBSCzVWaNxHHLgFsYc9yYekRbi8tqPh3m/r9gwwJmrpjJ1r1b+ce4mklqfZdiJiyqqK41pZcd6p8BKE9xBoR5y6JtuvNrqtTycg6/0ErorfQYtw1V6HbngkNlndvy//4jL4rImgZLNsYkqdSsrJA1m+DmsEhrLXFJKhEac9yYkDEsGLKAB4bMZMteJ+EOW1PJzxdpde3nmTznC99b5lcjinffkVdqG/+rBny5bS9973udVb8/p4EiSU5RJRsRuR84H6gCtgETVbVIRDoALwCDgSdU9UZ3/jbA88BPgEqgUFXvrGX9xwBrgXxVnR5NrMY0Nt9dPpLMh+fWOKu9PrUXSJ4EEy5vIlqwYQEz287kxt5bq5vtNrjNdmsGZFB6sJQKdXbSF10q60xAfrWiA2mQkt6a1D3hDiOvmcYkpYrOffegCru09WFz7z5QyX++/CkPXNAn0l3RZETVZyMi7VV1t/v4ZuAkVb1eRNoC/YEcICco2QxV1YUi0hJ4C/iDqv4zxPpfxElkH4STbKzPxjR2gaal2n7R15ZkWqe0pmVKy/g2iyWZ4H0WLJxaUaDs+n9qjas/S1oaqgoVh7KVpFSR0W1fjSHWnfruIePY/ezS1gw4OMc3jhQR1v/xpzF61bGV9H02gUTjaovbL6mqe4HFInJ80Pz7gIXu44MisgLwvbysiFwAbAD8xx4a04Qs2LCAaR9Oo/hAcY3yJb1TWNK77uUbW80llg6rBQUlnlD7cEnvQ4nopvnKjvbwdp9Do+gqOmdy7O13Axwaoda6gs59d7uX0zn09be/KoXuB56pNc7KJjQYqz6i7rMRkanAlUAJcHoEy2UCY4GZPtPaAncAo4Hb6ljPJGASwDHHHBPu5o1JuLp+kdelOSeYUOpKPF7D1lTyq9e0uomt026nye2vP3Vqjymyn6m9WzDmuDGHLtu/ah6VL02qsZ5yFe6o+FWdsaVIQ/UeJac6m9FE5E3gaJ9J96jqK5757gLSVfU+T9lEYFCgGc1TngoUAv9S1Rk+25wOfKiq80QkHyi1ZjTTlCzYsID8pfmUVUZ2Dk0LacEfhv/BEkyE/BLPo49W0MnnSgLb28NvfuP8Dm+T2oYPLv+g5gyr5rGr8D/JOLiNIu3AgxXjmV81vM4Yrjj5mKTts0mKZjRVPTPMdT0DLADuq2tGYDbwpV+icQ0FLhGRB4FMoEpEylT1kTBjMSZpLdiwgLsX302V+pwGX4v0lHTyT823RFMPwTWeaR9Oo8PuH3zn7eBJQPsqfK5303c8R7q3NOgCzHL/Al7++Dvu+cen7D3ojE4T4PIkTjQNJdrRaD1U9Uv36ThgXRjLPABkAL8MNY+qnuaZPx+nZmOJxjRqofplwmFNZrETSDzvPdybjiWHJ/wd7aNb/wX9f8wF/X8c3UqaoGj7bKaJSE+cEWPfANcHJojIRqA90NLt7D8Lp0ftHpyktEKcNsxHVPVvIjIOp8mt8dwEwpgw1afZLKNlBncNvcsSTJxUTprAgRlza4w+K0s9NDLNxFa0o9EurmVatxCTfN9JVZ0PzPcpz69PbMYki0ibzawW0zAO3a56Hpkllb5Dyyf0nJCw+JoauzaaMXEQ6Ugz6/hPvAfef4Dnv3ieKq2ihbTg0hMu5T9P/k8WzSmoTkjFGSmUTxrfqO7CGQ67LXSELNmYZBBpk5l1/CevRXMKyAxqajuQBsWTG9dtn+vSEMnG/xrZxph6CTSZhZtoMlpmWKJJYmmz59VINACtyp1yExm7EKcxMRDpSDNrNmscMkv8L64ZqtyEZjUbY6IUaDYLN9Gkp6RbomkkijP8r0MXqtyEZsnGmChE2myW1TbLms0akfJJ4zmQVrPsQJpTbiJjzWjG1FOgRlPXkGZrMmu8godHJ3I0WklhoeeW1Vl0vnXyoWu2NQI2Gs2Yegj33BkbaWZioaSwkC33Tql5y+r0dLLuL4hJwrHRaMYkoXBrNDbSzMTKtodn1Eg0AFpWxraHZyQmoHqwZjRjIhBOjcaazUysVWzxPzk4VHkyspqNMWEKp0ZjI81MPKRmZUVUnows2RgThnBGnbWQFtZsZuKi862TkfT0GmWSnk7nWycnJqB6sGY0Y+oQbo3GEo2Jl8AggMY8Gs2SjTF1+OMHf7QajUm4jLFjG1VyCWbNaMbUYsGGBZQcLAk53fpojAmPJRtjQgj004RiNRpjwmfJxhgf4fTTWI3GmPBZn40xQcI5lyazVaYlGmMiYDUbYzzCHXl255A7GzAqYxo/SzbGeMxcMdNGnhkTB5ZsjPHYundryGk28syY+rNkY4zH0W2P9i23Go0x0bFkY4zHLQNuIT2l5mVBrEZjTPRsNJoxHoGEMnPFTLbu3crRbY/mlgG3WKIxJkpRJRsRuR84H6gCtgETVbVIRDoALwCDgSdU9UZ3/jbA88BPgEqgUFV9h/WISF/gr0B7d/2DVTW8e+8aE4Uxx42x5GJMjEXbjPaQqvZV1VzgVSBwr9Qy4F7gNp9lpqtqL6A/MExEzg2eQURSgf8FrlfV3kAeUB5lrMYYYxIkqpqNqu72PG0LqFu+F1gsIscHzb8PWOg+PigiK4AuPqs+C1ilqp+48+6ob4zl5eVs3ryZsjKrFBlIT0+nS5cupKWlJToUY5qVqPtsRGQqcCVQApwewXKZwFhgps/kEwAVkX8BnYBnVfXB+sS3efNmjjjiCLp164aI1GcVpolQVXbs2MHmzZvp3r17osMxplmpsxlNRN4UkdU+f+cDqOo9qtoVeBq4MZyNus1kc4FZqrrBZ5ZUYDhwufv/hSIyKsS6JonIMhFZtn379sOml5WV0aFDB0s0BhGhQ4cOVss1JgHqrNmo6plhrusZYAFwXxjzzga+VNUZIaZvBt5R1R8AROQ1YADwlk98s931MWjQIPVbmSUaE2DHgjGJEdUAARHp4Xk6DlgXxjIPABnA5Fpm+xfQV0TauLWgkcDaKEI1xhiTQNH22UwTkZ44Q5O/Aa4PTBCRjTjDlluKyAU4nf67gXtwktIK91fmI6r6NxEZBwxS1SmquktE/gR8hDPo4DVVXRBlrE1Ku3btKC0tTXQYxhgTlmhHo11cy7RuISb5tmOo6nxgvuf5/+IMf25QL3/8HQ/963OKiveTndma28/uyQX9f9zQYRhjTJNil6vxePnj77jrpU/5rng/CnxXvJ+7XvqUlz/+Lqr17t27lzFjxtCvXz9ycnJ47rnnWL58OSNHjmTgwIGcffbZbNmyBYD169dzzjnnMHDgQE477TTWrXNaJr/++mtOOeUUBg8ezL333hvtSzUm6SyaU8CSoTms6XUiS4bmsGhOQaJDMjFkycbjoX99zv7yyhpl+8sreehfn0e13tdff53s7Gw++eQTVq9ezTnnnMNNN93ECy+8wPLly7nmmmu45557AJg0aRL//d//zfLly5k+fTq//vWvAbjlllu44YYb+Oijjzj6aP+LRRrTWC2aU0DmjLkcVVJJC+CokkoyZ8y1hNOE2LXRPIqK90dUHq4+ffpw2223cccdd3Deeedx5JFHsnr1akaPHg1AZWUlWVlZlJaWsnTpUi699NLqZQ8cOADAkiVLePHFFwH4xS9+wR133BFVTMYkk7TZ82gVdI2QVuVOOddO8V/INCqWbDyyM1vznU9iyc5sHdV6TzjhBJYvX85rr73GXXfdxejRo+nduzf//ve/a8y3e/duMjMzWblype96bNiuaaoySyojKjeNjzWjedx+dk9ap6XUKGudlsLtZ/eMar1FRUW0adOGK664gttuu40PPviA7du3Vyeb8vJy1qxZQ/v27enevTvPP/884Jzx/sknnwAwbNgwnn32WQCefvrpqOIxJtkUZ6REVG4aH0s2Hhf0/zF/vKgPP85sjQA/zmzNHy/qE/VotE8//ZQhQ4aQm5vL1KlTKSgo4IUXXuCOO+6gX79+5ObmsnTpUsBJJHPmzKFfv3707t2bV155BYCZM2fy6KOPMnjwYEpKSqJ9qcYklfJJ4zkQdLm6A2lOuWkaRNX3pPtGadCgQbps2bIaZZ999hknnnhigiIyyciOieS0aE4BabPnkVlSSXFGCuWTxpNn/TUNQkSWq+qgeG7D+myMMUkh79opNhigCbNmNGOMMXFnycYYY0zcWbIxxhgTd5ZsjDHGxJ0lG2OMMXFnyaYZyM/PZ/r06YeVv/zyy6xdG/ltgjZu3MgzzzxT/fyJJ57gxhvDukmrMaaZsmQTbNU8eDgH8jOd/1fNa5DNVlRUNMh2vGpLNrXFE5xsjDGmLpZsvFbNg8KboWQToM7/hTdHnXDuv/9+evXqxejRo7nsssuqaxl5eXncfffdjBw5kpkzZ/LWW2/Rv39/+vTpwzXXXFN9Ec5u3brxww8/ALBs2TLy8vIAp8ZyzTXXkJeXx3HHHcesWbOqtzl16lR69uzJmWeeyeefH37V6qVLlzJ//nxuv/12cnNzWb9+/WHxTJw4kRdeeKF6mXbt2gFw55138t5775Gbm8vDDz8MOJfkOeecc+jRowe/+93votpfxpimx07q9HqrAMqDLsRZvt8p71u/y2YsW7aMF198kY8//piKigoGDBjAwIEDq6cXFxfzzjvvUFZWRo8ePXjrrbc44YQTuPLKK3nssceYPHlyretft24dCxcuZM+ePfTs2ZMbbriBVatW8eyzz4bcJsCpp57KuHHjOO+887jkkksOiwdg4sSJvtucNm0a06dP59VXXwWcZrSVK1fy8ccf06pVK3r27MlNN91E165d67HHjDFNkdVsvEo2R1YehsWLF3P++efTunVrjjjiCMaOHVtj+oQJEwD4/PPP6d69OyeccAIAV111Fe+++26d6x8zZgytWrWiY8eOdO7cme+//5733nuPCy+8kDZt2tC+fXvGjRsXdryBeCI1atQoMjIySE9P56STTuKbb76p13qMMU2TJRuvjC6RlYehrmvPtW3bts75UlNTqaqqAqCsrKzGtFatWlU/TklJqe5rqe/tCALxBG9XVTl48GDI5ULFYYwxYMmmplFTIC3o3jVprZ3yeho+fDiFhYWUlZVRWlrKggULfOfr1asXGzdu5KuvvgLg73//OyNHjgScPpvly5cDVN9ArTYjRozgH//4B/v372fPnj0UFhb6znfEEUewZ8+ekOvxbveVV16hvLw8rOWMMSaYJRuvvuNh7CzI6AqI8//YWfXurwEYPHgw48aNo1+/flx00UUMGjSIjIyMw+ZLT0/n8ccf59JLL6VPnz60aNGC66+/HoD77ruPW265hdNOO42UlLrv7zFgwAAmTJhAbm4uF198MaeddprvfD/72c946KGH6N+/P+vXrz9s+nXXXcc777zDkCFD+OCDD6prPX379iU1NZV+/fpVDxAwxpja2C0GGkBpaSnt2rVj3759jBgxgtmzZzNgwICExtScJcMxYUwysVsMNBGTJk1i7dq1lJWVcdVVV1miMcY0O5ZsGoCdAGmMae6sz8YYY0zcRZVsROR+EVklIitF5A0RyXbLO4jIQhEpFZFHPPO3EZEFIrJORNaIyLQQ600TkSdF5FMR+UxE7oomTmOMMYkVbc3mIVXtq6q5wKtAYIxwGXAvcJvPMtNVtRfQHxgmIuf6zHMp0EpV+wADgV+JSLcoYzXGGJMgUSUbVd3tedoWULd8r6ouxkk63vn3qepC9/FBYAXgd8akAm1FJBVoDRwEdvvMZ4wxphGIus9GRKaKyCbgcg7VbMJZLhMYC7zlM/kFYC+wBfgWpza0M8R6JonIMhFZtn379kjDb3QWLVrEeeedB8D8+fOZNs23JRJwrnP25z//ufp5UVFRjeugNaSVK1fy2muvJWTbxpjEqzPZiMibIrLa5+98AFW9R1W7Ak8DYd3UxK2xzAVmqeoGn1mGAJVANtAd+K2IHOe3LlWdraqDVHVQp06dwtl8rRZsWMBZL5xF3yf7ctYLZ7Fgg/8Z/7FWWVkZ8TLjxo3jzjvvDDk9ONlkZ2fXuIpzQ7JkY0zzVmeyUdUzVTXH5++VoFmfAS4Oc7uzgS9VdUaI6T8HXlfVclXdBiwB4nrCETiJJn9pPlv2bkFRtuzdQv7S/KgSzsaNG+nVqxdXXXUVffv25ZJLLmHfvn2AczmYgoIChg8fzvPPP88bb7zBKaecwoABA7j00kspLS0F4PXXX6dXr14MHz6cl156qXrd3puWff/991x44YX069ePfv36sXTpUu68807Wr19Pbm4ut99+Oxs3biQnJwdwrrF29dVX06dPH/r378/ChQur13nRRRfVebuAgoICBg8eTE5ODpMmTaq+tlteXh6BE2t/+OEHunXrxsGDB5kyZQrPPfccubm5PPfcc+zcuZMLLriAvn37cvLJJ7Nq1ap672NjTPKLdjRaD8/TccC6MJZ5AMgAJtcy27fAGeJoC5wczrqjNXPFTMoqa17osqyyjJkrZka13s8//5xJkyaxatUq2rdvX6O2kZ6ezuLFiznzzDN54IEHePPNN1mxYgWDBg3iT3/6E2VlZVx33XUUFhby3nvvsXXrVt9t3HzzzYwcOZJPPvmEFStW0Lt3b6ZNm8ZPfvITVq5cyUMPPVRj/kcffRSATz/9lLlz53LVVVdVX+Rz5cqVPPfcc3z66ac899xzbNq06bDt3XjjjXz00UesXr2a/fv3V99uwE/Lli0pKChgwoQJrFy5kgkTJnDffffRv39/Vq1axR/+8AeuvPLKiPerMabxiLbPZprbpLYKOAu4JTBBRDYCfwImishmETlJRLoA9wAnASvcIdO/dOcfJyIF7uKPAu2A1cBHwOOqGvefvlv3+n+RhyoPV9euXRk2bBgAV1xxBYsXL66eFrik//vvv8/atWsZNmwYubm5PPnkk3zzzTesW7eO7t2706NHD0SEK664wncbb7/9NjfccAPgXHXZ7/prXosXL+YXv/gF4FwE9Nhjj+WLL74AwrtdwMKFCxk6dCh9+vTh7bffZs2aNRHtE+/2zzjjDHbs2EFJSUlE6zDGNB5RXUFAVUM2m6lqtxCTfK99r6rzgfnu41Kc4c8N6ui2R7Nl7xbf8mgEX+7f+9x7i4HRo0czd+7cGvOuXLmy3rcLqE1t18Sr63YBZWVl/PrXv2bZsmV07dqV/Pz86lpRbbdDqGv78XidxpjkYFcQ8LhlwC2kp6TXKEtPSeeWAbeEWCI83377Lf/+978BmDt3LsOHDz9snpNPPpklS5ZU32Jg3759fPHFF/Tq1Yuvv/66+qrMwckoYNSoUTz22GOAM9hg9+7dtd4KYMSIETz99NMAfPHFF3z77bf07NkzrNcTSCIdO3aktLS0xqAD720JvOXBsXi3v2jRIjp27Ej79u3D2r4xpvGxZOMx5rgx5J+aT1bbLAQhq20W+afmM+a4MVGt98QTT+TJJ5+kb9++7Ny5s7q5y6tTp0488cQTXHbZZdWd5uvWrSM9PZ3Zs2czZswYhg8fzrHHHuu7jZkzZ7Jw4UL69OnDwIEDWbNmDR06dGDYsGHk5ORw++2315j/17/+NZWVlfTp04cJEybwxBNP1KjR1CYzM5PrrruOPn36cMEFFzB48ODqabfddhuPPfYYp556Kj/88EN1+emnn87atWurBwjk5+ezbNky+vbty5133smTTz4Z1raNMY2T3WIgzjZu3Mh5553H6tWrExaDqSnRx4QxyaYhbjFgNRtjjDFxZ8kmzrp162a1GmNMs2fJxhhjTNxZsjHGGBN3lmyMMcbEnSUbY4wxcWfJppFJllsMzJgxo/qCorHifW3GmKbFkk2QksJCvjxjFJ+deBJfnjGKksLCBtluY7vFQDySTbSCL6tjjEkelmw8SgoL2XLvFCqKikCViqIittw7JaqE0xRvMTBr1iyKioo4/fTTOf300wFCxh7qVgRfffUVZ555Jv369WPAgAHVl+MpLS3lkksuoVevXlx++eXV8y9fvpyRI0cycOBAzj77bLZsca5hl5eXx913383IkSOZOXMmzz//PDk5OfTr148RI0bU+30zxsSYqjaZv4EDB2qwtWvXHlYWyhenn6Fre/Y67O+L088Iex3Bvv76awV08eLFqqp69dVX60MPPaSqqscee6z+13/9l6qqbt++XU877TQtLS1VVdVp06bp73//e92/f7926dJFv/jiC62qqtJLL71Ux4wZo6qqjz/+uP7mN79RVdXx48frww8/rKqqFRUVWlxcrF9//bX27t27RiyB59OnT9eJEyeqqupnn32mXbt21f379+vjjz+u3bt31+LiYt2/f78ec8wx+u233x72uo499ljdvn17rbGrqu7YsaN6mSuuuELnz5+vqqpDhgzRl156SVVV9+/fr3v37tWFCxdq+/btddOmTVpZWaknn3yyvvfee3rw4EE95ZRTdNu2baqq+uyzz+rVV1+tqqojR47UG264oXobOTk5unnzZlVV3bVrl+97EskxYUxzACzTOH8/R3XV56amYsvhV3yurTxcwbcYmDVrFrfddhvgf4sBgIMHD3LKKafUuMVAYPnZs2cfto23336bp556Cjh0i4Fdu3aFjGnx4sXcdNNNQOhbDADVtxjo2rVryHWFih2cWxE8+OCD7Nu3j507d9K7d2/y8vL47rvvuPDCCwHnnj4BQ4YMoUuXLgDk5uayceNGMjMzWb16NaNHjwacJsesrKzqZQL7EGDYsGFMnDiR8ePHc9FFF4WM2RjTsCzZeKRmZTlNaD7l0WhqtxjwW5df7KFuRRDptlWV3r17V185O1hgHwL85S9/4YMPPmDBggXk5uaycuVKOnToUGv8dVk0p4C02fPILKmkOCOF8knjybt2SlTrNKa5sT4bj863TkbSa95iQNLT6Xzr5KjW29RuMQA1bxkQKvZQtyJo3749Xbp04eWXXwbgwIEDtQ426NmzJ9u3b6/eh+Xl5SFv1rZ+/XqGDh1KQUEBHTt29L3LaCQWzSkgc8ZcjiqppAVwVEklmTPmsmhOQZ3LGmMOsWTjkTF2LFn3F5CanQ0ipGZnk3V/ARljx0a13qZ2iwGASZMmce6553L66aeHjL22WxH8/e9/Z9asWfTt25dTTz015O2uwbmt9AsvvMAdd9xBv379yM3NZenSpb7z3n777fTp04ecnBxGjBhBv379wn5NftJmz6NVec2yVuVOuTEmfHaLgTizWwwkn0iOiTW9TvT9RVYF9F73WUzjMiZR7BYDxiRYcUZKROXGGH+WbOLMbjHQuJVPGs+BtJplB9KccmNM+JpFsmlKTYUmOpEeC3nXTqF48mXszEihCtiZkULx5MtsNJoxEWryQ5/T09PZsWMHHTp0iMsQYtN4qCo7duyocV5POPKunQKWXIyJSpNPNl26dGHz5s1s37490aGYJJCenl590qgxpuE0+WSTlpZG9+7dEx2GMcY0a82iz8YYY0xiWbIxxhgTd5ZsjDHGxF2TuoKAiGwHvqnn4h2BH2IYTqwle3yQ/DEme3yQ/DEme3yQ/DEmY3zHqmqneG6gSSWbaIjIsnhfriEayR4fJH+MyR4fJH+MyR4fJH+MyR5fvFgzmjHGmLizZGOMMSbuLNkccvjtL5NLsscHyR9jsscHyR9jsscHyR9jsscXF9ZnY4wxJu6sZmOMMSbuLNkYY4yJuyabbETkf0Rkm4is9pTlisj7IrJSRJaJyBC3PE1EnhSRT0XkMxG5y7NMSxGZLSJfiMg6Ebk4mWIUkSPceQN/P4jIjGSJz512mVu+SkReF5GOsYgvxjFOcONbIyIPJii+liLyuBvfJyKS51lmoFv+lYjMkhhewjyGMU4VkU0iUhqr2GIVn4i0EZEF7md4jYhMS7YY3Wmvu2VrROQvItJ07tKnqk3yDxgBDABWe8reAM51H/8UWOQ+/jnwrPu4DbAR6OY+/z3wgPu4BdAx2WIMWudyYESyxIdzsddtgf0GPAjkJ9M+BDoA3wKd3GlPAqMSEN9vgMfdx53d97KF+/xD4BRAgH8Glk+yGE8GsoDSWMUWq/jc9/t0t7wl8F6S7sP27v8CvAj8LJb7MpF/TbZmo6rvAjuDi4H27uMMoMhT3lZEUoHWwEFgtzvtGuCP7jqrVDVmZ/7GMEYARKQHzsH7XhLFJ+5fW/fXeHvPMskS43HAF6oauA/Fm0BMarARxncS8Ja73DagGBgkIlk4X0L/Vueb6CnggljEF6sY3efvq+qWWMUVy/hUdZ+qLnTLDwIrgJjdayKG+zDwmU7FSYpNZwRXorNdPP9wfrV6f2mciPMLdhPwHc4lGgDSgGeB7cBeYJJbnunO+yecg/N54EfJFGPQuqYA05MtPuASnC/1LcC7QEoyxQgcCWzmUE3sRaAwAfFNco+xVKA7zpfQxThfRG96lj8NeDVB+9A3xqB1xbRmE4f4MoENwHHJGCPwL2AX8EysPyuJ/GuyNZsQbgBuVdWuwK3AHLd8CFAJZOO8+b8VkeNwDoYuwBJVHQD8G5ieZDF6/QyYm0zxiUiau0x/d9oq4K7D1prAGFV1l7vMczi1wo1ARQLi+x+cpLcMmAEsdePw65+J9y/eSGNsaPWKz63VzgVmqeqGZIxRVc/GaY5sBZwR5xgbTqKzXTz/OPyXRgmHzi0SYLf7+FHgF575/gcY786zl0PtqV2BNckUo+d5P5ymoGTbh4OBtzzlI4DXkilGn/VNAh5s6Ph8lluK0+SSBazzlF8G/DUR+zBUjEFlDVGzqVd87ns+K9bxxXofuuVXAY/EI9ZE/DW3mk0RMNJ9fAbwpfv4W+AMcbTF6ehcp847XgjkufONAtYmU4ye5S4j/rWa+sT3HXCSiASuKDsa+CzJYkREOrv/Hwn8GvhbQ8fnjphq6z4eDVSo6lp1+kH2iMjJbr/XlcArcYwv4hjjHEtM4hORB3D6TiYnY4wi0s7tnwvUwH5Kzc9445bobBevP5wv3i1AOU6V9VpgOM7Ij0+AD4CB7rztcNpQ1+Akk9s96zkWp59hFU6n3jHJFqM7fQPQK0n34fU4CWYVTvLukIQxznXL1hLDEUARxtcN+NzdV2/itvG70wYBq4H1wCO4v5iTLMYH3eWr3P/zkyU+nOZwdctXun+/TKZ9CPwI+Mj9nKwB/htIjeVnOpF/drkaY4wxcdfcmtGMMcYkgCUbY4wxcWfJxhhjTNxZsjHGGBN3lmyMMcbEnSUbY4wxcWfJxhhjTNz9fxgChyxz4raJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "vis_results('pittsburgh', 'train', 32, pitts_net, 20)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}