LymphomaClassificationUsingResNet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LymphomaClassificationUsingResNet.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "3ad8689e91734af59bea213d026b30ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_603f56bffeb74d158736f0425d6876a5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c7efaa1e5a7a45ff9bce10e12ee50ddc",
              "IPY_MODEL_fc7d5803b3c043ba8b4f646678383051"
            ]
          }
        },
        "603f56bffeb74d158736f0425d6876a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c7efaa1e5a7a45ff9bce10e12ee50ddc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d30c6790fef64c91a852f0853130a2ab",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 46827520,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 46827520,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bf2efa9565954877bd945c3b822f6f7d"
          }
        },
        "fc7d5803b3c043ba8b4f646678383051": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2f114f48d76c4746a114be911fc0dbd2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 44.7M/44.7M [11:56&lt;00:00, 65.3kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4cd780d9042c4e29bc89a73e2fb6b909"
          }
        },
        "d30c6790fef64c91a852f0853130a2ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bf2efa9565954877bd945c3b822f6f7d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2f114f48d76c4746a114be911fc0dbd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4cd780d9042c4e29bc89a73e2fb6b909": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KQJBnILtnnl8",
        "outputId": "9679cb6d-9636-495b-8f94-635bbd5d2793"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "#!cd /content/gdrive/My\\ Drive/BioImage/Project\n",
        "!pip install --upgrade --force-reinstall --no-deps kaggle\n",
        "import os\n",
        "os.environ['KAGGLE_USERNAME'] = \"serenaabraham\" \n",
        "os.environ['KAGGLE_KEY'] = \"0aa4e1881376dfb2149dcfb30ffe7975\""
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n",
            "Collecting kaggle\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/33/365c0d13f07a2a54744d027fe20b60dacdfdfb33bc04746db6ad0b79340b/kaggle-1.5.10.tar.gz (59kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 3.1MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: kaggle\n",
            "  Building wheel for kaggle (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for kaggle: filename=kaggle-1.5.10-cp36-none-any.whl size=73269 sha256=f77503b3d28a67aa5b7c07e4e5d940ae9d485b50d09087cea14b460ebdf7d517\n",
            "  Stored in directory: /root/.cache/pip/wheels/3a/d1/7e/6ce09b72b770149802c653a02783821629146983ee5a360f10\n",
            "Successfully built kaggle\n",
            "Installing collected packages: kaggle\n",
            "  Found existing installation: kaggle 1.5.10\n",
            "    Uninstalling kaggle-1.5.10:\n",
            "      Successfully uninstalled kaggle-1.5.10\n",
            "Successfully installed kaggle-1.5.10\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YLi2Q2sPn401",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "96b05c90-34e7-4fe0-8d1f-b08774465203"
      },
      "source": [
        "!kaggle datasets download -d andrewmvd/malignant-lymphoma-classification\n",
        "!unzip -q /content/malignant-lymphoma-classification.zip"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading malignant-lymphoma-classification.zip to /content\n",
            " 99% 1.33G/1.34G [01:27<00:01, 9.67MB/s]\n",
            "100% 1.34G/1.34G [01:27<00:00, 16.6MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YE7upblvpRnr"
      },
      "source": [
        "from skimage import io\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "for fn in os.listdir('inputs/MCL'):\n",
        "  if(fn.endswith('.tif')):\n",
        "    original = io.imread('inputs/MCL/' + fn, plugin='tifffile')\n",
        "    height, width, length = original.shape\n",
        "    width_cutoff=width//2\n",
        "    img1=original[:,:width_cutoff,:]\n",
        "    img2=original[:,width_cutoff:,:]\n",
        "    plt.imsave('inputs/MCL/' + fn[:-4] + '1.png', img1)\n",
        "    plt.imsave('inputs/MCL/' + fn[:-4] + '2.png', img2)\n",
        "    os.remove('inputs/MCL/'+fn)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2ZNjueiFy0VN",
        "outputId": "293a7558-3bce-426b-e009-137be1eb67b6"
      },
      "source": [
        "!pip install split_folders\n",
        "import splitfolders\n",
        "splitfolders.ratio('inputs', output='output', seed=1337, ratio=(.8, 0.1,0.1)) "
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting split_folders\n",
            "  Downloading https://files.pythonhosted.org/packages/b8/5f/3c2b2f7ea5e047c8cdc3bb00ae582c5438fcdbbedcc23b3cc1c2c7aae642/split_folders-0.4.3-py3-none-any.whl\n",
            "Installing collected packages: split-folders\n",
            "Successfully installed split-folders-0.4.3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Copying files: 748 files [00:03, 211.54 files/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B0KTl72CzXO8"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import torchvision.datasets as datasets\n",
        "from torch.optim.lr_scheduler import StepLR\n",
        "from functools import partial\n",
        "from dataclasses import dataclass\n",
        "from collections import OrderedDict\n",
        "from torchvision import datasets, models, transforms\n",
        "\n",
        "\n",
        "\n",
        "import os\n",
        "import random\n",
        "import numpy as np\n",
        "import time\n",
        "\n",
        "SEED = 1234\n",
        "\n",
        "random.seed(SEED)\n",
        "np.random.seed(SEED)\n",
        "torch.manual_seed(SEED)\n",
        "torch.cuda.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "\n",
        "def load_dataset():\n",
        "    train_data_path = './output/train'\n",
        "    test_data_path = './output/test'\n",
        "    valid_data_path = './output/val'\n",
        "\n",
        "    train_dataset = torchvision.datasets.ImageFolder(root=train_data_path,transform=torchvision.transforms.ToTensor())\n",
        "    test_dataset = torchvision.datasets.ImageFolder(root=test_data_path,transform=torchvision.transforms.ToTensor())\n",
        "    valid_dataset = torchvision.datasets.ImageFolder(root=valid_data_path,transform=torchvision.transforms.ToTensor())\n",
        "\n",
        "    train_iterator = torch.utils.data.DataLoader(train_dataset,batch_size=32,num_workers=0,shuffle=True)\n",
        "    test_iterator = torch.utils.data.DataLoader(test_dataset,batch_size=32,num_workers=0,shuffle=False)\n",
        "    valid_iterator = torch.utils.data.DataLoader(valid_dataset,batch_size=32,num_workers=0,shuffle=True)\n",
        "\n",
        "    return train_iterator, test_iterator, valid_iterator\n",
        "\n",
        "\n",
        "\n",
        "def calculate_accuracy(fx, y):\n",
        "    preds = fx.max(1, keepdim=True)[1]\n",
        "    correct = preds.eq(y.view_as(preds)).sum()\n",
        "    acc = correct.float()/preds.shape[0]\n",
        "    return acc\n",
        "\n",
        "\n",
        "\n",
        "def train(model, device, iterator, optimizer, criterion):\n",
        "\n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "\n",
        "    model.train()\n",
        "\n",
        "    for (x, y) in iterator:\n",
        "\n",
        "        x = x.to(device)\n",
        "        y = y.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        fx = model(x)\n",
        "\n",
        "        loss = criterion(fx, y)\n",
        "\n",
        "        acc = calculate_accuracy(fx, y)\n",
        "\n",
        "        loss.backward()\n",
        "\n",
        "        optimizer.step()\n",
        "\n",
        "        epoch_loss += loss.item()\n",
        "        epoch_acc += acc.item()\n",
        "\n",
        "    #scheduler.step()\n",
        "\n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def evaluate(model, device, iterator, criterion):\n",
        "\n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for (x, y) in iterator:\n",
        "\n",
        "            x = x.to(device)\n",
        "            y = y.to(device)\n",
        "\n",
        "            fx = model(x)\n",
        "\n",
        "            loss = criterion(fx, y)\n",
        "\n",
        "            acc = calculate_accuracy(fx, y)\n",
        "\n",
        "            epoch_loss += loss.item()\n",
        "            epoch_acc += acc.item()\n",
        "\n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "3ad8689e91734af59bea213d026b30ce",
            "603f56bffeb74d158736f0425d6876a5",
            "c7efaa1e5a7a45ff9bce10e12ee50ddc",
            "fc7d5803b3c043ba8b4f646678383051",
            "d30c6790fef64c91a852f0853130a2ab",
            "bf2efa9565954877bd945c3b822f6f7d",
            "2f114f48d76c4746a114be911fc0dbd2",
            "4cd780d9042c4e29bc89a73e2fb6b909"
          ]
        },
        "id": "M6988LcqzcKg",
        "outputId": "971567a2-65a4-47fb-d88d-ab645d8a29ce"
      },
      "source": [
        "from torchvision import datasets, models, transforms\n",
        "model=models.resnet18(pretrained=True)\n",
        "#set_parameter_requires_grad(model, True)\n",
        "num_ftrs = 512\n",
        "num_classes = 3\n",
        "model.fc = nn.Linear(num_ftrs, num_classes)\n",
        "import time\n",
        "params_to_update = []\n",
        "for name,param in model.named_parameters():\n",
        "  if param.requires_grad == True:\n",
        "    params_to_update.append(param)\n",
        "    print(\"\\t\",name)\n",
        "learningRate = 1e-2\n",
        "weightDecay = 5e-5\n",
        "numEpochs = 20\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.SGD(params_to_update, lr=learningRate, momentum=0.9)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet18-5c106cde.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-5c106cde.pth\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3ad8689e91734af59bea213d026b30ce",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=46827520.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\t conv1.weight\n",
            "\t bn1.weight\n",
            "\t bn1.bias\n",
            "\t layer1.0.conv1.weight\n",
            "\t layer1.0.bn1.weight\n",
            "\t layer1.0.bn1.bias\n",
            "\t layer1.0.conv2.weight\n",
            "\t layer1.0.bn2.weight\n",
            "\t layer1.0.bn2.bias\n",
            "\t layer1.1.conv1.weight\n",
            "\t layer1.1.bn1.weight\n",
            "\t layer1.1.bn1.bias\n",
            "\t layer1.1.conv2.weight\n",
            "\t layer1.1.bn2.weight\n",
            "\t layer1.1.bn2.bias\n",
            "\t layer2.0.conv1.weight\n",
            "\t layer2.0.bn1.weight\n",
            "\t layer2.0.bn1.bias\n",
            "\t layer2.0.conv2.weight\n",
            "\t layer2.0.bn2.weight\n",
            "\t layer2.0.bn2.bias\n",
            "\t layer2.0.downsample.0.weight\n",
            "\t layer2.0.downsample.1.weight\n",
            "\t layer2.0.downsample.1.bias\n",
            "\t layer2.1.conv1.weight\n",
            "\t layer2.1.bn1.weight\n",
            "\t layer2.1.bn1.bias\n",
            "\t layer2.1.conv2.weight\n",
            "\t layer2.1.bn2.weight\n",
            "\t layer2.1.bn2.bias\n",
            "\t layer3.0.conv1.weight\n",
            "\t layer3.0.bn1.weight\n",
            "\t layer3.0.bn1.bias\n",
            "\t layer3.0.conv2.weight\n",
            "\t layer3.0.bn2.weight\n",
            "\t layer3.0.bn2.bias\n",
            "\t layer3.0.downsample.0.weight\n",
            "\t layer3.0.downsample.1.weight\n",
            "\t layer3.0.downsample.1.bias\n",
            "\t layer3.1.conv1.weight\n",
            "\t layer3.1.bn1.weight\n",
            "\t layer3.1.bn1.bias\n",
            "\t layer3.1.conv2.weight\n",
            "\t layer3.1.bn2.weight\n",
            "\t layer3.1.bn2.bias\n",
            "\t layer4.0.conv1.weight\n",
            "\t layer4.0.bn1.weight\n",
            "\t layer4.0.bn1.bias\n",
            "\t layer4.0.conv2.weight\n",
            "\t layer4.0.bn2.weight\n",
            "\t layer4.0.bn2.bias\n",
            "\t layer4.0.downsample.0.weight\n",
            "\t layer4.0.downsample.1.weight\n",
            "\t layer4.0.downsample.1.bias\n",
            "\t layer4.1.conv1.weight\n",
            "\t layer4.1.bn1.weight\n",
            "\t layer4.1.bn1.bias\n",
            "\t layer4.1.conv2.weight\n",
            "\t layer4.1.bn2.weight\n",
            "\t layer4.1.bn2.bias\n",
            "\t fc.weight\n",
            "\t fc.bias\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 550
        },
        "id": "qCpcKYVgzdYo",
        "outputId": "5af676c9-3396-4cac-cefd-a4203a0759e4"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "\n",
        "    train_iterator, test_iterator, valid_iterator = load_dataset()\n",
        "\n",
        "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "    #model=Net()\n",
        "    #model=MobileNet()\n",
        "    model.to(device)\n",
        "\n",
        "    optimizer = optim.Adam(model.parameters())\n",
        "    scheduler = StepLR(optimizer, step_size=5, gamma=0.2)\n",
        "\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "    trainSet=[]\n",
        "    validSet=[]\n",
        "\n",
        "\n",
        "    ### Training ###\n",
        "    EPOCHS = 14\n",
        "    SAVE_DIR = 'models'\n",
        "    MODEL_SAVE_PATH = os.path.join(SAVE_DIR, 'mlp-mnist.pt')\n",
        "\n",
        "    best_valid_loss = float('inf')\n",
        "    time_curr = time.time()\n",
        "\n",
        "    if not os.path.isdir(f'{SAVE_DIR}'):\n",
        "        os.makedirs(f'{SAVE_DIR}')\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        train_loss, train_acc = train(model, device, train_iterator, optimizer, criterion)\n",
        "        valid_loss, valid_acc = evaluate(model, device, valid_iterator, criterion)\n",
        "\n",
        "        if valid_loss < best_valid_loss:\n",
        "            best_valid_loss = valid_loss\n",
        "            torch.save(model.state_dict(), MODEL_SAVE_PATH)\n",
        "\n",
        "        print(\"|Epoch: {0:d} | Train loss: {1:.3f} | Train Acc: {2:.2f}% | Val Loss : {3:.3f} | Val Acc: {4:.2f}% | Time used: {5:d}s |\".format(epoch+1,\n",
        "            train_loss, train_acc*100, valid_loss, valid_acc*100, int(time.time()-time_curr)))\n",
        "        \n",
        "        trainSet.append(train_acc*100)\n",
        "        validSet.append(valid_acc*100) \n",
        "\n",
        "        time_curr = time.time()\n",
        "    plt.plot(trainSet, validSet, 'go', markersize=5.0)\n",
        "    plt.xlabel(\"training accuracy\")\n",
        "    plt.ylabel(\"validation accuracy\")\n",
        "    plt.title(\"Training vs Validation Accuracy\")\n",
        "    plt.show()\n",
        "\n",
        "\n",
        "    ### Testing ###\n",
        "    model.load_state_dict(torch.load(MODEL_SAVE_PATH))\n",
        "\n",
        "    test_loss, test_acc = evaluate(model, device, test_iterator, criterion)\n",
        "\n",
        "    print('| Test Loss: {0:.3f} | Test Acc: {1:.2f}% |'.format(test_loss,test_acc*100))\n"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "|Epoch: 1 | Train loss: 0.929 | Train Acc: 61.32% | Val Loss : 1.992 | Val Acc: 43.40% | Time used: 50s |\n",
            "|Epoch: 2 | Train loss: 0.627 | Train Acc: 73.59% | Val Loss : 4.177 | Val Acc: 34.03% | Time used: 50s |\n",
            "|Epoch: 3 | Train loss: 0.435 | Train Acc: 84.69% | Val Loss : 1.377 | Val Acc: 74.31% | Time used: 50s |\n",
            "|Epoch: 4 | Train loss: 0.482 | Train Acc: 81.97% | Val Loss : 1.485 | Val Acc: 67.59% | Time used: 50s |\n",
            "|Epoch: 5 | Train loss: 0.331 | Train Acc: 88.23% | Val Loss : 0.723 | Val Acc: 74.42% | Time used: 50s |\n",
            "|Epoch: 6 | Train loss: 0.237 | Train Acc: 90.95% | Val Loss : 1.726 | Val Acc: 64.35% | Time used: 50s |\n",
            "|Epoch: 7 | Train loss: 0.317 | Train Acc: 88.31% | Val Loss : 1.031 | Val Acc: 74.42% | Time used: 50s |\n",
            "|Epoch: 8 | Train loss: 0.293 | Train Acc: 88.23% | Val Loss : 1.078 | Val Acc: 63.89% | Time used: 50s |\n",
            "|Epoch: 9 | Train loss: 0.308 | Train Acc: 90.21% | Val Loss : 0.421 | Val Acc: 83.80% | Time used: 50s |\n",
            "|Epoch: 10 | Train loss: 0.220 | Train Acc: 91.69% | Val Loss : 0.850 | Val Acc: 71.30% | Time used: 50s |\n",
            "|Epoch: 11 | Train loss: 0.108 | Train Acc: 97.12% | Val Loss : 1.332 | Val Acc: 69.10% | Time used: 50s |\n",
            "|Epoch: 12 | Train loss: 0.173 | Train Acc: 94.24% | Val Loss : 3.410 | Val Acc: 52.43% | Time used: 50s |\n",
            "|Epoch: 13 | Train loss: 0.129 | Train Acc: 95.88% | Val Loss : 1.159 | Val Acc: 64.93% | Time used: 50s |\n",
            "|Epoch: 14 | Train loss: 0.157 | Train Acc: 95.06% | Val Loss : 0.585 | Val Acc: 86.92% | Time used: 50s |\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfZhcdXn/8fcnG7chGyAkBBICMdQQKIpsMLKCD0VBcesqaBFbDQ3UFiutFawV7c8WUduftrZqtUUR5BkBqSgXZQmKQuWhqxt2QRTUIASICUQMD7uASzZ3/zjfgdnJPpysOTszOZ/Xdc0153znPNxzdvaeM/c553sUEZiZWXlMq3cAZmY2tZz4zcxKxonfzKxknPjNzErGid/MrGSc+M3MSsaJ30YlqVvSyu09bbORFJKWpOEvSfr7PNNOYj3vknT9ZOM02xbyefw7DkkDVaMzgd8Aw2n8PRFxydRHVV+SrgN+EBH/UNN+DPBlYO+I2DzO/AHsFxFrcqwr17SSFgP3AS8Yb93bk6R9gXuBL0fEe6dinda4vMe/A4mIWZUH8ADw5qq255K+pOn1i3LKXQCskKSa9hOAS6Yq8TaAPwE2Ae+Q9DtTuWJJLVO5PpuYE38JSDpC0kOSTpe0AThP0m6SrpG0UdKmNLx31Tw3SvqzNHyipJslfSZNe5+kzklOu6+k/5H0pKTvSPoPSRePEffdkrqqxqeneA+RNEPSxZIelfSYpB9K2nOUxXwTmAu8umo5uwFdwIWSDpV0W1rGeklflNQ6RjznS/pk1fjfpnl+KelPa6Z9k6Q+SU9IelDSx6pe/p/0/JikAUmHVbZb1fyHp/f0eHo+vGZ7f0LSLWk7Xi9p99FiTtOLLPF/FHgWeHPN68dI6k+x3ivpjal9jqTz0vvbJOmbqX1ErKmtuiR2vqSzJF0raRB47QTbA0mvknRr+js8mNbxckkPV39xSHqbpDvGeq+WjxN/ecwH5gAvBE4m+9ufl8YXAU8DXxxn/g7gp8DuwD8D546yF51n2kuBH5Al44+R7XmP5WvAH1eNHw38KiJuB1YCuwL7pGX9RXoPI0TE08AVZImv4njgnoi4g6wUdlqK9TDgSOCUcWICICXHDwKvB/YDjqqZZDCtczbwJuC9ko5Nr70mPc9Ov8Zuq1n2HOC/gX9P7+3fgP+WNLdqsncCJwF7AK0plrG8CtgbuIxsWzx3PEbSocCFwN+mWF8D3J9evoisZPjitJ7PjrOOWu8E/hHYGbiZcbaHpBcC3cAXgHlAO9AfET8EHgXeULXcE1K89tuICD92wAfZP+9RafgIYAiYMc707cCmqvEbgT9LwycCa6pemwkEMH9bpiX7gtkMzKx6/WLg4jFiWgI8WZkeuAT4hzT8p8CtwEtzbItXAY9V3j9wC3DaGNOeClxVNR7AkjR8PvDJNPxV4FNV0y2tnnaU5X4O+GwaXpymnV71+onAzWn4BLLjEtXz3wacWLW9P1r12inAdeO8/3OAb6bhw8j2+vdI41+uxFUzzwJgC7DbKK89F+s42+nCCf4m1dvjI9XbvGa608lKcpDtuDwFLKj3/1ezP7zHXx4bI+KZyoikmZK+LGmtpCfIyg+zNXY9dkNlICKeSoOztnHavYBfV7UBPDhWwJEdJL0beLOkmcBbyH4xQLY3ugq4LJUi/lnSC8ZYzs3Ar4BjJb0IOLSyHElLU5lrQ9oO/0S29z+RvWpiX1v9oqQOSd9LpanHyX6R5FluZdlra9rWAgurxjdUDT/FGH8LSTsBbyf70iSyXxcPkO2RQ/aL6d5RZt2H7G+1KWfMtUb8XSfYHmPFANmOwZsltZH9Uvt+RKyfZEyWOPGXR+3pW38D7A90RMQuPF9+GKt8sz2sB+akJF6xzwTzVMo9xwA/SV8GRMSzEXFmRBwIHE5Ws/+TsRfDhen1FcCqiHg4tZ8F3EN2Ns4uwN+Rbxusr4l9Uc3rlwJXA/tExK7Al6qWO9GpdL8kK8FVWwSsyxFXrbcCuwD/mb7cNpB9gVTKPQ8CLxplvgfJ/lazR3ltkOyXHACS5o8yTe17HG97jBUDEbGO7NfO28h+CV002nS2bZz4y2tnspr4Y6mmfEbRK4yItUAv8DFJrZIOo+ZA4yguI6vxvpfn9/aR9FpJB6VfKE+QlS+2jLOcC8nq8H9OdqZPxc5p/gFJB6T15HEFcKKkA9MXWe3225lsj/mZVEd/Z9VrG1OsvzvGsq8Flkp6p7ID2u8ADgSuyRlbtZVkZamDyMp57cArgYMlHQScC5wk6UhJ0yQtlHRA2qvuJvvC2E3SCyRVdg7uAF4sqV3SDLJjNRMZb3tcAhwl6fj0fudKaq96/ULgQ+k9fGMS28BqOPGX1+eAnchKIP8LXDdF630XWZ35UeCTwOVk1xuMKiWg28j26i+vemk+cCVZ0r4buIlx9gYj4n6yYwJtZHueFR8kS0JPAl+pWceYIqKbbBt+F1iTnqudAnxc0pPAP5B9UVTmfYrswOct6SyWV9Qs+1GyXzB/Q7adPgR0RcSv8sRWIWkh2cHqz0XEhqrHarK/98qI+AHZQeLPAo+TbcfKr40TyL5Q7wEeITv+QUT8DPg48B3g52QHbycy3vZ4APiD9H5/DfQDB1fNe1WK6aqaMqFNki/gsrqSdDnZGTaF/+Kw5iXpXrKLEL9T71h2BN7jtymVzs1+USorvJGsdv/NesdljUvSH5IdM6j9VWWTVKYrOK0xzCer084FHgLeGxF99Q3JGpWkG8mOb5wQEeMdw7Ft4FKPmVnJuNRjZlYyTVHq2X333WPx4sX1DsPMrKmsXr36VxExr7a9KRL/4sWL6e3trXcYZmZNRVLtFeCASz1mZqXjxG9mVjJO/GZmJePEb2ZWMk78ZmYl0xRn9ZiZNYvhLcN0r+mmb30fyxYso3NJJy3TGuu2w078ZmbbyfCWYY6++Gh61vUwODRIW2sbHQs7WLViVUMlf5d6zMy2k+413fSs62FgaIAgGBgaoGddD91ruusd2ghO/GZm20nf+j4GhwZHtA0ODdK/ob9OEY3Oid/MbDtZtmAZba1tI9raWtton98+xhz14cRvZraddC7ppGNhB7NaZyHErNZZdCzsoHNJZ71DG8EHd83MtpOWaS2sWrGK7jXd9G/op31+u8/qMTPb0bVMa6FraRddS7vqHcqYXOoxMysZJ34zs5Jx4jczKxknfjOzkvHBXTNrGs3QD04zcOI3s6bQLP3gNAOXesysKTRLPzjNwInfzJpCs/SD0wyc+M2sKTRLPzjNwInfzJpCs/SD0wx8cNfMmkKz9IPTDJz4zaxpNEM/OM3ApR4zs5Jx4jczKxknfjOzknHiNzMrGSd+M7OSceI3MyuZQhO/pNMk/VjSXZK+JmmGpH0l9UhaI+lySa1FxmBmZiMVlvglLQT+GlgeES8BWoA/Aj4NfDYilgCbgHcXFYOZmW2t6FLPdGAnSdOBmcB64HXAlen1C4BjC47BzMyqFJb4I2Id8BngAbKE/ziwGngsIjanyR4CFo42v6STJfVK6t24cWNRYZqZlU6RpZ7dgGOAfYG9gDbgjXnnj4izI2J5RCyfN29eQVGamZVPkaWeo4D7ImJjRDwLfAN4JTA7lX4A9gbWFRiDmZnVKDLxPwC8QtJMSQKOBH4CfA84Lk2zEvhWgTGYmVmNImv8PWQHcW8HfpTWdTZwOvABSWuAucC5RcVgZmZbK7Rb5og4AzijpvkXwKFFrtfMzMbmK3fNzErGid/MrGSc+M3MSsaJ38ysZJz4zcxKxonfzKxknPjNzErGid/MrGSc+M3MSsaJ38ysZJz4zcxKxonfzKxkCu2kzcwaz/CWYbrXdNO3vo9lC5bRuaSTlmktU7Lea352DV//8ddB8PYD307X0q4pWbeN5MRvViLDW4Y5+uKj6VnXw+DQIG2tbXQs7GDVilWFJuDhLcO84aI3cNPamxiOYQAuu+syfv+Fv8/1J1zv5D/FXOoxK5HuNd30rOthYGiAIBgYGqBnXQ/da7oLX++tD936XNIHGI5hbn3o1sLXbVtz4jcrkb71fQwODY5oGxwapH9Df+HrfWbzM1u1P7P5mcLXbVtz4jcrkWULltHW2jaira21jfb57YWvd8b0GVu1z5g+o/B129ac+M1KpHNJJx0LO5jVOgshZrXOomNhB51LOgtf7+F7H06Lnq/lt6iFw/c+vPB129YUEfWOYULLly+P3t7eeodhtkOonNXTv6Gf9vntU35Wz5U/uRKA4w48zmf1FEzS6ohYvlW7E7+Z2Y5prMQ/YalH0kHFhGRmZvWQp8b/n5J+IOkUSbsWHpGZmRVqwsQfEa8G3gXsA6yWdKmk1xcemZmZFSLXWT0R8XPgo8DpwO8D/y7pHklvKzI4MzPb/ibsskHSS4GTgDcB3wbeHBG3S9oLuA34RrEhmplNTr36JWp0efrq+QJwDvB3EfF0pTEifinpo4VFZmb2W6hXv0TNIE+p503ApZWkL2mapJkAEXFRkcGZmU1WvfolagZ5Ev93gJ2qxmemNjOzhlWvfomaQZ7EPyMiBiojaXjmRDNJ2l9Sf9XjCUmnSpoj6duSfp6ed/tt3oCZ2Wjq1S9RM8iT+AclHVIZkfQy4OlxpgcgIn4aEe0R0Q68DHgKuAr4MHBDROwH3JDGzcy2q3r1S9QM8hzcPRX4uqRfAgLmA+/YxvUcCdwbEWslHQMckdovAG4kO03UzGy7aZnWwqoVq+rSL1Gjy9VXj6QXAPun0Z9GxLPbtBLpq8DtEfFFSY9FxOzULmBTZbxmnpOBkwEWLVr0srVr127LKs3MSu+36qRN0kuAA4HnOtSOiAtzrrgV+CXw4oh4uDrxp9c3RcS4dX530mZmZbK9rj8YK/HnuYDrDLLSzIHAtUAncDOQK/Gn6W+PiIfT+MOSFkTEekkLgEdyLsfMbIc3Fdcf5Dm4exxZjX5DRJwEHAxsS2dtfwx8rWr8amBlGl4JfGsblmVmtkObiusP8iT+pyNiC7BZ0i5ke+j75Fm4pDbg9Yzs1uFTwOsl/Rw4Ko2bmRlTc/1BnrN6eiXNBr4CrAYGyPromVBEDAJza9oeJfsFYWZmNSrXHwwMPXf51Ha//mDcxJ/Ouvn/EfEY8CVJ1wG7RMSd2y0CM9uKOxcrr8r1B7U1/u15/cG4iT8iQtK1wEFp/P7ttmYzG5U7Fyu3qbj+IE+p53ZJL4+IH263tZrZmKoP7gEjDu51Le2qc3Q2FVqmtdC1tKuwv3eeg7sdwG2S7pV0p6QfSXKpx6wg7lzMipZnj//owqMws+dMxcE9K7c8e/wxxsPMCuDOxaxoefb4/5ss0Yusy4Z9gZ8CLy4wLrPScudiVrQJE39EHFQ9nrpoPqWwiMys8IN7Vm55Sj0jRMTtZAd8zcysCeXppO0DVaPTgEPIets0M7MmlKfGv3PV8Gaymv9/FROOmdmOq1GuyM5T4z9zKgIxM9uRNdIV2RPW+NMN0atvnLKbpFXFhmVmtmOZiu6W88pzcHde6qQNgIjYBOxRXEhmZttmeMsw1/zsGj5x0ye45mfXMLxluN4hbaWRrsjOU+MflrQoIh4AkPRCfAGXmW2jourbjVRCGU8jXZGdJ/H/P+BmSTeRXcT1atJN0M3M8igyOTdLp3ZT0d1yXnkO7l6XLtp6RWo6NSJ+VWxYZrYjKTI5j1dCaaTE30hXZOc5j/+twHcj4po0PlvSsRHxzcKjM7MdQpHJuZFKKBNplCuy8xzcPSMiHq+MpAO9ZxQXkpntaCrJudr2Ss7u1G7b5anxj/blkGc+MzOg2Pp2I5VQmoUixj9BR9JXgceA/0hNfwnMiYgTiw3tecuXL4/e3t6pWp2ZFaByVo+T89SRtDoilm/VniPxtwF/DxyVmr4NfDIiBseea/ty4jcz23ZjJf48Z/UMAh8uJCozM5tyec7qmQd8iOzGKzMq7RHxugLjMjOzguQ5q+cS4B6yO2+dCdwP/LDAmMzMrEB5Ev/ciDgXeDYiboqIPwW8t29m1qTynJb5bHpeL+lNZDdhmVNcSGZmVqQ8if+TknYF/gb4ArALcFqhUZmZWWHynNVzTRp8HHhtseGYmVnRtvlm69si9etzpaR7JN0t6TBJc9LNXX6enncrMgYzMxup0MQPfB64LiIOAA4G7ia7JuCGiNgPuAFfI2BmNqUKS/zpuMBrgHMBImIodfB2DHBBmuwC4NiiYjAzs63luYDrd4A/BBZXTx8RH59g1n2BjcB5kg4GVgPvB/aMiPVpmg3AnmOs92TSDV8WLVo0UZhmZpZTnj3+b5HtpW8GBqseE5kOHAKcFRHL0jwjyjqRdRQ0amdBEXF2RCyPiOXz5s3LsTozM8sjz+mce0fEGyex7IeAhyKiJ41fSZb4H5a0ICLWS1oAPDKJZZuZ2STl2eO/VdJB27rgiNgAPChp/9R0JPAT4GpgZWpbSfaLwszMpkiePf5XASdKug/4DdkN1yMiXppj3vcBl0hqBX4BnET2ZXOFpHcDa4HjJxW5mZlNSp7EP+lb5EREP7BVX9Bke/9mZlYHE5Z6ImItMBt4c3rMTm1mZtaEJkz8kt5P1jXzHulxsaT3FR2YmZkVI0+p591AR+VWi5I+DdxG1mGbmZk1mTxn9QgYrhofTm1mZtaE8uzxnwf0SLoqjR9L6obBzMyaT55umf9N0o1kp3UCnBQRfYVGZWZmhRkz8UvaJSKekDSH7D6791e9Nicifl18eGZmtr2Nt8d/KdBF1rladX86SuO/W2BcZmZWkDETf0R0ped9py4cMzMrWp7z+G/I02ZmZs1hvBr/DGAmsHu6PWLlFM5dgIVTEJuZmRVgvBr/e4BTgb3I6vyVxP8E8MWC4zIzs4KMV+P/PPB5Se+LCF+la2a2g8hzHv8XJL0EOBCYUdV+YZGBmZlZMfLcc/cM4AiyxH8tWTfNNwNO/GZmTShPXz3HkfWfvyEiTgIOBnYtNCozMytMnsT/dERsATZL2oXsHrn7FBuWmZkVJU8nbb2SZgNfITu7Z4CsW2YzM2tCeQ7unpIGvyTpOmCXiLiz2LDMzKwo413Adch4r0XE7cWEZGbW2Ia3DNO9ppu+9X0sW7CMziWdtExrqXdYuY23x/+v6XkG2Q3T7yC7iOulQC9wWLGhmZk1nuEtwxx98dH0rOthcGiQttY2OhZ2sGrFqqZJ/mMe3I2I10bEa4H1wCERsTwiXgYsA9ZNVYBmZo2ke003Pet6GBgaIAgGhgboWddD95rueoeWW56zevaPiB9VRiLiLuD3igvJzKxx9a3vY3BocETb4NAg/Rv66xTRtstzVs+dks4BLk7j7wJ8cNfMSmnZgmW0tbYxMDTwXFtbaxvt89vrGNW2ybPHfxLwY+D96fGT1GZmVjqdSzrpWNjBrNZZCDGrdRYdCzvoXNJZ79ByU0RMPFWdLV++PHp7e+sdhpkZ8PxZPf0b+mmf396wZ/VIWh0Ry2vbxzud84qIOF7Sjxh560UAIuKl2zlGM7Om0DKtha6lXXQt7ap3KJMyXo3//em5Od+ZmZmNarz++Nen57WTXbik+4EngWFgc0QslzQHuBxYDNwPHB8Rmya7DjMz2zZjHtyV9KSkJ0Z5PCnpiW1Yx2sjor2qzvRh4IaI2A+4IY2bmdkUGW+Pf+eC1nkMWf/+ABcANwKnF7QuMzOrked0TgAk7SFpUeWRc7YArpe0WtLJqW3PShkJ2ADsOcb6TpbUK6l348aNecM0M7MJ5LkD11vI+u3Zi6wv/hcCdwMvzrH8V0XEOkl7AN+WdE/1ixERkkY9nzQizgbOhux0zhzrMjOzHPLs8X8CeAXws4jYl+xuXP+bZ+ERsS49PwJcBRwKPCxpAUB6fmQScZuZ2STlSfzPRsSjwDRJ0yLie2S9dY5LUpuknSvDwBuAu4CrgZVpspXAtyYVuZmZTUqevnoekzQL+B/gEkmPAIMTzANZ7f4qSZX1XBoR10n6IXCFpHcDa4HjJxe6mZlNRp7EfwzwNHAaWQdtuwIfn2imiPgF2Y3Za9sfJSsXmZlZHeRJ/O8BLk/1+gsKjsfMzAqWp8a/M9kpmd+X9FeSRj390szMmsOEiT8izoyIFwN/CSwAbpL0ncIjMzOzQuS+gIvstMsNwKPAHsWEY2ZmRZsw8Us6RdKNZP3qzAX+3F0ym5k1rzwHd/cBTo2I5rmhpJmZjWnCxB8RH5mKQMzMbGpsS43fzMx2AE78ZmYl48RvZlYyTvxmZiXjxG9mVjJO/GZmJePEb2ZWMk78ZmYl48RvZlYyebpsaErDW4bpXtNN3/o+li1YRueSTlqmtdQ7LDOzutshE//wlmGOvvhoetb1MDg0SFtrGx0LO1i1YpWTv5mV3g5Z6ule003Puh4GhgYIgoGhAXrW9dC9prveoZmZ1d0Omfj71vcxODTyfvCDQ4P0b3AHo2ZmO2TiX7ZgGW2tbSPa2lrbaJ/fXqeIzMwaxw6Z+DuXdNKxsINZrbMQYlbrLDoWdtC5pLPeoZmZ1d0OeXC3ZVoLq1asontNN/0b+mmf3+6zeszMkh0y8UOW/LuWdtG1tKveoZiZNZQdstRjZmZjc+I3MysZJ34zs5Jx4jczK5nCE7+kFkl9kq5J4/tK6pG0RtLlklqLjsHMzJ43FXv87wfurhr/NPDZiFgCbALePQUxmJlZUmjil7Q38CbgnDQu4HXAlWmSC4Bji4zBzMxGKnqP/3PAh4AtaXwu8FhEbE7jDwELC47BzMyqFJb4JXUBj0TE6knOf7KkXkm9Gzdu3M7RmZmVV5F7/K8E3iLpfuAyshLP54HZkipXDO8NrBtt5og4OyKWR8TyefPmFRimmVm5FJb4I+IjEbF3RCwG/gj4bkS8C/gecFyabCXwraJiMDOzrdXjPP7TgQ9IWkNW8z+3DjGYmZXWlHTSFhE3Ajem4V8Ah07Fes3MbGu+ctfMrGSc+M3MSsaJ38ysZJz4zcxKxonfzKxknPjNzErGid/MrGSc+M3MSsaJ38ysZJz4zcxKxonfzKxknPjNzErGid/MrGSc+M3MSmZKumU2qzW8ZZjuNd30re9j2YJldC7ppGVaS73DMisFJ36bcsNbhjn64qPpWdfD4NAgba1tdCzsYNWKVU7+ZlPApR6bct1ruulZ18PA0ABBMDA0QM+6HrrXdNc7NLNScOK3Kde3vo/BocERbYNDg/Rv6K9TRGbl4sRvU27ZgmW0tbaNaGtrbaN9fnudIjIrFyd+m3KdSzrpWNjBrNZZCDGrdRYdCzvoXNJZ79DMSsEHd23KtUxrYdWKVXSv6aZ/Qz/t89t9Vo/ZFHLit7pomdZC19IuupZ21TsUs9JxqcfMrGSc+M3MSsaJ38ysZJz4zcxKxonfzKxkFBH1jmFCkjYCa6uadgd+VadwtkWzxAnNE2uzxAnNE2uzxAnNE2ujxPnCiJhX29gUib+WpN6IWF7vOCbSLHFC88TaLHFC88TaLHFC88Ta6HG61GNmVjJO/GZmJdOsif/segeQU7PECc0Ta7PECc0Ta7PECc0Ta0PH2ZQ1fjMzm7xm3eM3M7NJcuI3MyuZhk/8kmZLulLSPZLulnSYpDmSvi3p5+l5t3rHCWPG+jFJ6yT1p8cf1DnG/ati6Zf0hKRTG3GbjhNrQ23TFOtpkn4s6S5JX5M0Q9K+knokrZF0uaTWescJY8Z6vqT7qrZp3e+KI+n9KcYfSzo1tTXc5xTGjLXhPqcVDV/jl3QB8P2IOCf948wE/g74dUR8StKHgd0i4vS6BsqYsZ4KDETEZ+ob3dYktQDrgA7gL2nAbVpRE+tJNNA2lbQQuBk4MCKelnQFcC3wB8A3IuIySV8C7oiIsxo01iOAayLiynrGVyHpJcBlwKHAEHAd8BfAyTTY53ScWFfQQJ/Tag29xy9pV+A1wLkAETEUEY8BxwAXpMkuAI6tT4TPGyfWRnYkcG9ErKUBt2mN6lgb0XRgJ0nTyb7w1wOvAyqJtJG2aW2sv6xzPKP5PaAnIp6KiM3ATcDbaMzP6VixNqyGTvzAvsBG4DxJfZLOkdQG7BkR69M0G4A96xbh88aKFeCvJN0p6auN8tM0+SPga2m4EbdptepYoYG2aUSsAz4DPECW8B8HVgOPpUQA8BCwsD4RPm+0WCPi+vTyP6Zt+llJv1O3IDN3Aa+WNFfSTLJfT/vQmJ/TsWKFBvqcVmv0xD8dOAQ4KyKWAYPAh6sniKxW1Qj1qrFiPQt4EdBO9o/2r3WLsEoqRb0F+Hrtaw20TYFRY22obZr+oY8h+/LfC2gD3ljPmMYyWqySVgAfAQ4AXg7MAepaPomIu4FPA9eTlU76geGaaRriczpOrA31Oa3W6In/IeChiOhJ41eSJdeHJS0ASM+P1Cm+aqPGGhEPR8RwRGwBvkJWB2wEncDtEfFwGm/EbVoxItYG3KZHAfdFxMaIeBb4BvBKYHYqpwDsTXaMot5Gi/XwiFgfmd8A51H/bUpEnBsRL4uI1wCbgJ/RoJ/T0WJtwM/pcxo68UfEBuBBSfunpiOBnwBXAytT20rgW3UIb4SxYq18SJO3kv0sbAR/zMjSScNt0yojYm3AbfoA8ApJMyWJ5z+n3wOOS9M0yjYdLda7q5KpyOrm9d6mSNojPS8iq5lfSoN+TkeLtQE/p89phrN62oFzgFbgF2RndEwDrgAWkXXXfHxE/LpuQSZjxPrvZD/1ArgfeE9VjbIu0rGHB4DfjYjHU9tcGnObjhbrRTTeNj0TeAewGegD/oyspn8ZWemkD1iR9qjraoxYu4F5gMhKFX8REQN1CxKQ9H1gLvAs8IGIuKGBP6ejxdpwn9OKhk/8Zma2fTV0qcfMzLY/J34zs5Jx4jczKxknfjOzknHiNzMrGSd+azjKejk9ZZLzXitp9gTTfFzSUZOLzqz5+XROaziSFpP1FPmSUV6bXtX/TWmU9X1bMbzHb43oU8CLUh/m/yLpCEnfl3Q12RWxSPqmpNWp//OTKzNKul/S7pIWK7snwlfSNNdL2ilNc76k46qmP1PS7ZJ+JOmA1D5PWX/vP04d7q2VtHttoJLOktSbpjuzqv3lkm6VdIekH0jaWVKLpM8o67f9Tknvq445DS+XdGMa/pikiyGN2VEAAALRSURBVCTdAlyU3tP3U6y3Szq8an2np/jvkPQpSS+SdHvV6/tVj1vJRYQffjTUA1gM3FU1fgRZp3f7VrXNSc87kV0KPzeN3w/snpaxGWhP7VeQXTkLcD5wXNX070vDpwDnpOEvAh9Jw28ku/py91FircTRAtwIvJTnr9x+eXptF7JO/N5L1ofT9Jp5768sG1gO3JiGP0bWy+dOaXwmMCMN7wf0puFO4FZgZs1yv1f1/v+p8j798KPSgZRZo/tBRNxXNf7Xkt6ahvchS4SP1sxzX0T0p+HVZF8Go/lG1TSVftRfRda/ChFxnaRNY8x7fPrFMR1YABxI9iWxPiJ+mOZ/AiAdV/hSpJJN5Otq4OqIeDoNvwD4YuoaZBhYmtqPAs6LiKdqlnsOcJKkD5B10dAwnYRZfTnxW7MYrAxIOoIs2R0WEU+l0siMUeap7hdnmOzXwWh+UzVN7v8JSfsCHyTbs98k6fwx4pjIZp4vu9bOP1g1fBrwMHBwmv6ZCZb7X8AZwHeB1RFR+8VoJeUavzWiJ4Gdx3l9V2BTSvoHAK8oIIZbgOMBJL0BGO0mGruQJebHJe1JVnIB+CmwQNLL0/w7p+6Zvw28p9JVs6Q5afr7gZel4T8cJ6ZdyX5JbAFOICsvkZZ7krKbgDy33Ih4BlhF1i/8ebnfue3wnPit4aQ901vSQdB/GWWS64Dpku4mOxD8vwWEcSbwBkl3AW8nu9vTkzVx3kHWu+U9ZF0G35Lah8hKK1+QdAdZYp5BVnp5ALgztb+zal2fl9RLzc1GavwnsDLNewDp10BEXEfWXXGvpH6yXyEVlwBbyG4SYgb4dE6zUSm79eBwRGyWdBjZndXa6x3XtpL0QWDXiPj7esdijcM1frPRLQKukDQNGAL+vM7xbDNJV5Hd+u919Y7FGov3+M3MSsY1fjOzknHiNzMrGSd+M7OSceI3MysZJ34zs5L5P2az5mfxxtx4AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "| Test Loss: 0.652 | Test Acc: 76.04% |\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UIZTNW5wI5AP",
        "outputId": "a694ad81-b1ec-488b-9254-3b808b856dc2"
      },
      "source": [
        "trainSet"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[71.10745624492043,\n",
              " 75.56390981925162,\n",
              " 84.8527569519846,\n",
              " 87.99342105263158,\n",
              " 90.44486221514249,\n",
              " 91.93295742336073,\n",
              " 92.26190479178177,\n",
              " 92.09743110757125,\n",
              " 94.56453637072914,\n",
              " 92.67700502746983,\n",
              " 95.88815789473685,\n",
              " 95.55921052631578,\n",
              " 95.39473684210526,\n",
              " 96.05263157894737]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 332
        },
        "id": "XT0i8eFRzpc6",
        "outputId": "295b5787-be06-43d3-ffd9-56c9528e97a3"
      },
      "source": [
        "#print(trainSet)\n",
        "print(trainSet)\n",
        "trainSetnew=[i/100 for i in trainSet]\n",
        "validSetnew=[i/100 for i in validSet]\n",
        "plt.plot(trainSetnew,'r')\n",
        "plt.plot(validSetnew, 'b')\n",
        "plt.legend([\"Training Accuracy\",\"Validation Accuracy\"])\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.title(\"Training Accuracy vs Epochs\")\n",
        "plt.show()\n",
        "#print(trainSet)\n",
        "\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[61.31735600923237, 73.5902256087253, 84.68828326777408, 81.97055145313865, 88.22838350346214, 90.95394736842105, 88.31453637072914, 88.22838350346214, 90.20989976431194, 91.69016292220668, 97.11779450115404, 94.23558900230809, 95.88032584441336, 95.05795742336073]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZzV8/7A8de7aV+0Cy2KCGlTCtnjCilRKoqE6NqSa732y+/i2uoiN0pCi61uSN1CtooWM6mEpDSppLSvM/P+/fE+U9M0y5mZ8z3fM837+Xicx5zle77f9zkz831/P7uoKs4550quUmEH4JxzLlyeCJxzroTzROCccyWcJwLnnCvhPBE451wJ54nAOedKOE8ErshE5CMRuSrW27oDi4j0FZEvw47D7c8TQQklIluy3DJEZHuWx1cUZF+qer6qvhbrbQtDRBpFPs/QoI5xIBCRMyPf05Zst5PDjs3FX+mwA3DhUNXKmfdFZBlwrapOy76diJRW1bR4xlZEVwJ/Aj1EZKCq7ozXgUUkSVXT43W8GPhNVeuFHYQLn5cI3D4iV4qpInKXiKwGXhWR6iLygYisFZE/I/frZXnPdBG5NnK/r4h8KSJPRbb9RUTOL+S2jUTkcxHZLCLTROQFEXkjj9gFSwT3AbuBi7K93kVEkkVkk4j8LCIdI8/XEJFXReS3SBwTssaXbR8qIo0j90eKyFARmSQiW4GzRORCEfk2cowVIvJQtvefKiIzRGRD5PW+InKiiKwRkaQs210iIik5fMZ2IrI627ZdRWR+5H5bEZkTOf4aEXkmt+8rL5Hf0z9F5JvIvv4rIjWyvN5ZRBZGPsd0ETk2y2v1ReS9yN/LOhF5Ptu+c/t99xWRpZHf9y8FLZm6wvNE4HJyCFADOBzoj/2dvBp53ADYDjyf67uhHfADUAt4EhgeOUkXdNvRwDdATeAhoE8+cZ8K1APGAm8Be9oiRKQtMAq4A6gGnA4si7z8OlARaAocDDybz3Gyuhx4DKgCfAlsxZJRNeBCYICIXByJ4XDgI+DfQG2gJZCsqrOBdcBfsuy3TyTefajq15FjnJ0thtGR+4OBwap6EHBk5HsorCuBfsChQBowJPI5jgbGAAMjn2MS8L6IlI0kqA+A5UBDoC72+8iU4+9bRCpF9n++qlYBTgGSixC7KwhV9VsJv2EnxHMi988EdgHl89i+JfBnlsfTsaolgL7AkiyvVQQUOKQg22IJJw2omOX1N4A38ojrFWBC5P7JWKng4Mjj/wDP5vCeQ4EMoHoOr/UFvsz2nAKNI/dHAqPy+W6fyzwucA8wPpft7gLejNyvAWwDDs1l20eBEZH7VbDEcHjk8efAw0CtfOI6M/K5N2S7Vcrye3o8y/bHRf4ukoD7gbeyvFYKWBnZ58nAWqB0Lt9nbr/vSpHjXwpUCPt/oqTdvETgcrJWVXdkPhCRiiLyHxFZLiKbsJNNtazVE9mszryjqtsidysXcNvDgPVZngNYkVvAIlIB6A68GdnXTOBX7GoZoD7wcw5vrR85zp+57Tsf+8QUqbr5NFItshG4Abv6zSsGsCR3UeTK+DLgC1Vdlcu2o4FLRKQccAkwT1WXR167BjgaWCwis0WkUx6x/6aq1bLdtuby2ZYDZSKf5bDIYwBUNSOybd3IZ1yuubcr5fj7jhy3B/Z9rRKRD0XkmDxidzHkicDlJPuUtLcDTYB2alUOp0eez626JxZWATVEpGKW5+rnsX1X4CDgxUgd+mrsxJRZPbQCqyrJbkXkONVyeG0rdtUKgIgcksM22b+r0cBEoL6qVgVeYu/3lFsMqOpKYCZ2Yu+DVVflSFUXYSfi89m3WghV/UlVe2FVXE8A70SSS2Fk/b4bYCWsP4DfsGpCYE/bTH2sVLACaCAiBe6IoqpTVPVcrJS2GHi5kHG7AvJE4KJRBWsX2BBpMHww6ANGrnDnAA9F6p5PJlvjbzZXASOAZljVVUugPdBCRJoBw4GrRaSDiJQSkboickzkqvsjLIFUF5EyIpKZ6FKApiLSUkTKY+0U+amClTB2RNolLs/y2pvAOSJymYiUFpGaItIyy+ujgDsjn+G9fI4zGrgVS8pvZz4pIr1FpHbkKn1D5OmMKOLOSW8ROS6SjB8B3lHrFfUWcGHkuyyDXSjsBGZgbTqrgMdFpJKIlBeR9vkdSETqiDXmV4rsa0sR4nYF5InAReM5oAJ2NTgLmByn416B1Tmvw+rFx2EniX2ISF2gA/Ccqq7OcpsbifUqVf0GuBprCN4IfMbeq9o+2NXuYuB3rBEUVf0ROwFOA37CGoPz81fgERHZDDxAlsZaVf0VuAA7ca7HGkNbZHnv+EhM47NVieVkDHAG8Imq/pHl+Y7AQhHZgjUc91TV7bns4zDZfxzBpVlefx1rB1kNlAduiXyOH4DeWKP3H1iCvkhVd0USxUVAY6xqLhWr8slPKWAQVtpYH/lsA6J4n4sBUfWFaVzxICLjgMWqGniJJCwi8jNwveYwpiPOcUzHGuZfCTMOFx9eInAJK9K//shIVU5HoAswIey4ghK5Glfgk7BjcSWLjyx2iewQrK68JlbFMEBVvw03pGBErsCPA/pE6vedi5vAqoZEZATQCfhdVY/P4XXB6jAvwPpM91XVeYEE45xzLldBVg2NxBqucnM+cFTk1h/wScKccy4EgVUNqernItIwj026YKMyFZglItVE5NA8BtEAUKtWLW3YMK/dOuecy27u3Ll/qGrtnF4Ls42gLvuOXEyNPLdfIhCR/lipgQYNGjBnzpy4BOiccwcKEVme22vFoteQqg5T1Taq2qZ27RwTmnPOuUIKMxGsZN8h7PUizznnnIujMBPBRODKyBS0JwEb82sfcM45F3uBtRGIyBhsWtpaIpKKzU9TBkBVX8LmML8AWIJ1H706qFicc87lLsheQ73yeV2BG4M6vnPOuegUi8Zi55xzwfFE4JxzJZwnAudc4vjtNxg6FH78MexIShRPBM65xPDBB9C8Ofz1r9CkCZx2GowcCVu35vtWVzSeCJxz4dqxA269FS66COrVgy++gMcfh99/h6uvhkMOgeuug1mzwNdPCYQnAudceL7/Hk46CYYMsWQwaxaceircdRcsXmxJoVs3GD0aTj4Zjj8ennkG1q4NO/IDSrFboaxNmzbqcw05V8ypwogRcMstULEivPoqdOqU+/abNsFbb8Hw4ZYsSpeGzp3hmmvgvPMgKSl+sWe3cSN8843FtWIFHHYY1K9vpZvMW9Wq4cUXISJzVbVNjq95InDOxdWGDXD99XZi79ABRo2yk2e0Fi60JPL661YyqFsXrroK+vWDI48MLm6AjAwrxcyaBTNn2s9Fi/ZWWdWqBevW7V+FVaXKvokh85Y1YVSrBiKBhe6JwDmXGGbMgMsvh9RUePRRuPNOKFXIGupdu6yBefhwmDzZTtJnnmmlhEsusZJGUa1fD19/vfek//XXVjoBqF7dqrVOOsmqrU480U7mu3bBqlX2GVNTrZSQeT/ztmqVxZtVxYr7J4fsCaNGjUInC08EzrlwpadbA/CDD0KDBjBmDLRrF7v9r1wJr71mJYWff7aqmF69LCm0bh3dyTMtDRYssBN+5hV/ZjfWUqWgWTM74Wee/I8+uvBX8Glp+yaL7LcVK6wrbXr6vu8bPNiq0wrBE4FzLjypqdCnD0yfbifnoUODqzPPyIDPP7dSwjvvWI+k5s2t2qh3b6hZc++2v/++bxXP7Nl7u6rWrr33pH/yydCmDVSuHEzMuUlPhzVr9k0QZ51lCakQPBE458Lx3//aSXjnTnjhBbjyykDrwfexcaOVPIYPhzlzoGxZ6NIFypSxE//SpbZd6dLQsuXek/5JJ0GjRvGLM048ETjn4mvHDrjjDnj+eWjVCsaOtaqUsMyfb9VGb7xhCeHkk/ee9Fu3hgoVwostTjwROOfiZ9Ei6NkTvvsOBg2C//s/KFcu7KhM5vnuALvaj0ZeiSDMNYudcwcSVXj5ZRg40OrTJ02C888PO6p9lcAEEA0fWeycK7o//4Tu3W18wKmnWlVMoiUBlytPBM65ovnyS2jRwhqGn3zS+vQfckjYUbkC8ETgnCuc9HR45BE44wxrgJ0xwxqICztAzIXG2wiccwW3YoX1y//8c/v54os2jYIrljwROOeis307pKTYAKx//AN277Z5gvr0CTsyV0SeCJxz+9u6FZKTYd48mDvXbt9/v3fKg7Zt4c03oXHjcON0MeGJwBVfu3fbHDOHH+7dAoti82Y76Wee8OfNs7UAMidFq1PHBl117QonnGD369Xz7/wAEmgiEJGOwGAgCXhFVR/P9vrhwAigNrAe6K2qqUHG5Io5VZsu4PXXbbTq2rVwzDFwxRU2j03Q0xAXdxs3wrff7nul/+OPewdaHXaYnei7d7efrVvDoYf6Sf8AF9jIYhFJAn4EzgVSgdlAL1VdlGWbt4EPVPU1ETkbuFpV86xw9JHFJdSvv9r0AK+/bler5crZwiTt2sHEidZoCTZlwOWXQ48ecPDB4cYctg0b9j3hz5sHP/209/V69fae7DOv9L3b5wErlCkmRORk4CFVPS/y+B4AVf1nlm0WAh1VdYWICLBRVQ/Ka7+eCEqQTZvg3XetQXL6dHvutNOscbJ7d5v7PdOKFTbB2OjR1qCZlATnnGNJoWvXktGj5Y8/4JNPYNo0+PRTWLJk72uHH77vCf+EEzxRljBhJYJu2En+2sjjPkA7Vb0pyzajga9VdbCIXAK8C9RS1XW57dcTwQEuLQ2mTrWT/4QJNnnZUUfZyb93b5sVMj8LF1pCGD0ali2D8uWt9HDFFdCxo/V5PxBs22Zr+n78sZ38v/3Wnq9a1RZoOekkO+GfcIKtnOVKtEROBIcBzwONgM+BS4HjVXVDtn31B/oDNGjQoPXy5csDidmFRNUaK0eNsqv6NWtsJaaePS0BtGtXuDpqVevqOHo0jBtnV8zVq1tp4vLLrXRRnAY/paVZFc+0aXabMcNWwypbFk45xUpA55xjV/ylvR+I21fCVg1l274ysFhV6+W1Xy8RHEBSU60L4uuv21V82bK2gHmfPnDBBbG9ct+9206eb75pJY2tW62OvFcvSwotWiReg6gq/PDD3iv+Tz+1xl6wqZ07dLAT/6mnQqVK4cbqEl5YiaA01ljcAViJNRZfrqoLs2xTC1ivqhki8hiQrqoP5LVfTwTF3ObN8N57dvL/5BM72Z1yip38L7vMSgJB27oV3n/fksLkyXalfdxxlhAuvzy66qegrFq198Q/bZp1jwVo2BDOPddO/GedZStoOVcAoa1HICIXAM9h3UdHqOpjIvIIMEdVJ0aqj/4JKFY1dKOq7sxrn54IiqH0dDu5jRoF48db3fYRR+yt9w9zUNK6dfD221Z99MUX9tzJJ1t7QvfuwTeobtoEn3229+S/MHKdVKPG3iv+c86x78u5IvCFaVw4MjJsNsohQ+xKt1o169bZp4+VAhKtKubXX/f2PJo/33oeVa9uP7PeSpXa/7lob1nf+9tv8PXXlijLl4fTT9974m/Roni1X7iE54nAxd/WrbY+7XvvWU+da6+FCy+0E15xsGCBLX6+dq2dqIO4VasGZ59tJ/6TTy4+340rlnyFMhdfqam2SPi338Izz9iKVYl29Z+f44+3m3MlgCcCF1uzZ1sS2LzZGmQvvDDsiJxz+fBKSBc7b71l9dzlyln/fU8CzsVEejr072//VkHwROCKTtVWqurRwwYzff21V6s4F0NffgkvvwxBjaX1qiFXNNu3Q79+NhPolVfCsGFWInDOxcy4cVChgo23DIInAld4q1bBxRdbu8Djj8Oddxa/RmHnElxamnVg69QJKlcO5hieCFzhJCfDRRfB+vXWRfTii8OOyLkD0vTp1ou5R4/gjuFtBK7gJkyA9u3t/ldfeRJwLkDjxllJ4IILgjuGJwIXPVV44gm45BJrDP7mG2jZMuyonDtg7d5tBe7Ona2NICieCIqDGTPg6adtCoSw7NwJV18Nd99tZdTp020JQ+dcYKZNs9rXIKuFwBNB4vv9dxug9be/2QyUZ50FI0bYZGXxsnatTYD22mvw8MM2F0+QlyfOOcCqhapWhfPOC/Y4nggSmSr89a920p80yU7CK1fCNddAnTo2l/6kSdatICgLFkDbtrYgyrhx8MAD3jPIuTjYudOa4y6+OPge2Z4IEtm4cbZm7yOPwPnnw/3320Ils2ZZMpg61Ubv1q1r8/nMnWvJI1YmTbJZQnfutMXhL7ssdvt2gdm0yaoSFi7Mf1uXuKZMsXWIgq4WAk8EiWv1arjxRlum8fbb9z4vYs89/7xNYzxhgi25OHQotGkDTZvCP/9ZtPYEVXj2Wese2rixNQqfeGLRP5OLi9GjbbaP666zmcBd8TRunC1Lcc45wR/LE0EiUrWJRbZts3r53NafLVvW2g/eeccSx3/+AzVrwr33Fr49YdcuuP56GDTIyqRffGFLOrpiY+RIa8KZOdPWAnLFz/btMHGiddArUyb443kiSERvvGEzdz72GDRpEt17qle35PHFF7B0aeHaE9ats1apl1+Gv//dVu7ytXCLle+/t6meHn7Ylji4807YsCHsqFxBTZoEW7bEp1oIAFUtVrfWrVvrAS01VbVqVdX27VXT0oq2r4wM1VmzVG+8UbVmTVVQPfhg1VtvVZ0zx17P9P33qo0bq5Ytq/r660U7rgvNXXepJiWprlqlOm+eaqlSqjffHHZUrqC6d1etXVt19+7Y7RNbIjjH86qvUJZIVK3xd/p0Wyoxlmv57toFH31ki8a//749PvZYWzbyyCOtNFGunLU5nHxy7I7r4iY9HRo0gBNOsF8xWDPTSy/BvHm2+qVLfFu22FLZffvCiy/Gbr95rVDmVUOJZMQIO1k/8UTsF3TPqz2hRw87g3zzjSeBYmzqVOs/0Lfv3ucefdQaHG+6KbYdylxwPvjA2gjiVi2Er1mcOH791aZtaN0aPv44fguX//KLdUft1AmqVInPMV0gevbcmwyy9jsfPtyWjB41ygqALrF17WrtPCtWQFJS7PbrJYJEp2qNuhkZViqIVxIAaNTIGpI9CRRrf/5ptXpXXLH/4KOrr7Yex3fcYf3SXeLatMkqBbp3j20SyI8ngkTwn//YpCJPPWUnZucKaNw4G/eXtVooU6lSNuzk99/hoYfiHZkriP/+136P8awWgoATgYh0FJEfRGSJiNydw+sNRORTEflWROaLSIATrSaoX36xeYTOOcf67ztXCCNHQrNm0KpVzq+3aWP9Af79b/juu7iG5gpg3DioXx9OOim+xw0sEYhIEvACcD5wHNBLRI7Lttl9wFuq2groCcSwjbwYyMiwZR6Tkqwi1+fwcYWQOXagb9+8/4Qee8wmMPOG48T055/wv//ZTC7xrB2GYEsEbYElqrpUVXcBY4Eu2bZR4KDI/arAbwHGk3heeMG6ij77rPXaca4QRo60a4krrsh7u5o1bfaRzz+HMWPiEporgAkTbP2BeFcLQYC9hkSkG9BRVa+NPO4DtFPVm7JscyjwP6A6UAk4R1Xn5rCv/kB/gAYNGrRevnx5IDHH1ZIl0Lw5nHkmfPihlwZcoaSl2TVEmzY2JUF+0tOt2mHlSli8GA46KP/3uPjo2BF++slODUGcDhK511AvYKSq1gMuAF4Xkf1iUtVhqtpGVdvUrl077kHGXHq6lePLlbPpHDwJuEKaOhVWrcq5kTgnSUlWEF292ia1dYnhjz+sv8hll4VzOggyEawE6md5XC/yXFbXAG8BqOpMoDxQK8CYEsPgwbbW75AhNoW0c4U0cqRV+XTqFP172ra13sqDB/tU1Ynivffs+jCMaiEINhHMBo4SkUYiUhZrDM5eeP0V6AAgIsdiiWBtgDGFb/Fim9Ctc2fo3TvsaFwxljl24PLLbeB4QfzznzZ0xBuOE8O4cXD00eFNAxJYIlDVNOAmYArwPdY7aKGIPCIinSOb3Q5cJyIpwBigrxa3oc4FkZZmZfiKFW3sgFcJuSIYO9amjIq2WiirWrWsF9H06XYScuFZs8Z+Dz16hHdK8Ckm4umJJ2zx9zFjbD4A54qgXTubkyYlpXAnkPR0qyZavdoKqj64PBwvvGAlswULbF2poCRyY3HJsXChrfd76aXhVQS6A8aiRTZH4NVXF/4qMrPh+Lff4B//iG18LnrjxlkCCDIJ5McTQTzs3g1XXWWjeYYO9SohV2SZC9flN3YgPyedZGMan33WBqa5+Fq5Er78MvxrQ08E8fDEE7aw/NChcCB0f3WhSkuzZSUuuMDmrS+qxx+HypXh5pu94Tje3n7bvnNPBAe6lBTrsN2zp1ULOVdEBR07kJ/atW3dgo8/tuUqXPyMGwctW1qPoTB5IgjSrl1WJVSjhk3/6FwMZI4duPDC2O3zhhvshDRokK2Q5YK3fLktBRJ2aQA8EQTrscesRDBsmP3nOldE69fvXXegoGMH8pLZcJyaan+2LnhvvWU/L7ss3DjAE0Fw5s61/6grr7TBY87FQFHGDuTnlFOsAPv00/DDD7Hfv9vXuHFw4olwxBFhR+KJIBg7d9p/VJ068NxzYUfjDiAjR9pchS1bBrP/J56w8Y633OINx0FassSuFROhWgg8EQTj4Ydt3MArr0D16mFH4w4QCxfC7Nn5rztQFHXqWN+G//0Pxo8P5hgusaqFwBNB7H39tV1WXXMNnH9+2NG4A0isxg7k569/tVLHbbfBtm3BHqukGjfOquLq189/23jwRBBL27fb5VrdulbR6khOhp9/DjuKwpkxwxpnE0Hm2IELL4zN2IG8lC5tndx+/RX+7/+CPVZJtHgxzJ+fONVC4Ikgtu6/337LI0bYKOIY+cc/bFbCJUtitsu4GD3aGsNOPdUWTi9OJk+G9u1teue0tLCjsaqa1auDaSTOyWmn2eS4//qXLZbiYmfcOKva69Yt7Ej28kQQK199Bc88Yx2yzzknprt+/327gjjlFCgu8+0NHmxVGG3a2HTJffrYEs3FwW+/WbyHHAIzZyZGd8qRI23G0AsuiN8xn3zS1k669VZvOI4VVUsEp58Ohx0WdjRZqGqxurVu3VoTzrZtqkcdpdqwoeqmTTHd9e7dquXLq158se2+UiXVyZNjeoiYyshQvfdeVVDt2lV1+3bVl16yx48/HnZ0+UtLUz3jDNWKFVW//161Tx/VUqVUv/oqvJjWrVMtW1b11lvjf+xnnrHf3YQJ8T/2gWj+fPs+X3wx/scG5mgu59XQT+wFvSVkInjgAfsqp02L+a4XLbJdv/aa6m+/qbZooVq6tOqoUTE/VJHt3q167bUW73XX2UlV1ZLDZZepJiWFe0KNxoMP7v2+VVU3brQE3KiR3Q/DCy9YTN9+G/9j79ql2rSpfQfbtsX/+Aeav//dLizWrIn/sT0RBOnHH+1y7YorAtn96NH2W0pJsccbNqiefbY99+STdpJNBNu3WwkAVO+7b/+4NmxQPeII1fr17Qo3EX38saqI6lVX7fv8l1/aP2/25+PlxBPtAiAs06fb7/WBB8KL4UCQkaHauLHqOeeEc3xPBEHJyFD9y19UDzpIddWqQA5x112qZcqo7ty597kdO1R79LDf3sCBqunpgRw6ahs2WHUKqA4enPt2s2fbZ+ncOXESWKY1a1QPOUT1mGNUN2/e//XMQt/YsfGNa8ECO+6zz8b3uNn16qVarpzqkiXhxlGczZ1rv8uXXw7n+J4IgvLWW/YVDhkS2CHOO0+1Zcv9n09PtzpjsKSwY0dgIeRp9WqLr3Rp1TffzH/7556zmJ97LvjYopWebvm8fPm9Ja/sdu9WbddOtVo11V9/jV9sf/ubfbdhVCVktXKlauXKqp06hRtHcXbnnfa7DKtE7IkgCJs2qdatq9qqlZ0lAnLIIblXSWRkWPUQWHVRvOuwf/5Z9cgjrWH1o4+ie09GhpUIypSxEkIi+Oc/7Tv8z3/y3m7JEjsZnnHG3vaPIO3erVqnjmqXLsEfKxr/+pd9T++/H3YkxU9GhrWznH9+eDEUKREAFwGl8tsuXreESQS3324VyrNmBXaI1as1qmqBUaPsSqNFC2tQjofkZEtSNWoU/CtYt87aCo44wqqVwvTVV9aI3aNHdNVVr76qcesB9cEHdqzx44M/VjR27VI99lj7vW3fHnY0xcusWfa7HDkyvBiKmgjeAH4GngSOyW/7oG8JkQjmz7ezR//+gR5myhT7DX3ySf7bTp5sXUsbNlT94YdAw9LPPrNmkXr1rFdTYWSegC+7LLz2gsyEdOSR0ZemMjJUu3WzxDtnTrDxdeumWru2nYATxccf299knz5W551obT2J6rbbrE9JmBc+Ra4aAg4CrgdmATOB/kCVaN4b61voiSA9XbV9e9VatVT/+CPQQz3xhP2Goq1TnD3bThy1agVXUJkwwRoNjzmm6HXljz9un++ll2ITW0FkraIq6Al93TqrFWzSRHXLlmDi++MPO3EMHBjM/ovippusMAyqhx6qes01qu+9F/MhNAeM9HT7e+ncOdw4YtJGANQEBgLLgI+An4Cb83lPR+AHYAlwdw6vPwskR24/AhvyiyP0RJBZNzBiROCHuvxyu2ItiJ9+sqJ7xYqqH34Y23iGD7dulO3axSYHpqdbY3i5crk30gYls9E6r15OefnkEzsZXn99bOPK9PzzFl9ycjD7L6o1a6yao3t3Kx2CJa5zz7Xv1nsX7fXFF/b9RNOZIkhFrRrqDIwHvgPuAA6OPF8RWJbH+5IiVUpHAGWBFOC4PLa/GRiRXzyhJoJ16+xy+5RT4tJn87jjVC+6qODvW71a9YQTrOrl1VeLHkdGxt6r9/POy7l7ZWGtWWNXlU2axHa/ecnsxtqlS9GqNu64w76T//43drFlatMm595iiWjXLkuMt99uJUWbSMF+p4MG2WuJVL0VbzfdZD3Swi4xFTURvAacnstrHfJ438nAlCyP7wHuyWP7GcC5+cUTaiK4/no7u8bh8nXbNjvUffcV7v2bNtnVGaj+3/8V/oSXnm7/zGB9ybOOZ4iVTz+1ksaVV8Z+39llDmxr0KDo3fh27LBOY7VqxbaR/rvvNOG62BbEkiVW0vrLX6yUANvST9YAACAASURBVFZq6NbNShFhd4WNp7Q061Rx6aVhR1L0RNAIKJ/lcQWgYRTv6wa8kuVxH+D5XLY9HFgFJOXyen9gDjCnQYMGwX5bufn6a6sLuO22uBxu9mz77bzzTuH3sXOnDXgGuyopaJfHXbusURBUb7452ELQQw9p4L0qsk51MWNGbPa5aJFqhQpWUorV95M5duD332OzvzBt3my9nq691kp+YP9GbduqPvLIgd/g/Mkn9pnfeivsSIqeCOYAZbM8LgvMjuJ9BUkEdwH/zm+fGlaJIC3N6loOOyxunfVfecV+Oz/9VLT9pKfbiQXsiizabn9bt6pecIG979FHg/9nTUtTPfNMa9sobE+k/GROfvfEE7Hd74svapHaG7LKHDtw8cVF31eiyciwE/8jj1giyNrgfO21ljDiVT0YL9dfb735tm4NO5KiJ4LkHJ5LieJ9UVcNAd8Cp+S3Tw0rEWS23I0bF7dD3nSTDV6K1VXm00/bRzjjDNU//8x723XrrBmkVKn8B1nF0sqV1uvp+ONjP8FZSoo1SnfsGPuSTUaGjbgtV856FhdF5tiBkjDbZ2aDc7du+zY4X3xx+PXpsbB7t1Ub9uwZdiSmqIlgKtA5y+MuwMdRvK80sDRStZTZWNw0h+2OifREkvz2qWEkglWrVKtWtQr3OJZhTzvNTsaxNHq0NZI2a6aamprzNqmpNttk2bKq774b2+NHY/Jk+6uM5RCNzZut4fKww4Krblmzxq7kjz++aIOtEnHsQDzs3GnVKLfcYr//p58OO6KiyxwHlCgDAouaCI7Exg/8CqyINOo2zu99kfdeEOkW+jPw98hzj2RLLA8Bj0ezPw0jEfTubWfFoEdpZZGRYVdIf/1r7Pc9daqVNBo02L8KZvFie75KlegGsQXl7rvtL3PMmNjs78orrXQzfXps9pebSZN0z0SAhZHIYwfi6Ywz7O8wwJlb4qJfP/s/TpRR2EVKBLr3hF0ZqBzt9kHd4poIPv1U98yrHEdLl2pUc98U1rx5dvVao8beRtNvvrFi7MEH2+th2rXLSkNVqhS9jWTkSPsuH344NrHl5+ab7XiFWTzo3//WhB47EC///W9sLwTCsHOnTVDYp0/YkexV5EQAXAjcCTyQeYvmfUHc4pYIdu60iVUaNYr7ihzvvWe/ma+/Du4YP/9si6pVqGAnyUqV7KMW9cQbK8uXq1avbm30hZ1ZddEia3w+66z4TBKnan8qTZtal8GCVkO1bm3dUUu69HTVo4+2sRTFtUdRZlvPBx+EHcleeSWCfNcsFpGXgB7YgC8Buke6ex7Ynn0Wvv8ehgyBChXieuiUFChVCo4/PrhjHHGELbN8/PHw4INw5JH2uHHj4I5ZEA0a2Dq98+bBnXcW/P3bt8Nll0GlSvDmm5CUFPMQc1Shgh1v/Xq47rro1/r97juYOzd+i9MnslKl4LbbbH3uL78MO5rCGTcOqleHc88NO5Io5ZYhMm/A/Gw/KwNf5Pe+oG5xKREsX26XkiH14evSxUZoxsPmzapDh+bfkygsAwdqoRrc+ve3902ZEkxc+clc6zfa6r3bb7eG/LVrg42ruNi6VbVmzcSZgrsgtm+3as1+/cKOZF8UsbH4m8jPWcBhQDlgSX7vC+oWl0TQtaslgmXLgj9WDho2tGmRndXQtWlj9a3R/jrGjLG/7HvuCTa2vKSn25KEFSpYI3xedu2yNpuuXeMTW3Fx33021uDHH8OOpGDGjw/3IiQ3eSWCfKuGgPdFpBrwL2Ae1tVzdCxLJQnlww9h/Hh44AE4PP41YBs2wLJl0LJl3A+dkMqWhbFjISMDevaE3bvz3n7JEujfH9q3h0ceiU+MOSlVCl57zaqKrrgCdu3KfdspU2DNGq8Wyu7GG6FMGaulLU7GjYNateDss8OOpAByyxCWQChFloFeWGmgal7vCfoWaIlg2zZrMT322GAm1YnCZ5/Z1cSkSaEcPmGNG2ffy5135r7Njh3WuFyjRnyXk8xLZsP/3Xfnvs2ll5bMsQPR6NfPSlUBz/geM1u3WseLoGalLQoKWyJQ1QzghSyPd6rqxuDSUsj++U/45Rd44QW7FA1BcrL99BLBvi67DG64AZ58EiZPznmbO++0xuWRI6F+/biGl6uuXeHaa+GJJ+Czz/Z/fd06mDgReve2q1+3r0GDrOH/pZfCjiQ6H34IW7dCjx5hR1JAuWWIzBvwFHApUY78DfoWWInghx9sNM8VVwSz/yj162dXh8W121yQtm1Tbd7cxjusXLnva5n1snGaE7BANm+2rrr166uuX7/va5ljB+K9HkNx0rGjdcctbDfieLr0Uos1Xt2VC4IithFcD7wN7BSRTSKyWUQ2BZeaQqBqFZLly8NTT4UaSnKylQZEQg0jIVWoYPWv27fD5ZdDero9v3w5XH01tGkDjz8ebow5qVzZupSuWgUDBuzbpXTkSGjVCpo3Dy28hDdoEKxeDaMTvGVy82YrEXTrFr/uyrGSbyJQ1SqqWkpVy6rqQZHHB8UjuLh5+22YNg0eewwOOSS0MHbvhoULoUWL0EJIeMccA0OHWjXLP/5h31nPntaYPG5caDV6+TrxRHj4YYvxjTfsOR87EJ1zzrFE+cwz0Y/LCMPEibBjRzGsFoKoqoZOz+mW3/uCusW8amjTJpuNrFWr0MtzmQuSvPFGqGEUC337WtfCTp00YeZ7z09amk0mWKWKjeweNMjHDkQrc6qQROuSmWnHDtXGjW1ywzgsXlgo5FE1VDqKXHFHlvvlgbbAXKA4dY7K3UMPWZn9vfdCL8+lpNhPLxHk7/nnYdYs+OADq27p3j3siPKXlASvv26/39694eef4aKLrKuhy1uvXnDPPfD00/CXv4Qdzf6eftq6Lk+ZYl2Hi5toqoYuynI7Fzge+DP40OJg/nwYPNjmAmjXLuxoSE6GcuWgSZOwI0l8lSrBf/8L995r/4TFxeGHW9XWzJnw++9eLRStsmXh5pvhf/+zKrVE8uuv8OijcMkliZmkoiFawEo3ERFgoaoeF0xIeWvTpo3OmTOn6DvKyIDTToMff4TFi6FmzaLvs4j+8hfrTjh3btiRuKBdfTV8+in89JN3G43W+vXWLfiyy+DVV8OOZq9u3WDSJDuNNGgQdjS5E5G5qtomp9fyrRoSkX8DmdmiFNASG2FcvL32GsyYASNGJEQSULUSwUUXhR2Ji4fhw220sSeB6NWoYQl02DD4v/+DQw8NOyKYOhXefddKBImcBPITTW3WHKxNYC4wE7hLVXsHGlXQ1q2DO+6weQiuuirsaADrHrd2rbcPlBSlSllvZVcwAwdCWpqN+Qzbrl1WXdW4Mfztb2FHUzTRNBa/A+xQ1XQAEUkSkYqqui3Y0AJ07702qc+LLyZMy46PKHYuf40bw8UXWzvLPfdYW1FYnn0WfvjBqoXKlQsvjliI5iz4MZB1Qv4KwLRgwomDr7+Gl1+GW29NqFE8mT2GEigk5xLS7bdbe8Frr4UXQ2qqjWPp0gXOPz+8OGIlmkRQXlW3ZD6I3K8YXEgBSk+3voaHHmrdRhNIcjI0bAjVqoUdiXOJ7ZRToG1buyLPHF0eb7ffbsd+7rlwjh9r0SSCrSJyQuYDEWkNbA8upAANHQrffmt/QVWqhB3NPlJSvH3AuWiI2Il4yRIbRxJvH38Mb71lVVMNG8b/+EHIt/uoiJwIjAV+w5aqPATooaqhdHIsdPfR1autg367djbqI4Em89m2zfLS/fcnXEHFuYSUlmbtBQ0awOefx++4u3ZZO97OnTYdTHFq8C9S91FVnS0ixwCZw5x+UNV8lgdJQM8/bxOBPP98QiUBgAULbFiDlwici07p0taD6LbbYPZsm8spHoYMsaXM33+/eCWB/ESzeP2NQCVVXaCqC4DKIvLXaHYuIh1F5AcRWSIid+eyzWUiskhEFopIcPMLPvywXTocfXRghygs7zHkXMFdcw0cdFD8Rpb/9pudRjp1stuBJJo2gutUdUPmA1X9E7guvzeJSBK2qM35wHFALxE5Lts2RwH3AO1VtSkwsACxF0xSUkJMI5GTlBT7gz5Q6hudi4cqVWxZ0nfesanIg/a3v9lst4MHB3+seIsmESRFppUA9pzgo5nsty22yP1SVd2FtTN0ybbNdcALkeSCqv4eXdgHluRkqxZKsBor5xLeLbfY/82QIcEeZ/p0GDMG7roLjjgi2GOFIZpEMBkYJyIdRKQDMAb4KIr31QVWZHmcGnkuq6OBo0XkKxGZJSIdown6QJKRYXPfefuAcwWXOffQyy/DxoAW0d29G266yUrsd+dYwV38RZMI7gI+AW6I3L5j3wFmRVEaOAo4E+gFvCwi+/WkF5H+IjJHROasXbs2RodODEuXwpYt3j7gXGENGmSrg73ySjD7f/556yH03HO2St6BKJppqDOAr4FlWHXP2cD3Uex7JZB1CfF6keeySgUmqupuVf0F+BFLDNljGKaqbVS1Te3ataM4dPHhaxA4VzStW8MZZ1jd/e4Y92dctQoefNBGD3fuHNt9J5JcE4GIHC0iD4rIYuDfwK8AqnqWqj4fxb5nA0eJSCMRKQv0BCZm22YCVhpARGphVUVLC/wpirHkZGvHbto07EicK75uvx1WrLCZQGPpzjttzMCQIQd2G15eJYLF2NV/J1U9VVX/DUQ9oFtV04CbgClYCeItVV0oIo+ISGZunQKsE5FFwKfAHaq6rjAfpLhKSbFxbgdqkdO5eLjwQvs/evrp2K1r/MUXtr70HXfY4LUDWa4ji0XkYuwqvj3WYDwWeEVVG8UvvP3FbGGaBNGgga2P8+abYUfiXPH2n//ADTfAZ5/B6acXbV9paXDCCdYA/f33ULF4zq62j7xGFudaIlDVCaraEzgGu1ofCBwsIkNFpJguyJZY1q+34qy3DzhXdFdeaWtMxWKA2Ysv2pKYzz57YCSB/ETTWLxVVUer6kVYg++3WE8iV0SZDcXeY8i5oqtQAf76V5v+4ccfC7+fNWts3q+//AW6do1dfImsQKuyqOqfkR48HYIKqCTxHkPOxdaNN9pC90WZHvquu2D7dvj3vw/sBuKsEmN5rhIqORkOOQTq1Ak7EucODHXqQO/eMHKkrUhbUF99ZQve3H57Qk5LFhhPBCHyNQici71Bg+yKfujQgr0vLc1KFPXqwX33BRNbovJEEJJdu2y0orcPOBdbxx1nA8AyZ56P1ksv2cXZM8+EuxZyGDwRhGTxYhsF6SUC52Jv0CBr9B0zJrrtf//dSgEdOkC3bsHGlog8EYTE1yBwLjgdOkDz5nZ1H80As7vvhq1bS1YDcVaeCEKSkmIrHB2138xKzrmiylzXeMEC+N//8t525kx49VVb7ezYY+MTX6LxRBCS5GRo1syW3HPOxV7PnnDYYXkPMEtPtymmDzvMxg6UVJ4IQqDqPYacC1rZsnDzzTB1qq35kZNhw2DePEsWVarEN75E4okgBCtXWh9nbx9wLlj9+9sUEc8+u/9rf/wBf/87nHUW9OgR/9gSiSeCEPiIYufio0YN6NfPJnVctWrf1+65xxa0KakNxFl5IghBZo+h5s3DjcO5kmDgQBss9nyWVVS++QaGD7c1j30tEE8EoUhJsQWwDzoo7EicO/AdeaRNHjd0qHURTU+3EcR16tjqY87WDHZxlpzs7QPOxdPtt8N779k8QqVLw5w5tuiMX4wZTwRxtmULLFliE2M55+Lj5JOhXTt46ilbbOb00+Hyy8OOKnF41VCcffeddR/1EoFz8ZM5wOyXXywRPP+8NxBn5SWCOPMeQ86Fo2tXaN0azjvPBnO6vTwRxFlKClSrZmsVO+fiJ7NtwO3Pq4biLDnZSgNeLHXOJQpPBHGUnm5tBN4+4JxLJJ4I4ujnn60fs7cPOOcSSaCJQEQ6isgPIrJERO7O4fW+IrJWRJIjt2uDjCdsmQ3FXiJwziWSwBqLRSQJeAE4F0gFZovIRFVdlG3Tcap6U1BxJJLkZGuwOu64sCNxzrm9giwRtAWWqOpSVd0FjAW6BHi8hJeSYgtflCsXdiTOObdXkImgLrAiy+PUyHPZXSoi80XkHRGpn9OORKS/iMwRkTlr164NIta4yOwx5JxziSTsxuL3gYaq2hyYCryW00aqOkxV26hqm9q1a8c1wFj54w9bh8DbB5xziSbIRLASyHqFXy/y3B6quk5Vd0YevgK0DjCeUPmIYudcogoyEcwGjhKRRiJSFugJTMy6gYgcmuVhZ+D7AOMJlScC51yiCqzXkKqmichNwBQgCRihqgtF5BFgjqpOBG4Rkc5AGrAe6BtUPGFLTrYFsotpzZZz7gAW6FxDqjoJmJTtuQey3L8HuCfIGBJFSoq3DzjnElPYjcUlws6dsGiRVws55xKTJ4I4+P57WzPVSwTOuUTkiSAOMher9xKBcy4ReSKIg5QUqFgRGjcOOxLnnNufJ4I4SE62FZGSksKOxDnn9ueJIGCq3mPIOZfYPBEEbMUK+PNPbx9wziUuTwQB8zUInHOJzhNBwJKTbX3iZs3CjsQ553LmiSBgKSnWW6hy5bAjcc65nHkiCJivQeCcS3SeCAK0ebMtWO/tA865ROaJIEDz59tPLxE45xJZiUkEu3bB9OnxPab3GHLOFQclJhE8/DCcey58/nn8jpmcDDVqQN2cVmp2zrkEUWISwZ13whFHQPfukJoan2NmjigWic/xnHOuMAJdmCaRVK0KEyZA27Zw6aXw2WdQvnxwx0tPh+++gxtuCO4YrmTbvXs3qamp7NixI+xQXAIpX7489erVo0yZMlG/p8QkAoBjj4VRo+CSS+DGG+GVV4K7Wv/pJ9i+3dsHXHBSU1OpUqUKDRs2RLzY6QBVZd26daSmptKoUaOo31diqoYyde0K990HI0bAf/4T3HF8DQIXtB07dlCzZk1PAm4PEaFmzZoFLiWWuEQA8NBDcMEFcMst8NVXwRwjJQXKlLFSiHNB8STgsivM30SJTARJSfDmm3D44dCtG/z2W+yPkZwMxx0HZcvGft/OORdLJTIRAFSrBuPH2+jfbt1sgflY8jUI3IFu3bp1tGzZkpYtW3LIIYdQt27dPY937dqV53vnzJnDLbfcku8xTjnllFiFC8DAgQOpW7cuGRkZMd1vcRdoIhCRjiLyg4gsEZG789juUhFREWkTZDzZHX88jBwJM2fCrbfGbr+//w6rVnn7gDuw1axZk+TkZJKTk7nhhhu47bbb9jwuW7YsaWlpub63TZs2DBkyJN9jzJgxI2bxZmRkMH78eOrXr89nn30Ws/1ml9fnTlSB9RoSkSTgBeBcIBWYLSITVXVRtu2qALcCXwcVS166dYO774bHH4fWreG664q+Tx9R7OJu4MC9PRRipWVLeO65Ar2lb9++lC9fnm+//Zb27dvTs2dPbr31Vnbs2EGFChV49dVXadKkCdOnT+epp57igw8+4KGHHuLXX39l6dKl/PrrrwwcOHBPaaFy5cps2bKF6dOn89BDD1GrVi0WLFhA69ateeONNxARJk2axKBBg6hUqRLt27dn6dKlfPDBB/vFNn36dJo2bUqPHj0YM2YMZ511FgBr1qzhhhtuYOnSpQAMHTqUU045hVGjRvHUU08hIjRv3pzXX3+dvn370qlTJ7p167ZffPfffz/Vq1dn8eLF/Pjjj1x88cWsWLGCHTt2cOutt9K/f38AJk+ezL333kt6ejq1atVi6tSpNGnShBkzZlC7dm0yMjI4+uijmTlzJrVr1y70r68gguw+2hZYoqpLAURkLNAFWJRtu38ATwB3BBhLnh59FObNg5tusnUDTjqpaPvzHkOuJEtNTWXGjBkkJSWxadMmvvjiC0qXLs20adO49957effdd/d7z+LFi/n000/ZvHkzTZo0YcCAAfv1g//2229ZuHAhhx12GO3bt+err76iTZs2XH/99Xz++ec0atSIXr165RrXmDFj6NWrF126dOHee+9l9+7dlClThltuuYUzzjiD8ePHk56ezpYtW1i4cCGPPvooM2bMoFatWqxfvz7fzz1v3jwWLFiwp9vmiBEjqFGjBtu3b+fEE0/k0ksvJSMjg+uuu25PvOvXr6dUqVL07t2bN998k4EDBzJt2jRatGgRtyQAwSaCusCKLI9TgXZZNxCRE4D6qvqhiOSaCESkP9AfoEGDBjEPNCkJxoyBNm1ssNncuXDIIYXfX0oK1K9v00s4FxcFvHIPUvfu3UlKSgJg48aNXHXVVfz000+ICLt3787xPRdeeCHlypWjXLlyHHzwwaxZs4Z69erts03btm33PNeyZUuWLVtG5cqVOeKII/acfHv16sWwYcP22/+uXbuYNGkSzzzzDFWqVKFdu3ZMmTKFTp068cknnzBq1CgAkpKSqFq1KqNGjaJ79+7UqlULgBpR/DO3bdt2n777Q4YMYfz48QCsWLGCn376ibVr13L66afv2S5zv/369aNLly4MHDiQESNGcPXVV+d7vFgKrbFYREoBzwC357etqg5T1Taq2iaoLFmjho083rDBpqHIp60rT74GgSvJKlWqtOf+/fffz1lnncWCBQt4//33c+3fXq5cuT33k5KScqxnj2ab3EyZMoUNGzbQrFkzGjZsyJdffsmYMWOifn+m0qVL72lozsjI2KdRPOvnnj59OtOmTWPmzJmkpKTQqlWrPPv2169fnzp16vDJJ5/wzTffcP755xc4tqIIMhGsBOpneVwv8lymKsDxwHQRWQacBEyMd4NxVs2bw/Dh8OWXMGhQ4faxYwcsXuztA86BlQjqRmZdHDlyZMz336RJE5YuXcqyZcsAGDduXI7bjRkzhldeeYVly5axbNkyfvnlF6ZOncq2bdvo0KEDQ4cOBSA9PZ2NGzdy9tln8/bbb7Nu3TqAPVVDDRs2ZO7cuQBMnDgx1xLOxo0bqV69OhUrVmTx4sXMmjULgJNOOonPP/+cX375ZZ/9Alx77bX07t17nxJVvASZCGYDR4lIIxEpC/QEJma+qKobVbWWqjZU1YbALKCzqs4JMKZ89ewJf/sbvPACvPpqwd+/cKHNM+QlAufgzjvv5J577qFVq1aB9KapUKECL774Ih07dqR169ZUqVKFqlWr7rPNtm3bmDx5MhdeeOGe5ypVqsSpp57K+++/z+DBg/n0009p1qwZrVu3ZtGiRTRt2pS///3vnHHGGbRo0YJBkSvD6667js8++4wWLVowc+bMfUoBWXXs2JG0tDSOPfZY7r77bk6KNDzWrl2bYcOGcckll9CiRQt69Oix5z2dO3dmy5Ytca8WAmxuiqBuwAXAj8DPwN8jzz2CnfCzbzsdaJPfPlu3bq1B271btUMH1bJlVb/+umDvHT5cFVR/+imY2JzLtGjRorBDSAibN29WVdWMjAwdMGCAPvPMMyFHVDizZ8/WU089NSb7yulvA5ijuZxXA510TlUnAZOyPfdALtueGWQsBVG6NIwda43Hl1xijcd16kT33uRkW6j+iCOCjdE5Z15++WVee+01du3aRatWrbj++uvDDqnAHn/8cYYOHcqbb74ZyvHFEkXx0aZNG50zJz61R8nJcMopcOKJMG2azR2UnzPOgLS04OYwci7T999/z7E+mZXLQU5/GyIyV1VzbIMtsVNMRKNlS5uq+vPPrd0gP6rWddTbB5xzxUmJWo+gMC6/HObMgWeftZHHV16Z+7bLl8PGjd5jyDlXvHiJIApPPglnngnXX2/tBbnxEcXOueLIE0EUSpeGt96C2rWt8Xjt2py3S0mBUqVsmgrnnCsuPBFEqXZtm7Z6zRro0cMahLNLToajjoKKFeMfn3PxdtZZZzFlypR9nnvuuecYMGBAru8588wzyezsccEFF7Bhw4b9tnnooYd46qmn8jz2hAkTWLRo77RlDzzwANOmTStI+HkqadNVeyIogNatYdgw+PRTuOuu/V/3NQhcSdKrVy/Gjh27z3Njx47Nc+K3rCZNmkS1atUKdezsieCRRx7hnHPOKdS+siuJ01V7IiigK6+Em2+GZ56B0aP3Pr9xI/zyi7cPuHAMHGjtWLG8DRyY9zG7devGhx9+uGe+nWXLlvHbb79x2mmnMWDAANq0aUPTpk158MEHc3x/w4YN+eOPPwB47LHHOProozn11FP54Ycf9mzz8ssvc+KJJ9KiRQsuvfRStm3bxowZM5g4cSJ33HEHLVu25Oeff6Zv37688847AHz88ce0atWKZs2a0a9fP3ZGVp1q2LAhDz74ICeccALNmjVj8eLFOcaVOV31gAED9pmPaM2aNXTt2pUWLVrQokWLPWsljBo1iubNm9OiRQv69OkDsE88YNNVZ+77tNNOo3Pnzhx33HEAXHzxxbRu3ZqmTZvuM2He5MmTOeGEE2jRogUdOnQgIyODo446irWRuumMjAwaN26853FReCIohKefhtNOg2uv3dtAPH++/fQSgSspatSoQdu2bfnoo48AKw1cdtlliAiPPfYYc+bMYf78+Xz22WfMz/wHycHcuXMZO3YsycnJTJo0idmzZ+957ZJLLmH27NmkpKRw7LHHMnz4cE455RQ6d+7Mv/71L5KTkznyyCP3bL9jxw769u3LuHHj+O6770hLS9szjxBArVq1mDdvHgMGDMi1+ilzuuquXbvy4Ycf7plPKHO66pSUFObNm0fTpk33TFf9ySefkJKSwuDBg/P93ubNm8fgwYP58ccfAZuueu7cucyZM4chQ4awbt061q5dy3XXXce7775LSkoKb7/99j7TVQMxna7au48WQpky8PbbVlXUtat1L/UeQy5MYc1CnVk91KVLF8aOHcvw4cMBeOuttxg2bBhpaWmsWrWKRYsW0bx58xz38cUXX9C1a1cqRhrXOnfuvOe1BQsWcN9997Fhwwa2bNnCeeedl2c8P/zwA40aNeLoo48G4KqrruKFF15gYKR4c8kllwDQunVr3nvvvf3eX1Kn0dU6qQAAB9xJREFUq/ZEUEh16sB771nJoGdPqFvXGpQPPTTsyJyLny5dunDbbbcxb948tm3bRuvWrfnll1946qmnmD17NtWrV6dv3755TsGcl759+zJhwgRatGjByJEjmT59epHizZzKOrdprLNOVw02YV2FChXo1KlTgY5TmOmqK1asyJlnnlmg6apjNSWFVw0VQdu2MHSoTT/x+utWGhAJOyrn4qdy5cqcddZZ9OvXb08j8aZNm6hUqRJVq1ZlzZo1e6qOcnP66aczYcIEtm/fzubNm3n//ff3vLZ582YOPfRQdu/evc9Jr0qVKmzevHm/fTVp0oRly5axZMkSAF5//XXOOOOMqD9PSZ2u2hNBEfXrBwMGQEaGtw+4kqlXr16kpKTsSQQtWrSgVatWHHPMMVx++eW0b98+z/efcMIJ9OjRgxYtWnD++edz4okn7nntH//4B+3ataN9+/Ycc8wxe57v2bMn//rXv2jVqhU///zznufLly/Pq6++Svfu3WnWrBmlSpXihhtuiOpzlOTpqn3SuRjYtQseeAD69IGmTcOOxpUUPulcyTRnzhxuu+02vvjii1y3Keikc95GEANly8Ljj4cdhXPuQBfUdNVeNeScc8XE3XffzfLlyzn11FNjul9PBM4VY8WtatcFrzB/E54InCumypcvz7p16zwZuD1UlXXr1lG+fPkCvc/bCJwrpurVq0dqampMphhwB47y5ctTr169Ar3HE4FzxVSZMmX2GaHqXGF51ZBzzpVwngicc66E80TgnHMlXLEbWSwia4HlhXx7LeCPGIYTTx57ODz2+CuucUNix364quY4Z3WxSwRFISJzchtineg89nB47PFXXOOG4hu7Vw0551wJ54nAOedKuJKWCIblv0nC8tjD4bHHX3GNG4pp7CWqjcA559z+SlqJwDnnXDaeCJxzroQrMYlARDqKyA8iskRE7g47nmiJSH0R+VREFonIQhG5NeyYCkJEkkTkWxH5IOxYCkJEqonIOyKyWES+F5GTw44pWiJyW+RvZYGIjBGRgk1FGUciMkJEfheRBVmeqyEiU0Xkp8jP6mHGmJtcYv9X5G9mvoiMF5FqYcYYrRKRCEQkCXgBOB84DuglIseFG1XU0oDbVfU44CTgxmIUO8CtwPdhB1EIg4HJqnoM0IJi8hlEpC5wC9BGVY8HkoCe4UaVp5FAx2zP3Q18rKpHAR9HHieikewf+1TgeFVtDvwI3BPvoAqjRCQCoC2wRFWXquouYCzQJeSYoqKqq1R1XuT+ZuyEVDfcqKIjIvWAC4FXwo6lIESkKnA6MBxAVXep6oZwoyqQ0kAFESkNVAR+CzmeXKnq58D6bE93AV6L3H8NuDiuQUUpp9hV9X+qmhZ5OAso2HzQISkpiaAusCLL41SKyck0KxFpCLQCvg43kqg9B9wJZIQdSAE1AtYCr0aqtV4RkUphBxUNVV0JPAX8CqwCNqrq/8KNqsDqqOqqyP3VQJ0wgymCfsBHYQcRjZKSCIo9EakMvAsMVNVNYceTHxHpBPyuqnPDjqUQSgMnAENVtRWwlcStnthHpD69C5bMDgMqiUjvcKMqPLX+7cWuj7uI/B2r1o3tKvMBKSmJYCVQP8vjepHnigURKYMlgTdV9b2w44lSe6CziCzDquLOFpE3wg0paqlAqqpmlrzewRJDcXAO8IuqrlXV3cB7wCkhx1RQa0TkUIDIz99DjqdARKQv0Am4QovJQK2SkghmA0eJSCMRKYs1nk0MOaaoiIhgddXfq+ozYccTLVW9R1XrqWpD7Pv+RFWLxZWpqq4GVohIk8hTHYBFIYZUEL8CJ4lIxcjfTgeKSUN3FhOBqyL3rwL+G2IsBSIiHbHq0M6qui3seKJVIhJBpPHmJmAK9k/xlqouDDeqqLUH+mBX1MmR2wVhB1UC3Ay8KSLzgZbA/4UcT1QipZh3gHnAd9j/eMJOeyAiY4CZQBMRSRWRa4DHgXNF5CeshPN4mDHmJpfYnweqAFMj/6svhRpklHyKCeecK+FKRInAOedc7jwROOdcCeeJwDnnSjhPBM45V8J5InDOuRLOE4FzESKSnqWLbnIsZ6kVkYZZZ6l0LpGUDjsA5xLIdlVtGXYQzsWblwicy4eILBORJ0XkOxH5RkQaR55vKCKfROae/1hEGkSerxOZiz4lcsuc4iFJRF6OrBXwPxGpENn+lsh6E/NFZGxIH9OVYJ4InNurQraqoR5ZXtuoqs2wkaPPRZ77N/BaZO75N4EhkeeHAJ+pagtsjqLMUexHAS+oalNgA3Bp5Pm7gVaR/dwQ1IdzLjc+sti5CBHZoqqVc3h+GXC2qi6NTAC4WlVrisgfwKGqujvy/CpVrSUia4F6qrozyz4aAlMji60gIncBZVT1URGZDGwBJgATVHVLwB/VuX14icC56Ggu9wtiZ5b76exto7sQW0HvBGB2ZEEZ5+LGE4Fz0emR5efMyP0Z7F0G8grgi8j9j4EBsGfN5qq57VRESgH1VfVT4C6gKrBfqcS5IPmVh3N7VRCR5CyPJ6tqZhfS6pGZSHcCvSLP3YytYnYHtqLZ1ZHnbwWGRWajTMeSwipylgS8EUkWAgwpZstiugOAtxE4l49IG0EbVf0j7FicC4JXDTnnXAnnJQLnnCvhvETgnHMlnCcC55wr4TwROOdcCeeJwDnnSjhPBM45V8L9P1NAkyQjkvxEAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VKShF3tm6wGt"
      },
      "source": [
        "def evaluate2(model, device, iterator, criterion):\n",
        "\n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    expected=[]\n",
        "    calculated=[]\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for (x, y) in iterator:\n",
        "\n",
        "\n",
        "\n",
        "            x = x.to(device)\n",
        "            y = y.to(device)\n",
        "            \n",
        "            \n",
        "\n",
        "            fx = model(x)\n",
        "            preds = fx.max(1, keepdim=True)[1]\n",
        "            correct = preds.eq(y.view_as(preds)).sum()\n",
        "            \n",
        "            expected.append(y)\n",
        "            calculated.append(fx)\n",
        "\n",
        "            loss = criterion(fx, y)\n",
        "\n",
        "            \n",
        "\n",
        "            acc = calculate_accuracy(fx, y)\n",
        "\n",
        "            epoch_loss += loss.item()\n",
        "            epoch_acc += acc.item()\n",
        "\n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator), expected, calculated"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qB_NHGJs7S-g"
      },
      "source": [
        "test_loss, test_acc, expected, calculated = evaluate2(model, device, test_iterator, criterion)"
      ],
      "execution_count": 12,
      "outputs": []
    }
  ]
}