ASRS_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ASRS classification.ipynb",
      "provenance": [],
      "mount_file_id": "1k2_nscpeUTZabpNsU6ne7sJ6wJZEZqFq",
      "authorship_tag": "ABX9TyPGHgKXw8VcPc9zcWFpjEsY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d5c0e957584f4827b612519ffa6301b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_37ca7a81424f494f8a0f7d1b030647cf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_58d5cfe4cb0f41239defbfcba119497f",
              "IPY_MODEL_96179f3462f84dcc909639c726dc8059"
            ]
          }
        },
        "37ca7a81424f494f8a0f7d1b030647cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "58d5cfe4cb0f41239defbfcba119497f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d17177d8262941b699eb9de63268ac89",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 798011,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 798011,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ae89d5ce25774f21a4ec0ae5792d4d41"
          }
        },
        "96179f3462f84dcc909639c726dc8059": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1853394880724b2e9929f5226ec7923d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 798k/798k [00:49&lt;00:00, 16.3kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_22539401210c4884901daafe5be27951"
          }
        },
        "d17177d8262941b699eb9de63268ac89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ae89d5ce25774f21a4ec0ae5792d4d41": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1853394880724b2e9929f5226ec7923d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "22539401210c4884901daafe5be27951": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2665412b59d74b238c9b088265e532b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_80bb6824663a49729fedd80cf481eef3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_59363e00f3a141ac803f808f8786af21",
              "IPY_MODEL_1813a06a1b1e432a8d06c8decfe2e470"
            ]
          }
        },
        "80bb6824663a49729fedd80cf481eef3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "59363e00f3a141ac803f808f8786af21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_02abafd3f9f3447fbdb27d275c1f1eec",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 760,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 760,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1f8dacbea60c4a308f1feeb6d95db064"
          }
        },
        "1813a06a1b1e432a8d06c8decfe2e470": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_faa6b837780349539677fe23e3dec212",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 760/760 [00:00&lt;00:00, 4.42kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_afd146581a3842c8995e17100795c874"
          }
        },
        "02abafd3f9f3447fbdb27d275c1f1eec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1f8dacbea60c4a308f1feeb6d95db064": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "faa6b837780349539677fe23e3dec212": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "afd146581a3842c8995e17100795c874": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b1416bcb2a714f9baf43a9e59ad2c391": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5fa49d36f410414b813b74e062fdb8a6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c5baf5f84fee44b7a5239e23929fd0d1",
              "IPY_MODEL_5ff7547a30294acfbe1660bfa1ec2f31"
            ]
          }
        },
        "5fa49d36f410414b813b74e062fdb8a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c5baf5f84fee44b7a5239e23929fd0d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6efa93678f6446ac99c04e87b7652813",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 467042463,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 467042463,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_af6a899406474400a0cbc2cd00dfc17d"
          }
        },
        "5ff7547a30294acfbe1660bfa1ec2f31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dff2cbe3d14d424ca5d863c8f59e74c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 467M/467M [00:16&lt;00:00, 28.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03ea15758e0945029565e88b5187d369"
          }
        },
        "6efa93678f6446ac99c04e87b7652813": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "af6a899406474400a0cbc2cd00dfc17d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dff2cbe3d14d424ca5d863c8f59e74c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03ea15758e0945029565e88b5187d369": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jaytimbadia/UnWrapper/blob/master/ASRS_classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PS6tZPX8IkLc",
        "outputId": "a8201d3d-e365-4fc6-c203-cce549a571b7"
      },
      "source": [
        "!pip install sentencepiece\r\n",
        "!pip install transformers\r\n",
        "!pip install tensorboardX\r\n",
        "!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\r\n",
        "!unzip ngrok-stable-linux-amd64.zip"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 3.8MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.94\n",
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/0c/7d5950fcd80b029be0a8891727ba21e0cd27692c407c51261c3c921f6da3/transformers-4.1.1-py3-none-any.whl (1.5MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5MB 5.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.8)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.4)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 35.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Collecting tokenizers==0.9.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 38.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=a236f61a1dc1b6fb426d8bd3657eab053ed1cda4c0f7b41989a07eaff642f248\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.1.1\n",
            "Collecting tensorboardX\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/af/0c/4f41bcd45db376e6fe5c619c01100e9b7531c55791b7244815bac6eac32c/tensorboardX-2.1-py2.py3-none-any.whl (308kB)\n",
            "\u001b[K     |████████████████████████████████| 317kB 4.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorboardX) (3.12.4)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from tensorboardX) (1.19.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from tensorboardX) (1.15.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tensorboardX) (51.0.0)\n",
            "Installing collected packages: tensorboardX\n",
            "Successfully installed tensorboardX-2.1\n",
            "--2020-12-23 08:31:56--  https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
            "Resolving bin.equinox.io (bin.equinox.io)... 52.2.242.115, 52.1.26.21, 35.175.47.233, ...\n",
            "Connecting to bin.equinox.io (bin.equinox.io)|52.2.242.115|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 13773305 (13M) [application/octet-stream]\n",
            "Saving to: ‘ngrok-stable-linux-amd64.zip’\n",
            "\n",
            "ngrok-stable-linux- 100%[===================>]  13.13M  6.54MB/s    in 2.0s    \n",
            "\n",
            "2020-12-23 08:31:58 (6.54 MB/s) - ‘ngrok-stable-linux-amd64.zip’ saved [13773305/13773305]\n",
            "\n",
            "Archive:  ngrok-stable-linux-amd64.zip\n",
            "  inflating: ngrok                   \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "afF7QX7L36T3",
        "outputId": "7696e770-7436-4372-8d52-e38e1f83d609"
      },
      "source": [
        "import json\r\n",
        "import pandas as pd\r\n",
        "import nltk\r\n",
        "nltk.download('stopwords')\r\n",
        "from nltk.corpus import stopwords\r\n",
        "import re\r\n",
        "from keras.preprocessing.sequence import pad_sequences\r\n",
        "from sklearn.model_selection import train_test_split\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "import math\r\n",
        "import numpy as np\r\n",
        "from transformers import AdamW, XLNetTokenizer, XLNetModel, XLNetLMHeadModel, XLNetConfig\r\n",
        "import torch\r\n",
        "from torch.nn import BCEWithLogitsLoss\r\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\r\n",
        "import os\r\n",
        "from tensorboardX import SummaryWriter\r\n",
        "from datetime import datetime"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pyfKstsIInK5",
        "outputId": "adc7cd61-8b15-46b9-91de-96dfdc18ca66"
      },
      "source": [
        "# Loading training and testing data json\r\n",
        "\r\n",
        "with open('drive/MyDrive/data/train.json', 'r') as f:\r\n",
        "    input_data = json.load(f)\r\n",
        "\r\n",
        "with open('drive/MyDrive/data/test.json', 'r') as f1:\r\n",
        "    testing_data = json.load(f1)\r\n",
        "\r\n",
        "train_labels = pd.read_csv('drive/MyDrive/data/training_labels.csv', header=None)\r\n",
        "\r\n",
        "train_data = pd.DataFrame(input_data)\r\n",
        "test_data = pd.DataFrame(testing_data)\r\n",
        "\r\n",
        "print('Number of data points in train text : ', train_data.shape[0])\r\n",
        "print('Number of features in train text: ', train_data.shape[1])\r\n",
        "print(\"Shape of train labels : \", train_labels.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of data points in train text :  21519\n",
            "Number of features in train text:  2\n",
            "Shape of train labels :  (21519, 22)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AqPz4RIXIzaD"
      },
      "source": [
        "# converting -1 label to 0 since we use sigmoid for multilabel instead of softmax and it lies in 0 to 1\r\n",
        "train_labels = train_labels.applymap(lambda x: 0 if x<0 else 1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R-Fp9gtwI4cF"
      },
      "source": [
        "stop_words = set(stopwords.words('english'))\r\n",
        "\r\n",
        "def preprocess(total_text, use_stopwords=False):\r\n",
        "    if type(total_text) is not int:\r\n",
        "        string = \"\"\r\n",
        "        # replace every special char with space\r\n",
        "        total_text = re.sub('[^a-zA-Z0-9\\n]', ' ', str(total_text))\r\n",
        "        # replace multiple spaces with single space\r\n",
        "        total_text = re.sub('\\s+',' ', str(total_text))\r\n",
        "        # converting all the chars into lower-case.\r\n",
        "        total_text = total_text.lower()\r\n",
        "        \r\n",
        "        if use_stopwords:\r\n",
        "            for word in total_text.split():\r\n",
        "            # if the word is a not a stop word then retain that word from the data\r\n",
        "                if not word in stop_words:\r\n",
        "                    string += word + \" \"\r\n",
        "        \r\n",
        "            return string\r\n",
        "        else:\r\n",
        "            return total_text\r\n",
        "\r\n",
        "train_data['cleaned_text'] = train_data.apply(lambda row: preprocess(row['text'], False), axis=1)\r\n",
        "test_data['cleaned_text'] = test_data.apply(lambda row: preprocess(row['text'], False), axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "d5c0e957584f4827b612519ffa6301b2",
            "37ca7a81424f494f8a0f7d1b030647cf",
            "58d5cfe4cb0f41239defbfcba119497f",
            "96179f3462f84dcc909639c726dc8059",
            "d17177d8262941b699eb9de63268ac89",
            "ae89d5ce25774f21a4ec0ae5792d4d41",
            "1853394880724b2e9929f5226ec7923d",
            "22539401210c4884901daafe5be27951"
          ]
        },
        "id": "SxXV0r82I_1u",
        "outputId": "2ded51f1-2f65-4cd2-9801-2ec32e98710e"
      },
      "source": [
        "tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased', do_lower_case=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d5c0e957584f4827b612519ffa6301b2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=798011.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hGHYakZNL7nD"
      },
      "source": [
        "train_text_list = train_data[\"cleaned_text\"].values\r\n",
        "test_text_list = test_data[\"cleaned_text\"].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xc2sCmNB6ott"
      },
      "source": [
        "# embedding size determination\r\n",
        "text_length = []\r\n",
        "for i in (train_text_list):\r\n",
        "    if 100 < len(i.split()) < 700:\r\n",
        "        text_length.append(len(i.split()))\r\n",
        "\r\n",
        "print(np.mean(text_length))\r\n",
        "plt.hist(text_length)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6RiNsz6GPJNL"
      },
      "source": [
        "def tokenize_inputs(text_list, tokenizer, num_embeddings=512):\r\n",
        "    \"\"\"\r\n",
        "    Tokenizes the input text input into ids. Appends the appropriate special\r\n",
        "    characters to the end of the text to denote end of sentence. Truncate or pad\r\n",
        "    the appropriate sequence length.\r\n",
        "    \"\"\"\r\n",
        "    # tokenize the text, then truncate sequence to the desired length minus 2 for\r\n",
        "    # the 2 special characters\r\n",
        "    tokenized_texts = list(map(lambda t: tokenizer.tokenize(t)[:num_embeddings-2], text_list))\r\n",
        "    # convert tokenized text into numeric ids for the appropriate LM\r\n",
        "    input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\r\n",
        "    # append special token \"<s>\" and </s> to end of sentence\r\n",
        "    input_ids = [tokenizer.build_inputs_with_special_tokens(x) for x in input_ids]\r\n",
        "    # pad sequences\r\n",
        "    input_ids = pad_sequences(input_ids, maxlen=num_embeddings, dtype=\"long\", truncating=\"post\", padding=\"post\")\r\n",
        "    return input_ids\r\n",
        "\r\n",
        "def create_attn_masks(input_ids):\r\n",
        "    \"\"\"\r\n",
        "    Create attention masks to tell model whether attention should be applied to\r\n",
        "    the input id tokens. Do not want to perform attention on padding tokens.\r\n",
        "    \"\"\"\r\n",
        "    # Create attention masks\r\n",
        "    attention_masks = []\r\n",
        "\r\n",
        "    # Create a mask of 1s for each token followed by 0s for padding\r\n",
        "    for seq in input_ids:\r\n",
        "        seq_mask = [float(i>0) for i in seq]\r\n",
        "        attention_masks.append(seq_mask)\r\n",
        "    return attention_masks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zfyKz5qzMnNc"
      },
      "source": [
        "# truncating seq between 100 and 700 since others are less in numbers and can act as an outlier.\r\n",
        "v= []\r\n",
        "v_tst = []\r\n",
        "filtered_labels = []\r\n",
        "list_labels = train_labels.values\r\n",
        "for i, j in zip(train_text_list, list_labels):\r\n",
        "    if 100 < len(i.split()) < 700:\r\n",
        "        v.append(i)\r\n",
        "        filtered_labels.append(j)\r\n",
        "\r\n",
        "for k in test_text_list:\r\n",
        "    if 100 < len(k.split()) < 700:\r\n",
        "        v_tst.append(k)        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w276cf9LNZQ4"
      },
      "source": [
        "# converting to dataframe\r\n",
        "train_df = pd.DataFrame(v)\r\n",
        "train_df = train_df.rename(columns={0: \"raw_text\"})\r\n",
        "\r\n",
        "test_df = pd.DataFrame(v_tst)\r\n",
        "test_df = test_df.rename(columns={0: \"raw_text\"})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8U1j_AmDOrZ1"
      },
      "source": [
        "train_text_list_ = train_df[\"raw_text\"].values\r\n",
        "test_text_list_ = test_df[\"raw_text\"].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KsJXD0rYNkT8"
      },
      "source": [
        "train_input_ids = tokenize_inputs(train_text_list_, tokenizer, num_embeddings=250)\r\n",
        "test_input_ids = tokenize_inputs(test_text_list_, tokenizer, num_embeddings=250)\r\n",
        "train_attention_masks = create_attn_masks(train_input_ids)\r\n",
        "test_attention_masks = create_attn_masks(test_input_ids)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1XDxeuquO6bm"
      },
      "source": [
        "# add input ids and attention masks to the dataframe\r\n",
        "train_df[\"features\"] = train_input_ids.tolist()\r\n",
        "train_df[\"masks\"] = train_attention_masks\r\n",
        "\r\n",
        "test_df[\"features\"] = test_input_ids.tolist()\r\n",
        "test_df[\"masks\"] = test_attention_masks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OFfKONAOaMEa"
      },
      "source": [
        "filtered_labels = np.array(filtered_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PIXq2GjAPq9K"
      },
      "source": [
        "train, valid, y_train, y_val = train_test_split(train_df, filtered_labels, test_size=0.2, random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VFd7wVYIZwcr",
        "outputId": "74f7cb15-36bc-4428-b1c6-367865f586f0"
      },
      "source": [
        "y_train.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(12692, 22)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XDrxM8NQaU_K"
      },
      "source": [
        "X_train = train[\"features\"].values.tolist()\r\n",
        "X_valid = valid[\"features\"].values.tolist()\r\n",
        "\r\n",
        "train_masks = train[\"masks\"].values.tolist()\r\n",
        "valid_masks = valid[\"masks\"].values.tolist()\r\n",
        "\r\n",
        "Y_train = y_train.tolist()\r\n",
        "Y_valid = y_val.tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GzqEu1iVa4k7"
      },
      "source": [
        "X_train = torch.tensor(X_train)\r\n",
        "X_valid = torch.tensor(X_valid)\r\n",
        "\r\n",
        "Y_train = torch.tensor(Y_train, dtype=torch.float32)\r\n",
        "Y_valid = torch.tensor(Y_valid, dtype=torch.float32)\r\n",
        "\r\n",
        "train_masks = torch.tensor(train_masks, dtype=torch.long)\r\n",
        "valid_masks = torch.tensor(valid_masks, dtype=torch.long)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "te6CbqBybb4O"
      },
      "source": [
        "batch_size = 16\r\n",
        "\r\n",
        "# Create an iterator of our data with torch DataLoader. This helps save on \r\n",
        "# memory during training because, unlike a for loop, \r\n",
        "# with an iterator the entire dataset does not need to be loaded into memory\r\n",
        "\r\n",
        "train_data = TensorDataset(X_train, train_masks, Y_train)\r\n",
        "train_sampler = RandomSampler(train_data)\r\n",
        "train_dataloader = DataLoader(train_data,\\\r\n",
        "                              sampler=train_sampler,\\\r\n",
        "                              batch_size=batch_size)\r\n",
        "\r\n",
        "validation_data = TensorDataset(X_valid, valid_masks, Y_valid)\r\n",
        "validation_sampler = SequentialSampler(validation_data)\r\n",
        "validation_dataloader = DataLoader(validation_data,\\\r\n",
        "                                   sampler=validation_sampler,\\\r\n",
        "                                   batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WQ-7HK2abikT"
      },
      "source": [
        "def train(model, num_epochs,\\\r\n",
        "          optimizer,\\\r\n",
        "          train_dataloader, valid_dataloader,\\\r\n",
        "          model_save_path,\\\r\n",
        "          train_loss_set=[], valid_loss_set = [],\\\r\n",
        "          lowest_eval_loss=None, start_epoch=0,\\\r\n",
        "          device=\"cpu\", writer=None\r\n",
        "          ):\r\n",
        "  \"\"\"\r\n",
        "  Train the model and save the model with the lowest validation loss\r\n",
        "  \"\"\"\r\n",
        "\r\n",
        "  model.to(device)\r\n",
        "\r\n",
        "  # trange is a tqdm wrapper around the normal python range\r\n",
        "  for i in trange(num_epochs, desc=\"Epoch\"):\r\n",
        "    # if continue training from saved model\r\n",
        "    actual_epoch = start_epoch + i\r\n",
        "\r\n",
        "    # Training\r\n",
        "\r\n",
        "    # Set our model to training mode (as opposed to evaluation mode)\r\n",
        "    model.train()\r\n",
        "\r\n",
        "    # Tracking variables\r\n",
        "    tr_loss = 0\r\n",
        "    num_train_samples = 0\r\n",
        "\r\n",
        "    # Train the data for one epoch\r\n",
        "    for step, batch in enumerate(train_dataloader):\r\n",
        "      # Add batch to GPU\r\n",
        "      batch = tuple(t.to(device) for t in batch)\r\n",
        "      # Unpack the inputs from our dataloader\r\n",
        "      b_input_ids, b_input_mask, b_labels = batch\r\n",
        "      # Clear out the gradients (by default they accumulate)\r\n",
        "      optimizer.zero_grad()\r\n",
        "      # Forward pass\r\n",
        "      loss = model(b_input_ids, attention_mask=b_input_mask, labels=b_labels)\r\n",
        "      # store train loss\r\n",
        "      tr_loss += loss.item()\r\n",
        "      num_train_samples += b_labels.size(0)\r\n",
        "      # Backward pass\r\n",
        "      loss.backward()\r\n",
        "      # Update parameters and take a step using the computed gradient\r\n",
        "      optimizer.step()\r\n",
        "      #scheduler.step()\r\n",
        "\r\n",
        "    # Update tracking variables\r\n",
        "    epoch_train_loss = tr_loss/num_train_samples\r\n",
        "    train_loss_set.append(epoch_train_loss)\r\n",
        "\r\n",
        "    print(\"Train loss: {}\".format(epoch_train_loss))\r\n",
        "    if writer != None:\r\n",
        "        writer.add_scalar(\"train_loss\", epoch_train_loss, actual_epoch)\r\n",
        "\r\n",
        "    # Validation\r\n",
        "\r\n",
        "    # Put model in evaluation mode to evaluate loss on the validation set\r\n",
        "    model.eval()\r\n",
        "\r\n",
        "    # Tracking variables \r\n",
        "    eval_loss = 0\r\n",
        "    num_eval_samples = 0\r\n",
        "\r\n",
        "    # Evaluate data for one epoch\r\n",
        "    for batch in valid_dataloader:\r\n",
        "      # Add batch to GPU\r\n",
        "      batch = tuple(t.to(device) for t in batch)\r\n",
        "      # Unpack the inputs from our dataloader\r\n",
        "      b_input_ids, b_input_mask, b_labels = batch\r\n",
        "      # Telling the model not to compute or store gradients,\r\n",
        "      # saving memory and speeding up validation\r\n",
        "      with torch.no_grad():\r\n",
        "        # Forward pass, calculate validation loss\r\n",
        "        loss = model(b_input_ids, attention_mask=b_input_mask, labels=b_labels)\r\n",
        "        # store valid loss\r\n",
        "        eval_loss += loss.item()\r\n",
        "        num_eval_samples += b_labels.size(0)\r\n",
        "\r\n",
        "    epoch_eval_loss = eval_loss/num_eval_samples\r\n",
        "    valid_loss_set.append(epoch_eval_loss)\r\n",
        "\r\n",
        "    print(\"Valid loss: {}\".format(epoch_eval_loss))\r\n",
        "    if writer != None:\r\n",
        "        writer.add_scalar(\"valid_loss\", epoch_eval_loss, actual_epoch)\r\n",
        "\r\n",
        "    if lowest_eval_loss == None:\r\n",
        "      lowest_eval_loss = epoch_eval_loss\r\n",
        "      # save model\r\n",
        "      save_model(model, model_save_path, actual_epoch,\\\r\n",
        "                 lowest_eval_loss, train_loss_set, valid_loss_set)\r\n",
        "    else:\r\n",
        "      if epoch_eval_loss < lowest_eval_loss:\r\n",
        "        lowest_eval_loss = epoch_eval_loss\r\n",
        "        # save model\r\n",
        "        save_model(model, model_save_path, actual_epoch,\\\r\n",
        "                   lowest_eval_loss, train_loss_set, valid_loss_set)\r\n",
        "    print(\"\\n\")\r\n",
        "\r\n",
        "  return model, train_loss_set, valid_loss_set\r\n",
        "\r\n",
        "\r\n",
        "def save_model(model, save_path, epochs, lowest_eval_loss, train_loss_hist, valid_loss_hist):\r\n",
        "  \"\"\"\r\n",
        "  Save the model to the path directory provided\r\n",
        "  \"\"\"\r\n",
        "  model_to_save = model.module if hasattr(model, 'module') else model\r\n",
        "  checkpoint = {'epochs': epochs, \\\r\n",
        "                'lowest_eval_loss': lowest_eval_loss,\\\r\n",
        "                'state_dict': model_to_save.state_dict(),\\\r\n",
        "                'train_loss_hist': train_loss_hist,\\\r\n",
        "                'valid_loss_hist': valid_loss_hist\r\n",
        "               }\r\n",
        "  torch.save(checkpoint, save_path)\r\n",
        "  print(\"Saving model at epoch {} with validation loss of {}\".format(epochs,\\\r\n",
        "                                                                     lowest_eval_loss))\r\n",
        "  return"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115,
          "referenced_widgets": [
            "2665412b59d74b238c9b088265e532b1",
            "80bb6824663a49729fedd80cf481eef3",
            "59363e00f3a141ac803f808f8786af21",
            "1813a06a1b1e432a8d06c8decfe2e470",
            "02abafd3f9f3447fbdb27d275c1f1eec",
            "1f8dacbea60c4a308f1feeb6d95db064",
            "faa6b837780349539677fe23e3dec212",
            "afd146581a3842c8995e17100795c874",
            "b1416bcb2a714f9baf43a9e59ad2c391",
            "5fa49d36f410414b813b74e062fdb8a6",
            "c5baf5f84fee44b7a5239e23929fd0d1",
            "5ff7547a30294acfbe1660bfa1ec2f31",
            "6efa93678f6446ac99c04e87b7652813",
            "af6a899406474400a0cbc2cd00dfc17d",
            "dff2cbe3d14d424ca5d863c8f59e74c9",
            "03ea15758e0945029565e88b5187d369"
          ]
        },
        "id": "uo18itAPcnnc",
        "outputId": "6467ada1-b1c0-4096-c006-b0e52624ce14"
      },
      "source": [
        "class XLNetForMultiLabelSequenceClassification(torch.nn.Module):\r\n",
        "  \r\n",
        "  def __init__(self, num_labels=2):\r\n",
        "    super(XLNetForMultiLabelSequenceClassification, self).__init__()\r\n",
        "    self.num_labels = num_labels\r\n",
        "    self.xlnet = XLNetModel.from_pretrained('xlnet-base-cased')\r\n",
        "    self.classifier = torch.nn.Linear(768, num_labels)\r\n",
        "\r\n",
        "    torch.nn.init.xavier_normal_(self.classifier.weight)\r\n",
        "\r\n",
        "  def forward(self, input_ids, token_type_ids=None,\\\r\n",
        "              attention_mask=None, labels=None):\r\n",
        "    # last hidden layer\r\n",
        "    last_hidden_state = self.xlnet(input_ids=input_ids,\\\r\n",
        "                                   attention_mask=attention_mask,\\\r\n",
        "                                   token_type_ids=token_type_ids)\r\n",
        "    # pool the outputs into a mean vector\r\n",
        "    mean_last_hidden_state = self.pool_hidden_state(last_hidden_state)\r\n",
        "    logits = self.classifier(mean_last_hidden_state)\r\n",
        "        \r\n",
        "    if labels is not None:\r\n",
        "      loss_fct = BCEWithLogitsLoss() # more stable than pure sigmoid\r\n",
        "      loss = loss_fct(logits.view(-1, self.num_labels),\\\r\n",
        "                      labels.view(-1, self.num_labels))\r\n",
        "      return loss\r\n",
        "    else:\r\n",
        "      return logits\r\n",
        "    \r\n",
        "  def freeze_xlnet_decoder(self):\r\n",
        "    \"\"\"\r\n",
        "    Freeze XLNet weight parameters. They will not be updated during training.\r\n",
        "    \"\"\"\r\n",
        "    for param in self.xlnet.parameters():\r\n",
        "      param.requires_grad = False\r\n",
        "    \r\n",
        "  def unfreeze_xlnet_decoder(self):\r\n",
        "    \"\"\"\r\n",
        "    Unfreeze XLNet weight parameters. They will be updated during training.\r\n",
        "    \"\"\"\r\n",
        "    for param in self.xlnet.parameters():\r\n",
        "      param.requires_grad = True\r\n",
        "    \r\n",
        "  def pool_hidden_state(self, last_hidden_state):\r\n",
        "    \"\"\"\r\n",
        "    Pool the output vectors into a single mean vector \r\n",
        "    \"\"\"\r\n",
        "    last_hidden_state = last_hidden_state[0]\r\n",
        "    mean_last_hidden_state = torch.mean(last_hidden_state, 1)\r\n",
        "    return mean_last_hidden_state\r\n",
        "    \r\n",
        "model = XLNetForMultiLabelSequenceClassification(num_labels=len(Y_train[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2665412b59d74b238c9b088265e532b1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=760.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b1416bcb2a714f9baf43a9e59ad2c391",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=467042463.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x7B0j8EidHK2"
      },
      "source": [
        "optimizer = AdamW(model.parameters(), lr=2e-5, weight_decay=0.01, correct_bias=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZQdeuDp0cw89",
        "outputId": "919061bb-ea4d-4362-ceb7-61fecfc98059"
      },
      "source": [
        "num_epochs=3\r\n",
        "import os\r\n",
        "from tqdm import tqdm, trange\r\n",
        "cwd = os.getcwd()\r\n",
        "model_save_path = output_model_file = os.path.join(cwd, \"drive/My Drive/Models/xlnet_asrs.bin\")\r\n",
        "writer = SummaryWriter(cwd + \"drive/My Drive/log/\" + datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\r\n",
        "model, train_loss_set, valid_loss_set = train(model=model,\\\r\n",
        "                                              num_epochs=num_epochs,\\\r\n",
        "                                              optimizer=optimizer,\\\r\n",
        "                                              train_dataloader=train_dataloader,\\\r\n",
        "                                              valid_dataloader=validation_dataloader,\\\r\n",
        "                                              model_save_path=model_save_path,\\\r\n",
        "                                              device=\"cuda\", writer=None)\r\n",
        "# It takes an hour to train just 3 epochs with pretty workable loss"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\rEpoch:   0%|          | 0/3 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train loss: 0.01204084963888503\n",
            "Valid loss: 0.009389177575270686\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rEpoch:  33%|███▎      | 1/3 [17:12<34:24, 1032.14s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Saving model at epoch 0 with validation loss of 0.009389177575270686\n",
            "\n",
            "\n",
            "Train loss: 0.00883569953340322\n",
            "Valid loss: 0.00870212494124759\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rEpoch:  67%|██████▋   | 2/3 [34:44<17:18, 1038.33s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Saving model at epoch 1 with validation loss of 0.00870212494124759\n",
            "\n",
            "\n",
            "Train loss: 0.00799812485804101\n",
            "Valid loss: 0.008669619300257997\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 100%|██████████| 3/3 [52:16<00:00, 1045.59s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Saving model at epoch 2 with validation loss of 0.008669619300257997\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S597hgPniYIY"
      },
      "source": [
        "### You can start model from a checkpoint with passing last checkpoint validation loss.\r\n",
        "import os\r\n",
        "cwd = os.getcwd()\r\n",
        "model_save_path = output_model_file = os.path.join(cwd, \"drive/My Drive/Models/xlnet_asrs.bin\")\r\n",
        "model, start_epoch, lowest_eval_loss, train_loss_hist, valid_loss_hist = load_model(model_save_path)\r\n",
        "\r\n",
        "num_epochs=3\r\n",
        "model, train_loss_set, valid_loss_set = train(model=model,\\\r\n",
        "                                              num_epochs=num_epochs,\\\r\n",
        "                                              optimizer=optimizer,\\\r\n",
        "                                              train_dataloader=train_dataloader,\\\r\n",
        "                                              valid_dataloader=validation_dataloader,\\\r\n",
        "                                              model_save_path=model_save_path,\\\r\n",
        "                                              train_loss_set=train_loss_hist,\\\r\n",
        "                                              valid_loss_set=valid_loss_hist,\\\r\n",
        "                                              lowest_eval_loss=lowest_eval_loss,\\\r\n",
        "                                              start_epoch=start_epoch,\\\r\n",
        "                                              device=\"cuda\", writer=None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wDCz0c0rsYys"
      },
      "source": [
        "# loading graph on tensorboard, can also do inline in notebook, done in below cell, so can skip.\r\n",
        "get_ipython().system_raw(\r\n",
        "    'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'\r\n",
        "    .format(\"./log\")\r\n",
        ")\r\n",
        "get_ipython().system_raw('./ngrok http 6006 &')\r\n",
        "!curl -s http://localhost:4040/api/tunnels | python3 -c \\\r\n",
        "    \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        },
        "id": "LuJHs5GndCF_",
        "outputId": "394739da-d865-4328-879b-54c4f71e3747"
      },
      "source": [
        "num_epochs = np.arange(len(train_loss_set))\r\n",
        "\r\n",
        "fig, ax = plt.subplots(figsize=(10, 5));\r\n",
        "ax.plot(num_epochs, np.array(train_loss_set), label=\"Train Loss\")\r\n",
        "ax.plot(num_epochs, np.array(valid_loss_set), 'g-', label=\"Valid Loss\")\r\n",
        "#ax1.plot(episode_record, lose_record, 'r-', label=\"Lose %\")\r\n",
        "ax.set_xlabel(\"Number of Epochs\")\r\n",
        "ax.set_ylabel(\"Loss\")\r\n",
        "ax.set_title(\"Loss vs Number of Epochs\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Loss vs Number of Epochs')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnQAAAFNCAYAAACJ9PI3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxV1bn/8c+TmQAJEMKUMMoYBFEioGLr0EEtiloHnKrW1qEC9fbWVm+vrT9vbau21SIOpUqdR6oVp9pBrYJKCYpgwhQRJJEhTAlTyPT8/jibEDIwmZOd4ft+vc7Lc9Zee59nh7R8WXuvtc3dEREREZGWKybsAkRERETky1GgExEREWnhFOhEREREWjgFOhEREZEWToFOREREpIVToBMRERFp4RToREQaiZm5mQ0M6buHmNlCM9tmZlPDqKE2M1tlZl8Luw6RtkCBTkQa1JL/Qjazk4KAdX+t9jlmdkVIZUXTT4C33L2ju0+rvdHM3jazUjPbXuP1cgh1ikgUKNCJSGu2A7jMzPqFXMchMbO4w9itL5B7gD6T3b1DjdeZh/E9ItIMKdCJyCEzs0Qzu8fMvghe95hZYrCtq5m9YmZbzWyzmb1rZjHBtp+aWWFwWXCZmZ1az7HHmtk6M4ut0XaOmS0K3o8xsxwzKzGz9Wb2+/2UuhV4BPhFA+dxq5k9UeNzv2BULy74/LaZ/dLM3tszomVmaWb2ZPD98+sJi2eY2Uoz22hmd+059+B43zWzJWa2xczeMLO+Nba5mV1vZiuAFQ3Ue5aZ5QY/27fNbFjQ/iZwMjA9qHPwfn4m9R33JDMrMLP/CepeZWaX1NieamaPmVmRma02s/+tdV7fD85rm5nlmdkxNQ4/yswWmVmxmT1rZknBPg3+nojIodP/eETkcPwMGAeMAo4CxgD/G2z7b6AASAe6A/8DuJkNASYDx7p7R+CbwKraB3b3eURG1k6p0Xwx8FTw/g/AH9w9BTgCeO4Atd4OfDv4/sMxCbgMyAi+733gz0AXYAl1w+I5QDZwDDAR+C6AmU0k8rM4l8jP5l3g6Vr7ng2MBbJqFxGEtKeBG4L9XwNeNrMEdz8lON6eEbjlh3GePYCuwXleDsyo8TO7F0gFBgBfBb4DXBnUdT5wa9CWApwFbKpx3AuA04D+wEjgiqC93t+Tw6hbRFCgE5HDcwlwm7tvcPci4P8RCT0A5UBPoK+7l7v7ux55aHQlkAhkmVm8u69y908bOP7TwEUAZtYROIO94accGGhmXd19u7t/sL9C3X0d8CBw22Ge65/d/VN3LwZeBz5193+6ewXwPHB0rf53uPtmd/8cuGfPeQDXAr929yXBvr8iMnrVt8a+vw723VVPHRcCr7r7P9y9HPgt0A44/hDOZVowIrbn9X+1tt/i7rvd/d/Aq8AFwUjpJOBmd9/m7quA37H3z/t7wJ3uPt8j8t19dc3vdPcv3H0z8DKRfwRAw78nInIYFOhE5HD0Amr+pb06aAO4C8gH/h5cerwJwN3ziYwu3QpsMLNnzKwX9XsKODe4jHsu8GGNkHAVMBhYGlzynHAQ9d4BfNPMjjroM9xrfY33u+r53KFW/zU13tf8ufQF/rAnTAGbASMyIlbfvrXt8zN396qgf0aDe9Q11d071XjdUmPbFnffUU/tXYF46v557/ne3kBDwRxgXY33O9n786r390REDo8CnYgcji+IBJQ9+gRtBKM4/+3uA4hcfvvRnnvl3P0pdx8f7OtEglYd7p5HJDSczr6XW3H3Fe5+EdAt2H+WmbXfX7HuvonIaFntEakdQHKNzz32d5yD1LvG++qfC5HwdU2tQNXO3d+rWep+jrvPz9zMLPiuwkaoGaBzrZ/jnto3EhlNq/3nved71xC5FH1I9vd7IiKHToFORA4k3sySarziiFz+/F8zSzezrsDPgScAzGyCmQ0MAkcxkUutVRZZJ+2UYNStlMjoVtV+vvcp4IfAV4hc2iQ4/qVmlh6MUG0Nmvd3nD1+T+Ty5LAabQuBr5hZHzNLBW4+iOMcyI1m1tnMegf1Pxu0PwjcbGbDg/NIDe4/O1jPAd8ys1PNLJ7IPWi7gff2v9sh+X9mlmBmJwITgOfdvTL47tvNrGNwifhHBH/ewEPAj81stEUMrHUZuV4N/Z404rmItCkKdCJyIK8RCV97XrcCvwRygEXAYuDDoA1gEPBPYDuRCQT3u/tbRO6f+w2REZ91REbY9hegniZyA/6b7r6xRvtpQK6ZbScyQWJSA/ec7cPdS4A7iUxm2NP2DyKBaxGwAHjlQMc5CC8Fx1pI5D60h4PvepHIiOIzZlYCfEJkBPKguPsy4FIiExQ2AmcCZ7p72SHUtmcW7J7Xghrb1gFbiIzKPQlc6+5Lg21TiIxmrgTmEAnbM4O6nicy8eQpYBvwV2r8jPejod8TETkMpntQRUTaNjM7CXjC3TPDrkVEDo9G6ERERERaOAU6ERERkRZOl1xFREREWjiN0ImIiIi0cAp0IiIiIi1cXNgFhKlr167er1+/sMsQEREROaAFCxZsdPf0+ra16UDXr18/cnJywi5DRERE5IDMbHVD23TJVURERKSFU6ATERERaeEU6ERERERaOAU6ERERkRZOgU5ERESkhVOgExEREWnhFOhEREREWjgFOhEREZEWToFOREREpIWLaqAzs9PMbJmZ5ZvZTfVsTzSzZ4Pt88ysX9CeZmZvmdl2M5teo3+ymb1qZkvNLNfMfnOgY4VpV1klf/z3p5RXVoVdioiIiLRiUQt0ZhYL3AecDmQBF5lZVq1uVwFb3H0gcDdwR9BeCtwC/LieQ//W3YcCRwMnmNnpBzhWaN7IXcevX1/KxX/6gPUlpWGXIyIiIq1UNEfoxgD57r7S3cuAZ4CJtfpMBB4N3s8CTjUzc/cd7j6HSLCr5u473f2t4H0Z8CGQub9jNfZJHYqzj87gD5NG8UlhCd+aNocPVm4KsxwRERFppaIZ6DKANTU+FwRt9fZx9wqgGEg7mIObWSfgTOBfX/ZY0TRxVAYvTT6BlHZxXPLQPB7896e4e9hliYiISCvSIidFmFkc8DQwzd1XHuK+V5tZjpnlFBUVRafAWgZ378jsyeP55vDu/Ob1pVzz+AJKSsub5LtFRESk9YtmoCsEetf4nBm01dsnCGmpwMFcl5wBrHD3ew71WO4+w92z3T07PT39IE/ly+uQGMd9Fx/DLROyeHPpBs66dw55X5Q02feLiIhI6xXNQDcfGGRm/c0sAZgEzK7VZzZwefD+POBNP8D1SDP7JZGwdsOXPVZTMzOuGt+fp68ex86ySs65fy6zFhSEXZaIiIi0cBbNzGNmZwD3ALHATHe/3cxuA3LcfbaZJQGPE5mxuhmYtOcSqpmtAlKABGAr8A2ghMh9ckuB3cHXTHf3h/Z3rIZkZ2d7Tk5OY57yQSvatpupT3/E+ys3cdGYPvzizCyS4mNDqUVERESaPzNb4O7Z9W5rZoNYTSrMQAdQUVnF7/6xnAfe/pQRGancf8kx9O6SHFo9IiIi0nztL9C1yEkRrUVcbAw/PW0of/pONqs27WDCvXN4a+mGsMsSERGRFkaBrhn4elZ3Xpkynl6d2nHlI/P5/d+XUVnVdkdORURE5NAo0DUTfdPa8+IPjuf80ZlMezOfK/78HzbvKAu7LBEREWkBFOiakaT4WO46/yju+PYI5n22mW9Ne5cPP98SdlkiIiLSzCnQNUMXHtuHF647ntgY48I/vs9j76/S0yVERESkQQp0zdSRGam8MmU8Jw5K5+cv5fLDZxayY3dF2GWJiIhIM6RA14x1Sk7goe9kc+M3h/DKoi84+7655G/YHnZZIiIi0swo0DVzMTHG9ScP5LHvjmXTjjImTp/Dq4vWhl2WiIiINCMKdC3E+EFdeXXqeAb36Mj1T33IbS/nUV5ZFXZZIiIi0gwo0LUgPVPb8ezVx3HF8f2YOfczLprxAeuKS8MuS0REREKmQNfCJMTFcOtZw5l20dHkrS1hwr3v8t6nG8MuS0REREKkQNdCnXVUL166/gRS28Vz6UPzuP/tfKr0dAkREZE2SYGuBRvUvSMvTR7PGSN6cufflnH14wso3lUedlkiIiLSxBToWrgOiXHce9HR/OLMLN5etoEz751D7hfFYZclIiIiTUiBrhUwM648oT/PXjOOsooqzr3/PZ7LWRN2WSIiItJEFOhakdF9u/DK1PGM7tuZn8xaxE1/WURpeWXYZYmIiEiUKdC1Ml07JPL4VWO5/uQjeGb+Gs578D3WbN4ZdlkiIiISRQp0rVBsjHHjN4fy0HeyWb1pJ9+a9i5vLl0fdlkiIiISJQp0rdjXsrrz6pQT6d0lme8+ksNv31hGpZY2ERERaXUU6Fq5PmnJ/OW647kwuzfT38rnOzPnsWn77rDLEhERkUakQNcGJMXHcsd5I7nz2yPJWbWFb02bw4LVW8IuS0RERBqJAl0bcsGxvfnLdceTEBfDhX98nz/P/Qx3XYIVERFp6aIa6MzsNDNbZmb5ZnZTPdsTzezZYPs8M+sXtKeZ2Vtmtt3Mptfa53YzW2Nm22u1X2FmRWa2MHh9L5rn1lIdmZHKy5PHc9KQdP7fy3lMefojduyuCLssERER+RKiFujMLBa4DzgdyAIuMrOsWt2uAra4+0DgbuCOoL0UuAX4cT2HfhkY08DXPuvuo4LXQ1/2HFqr1OR4ZlyWzU9OG8Jri9cy8b655G/YFnZZIiIicpiiOUI3Bsh395XuXgY8A0ys1Wci8GjwfhZwqpmZu+9w9zlEgt0+3P0Dd18bxbrbhJgY4wcnDeSJq8ayZUcZZ02fy8sffxF2WSIiInIYohnoMoCaz58qCNrq7ePuFUAxkPYlvvPbZrbIzGaZWe/6OpjZ1WaWY2Y5RUVFX+KrWofjB3bl1aknMqxnClOe/ohbZ+dSVlEVdlkiIiJyCFrTpIiXgX7uPhL4B3tH/vbh7jPcPdvds9PT05u0wOaqR2oSz1w9ju+e0J9H3lvFpBnvs7Z4V9hliYiIyEGKZqArBGqOkmUGbfX2MbM4IBXYdDhf5u6b3H3PAmsPAaMP5zhtVXxsDD8/M4vpFx/NsnXbmDBtDnPzN4ZdloiIiByEaAa6+cAgM+tvZgnAJGB2rT6zgcuD9+cBb/phrqNhZj1rfDwLWHI4x2nrJozsxUuTT6Bz+wQue3ge972VT5WeLiEiItKsRS3QBffETQbeIBKunnP3XDO7zczOCro9DKSZWT7wI6B6aRMzWwX8HrjCzAr2zJA1szvNrABIDtpvDXaZama5ZvYxMBW4Ilrn1toN7NaRl64/gQkje3HXG8v4/mM5FO8sD7ssERERaYC15YVls7OzPScnJ+wymi1357H3V/PLV/PokZrEA5eM5siM1LDLEhERaZPMbIG7Z9e3rTVNipBGZmZcfnw/nr3mOCoqnXMfeI9n538edlkiIiJSiwKdHNAxfTrzypTxjOnXhZ/+ZTE/mfUxpeWVYZclIiIiAQU6OShpHRJ59LtjmHLKQJ7LKeDc+99j9aYdYZclIiIiKNDJIYiNMf77G0OYeUU2hVt3MeHeOfwjb33YZYmIiLR5CnRyyE4Z2p1Xpoynb1oy338shzv/tpSKSj1dQkREJCwKdHJYendJZta1x3PRmN7c//anfGfmf9i4ffeBdxQREZFGp0Anhy0pPpZfnzuSu84byYLVW/jWtHfJWbU57LJERETaHAU6+dLOz+7NCz84nqT4WCbN+ICZcz6jLa9vKCIi0tQU6KRRDO+VyuzJ4zlpSDdueyWPyU9/xPbdFWGXJSIi0iYo0EmjSW0Xz4zLRvPT04by+uK1nDV9DivWbwu7LBERkVZPgU4aVUyMcd1JR/Dk98ZRsquciffN5aWFhWGXJSIi0qop0ElUHHdEGq9OPZGsnin88JmF/OKlTyir0NImIiIi0aBAJ1HTPSWJp68ex/fG9+fR91dz4Yz3+WLrrrDLEhERaXUU6CSq4mNj+N8JWdx/yTEsX7eNCffOYc6KjWGXJSIi0qoo0EmTOGNET2ZPGU/XDglcNnMe9/5rBVVVWtpERESkMSjQSZM5Ir0Df73+BM46qhe/+8dyrnp0Plt3loVdloiISIunQCdNKjkhjnsuHMX/TRzOnPyNTLh3DosLisMuS0REpEVToJMmZ2Zcdlw/nrvmOKqqnG8/+B5P/+dzPV1CRETkMCnQSWiO7tOZV6aeyNj+Xbj5hcXcOGsRu8oqwy5LRESkxVGgk1B1aZ/AI1eOYeqpg5i1oIBzH3iPVRt3hF2WiIhIi6JAJ6GLjTF+9PXB/PnKY1lbvIsz753D33PXhV2WiIhIixHVQGdmp5nZMjPLN7Ob6tmeaGbPBtvnmVm/oD3NzN4ys+1mNr3WPreb2Roz234wx5KW4+Qh3Xh58nj6dW3P1Y8v4NevL6GiUk+XEBEROZCoBToziwXuA04HsoCLzCyrVrergC3uPhC4G7gjaC8FbgF+XM+hXwbG1NPe0LGkBendJZnnrz2Oi8f24Y//XsmlD89jw7bSsMsSERFp1qI5QjcGyHf3le5eBjwDTKzVZyLwaPB+FnCqmZm773D3OUSC3T7c/QN3X1vP99V7rMY4EWlaSfGx/OqcEfzu/KNYuGYrE6bNYf6qzWGXJSIi0mxFM9BlAGtqfC4I2urt4+4VQDGQ9mW/rxGOJc3At0dn8uIPTiA5IZZJMz7goXdXamkTERGRerS5SRFmdrWZ5ZhZTlFRUdjlyAEM65nC7CnjOXVoN3756hKuf+pDtpWWh12WiIhIsxLNQFcI9K7xOTNoq7ePmcUBqcCmL/t9+zuWu89w92x3z05PTz/Mr5KmlJIUzx8vG83/nDGUN3LXM3H6XJat2xZ2WSIiIs1GNAPdfGCQmfU3swRgEjC7Vp/ZwOXB+/OAN/3wr6k15rGkmTEzrv7KETz5vbGUlFZw9n1z+etHtf99ICIi0jZFLdAF97FNBt4AlgDPuXuumd1mZmcF3R4G0swsH/gRUL20iZmtAn4PXGFmBXtmyJrZnWZWACQH7bce6FjSeowbkMZrU8czIiOVG55dyC1//YTdFXq6hIiItG3WlgexsrOzPScnJ+wy5DCUV1Zx1xvLmPHOSo7q3Yn7LzmGjE7twi5LREQkasxsgbtn17etzU2KkNYhPjaG/zljGA9eegyfbtjOhGnv8s5yTXIREZG2SYFOWrTTjuzJ7Mkn0K1jEpf/+T/84Z8rqKpqu6POIiLSNinQSYs3IL0DL15/PGePyuDufy7nu4/OZ8uOsrDLEhERaTIKdNIqJCfE8fsLjuKXZx/Je/mbmHDvHD5eszXsskRERJqEAp20GmbGpeP68vy1xwFw/oPv8+S81Xq6hIiItHoKdNLqHNW7E69MGc+4I9L42Yuf8N/Pf8yuMi1tIiIirZcCnbRKndsn8OcrjuWGrw3ixY8KOef+uXy2cUfYZYmIiESFAp20WrExxg1fG8wjV45hXUkpZ907h799si7sskRERBqdAp20el8dnM4rU8YzIL091z6xgF+/toSKyqqwyxIREWk0CnTSJmR2Tua5a4/j0nF9+OM7K7n4oXlsKCkNuywREZFGoUAnbUZiXCy/PHsEd194FIsKtvKte+cwb+WmsMsSERH50hTopM055+hM/nr9CXRIjOPih+Yx451PtbSJiIi0aAp00iYN7ZHC7Mkn8PVh3fnVa0u57okPKSktD7ssERGRw6JAJ21Wx6R4Hrj0GH52xjD+sWQ9E6fPZem6krDLEhEROWQKdNKmmRnf/8oAnv7+OLbvruDs++bywocFYZclIiJySBToRIAx/bvw6tTxjMzsxI+e+5ifvbiY3RV6uoSIiLQMCnQigW4dk3jqe2O55qsDeHLe51zw4PsUbNkZdlkiIiIHpEAnUkNcbAw3nz6MBy8dzcqiHUy4dw5vL9sQdlkiIiL7pUAnUo/TjuzB7Cnj6ZGSxJWPzOfufyynskpLm4iISPOkQCfSgP5d2/PiD07gnKMz+MO/VnDlI/PZvKMs7LJERETqUKAT2Y92CbH87vyj+NU5I/jg002cee8cFq7ZGnZZIiIi+1CgEzkAM+PisX2Ydd1xAJz/4Hs8/sFqPV1CRESajagGOjM7zcyWmVm+md1Uz/ZEM3s22D7PzPoF7Wlm9paZbTez6bX2GW1mi4N9ppmZBe23mlmhmS0MXmdE89yk7RmZ2YlXpoznhIFdueWvn/Cj5z5mZ1lF2GWJiIhEL9CZWSxwH3A6kAVcZGZZtbpdBWxx94HA3cAdQXspcAvw43oO/QDwfWBQ8Dqtxra73X1U8Hqt0U5GJNC5fQIzLz+WH319MH9dWMjZ981lZdH2sMsSEZE2LpojdGOAfHdf6e5lwDPAxFp9JgKPBu9nAaeambn7DnefQyTYVTOznkCKu3/gketdjwFnR/EcROqIiTGmnjqIR68cQ9G23Zw1fS6vL14bdlkiItKGRTPQZQBranwuCNrq7ePuFUAxkHaAY9Z8LlPtY042s0VmNtPMOh9u4SIH4yuD03ll6okc0a0D1z35Ibe/mkd5ZVXYZYmISBvUmiZFPAAcAYwC1gK/q6+TmV1tZjlmllNUVNSU9UkrlNGpHc9dM47vHNeXP737GZf8aR4bSkoPvKOIiEgjimagKwR61/icGbTV28fM4oBUYNMBjplZ3zHdfb27V7p7FfAnIpd863D3Ge6e7e7Z6enph3A6IvVLjIvltolH8odJo1hcWMwZ0+bwwcr9/RqLiIg0rmgGuvnAIDPrb2YJwCRgdq0+s4HLg/fnAW/6ftaCcPe1QImZjQtmt34HeAmq76/b4xzgk8Y5DZGDM3FUBi9NPoGUdnFc8tA8/vjvT7W0iYiINImoBbrgnrjJwBvAEuA5d881s9vM7Kyg28NAmpnlAz8Cqpc2MbNVwO+BK8ysoMYM2R8ADwH5wKfA60H7ncFyJouAk4H/ita5iTRkcPeOzJ48nm8O786vX1/KNY8voKS0POyyRESklbO2PIKQnZ3tOTk5YZchrZC7M3PuKn792hIyO7fjgUtHM6xnSthliYhIC2ZmC9w9u75trWlShEizYWZcNb4/T189jp1llZxz/1xmLSg48I4iIiKHQYFOJIqO7deFV6eeyKjenfjx8x9z8wuLKS2vDLssERFpZRToRKIsvWMiT1w1lmu/egRP/+dzzn/wfdZs3hl2WSIi0ooo0Ik0gbjYGG46fSgzLhvNqk07mHDvHN5auiHsskREpJVQoBNpQt8Y3oNXpoynV6d2XPnIfH7/92VUVrXdiUkiItI4FOhEmljftPa8+IPjOW90JtPezOeKP/+HzTvKwi5LRERaMAU6kRAkxcdy13kj+c25I5j32WYmTHuXjz7fEnZZIiLSQinQiYTEzJg0pg8vXHc8MTHGBX98n8feX6WnS4iIyCFToBMJ2ZEZqbwyZTwnDkrn5y/lcsOzC9lZVhF2WSIi0oIo0Ik0A52SE3joO9nc+M0hvPzxF0ycPpf8DdvDLktERFoIBTqRZiImxrj+5IE89t2xbNpRxsTpc3h10dqwyxIRkRZAgU6kmRk/qCuvTBnP4B4duf6pD7nt5TzKK6vCLktERJoxBTqRZqhXp3Y8e/VxXHF8P2bO/YyLZnzAuuLSsMsSEZFmSoFOpJlKiIvh1rOGM+2io8lbW8KEe9/lvU83hl2WiIg0Qwp0Is3cWUf14qXrTyC1XTyXPjSP+9/Op0pPlxARkRoOKtCZWXsziwneDzazs8wsPrqlicgeg7p35KXJ4zl9RE/u/Nsyrn58AcW7ysMuS0REmomDHaF7B0gyswzg78BlwCPRKkpE6uqQGMf0i47mF2dm8fayDZw1fQ65XxSHXZaIiDQDBxvozN13AucC97v7+cDw6JUlIvUxM648oT/PXjOO0vJKzr3/PZ7PWRN2WSIiErKDDnRmdhxwCfBq0BYbnZJE5EBG9+3Cq1NPZHTfztw4axE3/WURpeWVYZclIiIhOdhAdwNwM/Ciu+ea2QDgreiVJSIH0rVDIo9fNZbrTz6CZ+av4bwH32PN5p1hlyUiIiGwQ30QeDA5ooO7l0SnpKaTnZ3tOTk5YZch8qX9M289//XcQgy4Z9IoThnaPeySRESkkZnZAnfPrm/bwc5yfcrMUsysPfAJkGdmNx7EfqeZ2TIzyzezm+rZnmhmzwbb55lZv6A9zczeMrPtZja91j6jzWxxsM80M7OgvYuZ/cPMVgT/7Xww5ybSGnwtqzuvTjmRzM7JfPeRHH77xjIqtbSJiEibcbCXXLOCEbmzgdeB/kRmujbIzGKB+4DTgSzgIjPLqtXtKmCLuw8E7gbuCNpLgVuAH9dz6AeA7wODgtdpQftNwL/cfRDwr+CzSJvRJy2ZF35wPBdm92b6W/l8Z+Y8Nm3fHXZZIiLSBA420MUH686dDcx293LgQP/8HwPku/tKdy8DngEm1uozEXg0eD8LONXMzN13uPscIsGumpn1BFLc/QOPXCt+LKip9rEerdEu0mYkxcdyx3kjuePbI5i/agvfmjaHBau3hF2WiIhE2cEGuj8Cq4D2wDtm1hc40D10GUDN9RQKgrZ6+7h7BVAMpB3gmAUNHLO7u68N3q8DdBORtFkXHtuHF647noS4GC784/s8MvczDvV+WRERaTkOKtC5+zR3z3D3MzxiNXBylGs7bMHoXb1/e5nZ1WaWY2Y5RUVFTVyZSNM5MiOVlyeP56Qh6dz6ch5Tn1nIjt0VYZclIiJRcLCTIlLN7Pd7gpCZ/Y7IaN3+FAK9a3zODNrq7WNmcUAqsOkAx8xs4Jjrg0uyey7NbqjvAO4+w92z3T07PT39AKcg0rKlJscz47JsbvzmEF5d9AUT75tL/oZtYZclIiKN7GAvuc4EtgEXBK8S4M8H2Gc+MMjM+ptZAjAJmF2rz2zg8uD9ecCbvp/rQsEl1RIzGxfMbv0O8FI9x7q8RrtImxYTY1x/8kCeuGosW3aUcdb0ubz88RdhlyUiIo3ooNahM7OF7j7qQG317HcGcA+Rp0rMdPfbzew2IMfdZ5tZEvA4cDSwGZjk7iuDfVcBKUACsBX4hrvnmVk2kefItiMy43aKu7uZpQHPAX2A1cAF7kZW1BMAACAASURBVL55f/VpHTppa9YVl3L9Ux+yYPUWrjyhHzefPoyEuIP9d52IiIRpf+vQHWygex+4MZh5ipmdAPzW3Y9r1EqbmAKdtEXllVX8+rWlzJz7Gcf06cR9lxxDz9R2YZclIiIH8KUXFgauBe4zs1XByNl04JpGqk9EmlB8bAw/PzOL6RcfzdJ125gwbQ5z8zeGXZaIiHwJBzvL9WN3PwoYCYx096OBU6JamYhE1YSRvZg9+QQ6t0/gsofncd9b+VTp6RIiIi3SId084+4lNZ7h+qMo1CMiTWhgt468dP0JfGtkL+56YxnffyyH4p3lYZclIiKH6MvcDW2NVoWIhKZ9YhzTJo3i1jOzeGdFEROmv8snhcVhlyUiIofgywQ6XZsRaSXMjCtO6M8zVx9HeYVz7gPv8ez8z8MuS0REDtJ+A52ZbTOzknpe24BeTVSjiDSR0X078+rU8Yzp14Wf/mUxP5n1MaXllWGXJSIiB7DfQOfuHd09pZ5XR3ePa6oiRaTppHVI5NHvjmHKKQN5LqeAc+9/j8837Qy7LBER2Q+tKCoidcTGGP/9jSHMvCKbwq27+Na97/LPvPVhlyUiIg1QoBORBp0ytDuvTBlP37RkvvdYDnf+bSkVlVVhlyUiIrUo0InIfvXuksysa4/nojG9uf/tT/nOzP+wcfvusMsSEZEaFOhE5ICS4mP59bkjufO8kSxYvYVvTXuXnFX7fVSyiIg0IQU6ETloF2T35oUfHE9SfCyTZnzAzDmfcTDPgxYRkehSoBORQzK8VyqzJ4/npCHduO2VPCY//RHbd1eEXZaISJumQCcihyy1XTwzLhvNT08byuuL13LW9DmsWL8t7LJERNosBToROSwxMcZ1Jx3BE98bS8mucibeN5eXFhaGXZaISJukQCciX8rxR3Tl1aknktUzhR8+s5BfvPQJZRVa2kREpCkp0InIl9Y9JYmnrx7HVeP78+j7q7lwxvt8sXVX2GWJiLQZCnQi0ijiY2O4ZUIW919yDMvXbWPCvXOYs2Jj2GWJiLQJCnQi0qjOGNGT2VPGk9Y+gctmzmP6myuoqtLSJiIi0aRAJyKN7oj0Dvz1+hM466he/Pbvy/neYzls3VkWdlkiIq2WAp2IREX7xDjuuXAU/zdxOO+uKGLCvXNYXFAcdlkiIq2SAp2IRI2Zcdlx/XjumuOoqnK+/eB7PP2fz/V0CRGRRhbVQGdmp5nZMjPLN7Ob6tmeaGbPBtvnmVm/GttuDtqXmdk3a7T/0Mw+MbNcM7uhRvutZlZoZguD1xnRPDcROXhH9+nMK1NPZGz/Ltz8wmJunLWIXWWVYZclItJqRC3QmVkscB9wOpAFXGRmWbW6XQVscfeBwN3AHcG+WcAkYDhwGnC/mcWa2ZHA94ExwFHABDMbWON4d7v7qOD1WrTOTUQOXZf2CTxy5RimnjqIWQsKOPeB91i1cUfYZYmItArRHKEbA+S7+0p3LwOeASbW6jMReDR4Pws41cwsaH/G3Xe7+2dAfnC8YcA8d9/p7hXAv4Fzo3gOItKIYmOMH319MH++4li+2LqLM++dw99z14VdlohIixfNQJcBrKnxuSBoq7dPENCKgbT97PsJcKKZpZlZMnAG0LtGv8lmtsjMZppZ58Y8GRFpPCcP7cYrU8bTr2t7rn58Ab95fSkVlXq6hIjI4WpRkyLcfQmRy7J/B/4GLAT23IjzAHAEMApYC/yuvmOY2dVmlmNmOUVFRdEvWkTq1btLMs9fexwXj+3Dg//+lEsfnkfRtt1hlyUi0iJFM9AVsu/oWWbQVm8fM4sDUoFN+9vX3R9299Hu/hVgC7A8aF/v7pXuXgX8icgl2jrcfYa7Z7t7dnp6+pc8RRH5MpLiY/nVOSP43flHsXDNVr417V3mr9ocdlkiIi1ONAPdfGCQmfU3swQikxxm1+ozG7g8eH8e8KZH1jOYDUwKZsH2BwYB/wEws27Bf/sQuX/uqeBzzxrHPYfI5VkRaQG+PTqTF39wAskJsUya8QEPvbtSS5uIiByCuGgd2N0rzGwy8AYQC8x091wzuw3IcffZwMPA42aWD2wmEvoI+j0H5AEVwPXuvufS6l/MLA0oD9q3Bu13mtkowIFVwDXROjcRaXzDeqYwe8p4fvzcx/zy1SV8+PkW7vj2SDomxYddmohIs2dt+V/B2dnZnpOTE3YZIlKDuzPjnZXc+cYy+nZJ5oFLRzOkR8ewyxIRCZ2ZLXD37Pq2tahJES3Nuu3rmJU3iyVFSyivLA+7HJEWwcy45qtH8OT3xlJSWsHZ983lrx/Vvv1WRERqitolV4F3Vr/DhbMuBCAuJo7BaYMZnj6crPSs6v8OShtEQmxCyJWKND/jBqTx2tTxTH7qI254diELVm/hfycMIzEuNuzSRESaHV1yjeIl113lu1iycQl5RXnkbsglb2MeeUV5fLr5U5zIzz0uJo5BXQaRlZ61T9AbnDaYxLjEqNUm0lKUV1Zx1xvLmPHOSo7q3Yn7LzmGjE7twi5LRKTJ7e+SqwJdCPfQ7SrfxbJNyyIhryiPvI2RwPfplk+p8sjiqrEWy8AuA/cJeVnpWQzpOoSkuKQmr1kkbK8vXsuNsxYRH2v8YdLRfGWwlh0SkbZFga4BzW1SRGlFKcs3La8OerlFkf/mb86nMpjkG2MxHNH5CIZ3G05W12BUr9twhqQNoV28Ri2kdVtZtJ3rnviQ5Ru2ccOpg5lyykBiYizsskREmoQCXQOaW6BryO6K3SzftHyfkJdXlMeKzSuoqKoAIkFvQOcBkZG8rpGQl5WexdCuQ0mOTw75DEQaz86yCn724ie8+FEhJw1J5+4LRtG5ve5DFZHWT4GuAS0l0DWkrLKMFZtW1Al6yzYtqw56htG/c/86l26HdR1G+4T2IZ+ByOFxd56c9zm3vZxHesdEHrj0GEZmdgq7LBGRqFKga0BLD3QNKa8sJ39z/j4hL7col2Ubl1FetXf5lH6d+u0T9IanD2dY+jA6JHQIsXqRg7dwzVauf/JDirbt5hdnZXHxmD6Y6RKsiLROCnQNaK2BriEVVRXkb87fJ+TlFeWxdONSyirLqvv1Se1TZ3mVYenDSElMCbF6kfpt3lHGDc8u5J3lRZx7TAa3nz2Cdgla2kREWh8Fuga0tUDXkIqqClZuWVlneZUlRUvYXbm7ul/vlN51Lt1mpWeRmpQaYvUiUFnl3PvmCv7wrxUM6d6RBy4dTf+uuqVARFoXBboGKNDtX2VVJZ9t/azO8ipLNi6htKK0ul9Gx4x9L90GEzI6JemeJmlaby/bwA3PLqSy0rnr/KM47cgeYZckItJoFOgaoEB3eCqrKlldvLrO8ipLNi5hZ/nO6n49O/Sss7xKVnoWXdp1CbF6ae0Ktuzk+ic/5OOCYq75ygBu/OYQ4mL1lEMRafkU6BqgQNe4qryK1VtX15l1m1eUx47yHdX9enToUWd5leHpw0lLTguxemlNdldU8n+v5PHEB58zpn8Xpl90NN1StCC3iLRsCnQNUKBrGlVexZriNfsEvT3/3V62vbpft/bd6sy6zUrPIr29ngggh+eFDwv4nxcX0zEpnukXHc3YAfpHg4i0XAp0DVCgC5e7U1BSsDfk1ZiQUbK7pLpf1+Su+0zE2PO+W/tuWqJCDmjpuhKue+JDPt+8k5tOG8r3Tuyv3xsRaZEU6BqgQNc8uTuF2wr3Lq8SBL3cDbkU7y6u7pfWLq1OyMtKz6JHhx76C1v2sa20nBufX8Tfctdx2vAe3Hn+SFKS4sMuS0TkkCjQNUCBrmVxd9ZuX7s35NWYebuldEt1v85JneuEvOHdhtOzQ08FvTbM3Xno3c/4zd+W0qdLMg9cegxDe2htRRFpORToGqBA1zq4O+t3rN8b8oJ79HKLctm8a3N1v05JnaonY9ScdZvRMUNBrw2Zt3ITk5/+iG2l5fzqnBGce0xm2CWJiBwUBboGKNC1bu7Ohh0b6jwZI7col407N1b3S0lMqTPrNis9i94pvRX0WqkN20qZ/NRH/OezzVwytg8/PzOLxDg9XUJEmjcFugYo0LVdRTuK6iyvkluUy4YdG6r7dEjoUOceveHpw+md2psY07pmLV1FZRV3vbGMP76zkqMyU7nvkmPI7JwcdlkiIg1SoGuAAp3UtnHnRpYULamzvMq67euq+7SPb8+w9GF1llfp26mvgl4L9LdP1nHj8x8TG2vcc+EoThrSLeySRETqFVqgM7PTgD8AscBD7v6bWtsTgceA0cAm4EJ3XxVsuxm4CqgEprr7G0H7D4HvAwb8yd3vCdq7AM8C/YBVwAXuvoX9UKCTg7V51+Y6s27zivL4YtsX1X2S45MZ1nVYnQkZ/Tv3V9Br5j7buIPrnljAsvXbmHrKIKaeOojYGF1uF5HmJZRAZ2axwHLg60ABMB+4yN3zavT5ATDS3a81s0nAOe5+oZllAU8DY4BewD+BwcAw4JmgvQz4G3Ctu+eb2Z3AZnf/jZndBHR295/ur0YFOvmytuzawpKNS+rMui3cVljdp11cO4Z2HVrnebf9O/UnNkb3bTUXu8oq+dlfF/PCh4V8ZXA691w4ii7tE8IuS0SkWliB7jjgVnf/ZvD5ZgB3/3WNPm8Efd43szhgHZAO3FSz755+QCZwmrtfFbTfAux29zvNbBlwkruvNbOewNvuPmR/NSrQSbQUlxbvE/T2XLpdU7Kmuk9ibCJDuw6t87zbAZ0HEBcTF2L1bZe78/R/1nDr7FzSOyZy/yXHcFTvTmGXJSIC7D/QRfNvjQxgTY3PBcDYhvq4e4WZFQNpQfsHtfbNAD4BbjezNGAXcAawJ5F1d/e1wft1QPfGOxWRQ5OalMq4zHGMyxy3T3vJ7hKWFC3ZZyLG3M/n8tTip6r7JMYmMqTrkDozbwd2GaigF2VmxsVj+3BkRgrXPfEh5z/4PrecmcWlY/toxrOINGst6m8Hd19iZncAfwd2AAuJ3GNXu5+bWb1Dj2Z2NXA1QJ8+faJYrUhdKYkpjM0cy9jMff9ts233NpZuXLrPaN4HBR/wzCfPVPeJj4mvDno1J2QM7DKQ+Fg99aAxjczsxCtTxvNfzy3klr9+woert3D7OUeSnNCi/i9TRNqQaP6/UyHQu8bnzKCtvj4FwSXXVCKTIxrc190fBh4GMLNfERm9A1hvZj1rXHLdQD3cfQYwAyKXXA/77EQaUcfEjhybcSzHZhy7T/uOsh0s3bh0n+VVcr7I4fnc53Eiv75xMXEMThtc53m3g9IGkRCre8AOV+f2Ccy8/Fimv5XP3f9cTt4XJTxw6TEMSO8QdmkiInVE8x66OCKTIk4lEsbmAxe7e26NPtcDI2pMijjX3S8ws+HAU+ydFPEvYJC7V5pZN3ffYGZ9iIzUjXP3rWZ2F7CpxqSILu7+k/3VqHvopKXaWb6zekSv5qjep5s/3SfoDeoyqM5aeoPTBpMYlxjyGbQs7ywv4ofPfER5pXPXeSM5fUTPsEsSkTYozGVLzgDuIbJsyUx3v93MbgNy3H22mSUBjwNHA5uBSe6+Mtj3Z8B3gQrgBnd/PWh/l8h9duXAj9z9X0F7GvAc0AdYTWTZkr3PfaqHAp20NrvKd7Fs07I6y6vkb86nyqsAiLVYBnYZWGd5lSFdh5AUlxTyGTRfhVt38YMnP+TjNVv5/on9+clpQ4mP1XI0ItJ0tLBwAxTopK0orShl+abldZZXyd+cT6VHbkONsRiO6HxEnVm3Q9KG0C6+Xchn0Dzsrqjk9leX8Nj7qxnTrwvTLz6abikKwSLSNBToGqBAJ23d7ordrNi8os7yKis2r6CiqgIAwxjQeUB10Nsz63Zo16Ekx7fNR2W9tLCQm/6ymPaJcUy/+GjGDUgLuyQRaQMU6BqgQCdSv7LKMlZsWlHnebfLNy2nvKociAS9/p3717l0O6zrMNontA/5DKJv+fptXPv4AlZv3slPvjmEq78yQEubiEhUKdA1QIFO5NCUV5aTvzl/n6CXW5TLso3LqoMeQL9O/eosrzIsfRgdElrXDNFtpeX89C+LeG3xOr6R1Z3fXnAUKUlaQkZEokOBrgEKdCKNo6Kqgk83f7rPaF5uUS5LNy6lrLKsul+f1D77hLys9CyGpQ8jJTElxOq/HHdn5txV/Pq1JWR2bscDl45mWM+Wez4i0nwp0DVAgU4kuiqqKli5ZWWd5VWWblxKaUVpdb/eKb3rLK+SlZ5FalJqiNUfmvmrNnP9kx9SUlrOlFMGMW5AF7J6ptIuQc/rFZHGoUDXAAU6kXBUVlXy2dbP9lleJXdDLks2Ltkn6GV0zNj30m0wIaNTUvN8vmrRtt3c8OxHzM3fBECMwaBuHRmRmcqIjFRGZKaS1TOFpHiFPBE5dAp0DVCgE2leKqsqWV28us7yKks2LmFn+c7qfj079KyzvEpWehZd2nUJsfoId2d9yW4WFxazuGAriwuLWVRQzKYdkUvPsTHGoG4dGJmZyojMTozISGVoj44KeSJyQAp0DVCgE2kZqryKz4s/r7O8Sl5RHjvKd1T369GhR+RybY3lVbLSs+ia3DXE6iMhb21xaRDyilkUhL0tOyMTSeJijCE9OlaP4o3M6MTgHh1IjFPIE5G9FOgaoEAn0rJVeRVritfUCXl5RXlsK9tW3a9b+2517s8bnj6c9PbpodXu7hRu3cXiguJI0AtG8op3RUJefKwxtEfK3su1GakM6dFRT6cQacMU6BqgQCfSOrk7BSUFdWbd5hXlUbK7pLpf1+SudUJeVnoW3dp3C2VNOXenYMsuFhUUs6hwK58EIW9baWSR54S4GIb16Fg9indkRiqDundQyBNpIxToGqBAJ9K2uDtfbPti7xp6NSZkFO8uru6X1i6t3lm3PTr0aPKg5+6s3rSzxijeVj4pLGH77kjIS4yLIatXSvUo3sjMThyR3p44hTyRVkeBrgEKdCICwT1u29fuHc2rEfS2lG6p7tc5qXPdS7fdhtOzQ88mDXpVVc6qTTv2uScvt7CYHWWR5/ImxccwvFdqjZCXyoD0DsTG6EkWIi2ZAl0DFOhEZH/cnfU71u8NeTVm3m7atam6X2pi6j5BLzMlkxiLadKXu1GwpZRl67axZO12lq7bztK12yktd8BITognq2cqR/bqzMjMTozI6MzA9I7EaiRPpMVQoGuAAp2IHA53p2hn0d6QV+MevaKdRWGXd8iMSCiMtRhiYuoPjLEW2+QhVa/ovgzT84dbmP0FurimLkZEpKUzM7q170a3/t04uf/J+2wr2lHE+h3rcXeqvKrJXpVeeVD9KiorKdpRytrinawr3sX6kl0Ubd9FZVUV4CTEQVqHeLp2SCCtQxxd2seTnBgTtfOpqKpo0p9TzZfTdgc09jAs9GDZnF+H8g+ZUT1GMSx9WGh/lgp0IiKNKL19eqjLoRyO8soqlq/fVj2rdnFhMUvXbKOgsgqA1HbxjMhI5cjgfrwRGalkdm7X4kd33B2naYO3Xq030N/xtTsU6EREJDzxsZFJFMN7pXLhsZG2sopIyFtUvU7eVh6es5Lyyshfgp2S4/eZdDEisxO9UpNaVMgzs+oRKml7GnvUOewFzBXoRESkjoS4GI4MRuX22F1RybJ1QcgLgt6Md1ZSURUJeWntE/YZxRuRmUqPlJYV8qTtMDNiLZZYWscTWRToRETkoCTGxTIysxMjMztVt5WWV7Jkbck+l2vvf3sjlUHI69ohkZGZweXaIOR1T0kK6xREWi0FOhEROWxJ8bEc3aczR/fpXN22q6ySvLUlLC7YyuLCEhYXbuXtZRsIMh7dOiYGo3idGJGZwoiMTqR3TAzpDERaBwU6ERFpVO0SYhndtzOj++4NeTvLKsj7oqTGPXnF/GvpBvasnNUzNWmfUbwRGamkdVDIEzlYCnQiIhJ1yQlxZPfrQna/LtVt23dXkFu4N+AtLijmH3nrq7dndGpXfS/engkYndsnhFG+SLMX1UBnZqcBfwBigYfc/Te1ticCjwGjgU3Ahe6+Kth2M3AVUAlMdfc3gvb/Ar4HOLAYuNLdS83sEeCrwJ4HMl7h7gujeX4iInL4OiTGMXZAGmMHpFW3lZSWkxtcpl1cGLls+7fcddXbMzu323u5Ngh5qcnxYZQv0qxELdCZWSxwH/B1oACYb2az3T2vRrergC3uPtDMJgF3ABeaWRYwCRgO9AL+aWaDgR7AVCDL3XeZ2XNBv0eC493o7rOidU4iIhJdKUnxHHdEGscdsTfkFe8qJ7cw8szaPbNrX1u8N+T1TUve53LtkRmppCQp5EnbEs0RujFAvruvBDCzZ4CJQM1ANxG4NXg/C5hukfntE4Fn3H038JmZ5QfH+zyouZ2ZlQPJwBdRPAcREQlZart4jh/YleMH7l3na+vOsn0u1S78fCuvLlpbvb1/1/bVI3gjMlMZ3iuFjgp50opFM9BlAGtqfC4AxjbUx90rzKwYSAvaP6i1b4a7v29mvyUS7HYBf3f3v9fod7uZ/Rz4F3BTEAhFRKSV6ZScwImD0jlx0N6ncmzeEYS8gq0sLiwmZ9VmZn8c+Te/WSTkRUbxIpdrh/dKoX2ibiWX1qFF/SabWWcio3f9ga3A82Z2qbs/AdwMrAMSgBnAT4Hb6jnG1cDVAH369GmiykVEJNq6tE/gq4PT+ergvSFv4/bd1aN4iwqK+WDlZv66cG/IG5jeoXoUb2RmKlk9U2mX0DoWmpW2JZqBrhDoXeNzZtBWX58CM4sDUolMjmho368Bn7l7EYCZvQAcDzzh7nvG2neb2Z+BH9dXlLvPIBL4yM7O1pOZRURasa4dEjl5SDdOHtKtum1DSek+l2vfzd/ICx9F/nqKMRjUrWN1wDsyI5WsnikkxSvkSfMWzUA3HxhkZv2JhLFJwMW1+swGLgfeB84D3nR3N7PZwFNm9nsikyIGAf8BqoBxZpZM5JLrqUAOgJn1dPe1wT14ZwOfRPHcRESkheqWksSpKUmcOqx7ddv6ktLgkWaRy7VvLd3ArAUFAMTGGIO7d2RERkr15dqhPToq5EmzErVAF9wTNxl4g8iyJTPdPdfMbgNy3H028DDweDDpYTOR0EfQ7zkiEygqgOvdvRKYZ2azgA+D9o8IRtuAJ80sHTBgIXBttM5NRERal+4pSXw9K4mvZ0VCnruztrh07+XawmL+uWQDz+VEQl5cjDGkR8cajzXrxJAeHUmIiwnzNKQNM/e2e9UxOzvbc3Jywi5DRERaAHencOuu6qVTFgfPry3eVQ5AQmwMQ3oEl2szIkFvSI+OxMcq5EnjMLMF7p5d7zYFOgU6ERE5PO5OwZZdLCooZlHhVj4JQt620goAEuJiGNYzhREZKYzM6MSIzFQGdetAnEKeHAYFugYo0ImISGNzd1Zv2lljFG8ruYUlbNsdCXmJcTFk9UqpHsUbmdmJI9LbK+TJASnQNUCBTkREmkJVlbNq04597snLLSxmR1klAO3iY8nqlcKIjNTg0WapDEjvQGyMhVy5NCcKdA1QoBMRkbBUVjmfbdzB4sKtLCoo5pPCYj4pLGFXeSTkJSfEcmSvPaN4kbXy+qe1J0Yhr81SoGuAAp2IiDQnlVXOp0XbqydeLCrYSt7aEkrLqwDokBjH8F4pe2fXZnaib5dkhbw2Yn+BrkU9KUJERKQ127Pm3eDuHfn26EwAKiqryC/aXj2Kt6igmEffX01ZRSTkdUyK48hee0fxRmSk0qdLMpFlWaWt0AidRuhERKSFKa+sYsX67ftcrl2ydhtllZGQl9ounhEZNS7XZqSS2bmdQl4Lp0uuDVCgExGR1qKsoorl67dVr4+3uHAry9Zto7wy8vd85+T4fQLeiMxO9EpNUshrQRToGqBAJyIirdnuikqWrdu2z+Xa5eu3UVEV+bs/rX1CrZCXSo8UhbzmSvfQiYiItEGJcbGMzOzEyMxO1W2l5ZUsXbeNxQVbg5G8Yu5/eyOVQcjr2iFxb8ALwl63lKSwTkEOkgKdiIhIG5IUH8uo3p0Y1XtvyNtVVkne2pLqUbzFhVt5e9kGgoxH95TEIOB1YkRmCiMyOpHeMTGkM5D6KNCJiIi0ce0SYhndtzOj+3aubttZVkHeFyV7L9cWFvOvpRvYc6dWz9Sk6lG8PbNr0zoo5IXl/7d371FWlecdx7+/meE6A8NlUGFQuWtQ8U4j8RZjG3NRNNUlxhjjMrEaL0lczYppu2yW7Vo1y2QlzUpaY9WltlYkVCO1RjHR2FiDghYZwaiIqENoRJBBEBDk6R/7nfEwmRkOw8ycs+f8PmudxT7vvpz3Oe/s4Zn33Xu/TujMzMzsjwwdWMNxE0Zx3IRRbWWbt7cmeRvbpjZbuOIPbesbRwzZLcE7orGekbUDS1H9iuOEzszMzIpSN6iGmRNHMXPih0nepm07WL5mU1svXlPzRh5e/n9t6w8cNaRtuHbG+HoOH1dP/dABpah+v+aEzszMzLpt+OABnDB5NCdMHt1W1rJ1B8tbE7w0f+1DTR8meQePHrrbcO3hjfUMH+wkb184oTMzM7MeVT9kALOmNDBrSkNb2cb33m8bpm1qbuF/39jIg8vWtq2f2FDbdlft4emhyHWDnKYUy9+UmZmZ9boRQwdy0tQxnDR1TFvZhi0pyUvX5C1ZvYEFz/8eAAkmpSTviPHZcO30scOpdZLXIX8rZmZmVhKjagdyyrQxnDLtwyTv7c3b23rxljW3sGjVBn6+9MMkb8qYurabLrIkr54hA6tLFULZ8EwRninCzMysrL21adtuw7XL1rSw7t3tAFQJpu43jCPGfzhcO33scAYP6H9Jnqf+6oQTOjMzs3z6w6ZtbTNdtA7Zvr35fQCqq8S0/YdxROPwbLi2sZ5Dxw5jUE2+kzwndJ1wQmdmZtY/RARrW7bt1ov3wpoWNmzJkrwB1VmS19qLN6NxBIccMIyBNVUlrnnxnNB1wgmdmZlZ/xURrNm4labW1pMMiwAADQBJREFUnrw0tVnL1h0ADKyu4pAD0nBturP2kAOGMaC6PJO8kiV0ks4A/hGoBm6NiBvbrR8E3AUcC6wHzo+I1Wndt4FLgQ+AayLikVT+DeDLQABNwCURsU3SRGAuMBp4FrgoIt7vqn5O6MzMzCpLRND8zlaWNbewbM3Gtvlr3922E4CBNVV8ZOxwZhQ8J2/qfnXUlEGSV5KETlI18DLwp0AzsBi4ICJWFGzzVWBGRFwuaQ5wTkScL2k6cA8wExgH/BKYBhwAPAlMj4itkuYBD0XEHWn5voiYK+lm4PmI+Oeu6uiEzszMzCKC19e/V9CLt5Hlazbx7vYsyRtUU8X0cSnJGz+CIxrrmTymts+TvK4Sut58bMlMYGVErEqVmAvMBlYUbDMb+E5ang/8WJJS+dyI2A68JmllOt4bqc5DJO0AhgK/T/ucBnw+HevOdNwuEzozMzMzSUxoqGVCQy1nHjkOgF27gtXrt+x2Td78Z5u587evAzBkQDXTxw1ve3zKzImjGD9yaMli6M2ErhF4s+B9M/AnnW0TETsltZANmTYCi9rt2xgRv5X0PbLEbiuwMCIWSmoANkbEzsLtO6qUpMuAywAOOuigfQjPzMzM+quqKjFpTB2TxtQx+6gspdi1K1j19haa1mxkWXN208W9i9/kjqdWc92nDuXyUyaXrL65erCwpJFkvXcTgY3AzyR9AXi42GNExC3ALZANufZGPc3MzKz/qaoSU/arY8p+dZxz9HgAPtgVvLpuM/VDSjsXbW8O/q4BDix4Pz6VdbiNpBqgnuzmiM72PR14LSLWRcQO4D5gVtpnRDpGZ59lZmZm1qNan3m3//DBJa1HbyZ0i4GpkiZKGgjMARa022YBcHFaPhd4LLK7NBYAcyQNSnevTgWeIRtq/aikoem6uU8AL6Z9Hk/HIB3zgV6MzczMzKxs9FpCl65nuwp4BHgRmBcRyyXdIOmstNltwOh008O1wHVp3+XAPLIbKB4GroyIDyLiabKbJ54je2RJFWn4FPgWcG061uh0bDMzM7N+zw8W9mNLzMzMLAe6emxJ6Z+SZ2ZmZmb7xAmdmZmZWc45oTMzMzPLOSd0ZmZmZjnnhM7MzMws55zQmZmZmeWcEzozMzOznKvo59BJWge83ssf0wC83cufUc4qOf5Kjh0qO37HXrkqOf5Kjh36Jv6DI2JMRysqOqHrC5KWdPYQwEpQyfFXcuxQ2fE79sqMHSo7/kqOHUofv4dczczMzHLOCZ2ZmZlZzjmh6323lLoCJVbJ8Vdy7FDZ8Tv2ylXJ8Vdy7FDi+H0NnZmZmVnOuYfOzMzMLOec0O0DSWdIeknSSknXdbB+kKR70/qnJU0oWPftVP6SpE/2Zb17QhGxXytphaRlkn4l6eCCdR9IWppeC/q25j2jiPi/JGldQZxfLlh3saRX0uvivq35visi9h8UxP2ypI0F63Ld9pJul/SWpBc6WS9JP0rfzTJJxxSsy3u77yn2C1PMTZKeknRkwbrVqXyppCV9V+ueU0T8p0pqKfj5vr5gXZfnTLkrIvZvFsT9QjrPR6V1uW57SQdKejz9f7Zc0tc62KY8zvuI8KsbL6AaeBWYBAwEngemt9vmq8DNaXkOcG9anp62HwRMTMepLnVMPRz7x4GhafmK1tjT+82ljqEP4v8S8OMO9h0FrEr/jkzLI0sdU0/G3m77q4Hb+1HbnwwcA7zQyfpPA78ABHwUeLo/tHuRsc9qjQn4VGvs6f1qoKHUMfRy/KcCD3ZQvlfnTDm+9hR7u23PBB7rL20PjAWOScvDgJc7+H1fFue9e+i6byawMiJWRcT7wFxgdrttZgN3puX5wCckKZXPjYjtEfEasDIdLy/2GHtEPB4R76W3i4DxfVzH3lRM23fmk8CjEbEhIt4BHgXO6KV69oa9jf0C4J4+qVkfiIj/BjZ0scls4K7ILAJGSBpL/tt9j7FHxFMpNuh/53wxbd+Zffl9URb2Mvb+ds6vjYjn0vK7wItAY7vNyuK8d0LXfY3AmwXvm/njRm7bJiJ2Ai3A6CL3LWd7W/9Lyf56aTVY0hJJiySd3RsV7GXFxv/nqft9vqQD93LfclV0/dMw+0TgsYLivLf9nnT2/eS93fdW+3M+gIWSnpV0WYnq1BdOkPS8pF9IOiyVVUzbSxpKlrD8R0Fxv2l7ZZdNHQ083W5VWZz3Nb11YDMASV8AjgNOKSg+OCLWSJoEPCapKSJeLU0Ne81/AvdExHZJf0HWU3taievU1+YA8yPig4KySmj7iibp42QJ3YkFxSemdt8PeFTS71KvT3/yHNnP92ZJnwZ+DkwtcZ362pnA/0REYW9ev2h7SXVkierXI2JTqevTEffQdd8a4MCC9+NTWYfbSKoB6oH1Re5bzoqqv6TTgb8GzoqI7a3lEbEm/bsK+DXZXzx5ssf4I2J9Qcy3AscWu2+Z25v6z6Hd0Es/aPs96ez7yXu7F0XSDLKf99kRsb61vKDd3wLuJ1+XmBQlIjZFxOa0/BAwQFIDFdL2SVfnfG7bXtIAsmTu7oi4r4NNyuO87+sLDPvLi6x3cxXZkFLrha6HtdvmSna/KWJeWj6M3W+KWEW+boooJvajyS4EntqufCQwKC03AK+QvwuEi4l/bMHyOcCitDwKeC19DyPT8qhSx9STsaftDiW7GFr9qe1T3SfQ+YXxn2H3i6Of6Q/tXmTsB5FdDzyrXXktMKxg+SngjFLH0gvxH9D6806WtLyRfg6KOmfK/dVV7Gl9Pdl1drX9qe1TG94F/LCLbcrivPeQazdFxE5JVwGPkN3FdHtELJd0A7AkIhYAtwH/Kmkl2Q/6nLTvcknzgBXATuDK2H1YqqwVGftNQB3ws+w+EN6IiLOAjwA/lbSLrIf4xohYUZJAuqnI+K+RdBZZ+24gu+uViNgg6e+AxelwN8TuwxNlrcjYIftZnxvpt1qS+7aXdA/Z3YwNkpqBvwUGAETEzcBDZHe8rQTeAy5J63Ld7lBU7NeTXSP8T+mc3xnZROX7A/enshrg3yPi4T4PYB8VEf+5wBWSdgJbgTnp57/Dc6YEIXRbEbFD9ofrwojYUrBrf2j7jwEXAU2SlqayvyL7A6asznvPFGFmZmaWc76GzszMzCznnNCZmZmZ5ZwTOjMzM7Occ0JnZmZmlnNO6MzMzMxyzgmdmZU1SSHp+wXv/1LSd3ro2HdIOrcnjrWHzzlP0ouSHm9XPkHSVklLC15f7MHPPVXSgz11PDMrX34OnZmVu+3A5yT9Q0S8XerKtJJUE9kczcW4FPhKRDzZwbpXI+KoHqyamVUg99CZWbnbCdwCfKP9ivY9bJI2p39PlfSEpAckrZJ0o6QLJT0jqUnS5ILDnC5piaSXJX027V8t6SZJiyUtS/Pxth73N5IWkD0YvH19LkjHf0HSd1PZ9WTzmt4m6aZig5a0WdIPJC2X9CtJY1L5UZIWpXrdL2lkKp8i6ZdpcvjnCmKskzRf0u8k3a30lNf0naxIx/lesfUys/LkhM7M8uAnwIWS6vdinyOBy8lmqLgImBYRM8nmGr26YLsJZFM1fQa4WdJgsh61log4Hjge+IqkiWn7Y4CvRcS0wg+TNA74LnAacBRwvKSzI+IGYAlwYUR8s4N6Tm435HpSKq8lm33jMOAJsqfzQzYN0bciYgbQVFB+N/CTiDgSmAWsTeVHA18HpgOTgI9JGk32ZP/D0nH+fk9fppmVNyd0Zlb2ImITWSJzzV7stjgi1kbEdrJ5hRem8iayJK7VvIjYFRGvkM25eSjwZ8AX01Q/T5NNaTU1bf9MRLzWwecdD/w6Italodi7gZOLqOerEXFUwes3qXwXcG9a/jfgxJTQjoiIJ1L5ncDJkoYBjRFxP0BEbIuI9wrq2xwRu4ClKfYWYBtZr+HnyKYrMrMcc0JnZnnxQ7Kes9qCsp2k32OSqsgmP2+1vWB5V8H7Xex+/XD7+Q+DbJLtqwuSrIkR0ZoQbqE0ujtPY+H38AHQeu3fTGA+8Fkgb/Nrmlk7TujMLBfSpNbzyJK6VquBY9PyWaQJw/fSeZKq0jVnk4CXyCZSv0LSAABJ0yTVdnUQ4BngFEkNkqqBC8iGSrurimzCd4DPA09GRAvwTsGw7EXAExHxLtAs6exU30GShnZ2YEl1QH1EPER2beKR+1BPMysDvsvVzPLk+8BVBe//BXhA0vNkvUzd6T17gywZGw5cHhHbJN1KNjT5XLqJYB1wdlcHiYi1kq4DHifr4fuviHigiM+fnIZ2W90eET8ii2WmpL8B3gLOT+svJrvWbyjZEPElqfwi4KeSbgB2AOd18ZnDyL63wamu1xZRTzMrY4robi++mZn1FkmbI6Ku1PUws3zwkKuZmZlZzrmHzszMzCzn3ENnZmZmlnNO6MzMzMxyzgmdmZmZWc45oTMzMzPLOSd0ZmZmZjnnhM7MzMws5/4fktiu3TVlt4gAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "yUsMUqpLvrKA",
        "outputId": "836605d3-4eaa-491e-86d8-5992153c0231"
      },
      "source": [
        "test_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>raw_text</th>\n",
              "      <th>features</th>\n",
              "      <th>masks</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>after takeoff on runway a loudnoise was hear c...</td>\n",
              "      <td>[99, 182, 1372, 31, 11013, 24, 6037, 488, 2337...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>taxi off the parkingramp the brake system fail...</td>\n",
              "      <td>[9507, 177, 18, 4169, 213, 1701, 18, 15878, 27...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>just prior to depart of aircraft flight xxx ra...</td>\n",
              "      <td>[125, 2114, 22, 16085, 20, 2005, 2115, 17, 272...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>selectivecalling from sfo airport commercialra...</td>\n",
              "      <td>[14140, 12611, 56, 40, 17, 23, 6571, 2212, 147...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>had taxi out to runway for takeoff per prevail...</td>\n",
              "      <td>[54, 9507, 78, 22, 11013, 28, 182, 1372, 451, ...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5264</th>\n",
              "      <td>i was attempt to make a short mile flight to r...</td>\n",
              "      <td>[17, 150, 30, 1681, 22, 144, 24, 573, 5125, 21...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5265</th>\n",
              "      <td>this incident was a groundproximitywarningsyst...</td>\n",
              "      <td>[52, 2510, 30, 24, 881, 3143, 6269, 98, 769, 2...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5266</th>\n",
              "      <td>we encounter sever downdraft and microburst ca...</td>\n",
              "      <td>[80, 6961, 17, 23, 3079, 151, 24827, 21, 4012,...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5267</th>\n",
              "      <td>after a weather reroute and minute of hold wes...</td>\n",
              "      <td>[99, 24, 2194, 17, 88, 26685, 21, 1901, 20, 85...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5268</th>\n",
              "      <td>clear instrumentlandingsystemrunway wind degre...</td>\n",
              "      <td>[791, 7033, 729, 56, 11580, 2169, 1550, 2436, ...</td>\n",
              "      <td>[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5269 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               raw_text  ...                                              masks\n",
              "0     after takeoff on runway a loudnoise was hear c...  ...  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...\n",
              "1     taxi off the parkingramp the brake system fail...  ...  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...\n",
              "2     just prior to depart of aircraft flight xxx ra...  ...  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...\n",
              "3     selectivecalling from sfo airport commercialra...  ...  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...\n",
              "4     had taxi out to runway for takeoff per prevail...  ...  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...\n",
              "...                                                 ...  ...                                                ...\n",
              "5264  i was attempt to make a short mile flight to r...  ...  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...\n",
              "5265  this incident was a groundproximitywarningsyst...  ...  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...\n",
              "5266  we encounter sever downdraft and microburst ca...  ...  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...\n",
              "5267  after a weather reroute and minute of hold wes...  ...  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...\n",
              "5268  clear instrumentlandingsystemrunway wind degre...  ...  [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ...\n",
              "\n",
              "[5269 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SuhI7GX7Qwea"
      },
      "source": [
        "Genarate predictions on test dataset and saving"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y2k5jIZUwDvZ"
      },
      "source": [
        "def generate_predictions(model, df, num_labels, device=\"cpu\", batch_size=32):\r\n",
        "  num_iter = math.ceil(df.shape[0]/batch_size)\r\n",
        "  \r\n",
        "  pred_probs = np.array([]).reshape(0, num_labels)\r\n",
        "  \r\n",
        "  model.to(device)\r\n",
        "  model.eval()\r\n",
        "  \r\n",
        "  for i in range(num_iter):\r\n",
        "    df_subset = df.iloc[i*batch_size:(i+1)*batch_size,:]\r\n",
        "    X = df_subset[\"features\"].values.tolist()\r\n",
        "    masks = df_subset[\"masks\"].values.tolist()\r\n",
        "    X = torch.tensor(X)\r\n",
        "    masks = torch.tensor(masks, dtype=torch.long)\r\n",
        "    X = X.to(device)\r\n",
        "    masks = masks.to(device)\r\n",
        "    with torch.no_grad():\r\n",
        "      logits = model(input_ids=X, attention_mask=masks)\r\n",
        "      logits = logits.sigmoid().detach().cpu().numpy()\r\n",
        "      pred_probs = np.vstack([pred_probs, logits])\r\n",
        "  \r\n",
        "  return pred_probs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nPFHJoNuwXab",
        "outputId": "b207c7c8-d019-416d-cea4-fafad873920e"
      },
      "source": [
        "pred_probs = generate_predictions(model, test_df, 22, device=\"cuda\", batch_size=32)\r\n",
        "pred_probs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[8.69318284e-03, 8.13918859e-02, 4.42971755e-03, ...,\n",
              "        7.02843489e-03, 7.82357994e-03, 1.13871720e-04],\n",
              "       [1.29542202e-02, 3.94528091e-01, 1.83771420e-02, ...,\n",
              "        6.47024775e-04, 2.73428694e-03, 2.18123270e-04],\n",
              "       [8.12007859e-02, 9.83171821e-01, 6.89474808e-04, ...,\n",
              "        1.07658643e-03, 9.51244205e-04, 1.00453673e-02],\n",
              "       ...,\n",
              "       [2.51499098e-03, 4.63218465e-02, 7.59470422e-05, ...,\n",
              "        9.91543639e-04, 1.60006527e-02, 9.12326039e-04],\n",
              "       [1.29880512e-03, 9.95421857e-02, 7.54158769e-04, ...,\n",
              "        1.03548472e-03, 4.73000109e-03, 1.80090102e-03],\n",
              "       [5.08053834e-03, 1.19083516e-01, 1.59518828e-03, ...,\n",
              "        2.31891917e-03, 2.35915128e-02, 3.61341168e-03]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "bhSJ3QP_wahD",
        "outputId": "58189873-c793-4340-a837-396fc244330a"
      },
      "source": [
        "u = pd.DataFrame(pred_probs)\r\n",
        "u = u.applymap(lambda x: 1 if x>=0.5 else 0)\r\n",
        "u"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5264</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5265</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5266</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5267</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5268</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5269 rows × 22 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      0   1   2   3   4   5   6   7   8   ...  13  14  15  16  17  18  19  20  21\n",
              "0      0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   1   0   0   0\n",
              "1      0   0   0   1   1   0   0   0   0  ...   0   0   0   0   0   1   0   0   0\n",
              "2      0   1   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "3      0   0   0   0   0   1   0   1   0  ...   0   0   0   0   0   0   0   0   0\n",
              "4      0   1   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "...   ..  ..  ..  ..  ..  ..  ..  ..  ..  ...  ..  ..  ..  ..  ..  ..  ..  ..  ..\n",
              "5264   0   0   0   0   0   0   0   0   0  ...   0   0   1   0   1   0   0   0   0\n",
              "5265   0   0   0   0   0   0   0   0   0  ...   0   1   1   0   0   0   0   0   0\n",
              "5266   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "5267   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "5268   0   0   0   0   0   0   1   0   0  ...   0   0   1   0   0   1   0   0   0\n",
              "\n",
              "[5269 rows x 22 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iykPof9jxAd7"
      },
      "source": [
        "u['text'] = test_df['raw_text']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LIXAgIDEyC1Q"
      },
      "source": [
        "u.to_csv(os.path.join(cwd, \"drive/My Drive/Output/result.csv\"))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2N3_KVGdQp3K"
      },
      "source": [
        "Accuracy evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QF4xq3J_zvWD"
      },
      "source": [
        "def load_model(save_path):\r\n",
        "  \"\"\"\r\n",
        "  Load the model from the path directory provided\r\n",
        "  \"\"\"\r\n",
        "  checkpoint = torch.load(save_path)\r\n",
        "  model_state_dict = checkpoint['state_dict']\r\n",
        "  model = XLNetForMultiLabelSequenceClassification(num_labels=model_state_dict[\"classifier.weight\"].size()[0])\r\n",
        "  model.load_state_dict(model_state_dict)\r\n",
        "\r\n",
        "  epochs = checkpoint[\"epochs\"]\r\n",
        "  lowest_eval_loss = checkpoint[\"lowest_eval_loss\"]\r\n",
        "  train_loss_hist = checkpoint[\"train_loss_hist\"]\r\n",
        "  valid_loss_hist = checkpoint[\"valid_loss_hist\"]\r\n",
        "  \r\n",
        "  return model, epochs, lowest_eval_loss, train_loss_hist, valid_loss_hist"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "duwSFn9yNpEg"
      },
      "source": [
        "import os\r\n",
        "cwd = os.getcwd()\r\n",
        "model_save_path = output_model_file = os.path.join(cwd, \"drive/My Drive/Models/xlnet_asrs.bin\")\r\n",
        "model, start_epoch, lowest_eval_loss, train_loss_hist, valid_loss_hist = load_model(model_save_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OZSZg518OS6B"
      },
      "source": [
        "pred_probs = generate_predictions(model, train_df.head(5000), 22, device=\"cuda\", batch_size=32)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fe7JWiM3Owmj",
        "outputId": "5c971f58-91a2-4b9f-9d1a-7c7faf200e18"
      },
      "source": [
        "# u = u.applymap(lambda x: 1 if x>=0.5 else 0)\r\n",
        "tensor_filtered_labels = torch.Tensor(filtered_labels)\r\n",
        "tensor_filtered_labels = tensor_filtered_labels[:5000]\r\n",
        "tensor_filtered_labels.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([5000, 22])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JBBKdUmJPdMy",
        "outputId": "7dc8b214-3601-45e7-d39a-4cc823ae3d42"
      },
      "source": [
        "u_infer = pd.DataFrame(pred_probs)\r\n",
        "u_infer = u_infer.applymap(lambda x: 1 if x>=0.5 else 0)\r\n",
        "tensor_infer_pred = torch.Tensor(u_infer.values)\r\n",
        "tensor_infer_pred.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([5000, 22])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7B1ByQm7Ptw9",
        "outputId": "f26998a4-6150-4532-f18a-09c387fc3c9c"
      },
      "source": [
        "perf_sum = (tensor_filtered_labels == tensor_infer_pred).sum()\r\n",
        "accuracy = (perf_sum / (tensor_infer_pred.shape[0] * tensor_infer_pred.shape[1]) ) * 100\r\n",
        "print('Accuracy evaluated gives : {}'.format(accuracy.item()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy evaluated gives : 95.24000549316406\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZtqA83SeQAd1"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}