gpt2_refact.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "img_capt_ref.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "vKmkWdf_mBKJ"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "6888e21d54d041dbb47650cece709b82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d0af580d9cb347dd84dfbd240f144151",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7a846b7d3f634f34920d7a66bed9fcc6",
              "IPY_MODEL_2accb4a9b9594f86a9775550c98119d6"
            ]
          }
        },
        "d0af580d9cb347dd84dfbd240f144151": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7a846b7d3f634f34920d7a66bed9fcc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d0af1ce479bb4c7cb7e70ee5fb866426",
            "_dom_classes": [],
            "description": "loss:8.1084, lr: 0.1414:   1%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 5174,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 49,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_96f8eeb3e7d640a88e63b7c2be0b54e7"
          }
        },
        "2accb4a9b9594f86a9775550c98119d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_10f671df8c7546f185a26d3bdfa7d12f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 49/5174 [01:18&lt;2:13:14,  1.56s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_de039b8fe69c4f2ca0285b0814c54dd0"
          }
        },
        "d0af1ce479bb4c7cb7e70ee5fb866426": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "96f8eeb3e7d640a88e63b7c2be0b54e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "10f671df8c7546f185a26d3bdfa7d12f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "de039b8fe69c4f2ca0285b0814c54dd0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8fcc0c40052c4641af2f2db01d0292ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_80ec936caf174e85b6cd617348005713",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c8d18a8ff5b9477fbeb2506b2bed637f",
              "IPY_MODEL_b55045558dfe4648bc87ac41446713a1"
            ]
          }
        },
        "80ec936caf174e85b6cd617348005713": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8d18a8ff5b9477fbeb2506b2bed637f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5c61af77b9b34c269a67e79282e09278",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 2026,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2026,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4d0ab4625f9349ea894883ec650da8e7"
          }
        },
        "b55045558dfe4648bc87ac41446713a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_04ef2dbce6db4894b6bc81ee3fe886bd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2026/2026 [1:04:31&lt;00:00,  1.78s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d5a50a68ace640b387aade194253fd20"
          }
        },
        "5c61af77b9b34c269a67e79282e09278": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4d0ab4625f9349ea894883ec650da8e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "04ef2dbce6db4894b6bc81ee3fe886bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d5a50a68ace640b387aade194253fd20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "847e2b8aa1634e59b29c8ec101843646": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e58c701843224c628812a22430cacde5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8b2da474bee74ca2b5f1983579ea0cbb",
              "IPY_MODEL_c1687ee0655e424090ecde9b4d1e5eda"
            ]
          }
        },
        "e58c701843224c628812a22430cacde5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8b2da474bee74ca2b5f1983579ea0cbb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ea8a9f62d54c45078eacfe6381566cca",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 40504,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 40504,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_00c6a1b56d814a39aae49fbf4cde9cf6"
          }
        },
        "c1687ee0655e424090ecde9b4d1e5eda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8081dda8342a4c47a1f6dc598ce868f5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 40504/40504 [07:33&lt;00:00, 89.24it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7194eb9038ac4644b4ec4b3fc40f8d2d"
          }
        },
        "ea8a9f62d54c45078eacfe6381566cca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "00c6a1b56d814a39aae49fbf4cde9cf6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8081dda8342a4c47a1f6dc598ce868f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7194eb9038ac4644b4ec4b3fc40f8d2d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dhxlk8Owl8Pl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import torchvision.models as models\n",
        "from torchvision.datasets import CocoCaptions\n",
        "\n",
        "import pandas as pd\n",
        "import os\n",
        "import sys\n",
        "import random\n",
        "\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from transformers.optimization import AdamW\n",
        "\n",
        "from PIL import Image\n",
        "import requests\n",
        "from io import BytesIO\n",
        "from IPython.display import display\n",
        "from multiprocessing.pool import ThreadPool\n",
        "\n",
        "from IPython.display import display\n",
        "from collections import namedtuple\n",
        "from time import perf_counter\n",
        "from tqdm.autonotebook import tqdm\n",
        "from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_hXyfiEolkWR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q transformers"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YSEjD4t8l975",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f4dbe6f9-f573-4af2-8df8-fd3ccbfcab6f"
      },
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(device)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "azDJ6-FIh5Kt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c0d08690-aa2f-4e6c-92f7-a94bff6897ab"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XS7aqhrKLnT-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "IMG_SIZE = 299\n",
        "PATH_TO_SAVE = 'drive/My Drive/'\n",
        "PATH_TO_MODELS = f'drive/My Drive/models/'\n",
        "\n",
        "sys.path.append(PATH_TO_MODELS)\n",
        "\n",
        "from image2text import Image2TextDescriptor\n",
        "from sampler import Sampler\n",
        "from train_model import train"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LanG6u4DNpgi",
        "colab_type": "text"
      },
      "source": [
        "# COCO Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bMTu01oVVQEJ",
        "colab_type": "text"
      },
      "source": [
        "## Загрузка данных"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dLDaZGfSTvJJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir -p data/COCO"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pUUMKuH8UDoR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "outputId": "34704e3b-e62f-4a37-ce8f-e6c2aed2c0f8"
      },
      "source": [
        "!wget http://images.cocodataset.org/annotations/annotations_trainval2014.zip\n",
        "!unzip -q -n annotations_trainval2014.zip -d data/COCO\n",
        "!rm annotations_trainval2014.zip"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-06-26 05:48:29--  http://images.cocodataset.org/annotations/annotations_trainval2014.zip\n",
            "Resolving images.cocodataset.org (images.cocodataset.org)... 52.216.146.67\n",
            "Connecting to images.cocodataset.org (images.cocodataset.org)|52.216.146.67|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 252872794 (241M) [application/zip]\n",
            "Saving to: ‘annotations_trainval2014.zip’\n",
            "\n",
            "annotations_trainva 100%[===================>] 241.16M  32.6MB/s    in 8.0s    \n",
            "\n",
            "2020-06-26 05:48:38 (30.0 MB/s) - ‘annotations_trainval2014.zip’ saved [252872794/252872794]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7xK3wApf_Geb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "outputId": "ea07d787-6ddb-438e-fcda-dd7a6cbbf287"
      },
      "source": [
        "!wget http://images.cocodataset.org/zips/train2014.zip\n",
        "!unzip -q -n train2014.zip -d data/COCO\n",
        "!rm train2014.zip"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-06-26 05:48:53--  http://images.cocodataset.org/zips/train2014.zip\n",
            "Resolving images.cocodataset.org (images.cocodataset.org)... 52.216.92.11\n",
            "Connecting to images.cocodataset.org (images.cocodataset.org)|52.216.92.11|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 13510573713 (13G) [application/zip]\n",
            "Saving to: ‘train2014.zip’\n",
            "\n",
            "train2014.zip       100%[===================>]  12.58G  33.0MB/s    in 6m 31s  \n",
            "\n",
            "2020-06-26 05:55:24 (33.0 MB/s) - ‘train2014.zip’ saved [13510573713/13510573713]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ynd1xXFpXgcE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "outputId": "127c5a76-36e9-4237-9202-28cb841ebe08"
      },
      "source": [
        "!wget http://images.cocodataset.org/zips/val2014.zip\n",
        "!unzip -q -n val2014.zip -d data/COCO\n",
        "!rm val2014.zip"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-06-26 06:01:54--  http://images.cocodataset.org/zips/val2014.zip\n",
            "Resolving images.cocodataset.org (images.cocodataset.org)... 52.216.233.35\n",
            "Connecting to images.cocodataset.org (images.cocodataset.org)|52.216.233.35|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 6645013297 (6.2G) [application/zip]\n",
            "Saving to: ‘val2014.zip’\n",
            "\n",
            "val2014.zip         100%[===================>]   6.19G  33.0MB/s    in 3m 15s  \n",
            "\n",
            "2020-06-26 06:05:08 (32.6 MB/s) - ‘val2014.zip’ saved [6645013297/6645013297]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s9CwmIsiVUGr",
        "colab_type": "text"
      },
      "source": [
        "## Создание даталодера"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JuDJ8SaSUarc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "transform = transforms.Compose([\n",
        "     transforms.Resize((IMG_SIZE, IMG_SIZE)),\n",
        "     transforms.ToTensor(),\n",
        "     transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
        "                          std=[0.229, 0.224, 0.225])\n",
        "])\n",
        "\n",
        "path2data = os.path.abspath('/content/data/COCO')"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jo7YT4_etvWf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def target_transform(target):\n",
        "    target = random.choice(target)\n",
        "    return target.lower().split()"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0RaIrzQOUag3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "2816e729-6b16-4768-9f2b-094d14a38c51"
      },
      "source": [
        "coco_train = CocoCaptions(root = path2data + '/train2014',\n",
        "                          annFile = path2data + '/annotations/captions_train2014.json',\n",
        "                          transform=transform,\n",
        "                          target_transform=target_transform)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading annotations into memory...\n",
            "Done (t=0.78s)\n",
            "creating index...\n",
            "index created!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oVb1THVzU-yz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "2879622d-b0b8-4682-bdfb-d0fea6a46f04"
      },
      "source": [
        "print('Number of samples: ', len(coco_train))\n",
        "img, target = coco_train[3] # load 4th sample\n",
        "\n",
        "print(\"Image Size: \", img.size())\n",
        "print(target)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of samples:  82783\n",
            "Image Size:  torch.Size([3, 299, 299])\n",
            "['a', 'lone', 'zebra', 'grazing', 'in', 'some', 'green', 'grass.']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1jf1gbo_fHa3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_coco_dataloader(dataset, batch_size, pad_elem, shuffle=True):\n",
        "    def pad(seq, max_len, pad_elem):\n",
        "        return seq + [pad_elem] * max(0, max_len - len(seq))\n",
        "\n",
        "    def collate_fn(batch_data):\n",
        "        list_imgs, list_ids = list(zip(*batch_data))\n",
        "\n",
        "        batch_img = torch.stack(list_imgs, 0)    \n",
        "\n",
        "        batch_len = max(map(len, list_ids))\n",
        "        batch_ids = torch.tensor(\n",
        "            [pad(ids, batch_len, pad_elem) \n",
        "             for ids in list_ids]\n",
        "        ).long()\n",
        "        batch_mask = batch_ids.ne(pad_elem).int()\n",
        "\n",
        "        return batch_img, batch_ids, batch_mask\n",
        "\n",
        "    return DataLoader(\n",
        "        dataset=dataset, batch_size=batch_size, shuffle=shuffle,\n",
        "        collate_fn=collate_fn, pin_memory=False, num_workers=0 \n",
        "    )"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v1ipA4IdmRRS",
        "colab_type": "text"
      },
      "source": [
        "# Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q8_kAVIZl9Y7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "descriptor = Image2TextDescriptor(device, SPECIAL_TOKENS=('img', 'desc', 'pad'))"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OjOo-JlYdaOE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "4bc60c8e-b5ff-4e80-8d29-54a76e286e75"
      },
      "source": [
        "####### COCO DATASET #######\n",
        "batch_size = 16\n",
        "\n",
        "transform = transforms.Compose([\n",
        "     transforms.Resize((IMG_SIZE, IMG_SIZE)),\n",
        "     transforms.ToTensor(),\n",
        "     transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
        "                          std=[0.229, 0.224, 0.225])\n",
        "])\n",
        "\n",
        "path2data = os.path.abspath('/content/data/COCO')\n",
        "\n",
        "coco_train = CocoCaptions(root = path2data + '/train2014',\n",
        "                          annFile = path2data + '/annotations/captions_train2014.json',\n",
        "                          transform=transform,\n",
        "                          target_transform=descriptor.target_transform)\n",
        "\n",
        "coco_trainloader = get_coco_dataloader(coco_train, batch_size, \n",
        "                                       descriptor.special_ids.pad)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading annotations into memory...\n",
            "Done (t=0.73s)\n",
            "creating index...\n",
            "index created!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qNex9gXjSvU_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "LAST_TIMESTEP = 0 # 3775 + 3800 + 3763 + 2841 + 1600 + 10000 # for inception: 3775 + 3800 + 3763 + 2841 + 1600 + 10000 # for resnet: 219 + 1020 + 903 + 1000 + 500 + 1471 + 313 + 642 # !!!!!!!!!"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G-wyVgw7l9Ti",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 389,
          "referenced_widgets": [
            "6888e21d54d041dbb47650cece709b82",
            "d0af580d9cb347dd84dfbd240f144151",
            "7a846b7d3f634f34920d7a66bed9fcc6",
            "2accb4a9b9594f86a9775550c98119d6",
            "d0af1ce479bb4c7cb7e70ee5fb866426",
            "96f8eeb3e7d640a88e63b7c2be0b54e7",
            "10f671df8c7546f185a26d3bdfa7d12f",
            "de039b8fe69c4f2ca0285b0814c54dd0"
          ]
        },
        "outputId": "a086e961-d75f-49c3-92ee-927285d3686f"
      },
      "source": [
        "train(descriptor, coco_trainloader, PATH_TO_SAVE, lr=1e-3, accum_interval=10, timestep=LAST_TIMESTEP, n_epochs=1)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6888e21d54d041dbb47650cece709b82",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=5174.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:123: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
            "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-25-24d346e073b5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdescriptor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoco_trainloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mPATH_TO_SAVE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccum_interval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimestep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mLAST_TIMESTEP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/content/drive/My Drive/models/train_model.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, dataloader, PATH_TO_SAVE, n_epochs, clip_value, accum_interval, lr, warmup, timestep, save_interval)\u001b[0m\n\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m         \u001b[0mtrain_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mPATH_TO_SAVE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_interval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msave_interval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     72\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Epoch #{epoch} finished'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m         \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPATH_TO_SAVE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'weights/full_trained_{epoch}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/drive/My Drive/models/train_model.py\u001b[0m in \u001b[0;36mtrain_epoch\u001b[0;34m(dataloader, model, optimizer, scheduler, PATH_TO_SAVE, save_interval)\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0midx_batch\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0msave_interval\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m             \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPATH_TO_SAVE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'weights/temp'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/drive/My Drive/models/image2text.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, path)\u001b[0m\n\u001b[1;32m     71\u001b[0m             \u001b[0;34m'tokenizer'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m         }\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m         \u001b[0;31m# print('!!! model is saved !!!')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(obj, f, pickle_module, pickle_protocol, _use_new_zipfile_serialization)\u001b[0m\n\u001b[1;32m    367\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    368\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 369\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mopened_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    370\u001b[0m         \u001b[0m_legacy_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopened_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_protocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_open_file_like\u001b[0;34m(name_or_buffer, mode)\u001b[0m\n\u001b[1;32m    232\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0m_is_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m'w'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, mode)\u001b[0m\n\u001b[1;32m    213\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_opener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    214\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 215\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_open_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    216\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    217\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'drive/My Drive/weights/temp'"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jVpbJH0aNsUI",
        "colab_type": "text"
      },
      "source": [
        "# Валидация"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u2zSQ8iZki-5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#descriptor = Image2TextDescriptor()\n",
        "descriptor.load(os.path.join(PATH_TO_SAVE, 'weights/temp_inception'))"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5LACN-6mAuo3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sampler = Sampler(descriptor)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WsFGT6H0L4_5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "f3a4c860-e064-45c1-f835-e2788f2b3742"
      },
      "source": [
        "####### COCO DATASET #######\n",
        "val_transform = transforms.Compose([\n",
        "                                    transforms.Resize((IMG_SIZE, IMG_SIZE)),\n",
        "                                    transforms.ToTensor(),\n",
        "                                    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
        "                                                         std=[0.229, 0.224, 0.225])\n",
        "])\n",
        "'''normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
        "                                 std=[0.229, 0.224, 0.225])\n",
        "to_img = transforms.ToPILImage()'''\n",
        "\n",
        "batch_size = 20\n",
        "\n",
        "path2data = os.path.abspath('/content/data/COCO')\n",
        "\n",
        "coco_val = CocoCaptions(root = path2data + '/val2014',\n",
        "                        annFile = path2data + '/annotations/captions_val2014.json',\n",
        "                        transform=val_transform,\n",
        "                        target_transform=descriptor.target_transform)\n",
        "\n",
        "coco_valloader = get_coco_dataloader(coco_val, batch_size, \n",
        "                                     descriptor.special_ids.pad, shuffle=False)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading annotations into memory...\n",
            "Done (t=1.20s)\n",
            "creating index...\n",
            "index created!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O9o-GhXFvOcd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "predictions = []\n",
        "captions = []"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9JLJl0z6l9Bk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "8fcc0c40052c4641af2f2db01d0292ec",
            "80ec936caf174e85b6cd617348005713",
            "c8d18a8ff5b9477fbeb2506b2bed637f",
            "b55045558dfe4648bc87ac41446713a1",
            "5c61af77b9b34c269a67e79282e09278",
            "4d0ab4625f9349ea894883ec650da8e7",
            "04ef2dbce6db4894b6bc81ee3fe886bd",
            "d5a50a68ace640b387aade194253fd20"
          ]
        },
        "outputId": "9a697330-bdda-4b86-e7b7-5f7f7dc9a23f"
      },
      "source": [
        "descriptor.eval()\n",
        "pbar = tqdm(enumerate(coco_valloader), total=len(coco_valloader), leave=False)\n",
        "with torch.no_grad():\n",
        "    for idx, batch in pbar:\n",
        "        imgs, desc_ids, mask = [x.to(descriptor.device) for x in batch]\n",
        "        \n",
        "        #processed_imgs = normalize(imgs)\n",
        "        sampler.initialize(imgs)\n",
        "        sys_desc_ids = sampler.run()\n",
        "        for sys_ids in sys_desc_ids:\n",
        "            predictions.append(descriptor.tokenizer.convert_ids_to_tokens(sys_ids))\n",
        "\n",
        "        '''if idx < 2:\n",
        "            for img, gold_ids, sys_ids in zip(imgs, desc_ids, sys_desc_ids):\n",
        "                print('gold description: ', descriptor.tokenizer.decode(gold_ids, skip_special_tokens=True))\n",
        "                print('system description: ', descriptor.tokenizer.decode(sys_ids, skip_special_tokens=True))\n",
        "                display(to_img(img))\n",
        "                print('\\n\\n')'''"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8fcc0c40052c4641af2f2db01d0292ec",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=2026.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XDFNT6k16-mP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e8899e31-c74c-4a22-d7ab-117aaa4fe883"
      },
      "source": [
        "len(predictions)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "40504"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yI-jlc5S7Cjy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "c9f375b2-326d-430d-dd39-327d8b8a73b5"
      },
      "source": [
        "coco_val = CocoCaptions(root = path2data + '/val2014',\n",
        "                        annFile = path2data + '/annotations/captions_val2014.json')"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading annotations into memory...\n",
            "Done (t=1.22s)\n",
            "creating index...\n",
            "index created!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zKwWgUsemf47",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.translate.bleu_score import sentence_bleu\n",
        "from nltk.translate.bleu_score import SmoothingFunction\n",
        "import numpy as np"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qjncJuMvmf1_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 82,
          "referenced_widgets": [
            "847e2b8aa1634e59b29c8ec101843646",
            "e58c701843224c628812a22430cacde5",
            "8b2da474bee74ca2b5f1983579ea0cbb",
            "c1687ee0655e424090ecde9b4d1e5eda",
            "ea8a9f62d54c45078eacfe6381566cca",
            "00c6a1b56d814a39aae49fbf4cde9cf6",
            "8081dda8342a4c47a1f6dc598ce868f5",
            "7194eb9038ac4644b4ec4b3fc40f8d2d"
          ]
        },
        "outputId": "7689a6ef-26b3-4711-d964-00c2fa82c915"
      },
      "source": [
        "def get_bleus(predictions, dataset):\n",
        "    score1 = 0\n",
        "    score4 = 0\n",
        "    smoother = SmoothingFunction()\n",
        "    for candidate, (_, captions) in tqdm(zip(predictions, dataset), total=len(predictions)):\n",
        "        references = list(map(lambda x: descriptor.tokenizer.convert_ids_to_tokens(descriptor.tokenizer.encode(x)), \n",
        "                              captions))\n",
        "        score1 += sentence_bleu(references, candidate, weights=(1, 0, 0, 0), smoothing_function=smoother.method1)\n",
        "        score4 += sentence_bleu(references, candidate, weights=(0, 0, 0, 1), smoothing_function=smoother.method1)\n",
        "    bleu1 = 100*score1/len(dataset)\n",
        "    bleu4 = 100*score4/len(dataset)        \n",
        "    print(\"BLEU 1:\", np.round(bleu1, 2), \n",
        "          \"BLEU 4:\", np.round(bleu4, 2))    \n",
        "    return bleu1, bleu4\n",
        "\n",
        "bleu1, bleu4 = get_bleus(predictions, coco_val)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "847e2b8aa1634e59b29c8ec101843646",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=40504.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "BLEU 1: 9.81 BLEU 4: 0.89\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CV-uogpirWKa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8__fjXymfx1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}