22.Evaluate_a_Hugging_Face_LLM_with_mlflow_evaluate.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ashishpatel26/LLM-Finetuning/blob/main/22.Evaluate_a_Hugging_Face_LLM_with_mlflow_evaluate.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5XBzAX9ZCns0"
      },
      "source": [
        "## Evaluate a Hugging Face LLM with `mlflow.evaluate()`\n",
        "\n",
        "This guide will show how to load a pre-trained Hugging Face pipeline, log it to MLflow, and use `mlflow.evaluate()` to evaluate builtin metrics as well as custom LLM-judged metrics for the model.\n",
        "\n",
        "For detailed information, please read the documentation on [using MLflow evaluate](https://mlflow.org/docs/latest/llms/llm-evaluate/index.html)."
      ]
    },
    {
      "cell_type": "raw",
      "metadata": {
        "id": "RAGL_cPtCns2"
      },
      "source": [
        "<a href=\"https://raw.githubusercontent.com/mlflow/mlflow/master/docs/source/llms/llm-evaluate/notebooks/huggingface-evaluation.ipynb\" class=\"notebook-download-btn\"><i class=\"fas fa-download\"></i>Download this Notebook</a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tLdbWfjgCns3"
      },
      "source": [
        "### Start MLflow Server\n",
        "\n",
        "You can either:\n",
        "\n",
        "- Start a local tracking server by running `mlflow ui` within the same directory that your notebook is in.\n",
        "- Use a tracking server, as described in [this overview](https://mlflow.org/docs/latest/getting-started/tracking-server-overview/index.html)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7MbdRXlaCns3"
      },
      "source": [
        "### Install necessary dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IIeRHgxbCns3",
        "outputId": "741352fe-ccf2-4639-851d-e11d86dc866d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m25.8/25.8 MB\u001b[0m \u001b[31m22.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m547.8/547.8 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m328.3/328.3 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m26.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.1/105.1 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m233.0/233.0 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m147.8/147.8 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m128.2/128.2 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.9/59.9 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.0/107.0 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.4/84.4 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m56.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m38.3/38.3 MB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.9/64.9 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.1/141.1 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.4/62.4 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m54.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.6/78.6 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m202.9/202.9 kB\u001b[0m \u001b[31m15.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.8/52.8 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.5/130.5 kB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m32.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m77.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m56.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for rouge_score (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "cudf-cu12 24.4.1 requires pyarrow<15.0.0a0,>=14.0.1, but you have pyarrow 15.0.2 which is incompatible.\n",
            "google-colab 1.0.0 requires requests==2.31.0, but you have requests 2.32.3 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "%pip install -q mlflow transformers torch torchvision evaluate datasets openai tiktoken fastapi rouge_score textstat"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Nf1zwQQiCns4"
      },
      "outputs": [],
      "source": [
        "# Necessary imports\n",
        "import warnings\n",
        "\n",
        "import pandas as pd\n",
        "from datasets import load_dataset\n",
        "from transformers import pipeline\n",
        "\n",
        "import mlflow\n",
        "from mlflow.metrics.genai import EvaluationExample, answer_correctness, make_genai_metric"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "GBHM32TcCns4"
      },
      "outputs": [],
      "source": [
        "# Disable FutureWarnings\n",
        "warnings.filterwarnings(\"ignore\", category=FutureWarning)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ[\"HF_TOKEN\"] = \"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\""
      ],
      "metadata": {
        "id": "MU2LplqtFGuH"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wk097gxHCns4"
      },
      "source": [
        "### Load a pretrained Hugging Face pipeline\n",
        "\n",
        "Here we are loading a text generation pipeline, but you can also use either a text summarization or question answering pipeline."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313,
          "referenced_widgets": [
            "bec0d3643d0f4239b44ab6cc1ddafbb5",
            "5aa6f64c914f4f5db0de42de11abb24a",
            "69a8ee72206c41e19edc104df7cba7bf",
            "ca6760391d22477a8ffc6765f3973094",
            "b819e261b79348edb84d82ac258353e2",
            "a8945c43b9024367b99f4f8c15695bf3",
            "4514e00544ae4be094f4072aef87178d",
            "dc11bfdb91ef494092324cff892e034c",
            "d741cbeabc5a442fbdf2d280a80638f3",
            "2133c76bf07c4ec5ac8f840a495b7121",
            "a22bb4b308e446d3a6d44a60d7032312",
            "acaae41525214422a2c6c2f5f5f4a327",
            "15ad66c5c4924957873e11b30b1e1c85",
            "78b12da7919f407898ea38c6eb734afd",
            "8d3cdaa609984a2cae72caecc09bc0d5",
            "c309c28eb9964154b4870289155364d8",
            "0e22b9a7721a40b19f0f8e6ac7b80d3b",
            "640d0f602404443c982e405a45179dd2",
            "1d9ccae90ffe4786998c60bf4a763198",
            "58b5f9ea7e694bbebf26448ed357148d",
            "6abae3e8eff84423ba7a5c383754e3a0",
            "7c1bc5edcee945b09fd9813af1f526c6",
            "6d7b7d3df459481d8334231b9bc9037b",
            "6bd84cf8fde74eaaa0fe0f0f2afab855",
            "3a556dd8213341d0b5c635f9016ffb7d",
            "b37ca98bd12c4ab98eeb08a9c0bc3fb8",
            "c6bc18b798ac4b3aa2d17d7f315a42b5",
            "6debcb842ce44ca28da939a4f50a6bfe",
            "872575ef8e5e41ceb39b19e1437d036d",
            "c5b6cbc5cfd54897854ecb657eee4da1",
            "0035c659b38d43908ef6916ff4336644",
            "2e5e1c3df3584d2ca452f18dec9401c0",
            "135c5de607a94af39bdde9cb3b223f9e",
            "2580f3279b76403ebb1dbbbba8d424bf",
            "d486180354c343779caed3f5e38abcef",
            "d0c95cebcfe44116bfab9e654dbc8e9d",
            "23e0e9bc80c34d8294d145c349fe0413",
            "8a3b5bebdfba42418ab67b33a8e5b3d2",
            "7e2dbccd527d49489624ac7fb0934e75",
            "71b685a43f704800815bc1a6c9283cb6",
            "d654a1cf9f1341c39f79c8c6625986f7",
            "10354da5be664163bb8d87de9fb84aaa",
            "b0747a467641432ca3ff4ebd1dc7b8fc",
            "0eb29aac50df42c88ce1664b845eea22",
            "57ba083b9fd44a4bad56cca0f40ca96b",
            "9bd1214a2aaf4b42972e684932207d0c",
            "3c23643b97834f3d83c9c530d5ade109",
            "ff9a4e053c63447481a4a8dbe26c10b5",
            "1da1c5738f854e42ab593d74d9ad7191",
            "61aa29b5b18d49d9868b63e18f2e1e63",
            "4db0c0b3304d43ecb0ee1468bc79ea06",
            "66ad8ecbb99b49eaa9e7bdb1a592f94e",
            "33a5a69352c64b32ade5eb5a9a0a88e2",
            "3b878ba5949743869d989b9fbda940d8",
            "34791495002a4de6b6317410e20bd9f6",
            "0c8693f0b20b41979fe9f6972babc2b4",
            "dbcb4f8bccef47b2a9d11f257a206e1d",
            "aaed8aba1bfa4360827892fba045e20d",
            "c734d30155124b8791aa643f2c78d8f5",
            "7e12943791ac44379af97a3a8ae921eb",
            "22daa5f7e09c49278c10962964143131",
            "ba8d2c6e7d494123919c9c0b6510424b",
            "a91a5cdd197b4976aef986791aed07d1",
            "2c9bdd4cd6614eef9132b9b66238dd38",
            "5203572d7dff4ba5a6487e8cd7ded86b",
            "6801c003f197449bb2083b1df027e94f"
          ]
        },
        "id": "nFLmjsqQCns4",
        "outputId": "9c6217ec-0aec-4b9d-fc26-1987e7c752aa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/1.23k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bec0d3643d0f4239b44ab6cc1ddafbb5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pytorch_model.bin.index.json:   0%|          | 0.00/16.0k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "acaae41525214422a2c6c2f5f5f4a327"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6d7b7d3df459481d8334231b9bc9037b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pytorch_model-00001-of-00002.bin:   0%|          | 0.00/9.94G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2580f3279b76403ebb1dbbbba8d424bf"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pytorch_model-00002-of-00002.bin:   0%|          | 0.00/3.36G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "57ba083b9fd44a4bad56cca0f40ca96b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0c8693f0b20b41979fe9f6972babc2b4"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "mpt_pipeline = pipeline(\"text-generation\", model=\"mosaicml/mpt-7b-chat\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6iQdOYGMCns5"
      },
      "source": [
        "### Log the model using MLflow"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_V631PDGCns5"
      },
      "source": [
        "We log our pipeline as an MLflow Model, which follows a standard format that lets you save a model in different \"flavors\" that can be understood by different downstream tools. In this case, the model is of the transformers \"flavor\"."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U4YHj7B7Cns5"
      },
      "outputs": [],
      "source": [
        "mlflow.set_experiment(\"Evaluate Hugging Face Text Pipeline\")\n",
        "\n",
        "# Define the signature\n",
        "signature = mlflow.models.infer_signature(\n",
        "    model_input=\"What are the three primary colors?\",\n",
        "    model_output=\"The three primary colors are red, yellow, and blue.\",\n",
        ")\n",
        "\n",
        "# Log the model using mlflow\n",
        "with mlflow.start_run():\n",
        "    model_info = mlflow.transformers.log_model(\n",
        "        transformers_model=mpt_pipeline,\n",
        "        artifact_path=\"mpt-7b\",\n",
        "        signature=signature,\n",
        "        registered_model_name=\"mpt-7b-chat\",\n",
        "    )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T9VV-ez_Cns5"
      },
      "source": [
        "### Load Evaluation Data\n",
        "\n",
        "Load in a dataset from Hugging Face Hub to use for evaluation.\n",
        "\n",
        "The data fields in the dataset below represent:\n",
        "\n",
        "- **instruction**: Describes the task that the model should perform. Each row within the dataset is a unique instruction (task) to be performed.\n",
        "- **input**: Optional contextual information that relates to the task defined in the `instruction` field. For example, for the instruction \"Identify the odd one out\", the `input` contextual guidance is given as the list of items to select an outlier from, \"Twitter, Instagram, Telegram\".\n",
        "- **output**: The answer to the instruction (with the optional `input` context provided) as generated by the original evaluation model (`text-davinci-003` from OpenAI)\n",
        "- **text**: The final total text as a result of applying the `instruction`, `input`, and `output` to the prompt template used, which is sent to the model for fine tuning purposes."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aVP6ofSuCns6",
        "outputId": "a7cf35ce-92f0-47dd-dfe7-340b4520bba0"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>instruction</th>\n",
              "      <th>input</th>\n",
              "      <th>output</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Give three tips for staying healthy.</td>\n",
              "      <td></td>\n",
              "      <td>1.Eat a balanced diet and make sure to include...</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>What are the three primary colors?</td>\n",
              "      <td></td>\n",
              "      <td>The three primary colors are red, blue, and ye...</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Describe the structure of an atom.</td>\n",
              "      <td></td>\n",
              "      <td>An atom is made up of a nucleus, which contain...</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>How can we reduce air pollution?</td>\n",
              "      <td></td>\n",
              "      <td>There are a number of ways to reduce air pollu...</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Describe a time when you had to make a difficu...</td>\n",
              "      <td></td>\n",
              "      <td>I had to make a difficult decision when I was ...</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Identify the odd one out.</td>\n",
              "      <td>Twitter, Instagram, Telegram</td>\n",
              "      <td>Telegram</td>\n",
              "      <td>Below is an instruction that describes a task,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Explain why the following fraction is equivale...</td>\n",
              "      <td>4/16</td>\n",
              "      <td>The fraction 4/16 is equivalent to 1/4 because...</td>\n",
              "      <td>Below is an instruction that describes a task,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Write a short story in third person narration ...</td>\n",
              "      <td></td>\n",
              "      <td>John was at a crossroads in his life. He had j...</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Render a 3D model of a house</td>\n",
              "      <td></td>\n",
              "      <td>&lt;nooutput&gt; This type of instruction cannot be ...</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Evaluate this sentence for spelling and gramma...</td>\n",
              "      <td>He finnished his meal and left the resturant</td>\n",
              "      <td>He finished his meal and left the restaurant.</td>\n",
              "      <td>Below is an instruction that describes a task,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                         instruction  \\\n",
              "0               Give three tips for staying healthy.   \n",
              "1                 What are the three primary colors?   \n",
              "2                 Describe the structure of an atom.   \n",
              "3                   How can we reduce air pollution?   \n",
              "4  Describe a time when you had to make a difficu...   \n",
              "5                          Identify the odd one out.   \n",
              "6  Explain why the following fraction is equivale...   \n",
              "7  Write a short story in third person narration ...   \n",
              "8                       Render a 3D model of a house   \n",
              "9  Evaluate this sentence for spelling and gramma...   \n",
              "\n",
              "                                          input  \\\n",
              "0                                                 \n",
              "1                                                 \n",
              "2                                                 \n",
              "3                                                 \n",
              "4                                                 \n",
              "5                  Twitter, Instagram, Telegram   \n",
              "6                                          4/16   \n",
              "7                                                 \n",
              "8                                                 \n",
              "9  He finnished his meal and left the resturant   \n",
              "\n",
              "                                              output  \\\n",
              "0  1.Eat a balanced diet and make sure to include...   \n",
              "1  The three primary colors are red, blue, and ye...   \n",
              "2  An atom is made up of a nucleus, which contain...   \n",
              "3  There are a number of ways to reduce air pollu...   \n",
              "4  I had to make a difficult decision when I was ...   \n",
              "5                                           Telegram   \n",
              "6  The fraction 4/16 is equivalent to 1/4 because...   \n",
              "7  John was at a crossroads in his life. He had j...   \n",
              "8  <nooutput> This type of instruction cannot be ...   \n",
              "9      He finished his meal and left the restaurant.   \n",
              "\n",
              "                                                text  \n",
              "0  Below is an instruction that describes a task....  \n",
              "1  Below is an instruction that describes a task....  \n",
              "2  Below is an instruction that describes a task....  \n",
              "3  Below is an instruction that describes a task....  \n",
              "4  Below is an instruction that describes a task....  \n",
              "5  Below is an instruction that describes a task,...  \n",
              "6  Below is an instruction that describes a task,...  \n",
              "7  Below is an instruction that describes a task....  \n",
              "8  Below is an instruction that describes a task....  \n",
              "9  Below is an instruction that describes a task,...  "
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset = load_dataset(\"tatsu-lab/alpaca\")\n",
        "eval_df = pd.DataFrame(dataset[\"train\"])\n",
        "eval_df.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NKzCu9_tCns6"
      },
      "source": [
        "### Define Metrics\n",
        "\n",
        "Since we are evaluating how well our model can provide an answer to a given instruction, we may want to choose some metrics to help measure this on top of any builtin metrics that `mlflow.evaluate()` gives us.\n",
        "\n",
        "Let's measure how well our model is doing on the following two metrics:\n",
        "\n",
        "- **Is the answer correct?** Let's use the predefined metric `answer_correctness` here.\n",
        "- **Is the answer fluent, clear, and concise?** We will define a custom metric `answer_quality` to measure this.\n",
        "\n",
        "We will need to pass both of these into the `extra_metrics` argument for `mlflow.evaluate()` in order to assess the quality of our model.\n",
        "\n",
        "#### What is an Evaluation Metric?\n",
        "\n",
        "An evaluation metric encapsulates any quantitative or qualitative measure you want to calculate for your model. For each model type, `mlflow.evaluate()` will automatically calculate some set of builtin metrics. Refer [here](https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate) for which builtin metrics will be calculated for each model type. You can also pass in any other metrics you want to calculate as extra metrics. MLflow provides a set of predefined metrics that you can find [here](https://mlflow.org/docs/latest/python_api/mlflow.metrics.html), or you can define your own custom metrics. In the example here, we will use the combination of predefined metrics `mlflow.metrics.genai.answer_correctness` and a custom metric for the quality evaluation."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0VUIT9pZCns6"
      },
      "source": [
        "Let's load our predefined metrics - in this case we are using `answer_correctness` with GPT-4."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fPyG_5jICns6"
      },
      "outputs": [],
      "source": [
        "answer_correctness_metric = answer_correctness(model=\"openai:/gpt-4\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RfYlEXZYCns6"
      },
      "source": [
        "Now we want to create a custom LLM-judged metric named `answer_quality` using `make_genai_metric()`. We need to define a metric definition and grading rubric, as well as some examples for the LLM judge to use."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WsB3OjaNCns6"
      },
      "outputs": [],
      "source": [
        "# The definition explains what \"answer quality\" entails\n",
        "answer_quality_definition = \"\"\"Please evaluate answer quality for the provided output on the following criteria:\n",
        "fluency, clarity, and conciseness. Each of the criteria is defined as follows:\n",
        "  - Fluency measures how naturally and smooth the output reads.\n",
        "  - Clarity measures how understandable the output is.\n",
        "  - Conciseness measures the brevity and efficiency of the output without compromising meaning.\n",
        "The more fluent, clear, and concise a text, the higher the score it deserves.\n",
        "\"\"\"\n",
        "\n",
        "# The grading prompt explains what each possible score means\n",
        "answer_quality_grading_prompt = \"\"\"Answer quality: Below are the details for different scores:\n",
        "  - Score 1: The output is entirely incomprehensible and cannot be read.\n",
        "  - Score 2: The output conveys some meaning, but needs lots of improvement in to improve fluency, clarity, and conciseness.\n",
        "  - Score 3: The output is understandable but still needs improvement.\n",
        "  - Score 4: The output performs well on two of fluency, clarity, and conciseness, but could be improved on one of these criteria.\n",
        "  - Score 5: The output reads smoothly, is easy to understand, and clear. There is no clear way to improve the output on these criteria.\n",
        "\"\"\"\n",
        "\n",
        "# We provide an example of a \"bad\" output\n",
        "example1 = EvaluationExample(\n",
        "    input=\"What is MLflow?\",\n",
        "    output=\"MLflow is an open-source platform. For managing machine learning workflows, it \"\n",
        "    \"including experiment tracking model packaging versioning and deployment as well as a platform \"\n",
        "    \"simplifying for on the ML lifecycle.\",\n",
        "    score=2,\n",
        "    justification=\"The output is difficult to understand and demonstrates extremely low clarity. \"\n",
        "    \"However, it still conveys some meaning so this output deserves a score of 2.\",\n",
        ")\n",
        "\n",
        "# We also provide an example of a \"good\" output\n",
        "example2 = EvaluationExample(\n",
        "    input=\"What is MLflow?\",\n",
        "    output=\"MLflow is an open-source platform for managing machine learning workflows, including \"\n",
        "    \"experiment tracking, model packaging, versioning, and deployment.\",\n",
        "    score=5,\n",
        "    justification=\"The output is easily understandable, clear, and concise. It deserves a score of 5.\",\n",
        ")\n",
        "\n",
        "answer_quality_metric = make_genai_metric(\n",
        "    name=\"answer_quality\",\n",
        "    definition=answer_quality_definition,\n",
        "    grading_prompt=answer_quality_grading_prompt,\n",
        "    version=\"v1\",\n",
        "    examples=[example1, example2],\n",
        "    model=\"openai:/gpt-4\",\n",
        "    greater_is_better=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jnTwC3IBCns6"
      },
      "source": [
        "### Evaluate"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A8-80QrqCns6"
      },
      "source": [
        "We need to set our OpenAI API key, since we are using GPT-4 for our LLM-judged metrics.\n",
        "\n",
        "In order to set your private key safely, please be sure to either export your key through a command-line terminal for your current instance, or, for a permanent addition to all user-based sessions, configure your favored environment management configuration file (i.e., .bashrc, .zshrc) to have the following entry:\n",
        "\n",
        "`OPENAI_API_KEY=<your openai API key>`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W0h11EH-Cns7"
      },
      "source": [
        "Now, we can call `mlflow.evaluate()`. Just to test it out, let's use the first 10 rows of the data. Using the `\"text\"` model type, toxicity and readability metrics are calculated as builtin metrics. We also pass in the two metrics we defined above into the `extra_metrics` parameter to be evaluated."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "referenced_widgets": [
            "d057748e00924cf0a195719c509f03a3",
            "4178d977ace3454a8af3a0e81972e76b",
            "f26345dbf35a45a69dc9e09c04c391a9",
            "309f7fb9ba764ab7b8fae3bed1ed15d7",
            "f5ec353b5c394f74bc4035e3afad4726",
            "fe85000e2aad4c40993382a8de039e09"
          ]
        },
        "id": "RKXtr-syCns7",
        "outputId": "9698d028-05d3-4153-bc73-8cff2b13cd9c"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d057748e00924cf0a195719c509f03a3",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading artifacts:   0%|          | 0/79 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2023/12/28 11:57:30 INFO mlflow.store.artifact.artifact_repo: The progress bar can be disabled by setting the environment variable MLFLOW_ENABLE_ARTIFACTS_PROGRESS_BAR to false\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4178d977ace3454a8af3a0e81972e76b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/66 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2023/12/28 12:00:25 INFO mlflow.models.evaluation.base: Evaluating the model with the default evaluator.\n",
            "2023/12/28 12:00:25 INFO mlflow.models.evaluation.default_evaluator: Computing model predictions.\n",
            "2023/12/28 12:02:23 INFO mlflow.models.evaluation.default_evaluator: Testing metrics on first row...\n",
            "Using default facebook/roberta-hate-speech-dynabench-r4-target checkpoint\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f26345dbf35a45a69dc9e09c04c391a9",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "309f7fb9ba764ab7b8fae3bed1ed15d7",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2023/12/28 12:02:43 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: token_count\n",
            "2023/12/28 12:02:43 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: toxicity\n",
            "2023/12/28 12:02:44 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: flesch_kincaid_grade_level\n",
            "2023/12/28 12:02:44 INFO mlflow.models.evaluation.default_evaluator: Evaluating builtin metrics: ari_grade_level\n",
            "2023/12/28 12:02:44 INFO mlflow.models.evaluation.default_evaluator: Evaluating metrics: answer_correctness\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f5ec353b5c394f74bc4035e3afad4726",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/10 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2023/12/28 12:02:53 INFO mlflow.models.evaluation.default_evaluator: Evaluating metrics: answer_quality\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fe85000e2aad4c40993382a8de039e09",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/10 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "with mlflow.start_run():\n",
        "    results = mlflow.evaluate(\n",
        "        model_info.model_uri,\n",
        "        eval_df.head(10),\n",
        "        evaluators=\"default\",\n",
        "        model_type=\"text\",\n",
        "        targets=\"output\",\n",
        "        extra_metrics=[answer_correctness_metric, answer_quality_metric],\n",
        "        evaluator_config={\"col_mapping\": {\"inputs\": \"instruction\"}},\n",
        "    )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l891i-ZcCns7"
      },
      "source": [
        "### View results"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J_hQjthqCns7"
      },
      "source": [
        "`results.metrics` is a dictionary with the aggregate values for all the metrics calculated. Refer [here](https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate) for details on the builtin metrics for each model type."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DGYObBktCns7",
        "outputId": "e827b63c-c1e9-47c4-d61c-2da71eae7830"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'toxicity/v1/mean': 0.00809656630299287,\n",
              " 'toxicity/v1/variance': 0.0004603014839856817,\n",
              " 'toxicity/v1/p90': 0.010559113975614286,\n",
              " 'toxicity/v1/ratio': 0.0,\n",
              " 'flesch_kincaid_grade_level/v1/mean': 4.9,\n",
              " 'flesch_kincaid_grade_level/v1/variance': 6.3500000000000005,\n",
              " 'flesch_kincaid_grade_level/v1/p90': 6.829999999999998,\n",
              " 'ari_grade_level/v1/mean': 4.1899999999999995,\n",
              " 'ari_grade_level/v1/variance': 16.6329,\n",
              " 'ari_grade_level/v1/p90': 7.949999999999998,\n",
              " 'answer_correctness/v1/mean': 1.5,\n",
              " 'answer_correctness/v1/variance': 1.45,\n",
              " 'answer_correctness/v1/p90': 2.299999999999999,\n",
              " 'answer_quality/v1/mean': 2.4,\n",
              " 'answer_quality/v1/variance': 1.44,\n",
              " 'answer_quality/v1/p90': 4.1}"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results.metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nT7FVqpRCns7"
      },
      "source": [
        "We can also view the `eval_results_table`, which shows us the metrics for each row of data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "referenced_widgets": [
            "2d968df3d2614ae49a15236fd1fd12f6"
          ]
        },
        "id": "O1nQJsbPCns7",
        "outputId": "f88d8ba9-27d5-46f5-b178-f73bbe9b8dee"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2d968df3d2614ae49a15236fd1fd12f6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading artifacts:   0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>instruction</th>\n",
              "      <th>input</th>\n",
              "      <th>text</th>\n",
              "      <th>output</th>\n",
              "      <th>outputs</th>\n",
              "      <th>token_count</th>\n",
              "      <th>toxicity/v1/score</th>\n",
              "      <th>flesch_kincaid_grade_level/v1/score</th>\n",
              "      <th>ari_grade_level/v1/score</th>\n",
              "      <th>answer_correctness/v1/score</th>\n",
              "      <th>answer_correctness/v1/justification</th>\n",
              "      <th>answer_quality/v1/score</th>\n",
              "      <th>answer_quality/v1/justification</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Give three tips for staying healthy.</td>\n",
              "      <td></td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>1.Eat a balanced diet and make sure to include...</td>\n",
              "      <td>Give three tips for staying healthy.\\n1. Eat a...</td>\n",
              "      <td>19</td>\n",
              "      <td>0.000446</td>\n",
              "      <td>4.1</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2</td>\n",
              "      <td>The output provided by the model only includes...</td>\n",
              "      <td>3</td>\n",
              "      <td>The output is understandable and fluent but it...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>What are the three primary colors?</td>\n",
              "      <td></td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>The three primary colors are red, blue, and ye...</td>\n",
              "      <td>What are the three primary colors?\\nThe three ...</td>\n",
              "      <td>19</td>\n",
              "      <td>0.000217</td>\n",
              "      <td>5.0</td>\n",
              "      <td>4.9</td>\n",
              "      <td>5</td>\n",
              "      <td>The output provided by the model is completely...</td>\n",
              "      <td>5</td>\n",
              "      <td>The model's output is fluent, clear, and conci...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Describe the structure of an atom.</td>\n",
              "      <td></td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>An atom is made up of a nucleus, which contain...</td>\n",
              "      <td>Describe the structure of an atom.\\nAn atom is...</td>\n",
              "      <td>18</td>\n",
              "      <td>0.000139</td>\n",
              "      <td>3.1</td>\n",
              "      <td>2.2</td>\n",
              "      <td>1</td>\n",
              "      <td>The output provided by the model is incomplete...</td>\n",
              "      <td>2</td>\n",
              "      <td>The output is incomplete and lacks clarity, ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>How can we reduce air pollution?</td>\n",
              "      <td></td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>There are a number of ways to reduce air pollu...</td>\n",
              "      <td>How can we reduce air pollution?\\nThere are ma...</td>\n",
              "      <td>18</td>\n",
              "      <td>0.000140</td>\n",
              "      <td>5.0</td>\n",
              "      <td>5.5</td>\n",
              "      <td>1</td>\n",
              "      <td>The output provided by the model is completely...</td>\n",
              "      <td>1</td>\n",
              "      <td>The output is entirely incomprehensible and ca...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Describe a time when you had to make a difficu...</td>\n",
              "      <td></td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>I had to make a difficult decision when I was ...</td>\n",
              "      <td>Describe a time when you had to make a difficu...</td>\n",
              "      <td>18</td>\n",
              "      <td>0.000159</td>\n",
              "      <td>5.2</td>\n",
              "      <td>2.9</td>\n",
              "      <td>1</td>\n",
              "      <td>The output provided by the model is completely...</td>\n",
              "      <td>2</td>\n",
              "      <td>The output is incomplete and lacks clarity, ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Identify the odd one out.</td>\n",
              "      <td>Twitter, Instagram, Telegram</td>\n",
              "      <td>Below is an instruction that describes a task,...</td>\n",
              "      <td>Telegram</td>\n",
              "      <td>Identify the odd one out.\\n\\n1. A car\\n2. A tr...</td>\n",
              "      <td>18</td>\n",
              "      <td>0.072345</td>\n",
              "      <td>0.1</td>\n",
              "      <td>-5.4</td>\n",
              "      <td>1</td>\n",
              "      <td>The output provided by the model is completely...</td>\n",
              "      <td>2</td>\n",
              "      <td>The output is not clear and lacks fluency. The...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Explain why the following fraction is equivale...</td>\n",
              "      <td>4/16</td>\n",
              "      <td>Below is an instruction that describes a task,...</td>\n",
              "      <td>The fraction 4/16 is equivalent to 1/4 because...</td>\n",
              "      <td>Explain why the following fraction is equivale...</td>\n",
              "      <td>23</td>\n",
              "      <td>0.000320</td>\n",
              "      <td>6.4</td>\n",
              "      <td>7.6</td>\n",
              "      <td>1</td>\n",
              "      <td>The output provided by the model is completely...</td>\n",
              "      <td>2</td>\n",
              "      <td>The output is not clear and does not answer th...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Write a short story in third person narration ...</td>\n",
              "      <td></td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>John was at a crossroads in his life. He had j...</td>\n",
              "      <td>Write a short story in third person narration ...</td>\n",
              "      <td>20</td>\n",
              "      <td>0.000247</td>\n",
              "      <td>10.7</td>\n",
              "      <td>11.1</td>\n",
              "      <td>1</td>\n",
              "      <td>The output provided by the model is completely...</td>\n",
              "      <td>1</td>\n",
              "      <td>The output is exactly the same as the input, a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Render a 3D model of a house</td>\n",
              "      <td></td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "      <td>&lt;nooutput&gt; This type of instruction cannot be ...</td>\n",
              "      <td>Render a 3D model of a house in Blender - Blen...</td>\n",
              "      <td>19</td>\n",
              "      <td>0.003694</td>\n",
              "      <td>5.2</td>\n",
              "      <td>2.7</td>\n",
              "      <td>1</td>\n",
              "      <td>The output provided by the model is completely...</td>\n",
              "      <td>2</td>\n",
              "      <td>The output is partially understandable but lac...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Evaluate this sentence for spelling and gramma...</td>\n",
              "      <td>He finnished his meal and left the resturant</td>\n",
              "      <td>Below is an instruction that describes a task,...</td>\n",
              "      <td>He finished his meal and left the restaurant.</td>\n",
              "      <td>Evaluate this sentence for spelling and gramma...</td>\n",
              "      <td>18</td>\n",
              "      <td>0.003260</td>\n",
              "      <td>4.2</td>\n",
              "      <td>6.4</td>\n",
              "      <td>1</td>\n",
              "      <td>The output provided by the model is completely...</td>\n",
              "      <td>4</td>\n",
              "      <td>The output is fluent and clear, but it is not ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                         instruction  \\\n",
              "0               Give three tips for staying healthy.   \n",
              "1                 What are the three primary colors?   \n",
              "2                 Describe the structure of an atom.   \n",
              "3                   How can we reduce air pollution?   \n",
              "4  Describe a time when you had to make a difficu...   \n",
              "5                          Identify the odd one out.   \n",
              "6  Explain why the following fraction is equivale...   \n",
              "7  Write a short story in third person narration ...   \n",
              "8                       Render a 3D model of a house   \n",
              "9  Evaluate this sentence for spelling and gramma...   \n",
              "\n",
              "                                          input  \\\n",
              "0                                                 \n",
              "1                                                 \n",
              "2                                                 \n",
              "3                                                 \n",
              "4                                                 \n",
              "5                  Twitter, Instagram, Telegram   \n",
              "6                                          4/16   \n",
              "7                                                 \n",
              "8                                                 \n",
              "9  He finnished his meal and left the resturant   \n",
              "\n",
              "                                                text  \\\n",
              "0  Below is an instruction that describes a task....   \n",
              "1  Below is an instruction that describes a task....   \n",
              "2  Below is an instruction that describes a task....   \n",
              "3  Below is an instruction that describes a task....   \n",
              "4  Below is an instruction that describes a task....   \n",
              "5  Below is an instruction that describes a task,...   \n",
              "6  Below is an instruction that describes a task,...   \n",
              "7  Below is an instruction that describes a task....   \n",
              "8  Below is an instruction that describes a task....   \n",
              "9  Below is an instruction that describes a task,...   \n",
              "\n",
              "                                              output  \\\n",
              "0  1.Eat a balanced diet and make sure to include...   \n",
              "1  The three primary colors are red, blue, and ye...   \n",
              "2  An atom is made up of a nucleus, which contain...   \n",
              "3  There are a number of ways to reduce air pollu...   \n",
              "4  I had to make a difficult decision when I was ...   \n",
              "5                                           Telegram   \n",
              "6  The fraction 4/16 is equivalent to 1/4 because...   \n",
              "7  John was at a crossroads in his life. He had j...   \n",
              "8  <nooutput> This type of instruction cannot be ...   \n",
              "9      He finished his meal and left the restaurant.   \n",
              "\n",
              "                                             outputs  token_count  \\\n",
              "0  Give three tips for staying healthy.\\n1. Eat a...           19   \n",
              "1  What are the three primary colors?\\nThe three ...           19   \n",
              "2  Describe the structure of an atom.\\nAn atom is...           18   \n",
              "3  How can we reduce air pollution?\\nThere are ma...           18   \n",
              "4  Describe a time when you had to make a difficu...           18   \n",
              "5  Identify the odd one out.\\n\\n1. A car\\n2. A tr...           18   \n",
              "6  Explain why the following fraction is equivale...           23   \n",
              "7  Write a short story in third person narration ...           20   \n",
              "8  Render a 3D model of a house in Blender - Blen...           19   \n",
              "9  Evaluate this sentence for spelling and gramma...           18   \n",
              "\n",
              "   toxicity/v1/score  flesch_kincaid_grade_level/v1/score  \\\n",
              "0           0.000446                                  4.1   \n",
              "1           0.000217                                  5.0   \n",
              "2           0.000139                                  3.1   \n",
              "3           0.000140                                  5.0   \n",
              "4           0.000159                                  5.2   \n",
              "5           0.072345                                  0.1   \n",
              "6           0.000320                                  6.4   \n",
              "7           0.000247                                 10.7   \n",
              "8           0.003694                                  5.2   \n",
              "9           0.003260                                  4.2   \n",
              "\n",
              "   ari_grade_level/v1/score  answer_correctness/v1/score  \\\n",
              "0                       4.0                            2   \n",
              "1                       4.9                            5   \n",
              "2                       2.2                            1   \n",
              "3                       5.5                            1   \n",
              "4                       2.9                            1   \n",
              "5                      -5.4                            1   \n",
              "6                       7.6                            1   \n",
              "7                      11.1                            1   \n",
              "8                       2.7                            1   \n",
              "9                       6.4                            1   \n",
              "\n",
              "                 answer_correctness/v1/justification  answer_quality/v1/score  \\\n",
              "0  The output provided by the model only includes...                        3   \n",
              "1  The output provided by the model is completely...                        5   \n",
              "2  The output provided by the model is incomplete...                        2   \n",
              "3  The output provided by the model is completely...                        1   \n",
              "4  The output provided by the model is completely...                        2   \n",
              "5  The output provided by the model is completely...                        2   \n",
              "6  The output provided by the model is completely...                        2   \n",
              "7  The output provided by the model is completely...                        1   \n",
              "8  The output provided by the model is completely...                        2   \n",
              "9  The output provided by the model is completely...                        4   \n",
              "\n",
              "                     answer_quality/v1/justification  \n",
              "0  The output is understandable and fluent but it...  \n",
              "1  The model's output is fluent, clear, and conci...  \n",
              "2  The output is incomplete and lacks clarity, ma...  \n",
              "3  The output is entirely incomprehensible and ca...  \n",
              "4  The output is incomplete and lacks clarity, ma...  \n",
              "5  The output is not clear and lacks fluency. The...  \n",
              "6  The output is not clear and does not answer th...  \n",
              "7  The output is exactly the same as the input, a...  \n",
              "8  The output is partially understandable but lac...  \n",
              "9  The output is fluent and clear, but it is not ...  "
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results.tables[\"eval_results_table\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2eNivp6VCns8"
      },
      "source": [
        "#### View results in UI"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lXYJaTbsCns8"
      },
      "source": [
        "Finally, we can view our evaluation results in the MLflow UI. We can select our experiment on the left sidebar, which will bring us to the following page. We can see that one run logged our model \"mpt-7b-chat\", and the other run has the dataset we evaluated."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fjqwj3UGCns8"
      },
      "source": [
        "![Evaluation Main](https://i.imgur.com/alymcBq.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HZw1ncvmCns8"
      },
      "source": [
        "We click on the Evaluation tab and hide any irrelevant runs."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WJasUoyrCns8"
      },
      "source": [
        "![Evaluation Filtering](https://i.imgur.com/sr7R9TL.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "58w3BSfcCns8"
      },
      "source": [
        "We can now choose what columns we want to group by, as well as which column we want to compare. In the following example, we are looking at the score for answer correctness for each input-output pair, but we could choose any other metric to compare."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gOzq-fk-Cns8"
      },
      "source": [
        "![Evaluation Selection](https://i.imgur.com/AeoYMEt.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3agUYv6tCns8"
      },
      "source": [
        "Finally, we get to the following view, where we can see the justification and score for answer correctness for each row."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H95MnoCFCns8"
      },
      "source": [
        "![Evaluation Comparison](https://i.imgur.com/axsHZxP.png)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.17"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "bec0d3643d0f4239b44ab6cc1ddafbb5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5aa6f64c914f4f5db0de42de11abb24a",
              "IPY_MODEL_69a8ee72206c41e19edc104df7cba7bf",
              "IPY_MODEL_ca6760391d22477a8ffc6765f3973094"
            ],
            "layout": "IPY_MODEL_b819e261b79348edb84d82ac258353e2"
          }
        },
        "5aa6f64c914f4f5db0de42de11abb24a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a8945c43b9024367b99f4f8c15695bf3",
            "placeholder": "​",
            "style": "IPY_MODEL_4514e00544ae4be094f4072aef87178d",
            "value": "config.json: 100%"
          }
        },
        "69a8ee72206c41e19edc104df7cba7bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc11bfdb91ef494092324cff892e034c",
            "max": 1234,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d741cbeabc5a442fbdf2d280a80638f3",
            "value": 1234
          }
        },
        "ca6760391d22477a8ffc6765f3973094": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2133c76bf07c4ec5ac8f840a495b7121",
            "placeholder": "​",
            "style": "IPY_MODEL_a22bb4b308e446d3a6d44a60d7032312",
            "value": " 1.23k/1.23k [00:00&lt;00:00, 25.4kB/s]"
          }
        },
        "b819e261b79348edb84d82ac258353e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a8945c43b9024367b99f4f8c15695bf3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4514e00544ae4be094f4072aef87178d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dc11bfdb91ef494092324cff892e034c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d741cbeabc5a442fbdf2d280a80638f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2133c76bf07c4ec5ac8f840a495b7121": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a22bb4b308e446d3a6d44a60d7032312": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "acaae41525214422a2c6c2f5f5f4a327": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_15ad66c5c4924957873e11b30b1e1c85",
              "IPY_MODEL_78b12da7919f407898ea38c6eb734afd",
              "IPY_MODEL_8d3cdaa609984a2cae72caecc09bc0d5"
            ],
            "layout": "IPY_MODEL_c309c28eb9964154b4870289155364d8"
          }
        },
        "15ad66c5c4924957873e11b30b1e1c85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0e22b9a7721a40b19f0f8e6ac7b80d3b",
            "placeholder": "​",
            "style": "IPY_MODEL_640d0f602404443c982e405a45179dd2",
            "value": "pytorch_model.bin.index.json: 100%"
          }
        },
        "78b12da7919f407898ea38c6eb734afd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1d9ccae90ffe4786998c60bf4a763198",
            "max": 16023,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_58b5f9ea7e694bbebf26448ed357148d",
            "value": 16023
          }
        },
        "8d3cdaa609984a2cae72caecc09bc0d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6abae3e8eff84423ba7a5c383754e3a0",
            "placeholder": "​",
            "style": "IPY_MODEL_7c1bc5edcee945b09fd9813af1f526c6",
            "value": " 16.0k/16.0k [00:00&lt;00:00, 312kB/s]"
          }
        },
        "c309c28eb9964154b4870289155364d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0e22b9a7721a40b19f0f8e6ac7b80d3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "640d0f602404443c982e405a45179dd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1d9ccae90ffe4786998c60bf4a763198": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58b5f9ea7e694bbebf26448ed357148d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6abae3e8eff84423ba7a5c383754e3a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c1bc5edcee945b09fd9813af1f526c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6d7b7d3df459481d8334231b9bc9037b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6bd84cf8fde74eaaa0fe0f0f2afab855",
              "IPY_MODEL_3a556dd8213341d0b5c635f9016ffb7d",
              "IPY_MODEL_b37ca98bd12c4ab98eeb08a9c0bc3fb8"
            ],
            "layout": "IPY_MODEL_c6bc18b798ac4b3aa2d17d7f315a42b5"
          }
        },
        "6bd84cf8fde74eaaa0fe0f0f2afab855": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6debcb842ce44ca28da939a4f50a6bfe",
            "placeholder": "​",
            "style": "IPY_MODEL_872575ef8e5e41ceb39b19e1437d036d",
            "value": "Downloading shards: 100%"
          }
        },
        "3a556dd8213341d0b5c635f9016ffb7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c5b6cbc5cfd54897854ecb657eee4da1",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0035c659b38d43908ef6916ff4336644",
            "value": 2
          }
        },
        "b37ca98bd12c4ab98eeb08a9c0bc3fb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e5e1c3df3584d2ca452f18dec9401c0",
            "placeholder": "​",
            "style": "IPY_MODEL_135c5de607a94af39bdde9cb3b223f9e",
            "value": " 2/2 [05:17&lt;00:00, 145.17s/it]"
          }
        },
        "c6bc18b798ac4b3aa2d17d7f315a42b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6debcb842ce44ca28da939a4f50a6bfe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "872575ef8e5e41ceb39b19e1437d036d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c5b6cbc5cfd54897854ecb657eee4da1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0035c659b38d43908ef6916ff4336644": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2e5e1c3df3584d2ca452f18dec9401c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "135c5de607a94af39bdde9cb3b223f9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2580f3279b76403ebb1dbbbba8d424bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d486180354c343779caed3f5e38abcef",
              "IPY_MODEL_d0c95cebcfe44116bfab9e654dbc8e9d",
              "IPY_MODEL_23e0e9bc80c34d8294d145c349fe0413"
            ],
            "layout": "IPY_MODEL_8a3b5bebdfba42418ab67b33a8e5b3d2"
          }
        },
        "d486180354c343779caed3f5e38abcef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7e2dbccd527d49489624ac7fb0934e75",
            "placeholder": "​",
            "style": "IPY_MODEL_71b685a43f704800815bc1a6c9283cb6",
            "value": "pytorch_model-00001-of-00002.bin: 100%"
          }
        },
        "d0c95cebcfe44116bfab9e654dbc8e9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d654a1cf9f1341c39f79c8c6625986f7",
            "max": 9943040275,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_10354da5be664163bb8d87de9fb84aaa",
            "value": 9943040275
          }
        },
        "23e0e9bc80c34d8294d145c349fe0413": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b0747a467641432ca3ff4ebd1dc7b8fc",
            "placeholder": "​",
            "style": "IPY_MODEL_0eb29aac50df42c88ce1664b845eea22",
            "value": " 9.94G/9.94G [03:55&lt;00:00, 42.0MB/s]"
          }
        },
        "8a3b5bebdfba42418ab67b33a8e5b3d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e2dbccd527d49489624ac7fb0934e75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71b685a43f704800815bc1a6c9283cb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d654a1cf9f1341c39f79c8c6625986f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10354da5be664163bb8d87de9fb84aaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b0747a467641432ca3ff4ebd1dc7b8fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0eb29aac50df42c88ce1664b845eea22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "57ba083b9fd44a4bad56cca0f40ca96b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9bd1214a2aaf4b42972e684932207d0c",
              "IPY_MODEL_3c23643b97834f3d83c9c530d5ade109",
              "IPY_MODEL_ff9a4e053c63447481a4a8dbe26c10b5"
            ],
            "layout": "IPY_MODEL_1da1c5738f854e42ab593d74d9ad7191"
          }
        },
        "9bd1214a2aaf4b42972e684932207d0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_61aa29b5b18d49d9868b63e18f2e1e63",
            "placeholder": "​",
            "style": "IPY_MODEL_4db0c0b3304d43ecb0ee1468bc79ea06",
            "value": "pytorch_model-00002-of-00002.bin: 100%"
          }
        },
        "3c23643b97834f3d83c9c530d5ade109": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_66ad8ecbb99b49eaa9e7bdb1a592f94e",
            "max": 3355599187,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_33a5a69352c64b32ade5eb5a9a0a88e2",
            "value": 3355599187
          }
        },
        "ff9a4e053c63447481a4a8dbe26c10b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3b878ba5949743869d989b9fbda940d8",
            "placeholder": "​",
            "style": "IPY_MODEL_34791495002a4de6b6317410e20bd9f6",
            "value": " 3.36G/3.36G [01:20&lt;00:00, 43.1MB/s]"
          }
        },
        "1da1c5738f854e42ab593d74d9ad7191": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61aa29b5b18d49d9868b63e18f2e1e63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4db0c0b3304d43ecb0ee1468bc79ea06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "66ad8ecbb99b49eaa9e7bdb1a592f94e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33a5a69352c64b32ade5eb5a9a0a88e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3b878ba5949743869d989b9fbda940d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "34791495002a4de6b6317410e20bd9f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0c8693f0b20b41979fe9f6972babc2b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dbcb4f8bccef47b2a9d11f257a206e1d",
              "IPY_MODEL_aaed8aba1bfa4360827892fba045e20d",
              "IPY_MODEL_c734d30155124b8791aa643f2c78d8f5"
            ],
            "layout": "IPY_MODEL_7e12943791ac44379af97a3a8ae921eb"
          }
        },
        "dbcb4f8bccef47b2a9d11f257a206e1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_22daa5f7e09c49278c10962964143131",
            "placeholder": "​",
            "style": "IPY_MODEL_ba8d2c6e7d494123919c9c0b6510424b",
            "value": "Loading checkpoint shards:   0%"
          }
        },
        "aaed8aba1bfa4360827892fba045e20d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a91a5cdd197b4976aef986791aed07d1",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2c9bdd4cd6614eef9132b9b66238dd38",
            "value": 0
          }
        },
        "c734d30155124b8791aa643f2c78d8f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5203572d7dff4ba5a6487e8cd7ded86b",
            "placeholder": "​",
            "style": "IPY_MODEL_6801c003f197449bb2083b1df027e94f",
            "value": " 0/2 [00:00&lt;?, ?it/s]"
          }
        },
        "7e12943791ac44379af97a3a8ae921eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "22daa5f7e09c49278c10962964143131": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ba8d2c6e7d494123919c9c0b6510424b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a91a5cdd197b4976aef986791aed07d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2c9bdd4cd6614eef9132b9b66238dd38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5203572d7dff4ba5a6487e8cd7ded86b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6801c003f197449bb2083b1df027e94f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}