New_Language_Training_(Colab).ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/New-Languages-for-NLP/repo-template/blob/main/New_Language_Training_(Colab).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "061ba15a"
      },
      "source": [
        "<img width=50% src=\"https://github.com/New-Languages-for-NLP/course-materials/raw/main/w2/using-inception-data/newnlp_notebook.png\" />\n",
        "\n",
        "For full documentation on this project, see [here](https://new-languages-for-nlp.github.io/course-materials/w2/using-inception-data/New%20Language%20Training.html)\n",
        " "
      ],
      "id": "061ba15a"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "psP7JdcRLR7n"
      },
      "source": [
        "# 1 Prepare the Notebook Environment"
      ],
      "id": "psP7JdcRLR7n"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a0ba9e5a",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title The Colab runtime comes with spaCy v2 and needs to be upgraded to v3.\n",
        "#@markdown This project uses the GPU by default, if you need to use just the CPU, just uncheck the box below.\n",
        "GPU = True #@param {type:\"boolean\"}\n",
        "\n",
        "# Install spaCy v3 and libraries for GPUs and transformers\n",
        "!pip install spacy --upgrade --quiet\n",
        "if GPU:\n",
        "    !pip install 'spacy[transformers,cuda111]' --quiet\n",
        "#!pip install wandb spacy-huggingface-hub --quiet"
      ],
      "id": "a0ba9e5a"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WfsEKZv6ErlG"
      },
      "source": [
        "The notebook will pull project files from your GitHub repository.  \n",
        "\n",
        "Note that you need to set the langugage (lang), treebank (same as the repo name), test_size and package name in the project.yml file in your repository.  "
      ],
      "id": "WfsEKZv6ErlG"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "7c0bda8e"
      },
      "outputs": [],
      "source": [
        "#@title Enter your language's repository name. \n",
        "#@markdown If the repo is private, please check the \"private_repo\" box and include an [access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token).\n",
        "private_repo = False #@param {type:\"boolean\"}\n",
        "repo_name = \"repo-template\" #@param {type:\"string\"}\n",
        "branch = \"main\" #@param {type:\"string\"}\n",
        "\n",
        "\n",
        "!rm -rf /content/newlang_project\n",
        "!rm -rf $repo_name\n",
        "if private_repo:\n",
        "    git_access_token = \"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\" #@param {type:\"string\"}\n",
        "    git_url = f\"https://{git_access_token}@github.com/New-Languages-for-NLP/{repo_name}/\"\n",
        "    !git clone $git_url  -b $branch\n",
        "    !cp -r ./$repo_name/newlang_project .  \n",
        "    !mkdir newlang_project/assets/\n",
        "    !mkdir newlang_project/configs/\n",
        "    !mkdir newlang_project/corpus/\n",
        "    !mkdir newlang_project/metrics/\n",
        "    !mkdir newlang_project/packages/\n",
        "    !mkdir newlang_project/training/\n",
        "    !mkdir newlang_project/assets/$repo_name\n",
        "    !cp -r ./$repo_name/* newlang_project/assets/$repo_name/\n",
        "    !rm -rf ./$repo_name\n",
        "else:\n",
        "    !python -m spacy project clone newlang_project --repo https://github.com/New-Languages-for-NLP/$repo_name --branch $branch\n",
        "    !python -m spacy project assets /content/newlang_project"
      ],
      "id": "7c0bda8e"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4dc13741"
      },
      "outputs": [],
      "source": [
        "# Install the custom language object from Cadet \n",
        "!python -m spacy project run install /content/newlang_project"
      ],
      "id": "4dc13741"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qU_AqRK6LZrF"
      },
      "source": [
        "# 2 Prepare the Data for Training"
      ],
      "id": "qU_AqRK6LZrF"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HPjD0SN6iAvc",
        "outputId": "04e7fed1-7613-423a-ce0f-7aaf73ae6953"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overwriting /usr/local/lib/python3.7/dist-packages/spacy/training/converters/conllu_to_docs.py\n"
          ]
        }
      ],
      "source": [
        "#@title (optional) cell to correct a problem when your tokens have no pos value\n",
        "%%writefile /usr/local/lib/python3.7/dist-packages/spacy/training/converters/conllu_to_docs.py\n",
        "import re\n",
        "\n",
        "from .conll_ner_to_docs import n_sents_info\n",
        "from ...training import iob_to_biluo, biluo_tags_to_spans\n",
        "from ...tokens import Doc, Token, Span\n",
        "from ...vocab import Vocab\n",
        "from wasabi import Printer\n",
        "\n",
        "\n",
        "def conllu_to_docs(\n",
        "    input_data,\n",
        "    n_sents=10,\n",
        "    append_morphology=False,\n",
        "    ner_map=None,\n",
        "    merge_subtokens=False,\n",
        "    no_print=False,\n",
        "    **_\n",
        "):\n",
        "    \"\"\"\n",
        "    Convert conllu files into JSON format for use with train cli.\n",
        "    append_morphology parameter enables appending morphology to tags, which is\n",
        "    useful for languages such as Spanish, where UD tags are not so rich.\n",
        "\n",
        "    Extract NER tags if available and convert them so that they follow\n",
        "    BILUO and the Wikipedia scheme\n",
        "    \"\"\"\n",
        "    MISC_NER_PATTERN = \"^((?:name|NE)=)?([BILU])-([A-Z_]+)|O$\"\n",
        "    msg = Printer(no_print=no_print)\n",
        "    n_sents_info(msg, n_sents)\n",
        "    sent_docs = read_conllx(\n",
        "        input_data,\n",
        "        append_morphology=append_morphology,\n",
        "        ner_tag_pattern=MISC_NER_PATTERN,\n",
        "        ner_map=ner_map,\n",
        "        merge_subtokens=merge_subtokens,\n",
        "    )\n",
        "    sent_docs_to_merge = []\n",
        "    for sent_doc in sent_docs:\n",
        "        sent_docs_to_merge.append(sent_doc)\n",
        "        if len(sent_docs_to_merge) % n_sents == 0:\n",
        "            yield Doc.from_docs(sent_docs_to_merge)\n",
        "            sent_docs_to_merge = []\n",
        "    if sent_docs_to_merge:\n",
        "        yield Doc.from_docs(sent_docs_to_merge)\n",
        "\n",
        "\n",
        "def has_ner(input_data, ner_tag_pattern):\n",
        "    \"\"\"\n",
        "    Check the MISC column for NER tags.\n",
        "    \"\"\"\n",
        "    for sent in input_data.strip().split(\"\\n\\n\"):\n",
        "        lines = sent.strip().split(\"\\n\")\n",
        "        if lines:\n",
        "            while lines[0].startswith(\"#\"):\n",
        "                lines.pop(0)\n",
        "            for line in lines:\n",
        "                parts = line.split(\"\\t\")\n",
        "                id_, word, lemma, pos, tag, morph, head, dep, _1, misc = parts\n",
        "                for misc_part in misc.split(\"|\"):\n",
        "                    if re.match(ner_tag_pattern, misc_part):\n",
        "                        return True\n",
        "    return False\n",
        "\n",
        "\n",
        "def read_conllx(\n",
        "    input_data,\n",
        "    append_morphology=False,\n",
        "    merge_subtokens=False,\n",
        "    ner_tag_pattern=\"\",\n",
        "    ner_map=None,\n",
        "):\n",
        "    \"\"\"Yield docs, one for each sentence\"\"\"\n",
        "    vocab = Vocab()  # need vocab to make a minimal Doc\n",
        "    for sent in input_data.strip().split(\"\\n\\n\"):\n",
        "        lines = sent.strip().split(\"\\n\")\n",
        "        if lines:\n",
        "            while lines[0].startswith(\"#\"):\n",
        "                lines.pop(0)\n",
        "            doc = conllu_sentence_to_doc(\n",
        "                vocab,\n",
        "                lines,\n",
        "                ner_tag_pattern,\n",
        "                merge_subtokens=merge_subtokens,\n",
        "                append_morphology=append_morphology,\n",
        "                ner_map=ner_map,\n",
        "            )\n",
        "            yield doc\n",
        "\n",
        "\n",
        "def get_entities(lines, tag_pattern, ner_map=None):\n",
        "    \"\"\"Find entities in the MISC column according to the pattern and map to\n",
        "    final entity type with `ner_map` if mapping present. Entity tag is 'O' if\n",
        "    the pattern is not matched.\n",
        "\n",
        "    lines (str): CONLL-U lines for one sentences\n",
        "    tag_pattern (str): Regex pattern for entity tag\n",
        "    ner_map (dict): Map old NER tag names to new ones, '' maps to O.\n",
        "    RETURNS (list): List of BILUO entity tags\n",
        "    \"\"\"\n",
        "    miscs = []\n",
        "    for line in lines:\n",
        "        parts = line.split(\"\\t\")\n",
        "        id_, word, lemma, pos, tag, morph, head, dep, _1, misc = parts\n",
        "        if \"-\" in id_ or \".\" in id_:\n",
        "            continue\n",
        "        miscs.append(misc)\n",
        "\n",
        "    iob = []\n",
        "    for misc in miscs:\n",
        "        iob_tag = \"O\"\n",
        "        for misc_part in misc.split(\"|\"):\n",
        "            tag_match = re.match(tag_pattern, misc_part)\n",
        "            if tag_match:\n",
        "                prefix = tag_match.group(2)\n",
        "                suffix = tag_match.group(3)\n",
        "                if prefix and suffix:\n",
        "                    iob_tag = prefix + \"-\" + suffix\n",
        "                    if ner_map:\n",
        "                        suffix = ner_map.get(suffix, suffix)\n",
        "                        if suffix == \"\":\n",
        "                            iob_tag = \"O\"\n",
        "                        else:\n",
        "                            iob_tag = prefix + \"-\" + suffix\n",
        "                break\n",
        "        iob.append(iob_tag)\n",
        "    return iob_to_biluo(iob)\n",
        "\n",
        "\n",
        "def conllu_sentence_to_doc(\n",
        "    vocab,\n",
        "    lines,\n",
        "    ner_tag_pattern,\n",
        "    merge_subtokens=False,\n",
        "    append_morphology=False,\n",
        "    ner_map=None,\n",
        "):\n",
        "    \"\"\"Create an Example from the lines for one CoNLL-U sentence, merging\n",
        "    subtokens and appending morphology to tags if required.\n",
        "\n",
        "    lines (str): The non-comment lines for a CoNLL-U sentence\n",
        "    ner_tag_pattern (str): The regex pattern for matching NER in MISC col\n",
        "    RETURNS (Example): An example containing the annotation\n",
        "    \"\"\"\n",
        "    # create a Doc with each subtoken as its own token\n",
        "    # if merging subtokens, each subtoken orth is the merged subtoken form\n",
        "    if not Token.has_extension(\"merged_orth\"):\n",
        "        Token.set_extension(\"merged_orth\", default=\"\")\n",
        "    if not Token.has_extension(\"merged_lemma\"):\n",
        "        Token.set_extension(\"merged_lemma\", default=\"\")\n",
        "    if not Token.has_extension(\"merged_morph\"):\n",
        "        Token.set_extension(\"merged_morph\", default=\"\")\n",
        "    if not Token.has_extension(\"merged_spaceafter\"):\n",
        "        Token.set_extension(\"merged_spaceafter\", default=\"\")\n",
        "    words, spaces, tags, poses, morphs, lemmas = [], [], [], [], [], []\n",
        "    heads, deps = [], []\n",
        "    subtok_word = \"\"\n",
        "    in_subtok = False\n",
        "    for i in range(len(lines)):\n",
        "        line = lines[i]\n",
        "        parts = line.split(\"\\t\")\n",
        "        id_, word, lemma, pos, tag, morph, head, dep, _1, misc = parts\n",
        "        if \".\" in id_:\n",
        "            continue\n",
        "        if \"-\" in id_:\n",
        "            in_subtok = True\n",
        "        if \"-\" in id_:\n",
        "            in_subtok = True\n",
        "            subtok_word = word\n",
        "            subtok_start, subtok_end = id_.split(\"-\")\n",
        "            subtok_spaceafter = \"SpaceAfter=No\" not in misc\n",
        "            continue\n",
        "        if merge_subtokens and in_subtok:\n",
        "            words.append(subtok_word)\n",
        "        else:\n",
        "            words.append(word)\n",
        "        if in_subtok:\n",
        "            if id_ == subtok_end:\n",
        "                spaces.append(subtok_spaceafter)\n",
        "            else:\n",
        "                spaces.append(False)\n",
        "        elif \"SpaceAfter=No\" in misc:\n",
        "            spaces.append(False)\n",
        "        else:\n",
        "            spaces.append(True)\n",
        "        if in_subtok and id_ == subtok_end:\n",
        "            subtok_word = \"\"\n",
        "            in_subtok = False\n",
        "        id_ = int(id_) - 1\n",
        "        head = (int(head) - 1) if head not in (\"0\", \"_\") else id_\n",
        "        tag = pos if tag == \"_\" else tag\n",
        "        morph = morph if morph != \"_\" else \"\"\n",
        "        dep = \"ROOT\" if dep == \"root\" else dep\n",
        "        lemmas.append(lemma)\n",
        "        if pos == \"_\":\n",
        "            pos = \"\"\n",
        "        poses.append(pos)\n",
        "        tags.append(tag)\n",
        "        morphs.append(morph)\n",
        "        heads.append(head)\n",
        "        deps.append(dep)\n",
        "\n",
        "    doc = Doc(\n",
        "        vocab,\n",
        "        words=words,\n",
        "        spaces=spaces,\n",
        "        tags=tags,\n",
        "        pos=poses,\n",
        "        deps=deps,\n",
        "        lemmas=lemmas,\n",
        "        morphs=morphs,\n",
        "        heads=heads,\n",
        "    )\n",
        "    for i in range(len(doc)):\n",
        "        doc[i]._.merged_orth = words[i]\n",
        "        doc[i]._.merged_morph = morphs[i]\n",
        "        doc[i]._.merged_lemma = lemmas[i]\n",
        "        doc[i]._.merged_spaceafter = spaces[i]\n",
        "    ents = get_entities(lines, ner_tag_pattern, ner_map)\n",
        "    doc.ents = biluo_tags_to_spans(doc, ents)\n",
        "\n",
        "    if merge_subtokens:\n",
        "        doc = merge_conllu_subtokens(lines, doc)\n",
        "\n",
        "    # create final Doc from custom Doc annotation\n",
        "    words, spaces, tags, morphs, lemmas, poses = [], [], [], [], [], []\n",
        "    heads, deps = [], []\n",
        "    for i, t in enumerate(doc):\n",
        "        words.append(t._.merged_orth)\n",
        "        lemmas.append(t._.merged_lemma)\n",
        "        spaces.append(t._.merged_spaceafter)\n",
        "        morphs.append(t._.merged_morph)\n",
        "        if append_morphology and t._.merged_morph:\n",
        "            tags.append(t.tag_ + \"__\" + t._.merged_morph)\n",
        "        else:\n",
        "            tags.append(t.tag_)\n",
        "        poses.append(t.pos_)\n",
        "        heads.append(t.head.i)\n",
        "        deps.append(t.dep_)\n",
        "\n",
        "    doc_x = Doc(\n",
        "        vocab,\n",
        "        words=words,\n",
        "        spaces=spaces,\n",
        "        tags=tags,\n",
        "        morphs=morphs,\n",
        "        lemmas=lemmas,\n",
        "        pos=poses,\n",
        "        deps=deps,\n",
        "        heads=heads,\n",
        "    )\n",
        "    doc_x.ents = [Span(doc_x, ent.start, ent.end, label=ent.label) for ent in doc.ents]\n",
        "\n",
        "    return doc_x\n",
        "\n",
        "\n",
        "def merge_conllu_subtokens(lines, doc):\n",
        "    # identify and process all subtoken spans to prepare attrs for merging\n",
        "    subtok_spans = []\n",
        "    for line in lines:\n",
        "        parts = line.split(\"\\t\")\n",
        "        id_, word, lemma, pos, tag, morph, head, dep, _1, misc = parts\n",
        "        if \"-\" in id_:\n",
        "            subtok_start, subtok_end = id_.split(\"-\")\n",
        "            subtok_span = doc[int(subtok_start) - 1 : int(subtok_end)]\n",
        "            subtok_spans.append(subtok_span)\n",
        "            # create merged tag, morph, and lemma values\n",
        "            tags = []\n",
        "            morphs = {}\n",
        "            lemmas = []\n",
        "            for token in subtok_span:\n",
        "                tags.append(token.tag_)\n",
        "                lemmas.append(token.lemma_)\n",
        "                if token._.merged_morph:\n",
        "                    for feature in token._.merged_morph.split(\"|\"):\n",
        "                        field, values = feature.split(\"=\", 1)\n",
        "                        if field not in morphs:\n",
        "                            morphs[field] = set()\n",
        "                        for value in values.split(\",\"):\n",
        "                            morphs[field].add(value)\n",
        "            # create merged features for each morph field\n",
        "            for field, values in morphs.items():\n",
        "                morphs[field] = field + \"=\" + \",\".join(sorted(values))\n",
        "            # set the same attrs on all subtok tokens so that whatever head the\n",
        "            # retokenizer chooses, the final attrs are available on that token\n",
        "            for token in subtok_span:\n",
        "                token._.merged_orth = token.orth_\n",
        "                token._.merged_lemma = \" \".join(lemmas)\n",
        "                token.tag_ = \"_\".join(tags)\n",
        "                token._.merged_morph = \"|\".join(sorted(morphs.values()))\n",
        "                token._.merged_spaceafter = (\n",
        "                    True if subtok_span[-1].whitespace_ else False\n",
        "                )\n",
        "\n",
        "    with doc.retokenize() as retokenizer:\n",
        "        for span in subtok_spans:\n",
        "            retokenizer.merge(span)\n",
        "\n",
        "    return doc"
      ],
      "id": "HPjD0SN6iAvc"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "563fdc94"
      },
      "outputs": [],
      "source": [
        "# Convert the conllu files from inception to spaCy binary format\n",
        "# Read the conll files with ner data and as ents to spaCy docs \n",
        "!python -m spacy project run convert /content/newlang_project"
      ],
      "id": "563fdc94"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9519c858"
      },
      "outputs": [],
      "source": [
        "# test/train split \n",
        "!python -m spacy project run split /content/newlang_project "
      ],
      "id": "9519c858"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4feefe6f"
      },
      "outputs": [],
      "source": [
        "# Debug the data\n",
        "!python -m spacy project run debug /content/newlang_project "
      ],
      "id": "4feefe6f"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "151-cj1dLgAD"
      },
      "source": [
        "# 3 Model Training "
      ],
      "id": "151-cj1dLgAD"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KKW2QGPTJ_CZ"
      },
      "source": [
        "If your project file uses Weights and Biases to monitor model training (`vars.wanb: true`), you'll need to create an account at [wandb.ai](https://wandb.ai/site) and get an API key.  "
      ],
      "id": "KKW2QGPTJ_CZ"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "38b490a4"
      },
      "outputs": [],
      "source": [
        "# train the model\n",
        "!python -m spacy project run train /content/newlang_project "
      ],
      "id": "38b490a4"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ynXr8vlXqCxv"
      },
      "source": [
        "If you get `ValueError: Could not find gold transition - see logs above.`  \n",
        "You may not have sufficent data to train on: https://github.com/explosion/spaCy/discussions/7282"
      ],
      "id": "ynXr8vlXqCxv"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "018362d8"
      },
      "outputs": [],
      "source": [
        "# Evaluate the model using the test data\n",
        "!python -m spacy project run evaluate /content/newlang_project "
      ],
      "id": "018362d8"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gg1sLlVrgiyu"
      },
      "outputs": [],
      "source": [
        "# Find the path for your meta.json file\n",
        "# You'll need to add newlang_project/ +  the path from the training step just after \"✔ Saved pipeline to output directory\"\n",
        "!ls newlang_project/training/urban-giggle/model-last"
      ],
      "id": "gg1sLlVrgiyu"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3zGCTURr9JE6"
      },
      "outputs": [],
      "source": [
        "#Update meta.json\n",
        "import spacy \n",
        "import srsly \n",
        "\n",
        "# Change path to match that from the training cell where it says \"✔ Saved pipeline to output directory\"\n",
        "meta_path = \"newlang_project/training/urban-giggle/model-last/meta.json\"\n",
        "\n",
        "# Replace values below for your project\n",
        "my_meta = { \n",
        "    \"lang\":\"yi\",\n",
        "    \"name\":\"yiddish_sm\",\n",
        "    \"version\":\"0.0.1\",\n",
        "    \"description\":\"Yiddish pipeline optimized for CPU. Components: tok2vec, tagger, parser, senter, lemmatizer.\",\n",
        "    \"author\":\"New Languages for NLP\",\n",
        "    \"email\":\"newnlp@princeton.edu\",\n",
        "    \"url\":\"https://newnlp.princeton.edu\",\n",
        "    \"license\":\"MIT\", \n",
        "    }\n",
        "meta = spacy.util.load_meta(meta_path)\n",
        "meta.update(my_meta)\n",
        "srsly.write_json(meta_path, meta)"
      ],
      "id": "3zGCTURr9JE6"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JM309FhNVAeb"
      },
      "source": [
        "### Download the trained model to your computer.\n"
      ],
      "id": "JM309FhNVAeb"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8e1d6f36"
      },
      "outputs": [],
      "source": [
        "# Save the model to disk in a format that can be easily  downloaded and re-used.\n",
        "!python -m spacy package newlang_project/training/urban-giggle/model-last newlang_project/export "
      ],
      "id": "8e1d6f36"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "8d32abf2",
        "outputId": "70bdb4ba-a62b-4601-ef2c-5f03a35ee1ce"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_08ad8c0e-1992-43fa-a20a-fe18e488c0b8\", \"yi_yiddish_sm-0.0.1.tar.gz\", 7877125)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "from google.colab import files\n",
        "# replace with the path in the previous cell under \"✔ Successfully created zipped Python package\"\n",
        "files.download('newlang_project/export/yi_yiddish_sm-0.0.1/dist/yi_yiddish_sm-0.0.1.tar.gz')\n",
        "\n",
        "# once on your computer, you can pip install yi_yiddish_sm-0.0.1.tar.gz\n",
        "# Be sure to add the file to the 4_trained_models folder in GitHub"
      ],
      "id": "8d32abf2"
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "New Language Training (Colab).ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.10"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}