chapter_06/chapter_06.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Uo7OQtUyJ0N9"
      },
      "outputs": [],
      "source": [
        "!python3 -m pip install spacy==3.7.4\n",
        "!python3 -m pip install spacy-transformers==1.3.4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!mkdir data \n",
        "!wget -P data https://raw.githubusercontent.com/PacktPublishing/Mastering-spaCy-Second-Edition./main/chapter_06/data/amazon_food_reviews.csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cvIoat6mJ-wj"
      },
      "outputs": [],
      "source": [
        "import spacy\n",
        "from spacy.training import Example\n",
        "\n",
        "nlp = spacy.blank(\"en\")\n",
        "review_text = 'This Hot chocolate is very good. It has just the right amount of milk chocolate flavor. The price is a very good deal and more than worth it!'\n",
        "doc = nlp(review_text)\n",
        "annotation = {\"cats\": {\"positive\": 1, \"negative\": 0}}\n",
        "example = Example.from_dict(doc, annotation)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fKvlOki4KZZK",
        "outputId": "4d62bc99-4eb4-41e8-84a3-ef9ed9b75e4b"
      },
      "outputs": [],
      "source": [
        "import pandas as pd \n",
        "import spacy\n",
        "from spacy.training import Example  \n",
        "\n",
        "df = pd.read_csv(\"data/amazon_food_reviews.csv\") \n",
        " \n",
        "df_train = df.sample(frac=0.8,random_state=200) \n",
        "df_test = df.drop(df_train.index) \n",
        "df_test.to_json(\"data/df_dev.json\") \n",
        "\n",
        "nlp = spacy.blank(\"en\") \n",
        "\n",
        "TRAIN_EXAMPLES = [] \n",
        "for _,row in df_train.iterrows(): \n",
        "    if row[\"positive_review\"] == 1: \n",
        "        annotation = {\"cats\": {\"positive\": 1, \"negative\": 0}} \n",
        "    else: \n",
        "        annotation = {\"cats\": {\"negative\": 1, \"positive\": 0}} \n",
        "    example = Example.from_dict(nlp(row[\"text\"]), annotation) \n",
        "    TRAIN_EXAMPLES.append(example) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [],
      "source": [
        "import spacy \n",
        "from spacy.training import Example \n",
        "\n",
        "nlp = spacy.blank(\"en\") \n",
        "textcat = nlp.add_pipe(\"textcat\") \n",
        "textcat.initialize(lambda: TRAIN_EXAMPLES, nlp=nlp) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 Losses {'textcat': 3.017495170235634}\n",
            "10 Losses {'textcat': 0.023405539178156687}\n",
            "20 Losses {'textcat': 0.0029252148409744105}\n",
            "30 Losses {'textcat': 5.756459904648903e-05}\n",
            "39 Losses {'textcat': 3.132792653764982e-10}\n"
          ]
        }
      ],
      "source": [
        "import spacy \n",
        "from spacy.util import minibatch \n",
        "import random \n",
        " \n",
        "nlp = spacy.blank(\"en\") \n",
        "textcat = nlp.add_pipe(\"textcat\") \n",
        "textcat.initialize(lambda: TRAIN_EXAMPLES, nlp=nlp) \n",
        "\n",
        "optimizer = nlp.resume_training() \n",
        "\n",
        "for epoch in range(40): \n",
        "    random.shuffle(TRAIN_EXAMPLES) \n",
        "    batches = minibatch(TRAIN_EXAMPLES, size=200) \n",
        "    losses = {} \n",
        "    for batch in batches: \n",
        "        nlp.update( \n",
        "                batch,    \n",
        "                losses=losses, \n",
        "                sgd=optimizer, \n",
        "            ) \n",
        "\n",
        "    if epoch % 10 == 0: \n",
        "            print(epoch, \"Losses\", losses) \n",
        "\n",
        "print(epoch, \"Losses\", losses) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Example 1 {'positive': 2.0541139747365378e-05, 'negative': 0.9999794960021973}\n",
            "Example 2 {'positive': 2.677466454770183e-06, 'negative': 0.9999973773956299}\n",
            "Example 3 {'positive': 0.9998534917831421, 'negative': 0.00014646562340203673}\n"
          ]
        }
      ],
      "source": [
        "text = \"Smoke Paprika My mother uses it for allot of dishes, but this particular one, doesn't compare to anything she had.  It is now being used for a decoration on the spice shelf and I will never use it and ruin a dish again. I have tried using just a little bit, thinking it was stronger than her's. And I am a decent cook. But this does not taste like the smoke paprika that I have had in the past.  Sorry I don't recommend this product at all.\" \n",
        "doc = nlp(text) \n",
        "print(\"Example 1\", doc.cats) \n",
        "\n",
        "text = \"Terrible Tasting for me The Teechino Caffeine-Free Herbal Coffee, Mediterranean Vanilla Nut tasted undrinkable to me. It lacked a deep, full-bodied flavor, which Cafix and Pero coffee-like substitute products have. I wanted to try something new, and for me, this substitute coffee drink wasn't my favorite.\" \n",
        "doc = nlp(text) \n",
        "print(\"Example 2\", doc.cats) \n",
        "\n",
        "text = \"Dishwater If I had a choice of THIS or nothing, I'd go with nothing. Of all the K-cups I've tasted - this is the worst. Very weak and if you close your eyes and think really hard about it, maybe you can almost taste cinnamon. Blech.\" \n",
        "doc = nlp(text) \n",
        "print(\"Example 3\", doc.cats) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of RobertaModel were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "TransformerData(wordpieces=WordpieceBatch(strings=[['<s>', 'D', 'ish', 'water', 'ĠIf', 'ĠI', 'Ġhad', 'Ġa', 'Ġchoice', 'Ġof', 'ĠTHIS', 'Ġor', 'Ġnothing', ',', 'ĠI', \"'d\", 'Ġgo', 'Ġwith', 'Ġnothing', '.', 'ĠOf', 'Ġall', 'Ġthe', 'ĠK', '-', 'c', 'ups', 'ĠI', \"'ve\", 'Ġtasted', 'Ġ-', 'Ġthis', 'Ġis', 'Ġthe', 'Ġworst', '.', 'ĠVery', 'Ġweak', 'Ġand', 'Ġif', 'Ġyou', 'Ġclose', 'Ġyour', 'Ġeyes', 'Ġand', 'Ġthink', 'Ġreally', 'Ġhard', 'Ġabout', 'Ġit', ',', 'Ġmaybe', 'Ġyou', 'Ġcan', 'Ġalmost', 'Ġtaste', 'Ġcinnamon', '.', 'ĠBle', 'ch', '.', '</s>']], input_ids=array([[    0,   495,  1173,  5412,   318,    38,    56,    10,  2031,\n",
            "            9, 10652,    50,  1085,     6,    38,  1017,   213,    19,\n",
            "         1085,     4,  1525,    70,     5,   229,    12,   438,  4489,\n",
            "           38,   348, 29143,   111,    42,    16,     5,  2373,     4,\n",
            "        12178,  3953,     8,   114,    47,   593,   110,  2473,     8,\n",
            "          206,   269,   543,    59,    24,     6,  2085,    47,    64,\n",
            "          818,  5840, 27053,     4, 16463,   611,     4,     2]]), attention_mask=array([[1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
            "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
            "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
            "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.]],\n",
            "      dtype=float32), lengths=[62], token_type_ids=None), model_output=ModelOutput([('last_hidden_state', array([[[-0.09543718,  0.09073567, -0.0704579 , ..., -0.07103769,\n",
            "         -0.01958286, -0.1347733 ],\n",
            "        [ 0.05629096,  0.1646752 ,  0.08961716, ...,  0.04972528,\n",
            "          0.1476407 ,  0.01797979],\n",
            "        [ 0.12693703,  0.06888243, -0.08189505, ..., -0.44657743,\n",
            "          0.05990743, -0.20180702],\n",
            "        ...,\n",
            "        [ 0.01926086,  0.19347799, -0.18029685, ..., -0.2244796 ,\n",
            "         -0.13424578, -0.1782346 ],\n",
            "        [-0.09126358,  0.08484644, -0.10294998, ..., -0.10173652,\n",
            "         -0.02157888, -0.17447217],\n",
            "        [-0.05183166,  0.09868944, -0.07387528, ...,  0.15020685,\n",
            "         -0.01248172, -0.09300787]]], dtype=float32)), ('pooler_output', array([[ 0.07138944, -0.52983415, -0.15019886, -0.54522556, -0.16226491,\n",
            "        -0.1030249 ,  0.10538666, -0.1154991 , -0.1524782 ,  0.18048625,\n",
            "        -0.22855058,  0.00432651, -0.05805686,  0.06345929,  0.02590265,\n",
            "        -0.18885307, -0.00417017,  0.3996874 , -0.4030982 ,  0.23093683,\n",
            "        -0.13660668,  0.00286832,  0.21331525,  0.3753915 , -0.23939177,\n",
            "        -0.27472496, -0.11228896, -0.15145841, -0.16397488,  0.22038971,\n",
            "        -0.20108207, -0.53421515,  0.18841116, -0.1672773 , -0.1281229 ,\n",
            "        -0.07055932,  0.14668897,  0.04159861,  0.15609993, -0.30084416,\n",
            "        -0.17729835, -0.19160926,  0.01332799,  0.19501399,  0.11706214,\n",
            "        -0.3513071 , -0.04314567,  0.08442674,  0.02705947,  0.06449229,\n",
            "        -0.31957433,  0.30215687,  0.01314408, -0.4602141 ,  0.05998074,\n",
            "        -0.19068138,  0.01047242,  0.13828014,  0.03598675,  0.28961962,\n",
            "        -0.31988212,  0.17072177,  0.19666561,  0.10892633, -0.3141892 ,\n",
            "         0.11515443,  0.36717674,  0.40853366, -0.25854576,  0.12773694,\n",
            "         0.04857499,  0.03386506, -0.0840795 , -0.19113432,  0.08052452,\n",
            "        -0.05670881, -0.3126126 , -0.05584762, -0.05476991, -0.0943405 ,\n",
            "        -0.42726114, -0.28319806, -0.04938131, -0.0738117 , -0.09776908,\n",
            "        -0.25071657, -0.23442054,  0.03751713,  0.32075647, -0.19038612,\n",
            "         0.16025615,  0.01962865,  0.2712819 ,  0.24018882,  0.00850993,\n",
            "         0.02515595,  0.17344262,  0.21106805,  0.12699464,  0.15758145,\n",
            "         0.39893675,  0.23451358,  0.15660049, -0.04542833,  0.2238772 ,\n",
            "         0.30406436,  0.39846236, -0.13289517, -0.13386941, -0.2276807 ,\n",
            "         0.40036464,  0.18026727,  0.01524845,  0.08045326, -0.42594475,\n",
            "        -0.3981541 , -0.09075548,  0.2065998 , -0.34766957,  0.21609339,\n",
            "         0.2627677 , -0.11828783, -0.34938973,  0.17445503,  0.12403768,\n",
            "         0.24033162,  0.20882867, -0.05315202,  0.2769543 ,  0.0037089 ,\n",
            "        -0.02020719, -0.18677726,  0.05948715, -0.25319058,  0.15295932,\n",
            "         0.10040046, -0.36945498,  0.11551033,  0.39251935, -0.0705733 ,\n",
            "        -0.114297  ,  0.07856452,  0.11022685,  0.14117742, -0.44123808,\n",
            "         0.06778562,  0.11169188, -0.09694305,  0.06841468,  0.2441586 ,\n",
            "        -0.16879189,  0.28962168,  0.04214196,  0.23231862,  0.27983695,\n",
            "         0.05692093, -0.05317679, -0.44329825,  0.3281168 , -0.1314892 ,\n",
            "         0.24245849, -0.04104838,  0.26183116,  0.3348679 , -0.10118031,\n",
            "         0.11882734, -0.03017393, -0.23944661, -0.01714892, -0.00982217,\n",
            "         0.21965627,  0.4018818 , -0.18107288,  0.04946343, -0.05477452,\n",
            "        -0.43520436, -0.07963122,  0.00626161,  0.08875798, -0.14155456,\n",
            "        -0.13277714,  0.23737259, -0.3085841 ,  0.2714427 , -0.11960008,\n",
            "         0.48815867, -0.3085467 , -0.15504223,  0.17489344,  0.19640183,\n",
            "         0.02405658,  0.4055825 ,  0.12713993,  0.1452214 , -0.16057073,\n",
            "        -0.17598899, -0.01836291, -0.18397333,  0.04440672,  0.43049508,\n",
            "         0.46546257, -0.4470381 ,  0.22430542,  0.04468072,  0.2726265 ,\n",
            "        -0.362267  ,  0.47198468,  0.1146284 , -0.31733537,  0.07426226,\n",
            "         0.01603457,  0.13627656,  0.14243661,  0.18983003,  0.14680131,\n",
            "        -0.03078773,  0.16142707, -0.43357974, -0.15114084, -0.24857211,\n",
            "        -0.10427305, -0.31301656, -0.04764402,  0.13214928, -0.20945463,\n",
            "         0.18097739, -0.05573878, -0.02655201, -0.23077647,  0.09014019,\n",
            "         0.24003486,  0.02633044,  0.12949632, -0.07659536,  0.1866014 ,\n",
            "        -0.21429461, -0.09304578,  0.11279031,  0.16029006,  0.02495415,\n",
            "        -0.199234  ,  0.03479766, -0.20751762,  0.11513771,  0.02145102,\n",
            "        -0.12611787,  0.15760918,  0.33692387, -0.10094368, -0.45074332,\n",
            "        -0.1227973 ,  0.24073988, -0.15141603, -0.44635385,  0.22753966,\n",
            "        -0.0946642 ,  0.2816373 ,  0.13550186, -0.07149048, -0.03409886,\n",
            "        -0.04764203, -0.01040211, -0.25833288,  0.10100552, -0.19503509,\n",
            "         0.16657482, -0.02916787, -0.0030874 ,  0.0099944 , -0.35740334,\n",
            "        -0.37937   , -0.03146988, -0.04681965,  0.36377126,  0.11041755,\n",
            "        -0.45444188, -0.04916992, -0.02935285, -0.04875735, -0.19488072,\n",
            "        -0.3540639 ,  0.24769928, -0.23044792,  0.09208608, -0.25459558,\n",
            "         0.16650507,  0.05237152, -0.01164846, -0.06432687,  0.1891891 ,\n",
            "        -0.16454168, -0.03587287,  0.52531284, -0.15506984,  0.24846055,\n",
            "         0.0443617 , -0.03504935, -0.09841166, -0.00130332,  0.11717711,\n",
            "        -0.06169544,  0.09020465, -0.02630465,  0.3676663 ,  0.13838312,\n",
            "        -0.07574268,  0.03999575, -0.02482067,  0.32599956, -0.14663108,\n",
            "         0.0733205 ,  0.00755116,  0.05974835, -0.13464719,  0.19095702,\n",
            "         0.19544269,  0.33230165,  0.48198858, -0.14850183, -0.20946552,\n",
            "         0.27647382, -0.23683015,  0.17136233, -0.14741506,  0.11506646,\n",
            "         0.07783838, -0.2525634 , -0.22151938, -0.29917485, -0.43070507,\n",
            "        -0.08564285,  0.10641912, -0.2502797 ,  0.40041927,  0.51111156,\n",
            "         0.28101182, -0.5016536 ,  0.15038835, -0.52066064,  0.10273883,\n",
            "         0.15060647, -0.50955236,  0.11674276, -0.11907756,  0.10887833,\n",
            "        -0.12057102,  0.03359156, -0.02298635,  0.1345632 ,  0.13687262,\n",
            "        -0.5763688 ,  0.21971603, -0.05461779,  0.14674242, -0.290893  ,\n",
            "        -0.3225445 ,  0.04494664, -0.16764207, -0.36117172,  0.03640951,\n",
            "        -0.05331704,  0.01669667,  0.18132839,  0.6823875 ,  0.08954787,\n",
            "        -0.16116343, -0.08997101, -0.1764717 , -0.14678861, -0.07066271,\n",
            "        -0.12941803, -0.1418736 , -0.02497988,  0.0248615 ,  0.11328579,\n",
            "        -0.10293448, -0.24696824, -0.25535828,  0.02864952,  0.39461872,\n",
            "         0.08205261,  0.21614525,  0.15085168, -0.14144024, -0.3786581 ,\n",
            "        -0.13557713,  0.26428768,  0.14273816, -0.21440701,  0.2189377 ,\n",
            "        -0.02768259, -0.1251971 ,  0.01927456, -0.05576678,  0.40223345,\n",
            "        -0.30285352,  0.20069683,  0.14904197, -0.02969512, -0.1197481 ,\n",
            "         0.28594685, -0.32453963, -0.20987493, -0.15677862,  0.09057948,\n",
            "         0.00818664,  0.3243117 , -0.06044299, -0.23118772,  0.07490049,\n",
            "        -0.10940415,  0.16848205, -0.18276806,  0.10610875, -0.16256489,\n",
            "        -0.15827411, -0.00872141,  0.19434221, -0.04947762,  0.35351992,\n",
            "        -0.3363623 ,  0.11787989, -0.200355  , -0.3725937 ,  0.46786338,\n",
            "        -0.02616578, -0.26693538, -0.01312433, -0.06352862, -0.05944832,\n",
            "        -0.03961051, -0.26986843, -0.14636518, -0.03665509, -0.30832475,\n",
            "        -0.04016347, -0.01119301, -0.23498088,  0.06926735, -0.06463224,\n",
            "         0.3448696 ,  0.17372777, -0.2956149 ,  0.06939513, -0.05610368,\n",
            "        -0.21000351, -0.27676845, -0.14898416,  0.13442786, -0.05589786,\n",
            "        -0.22245198, -0.26658246, -0.2145578 ,  0.25052947,  0.23669687,\n",
            "         0.24918684,  0.19672434,  0.1556796 ,  0.08447372,  0.167608  ,\n",
            "         0.26078945, -0.05562361,  0.27195805,  0.02873049,  0.2961171 ,\n",
            "        -0.23241194,  0.35391232, -0.3438149 , -0.03440802,  0.21950264,\n",
            "         0.09228703, -0.11660283,  0.269733  ,  0.3358436 , -0.42456752,\n",
            "         0.14629686, -0.41166997,  0.4694919 , -0.3012011 ,  0.26708996,\n",
            "         0.15735225,  0.02764079, -0.01776997,  0.0912431 ,  0.15175033,\n",
            "        -0.06309996, -0.09678482, -0.15028167,  0.11194002,  0.10254084,\n",
            "        -0.06592123, -0.02071927,  0.14516795,  0.06461126, -0.19893983,\n",
            "        -0.06592975,  0.2082175 , -0.09931302,  0.3668605 ,  0.19216321,\n",
            "        -0.21230263,  0.11892367, -0.04702885,  0.31033915, -0.17466235,\n",
            "        -0.07761194, -0.36445096, -0.12031273,  0.02962799, -0.06361165,\n",
            "        -0.00194254, -0.11424527, -0.01943416, -0.1308372 , -0.00988568,\n",
            "         0.1101778 , -0.13499038,  0.22858073, -0.27424493,  0.18188693,\n",
            "        -0.4052881 , -0.14906871,  0.06048644,  0.27791145,  0.06450716,\n",
            "        -0.10457471, -0.04201928, -0.06711765, -0.08256759, -0.37193677,\n",
            "        -0.1853729 , -0.04354429, -0.29276702,  0.29681855,  0.49406385,\n",
            "         0.10462417, -0.18012637,  0.21090873, -0.42731354,  0.07301139,\n",
            "        -0.00317121,  0.02768765, -0.2088751 , -0.07844127, -0.10485028,\n",
            "         0.01844217,  0.43078825, -0.04593091,  0.0701708 ,  0.44106296,\n",
            "         0.4274503 ,  0.29082742, -0.17787124,  0.2818959 ,  0.1462367 ,\n",
            "         0.01490885, -0.26801932,  0.23562525, -0.26446772, -0.06844375,\n",
            "         0.3690288 ,  0.36180153, -0.13701446, -0.5776253 ,  0.304018  ,\n",
            "        -0.08696296, -0.25861463, -0.35078338,  0.17441145,  0.20363462,\n",
            "         0.24621508, -0.07893791,  0.18393895, -0.24753106, -0.22123334,\n",
            "         0.06146426,  0.01680799,  0.02479079,  0.176193  ,  0.02569699,\n",
            "        -0.18122192, -0.25674632, -0.31619647,  0.1650966 , -0.02229935,\n",
            "         0.19115923,  0.17177297, -0.00328219,  0.19195186,  0.17195645,\n",
            "        -0.36821282, -0.18563887, -0.04523131,  0.03621037,  0.13912329,\n",
            "         0.08803152,  0.26237562,  0.07012612, -0.07654958, -0.33870304,\n",
            "        -0.23415324, -0.25126553,  0.09918566,  0.2871336 , -0.03844774,\n",
            "         0.18725902,  0.11597151,  0.25975162,  0.2198582 , -0.06382032,\n",
            "         0.06233317,  0.10823813,  0.2140148 ,  0.1019981 ,  0.16491924,\n",
            "        -0.12112191,  0.060285  ,  0.16570805, -0.27762967,  0.25141275,\n",
            "        -0.2702038 ,  0.23614947,  0.06248737, -0.0771132 ,  0.15698157,\n",
            "         0.21752512, -0.00133137, -0.12998843,  0.11355965, -0.1386454 ,\n",
            "        -0.2609733 ,  0.06344792, -0.1911768 ,  0.424593  , -0.29731977,\n",
            "        -0.16928221,  0.18952419,  0.0908818 , -0.10580875, -0.2519284 ,\n",
            "         0.05696485,  0.23259468,  0.06382195,  0.15505616, -0.13139372,\n",
            "        -0.2392472 , -0.02977477,  0.14687738,  0.2099821 ,  0.1739522 ,\n",
            "        -0.06083048,  0.22441757,  0.37534946,  0.10567271,  0.18100886,\n",
            "         0.03042917, -0.21986444,  0.12238146, -0.15353309,  0.09128058,\n",
            "        -0.07722534,  0.3663423 ,  0.3705305 , -0.22394244,  0.23760523,\n",
            "         0.05123867,  0.2357729 ,  0.05116655, -0.1775801 ,  0.5462243 ,\n",
            "        -0.0817308 , -0.1568157 ,  0.06311463, -0.01710267,  0.21550958,\n",
            "         0.03755516,  0.24769564, -0.02259508,  0.2982992 , -0.22123095,\n",
            "        -0.12426207,  0.09214423, -0.15417331, -0.06082478,  0.12307055,\n",
            "        -0.31246296,  0.05321455, -0.10307956, -0.25166014,  0.0116437 ,\n",
            "        -0.1168767 , -0.25519416,  0.0276995 , -0.36806428, -0.02998955,\n",
            "        -0.13355935, -0.57562715, -0.0750009 ,  0.2560643 , -0.258802  ,\n",
            "        -0.16606824,  0.1575741 , -0.01272241,  0.09982113, -0.23616059,\n",
            "        -0.07039167, -0.38752434,  0.02864374,  0.21163358,  0.32711145,\n",
            "         0.06030761, -0.33546972,  0.00511604, -0.15618041,  0.22782968,\n",
            "         0.15447868,  0.02870605, -0.33686915,  0.01395684,  0.33684063,\n",
            "        -0.12715533, -0.13909706,  0.20134893, -0.15035848, -0.27390534,\n",
            "        -0.14134389,  0.02851536,  0.2920627 , -0.24472034, -0.2315339 ,\n",
            "        -0.14548554,  0.13970402,  0.30270374, -0.26693493, -0.03738582,\n",
            "         0.09508105, -0.08090858,  0.14323293, -0.05990894,  0.23015039,\n",
            "         0.01849862,  0.29611674,  0.13161486, -0.31348783, -0.28579924,\n",
            "         0.06443253, -0.26732987, -0.0862911 ,  0.14392835, -0.25577784,\n",
            "         0.06728007, -0.29474708,  0.01504819,  0.3058025 ,  0.0273129 ,\n",
            "         0.00204119, -0.29799145, -0.4507914 , -0.03378237,  0.48799744,\n",
            "        -0.211652  , -0.30917516,  0.11212536, -0.00211922, -0.01531434,\n",
            "        -0.00795918, -0.16978641, -0.26059264]], dtype=float32))]), align=Ragged(data=array([[ 1],\n",
            "       [ 2],\n",
            "       [ 3],\n",
            "       [ 4],\n",
            "       [ 5],\n",
            "       [ 6],\n",
            "       [ 7],\n",
            "       [ 8],\n",
            "       [ 9],\n",
            "       [10],\n",
            "       [11],\n",
            "       [12],\n",
            "       [13],\n",
            "       [14],\n",
            "       [15],\n",
            "       [16],\n",
            "       [17],\n",
            "       [18],\n",
            "       [19],\n",
            "       [20],\n",
            "       [21],\n",
            "       [22],\n",
            "       [23],\n",
            "       [24],\n",
            "       [25],\n",
            "       [26],\n",
            "       [27],\n",
            "       [28],\n",
            "       [29],\n",
            "       [30],\n",
            "       [31],\n",
            "       [32],\n",
            "       [33],\n",
            "       [34],\n",
            "       [35],\n",
            "       [36],\n",
            "       [37],\n",
            "       [38],\n",
            "       [39],\n",
            "       [40],\n",
            "       [41],\n",
            "       [42],\n",
            "       [43],\n",
            "       [44],\n",
            "       [45],\n",
            "       [46],\n",
            "       [47],\n",
            "       [48],\n",
            "       [49],\n",
            "       [50],\n",
            "       [51],\n",
            "       [52],\n",
            "       [53],\n",
            "       [54],\n",
            "       [55],\n",
            "       [56],\n",
            "       [57],\n",
            "       [58],\n",
            "       [59],\n",
            "       [60]], dtype=int32), lengths=array([3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
            "       2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
            "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1], dtype=int32), data_shape=(-1,), starts_ends=None))\n"
          ]
        }
      ],
      "source": [
        "import spacy \n",
        "\n",
        "nlp = spacy.blank(\"en\")  \n",
        "\n",
        "config = { \n",
        "    \"model\": { \n",
        "        \"@architectures\": \"spacy-transformers.TransformerModel.v3\", \n",
        "        \"name\": \"roberta-base\" \n",
        "    } \n",
        "} \n",
        "\n",
        "nlp.add_pipe(\"transformer\", config=config) \n",
        "nlp.initialize()  \n",
        "\n",
        "doc = nlp(\"Dishwater If I had a choice of THIS or nothing, I'd go with nothing. Of all the K-cups I've tasted - this is the worst. Very weak and if you close your eyes and think really hard about it, maybe you can almost taste cinnamon. Blech.\") \n",
        "print(doc._.trf_data) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {},
      "outputs": [],
      "source": [
        "import pandas as pd \n",
        "import spacy \n",
        "from spacy.tokens import DocBin \n",
        " \n",
        "\n",
        "df = pd.read_csv(\"data/amazon_food_reviews.csv\") \n",
        "\n",
        "df_train = df.sample(frac=0.8,random_state=200) \n",
        "nlp = spacy.blank(\"en\") \n",
        "\n",
        "db = DocBin() \n",
        "\n",
        "for _,row in df_train.iterrows(): \n",
        "    doc = nlp(row[\"text\"]) \n",
        "    if row[\"positive_review\"] == 1: \n",
        "        doc.cats = {\"positive\": 1, \"negative\": 0} \n",
        "    else: \n",
        "        doc.cats = {\"positive\": 0, \"negative\": 1} \n",
        "    db.add(doc) \n",
        "\n",
        "db.to_disk(\"data/train.spacy\") "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [],
      "source": [
        "import spacy \n",
        "from spacy.tokens import DocBin \n",
        "from pathlib import Path  \n",
        "\n",
        "def convert_dataset(lang: str, input_path: Path, output_path: Path): \n",
        "    nlp = spacy.blank(lang) \n",
        "    db = DocBin() \n",
        "    df = pd.read_json(input_path) \n",
        "    for _,row in df.iterrows(): \n",
        "        doc = nlp.make_doc(row[\"Text\"]) \n",
        "        if row[\"positive_review\"] == 1: \n",
        "            doc.cats = {\"positive\": 1, \"negative\": 0} \n",
        "        else: \n",
        "            doc.cats = {\"negative\": 1, \"positive\": 0} \n",
        "        db.add(doc) \n",
        "    db.to_disk(output_path)  \n",
        "\n",
        "convert_dataset(\"en\", \"data/df_dev.json\", \"data/dev.spacy\") "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Usage: python -m spacy [OPTIONS] COMMAND [ARGS]...\n",
            "\n",
            "  spaCy Command-line Interface\n",
            "\n",
            "  DOCS: https://spacy.io/api/cli\n",
            "\n",
            "Options:\n",
            "  --install-completion [bash|zsh|fish|powershell|pwsh]\n",
            "                                  Install completion for the specified shell.\n",
            "  --show-completion [bash|zsh|fish|powershell|pwsh]\n",
            "                                  Show completion for the specified shell, to\n",
            "                                  copy it or customize the installation.\n",
            "  --help                          Show this message and exit.\n",
            "\n",
            "Commands:\n",
            "  apply           Apply a trained pipeline to documents to get predictions.\n",
            "  assemble        Assemble a spaCy pipeline from a config file.\n",
            "  benchmark       Commands for benchmarking pipelines.\n",
            "  convert         Convert files into json or DocBin format for training.\n",
            "  debug           Suite of helpful commands for debugging and profiling.\n",
            "  download        Download compatible trained pipeline from the default...\n",
            "  evaluate        Evaluate a trained pipeline.\n",
            "  find-function   Find the module, path and line number to the file the...\n",
            "  find-threshold  Runs prediction trials for a trained model with varying...\n",
            "  info            Print info about spaCy installation.\n",
            "  init            Commands for initializing configs and pipeline packages.\n",
            "  package         Generate an installable Python package for a pipeline.\n",
            "  pretrain        Pre-train the 'token-to-vector' (tok2vec) layer of...\n",
            "  project         Command-line interface for spaCy projects and templates.\n",
            "  train           Train or update a spaCy pipeline.\n",
            "  validate        Validate the currently installed pipeline packages and...\n"
          ]
        }
      ],
      "source": [
        "!python -m spacy --help"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;4mℹ Generated config template specific for your use case\u001b[0m\n",
            "- Language: en\n",
            "- Pipeline: textcat\n",
            "- Optimize for: efficiency\n",
            "- Hardware: CPU\n",
            "- Transformer: None\n",
            "\u001b[38;5;2m✔ Auto-filled config with all values\u001b[0m\n",
            "\u001b[38;5;2m✔ Saved config\u001b[0m\n",
            "config_without_transformer.cfg\n",
            "You can now add your data and train your pipeline:\n",
            "python -m spacy train config_without_transformer.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
          ]
        }
      ],
      "source": [
        "!python3 -m spacy init config config_without_transformer.cfg --lang \"en\" --pipeline \"textcat\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;4mℹ Saving to output directory: pipeline_without_transformer\u001b[0m\n",
            "\u001b[38;5;4mℹ Using CPU\u001b[0m\n",
            "\u001b[38;5;4mℹ To switch to GPU 0, use the option: --gpu-id 0\u001b[0m\n",
            "\u001b[1m\n",
            "=========================== Initializing pipeline ===========================\u001b[0m\n",
            "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
            "\u001b[1m\n",
            "============================= Training pipeline =============================\u001b[0m\n",
            "\u001b[38;5;4mℹ Pipeline: ['textcat']\u001b[0m\n",
            "\u001b[38;5;4mℹ Initial learn rate: 0.001\u001b[0m\n",
            "E    #       LOSS TEXTCAT  CATS_SCORE  SCORE \n",
            "---  ------  ------------  ----------  ------\n",
            "  0       0          0.25       43.34    0.43\n",
            "  0     200         42.36       43.34    0.43\n",
            "  0     400         35.47       46.51    0.47\n",
            "  0     600         31.62       57.25    0.57\n",
            "  0     800         31.34       64.85    0.65\n",
            "  0    1000         30.58       65.83    0.66\n",
            "  0    1200         29.32       65.01    0.65\n",
            "  0    1400         29.71       72.12    0.72\n",
            "  0    1600         25.19       73.64    0.74\n",
            "  1    1800          9.54       73.06    0.73\n",
            "  1    2000         11.48       70.14    0.70\n",
            "  1    2200         12.42       71.74    0.72\n",
            "  2    2400          4.45       73.80    0.74\n",
            "  2    2600          4.42       72.99    0.73\n",
            "  3    2800          3.17       74.19    0.74\n",
            "  3    3000          2.20       73.00    0.73\n",
            "  4    3200          0.94       72.72    0.73\n",
            "  5    3400          1.11       71.87    0.72\n",
            "  5    3600          0.63       72.94    0.73\n",
            "  6    3800          0.49       72.50    0.72\n",
            "  6    4000          0.48       73.24    0.73\n",
            "  7    4200          0.52       72.00    0.72\n",
            "  7    4400          0.38       71.61    0.72\n",
            "\u001b[38;5;2m✔ Saved pipeline to output directory\u001b[0m\n",
            "pipeline_without_transformer/model-last\n"
          ]
        }
      ],
      "source": [
        "!python3 -m spacy train config_without_transformer.cfg --paths.train \"data/train.spacy\" --paths.dev \"data/dev.spacy\" --output pipeline_without_transformer/ "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Example 1 {'positive': 0.6157549023628235, 'negative': 0.3842450678348541}\n",
            "Example 2 {'positive': 0.659024715423584, 'negative': 0.34097525477409363}\n",
            "Example 3 {'positive': 0.27787405252456665, 'negative': 0.7221259474754333}\n"
          ]
        }
      ],
      "source": [
        "import spacy  \n",
        "\n",
        "nlp = spacy.load(\"pipeline_without_transformer/model-best\")  \n",
        "\n",
        "text = \"Smoke Paprika My mother uses it for allot of dishes, but this particular one, doesn't compare to anything she had.  It is now being used for a decoration on the spice shelf and I will never use it and ruin a dish again. I have tried using just a little bit, thinking it was stronger than her's. And I am a decent cook. But this does not taste like the smoke paprika that I have had in the past.  Sorry I don't recommend this product at all.\" \n",
        "doc = nlp(text) \n",
        "print(\"Example 1\", doc.cats) \n",
        "\n",
        "text = \"Terrible Tasting for me The Teechino Caffeine-Free Herbal Coffee, Mediterranean Vanilla Nut tasted undrinkable to me. It lacked a deep, full-bodied flavor, which Cafix and Pero coffee-like substitute products have. I wanted to try something new, and for me, this substitute coffee drink wasn't my favorite.\" \n",
        "doc = nlp(text) \n",
        "print(\"Example 2\", doc.cats) \n",
        "\n",
        "text = \"Dishwater If I had a choice of THIS or nothing, I'd go with nothing. Of all the K-cups I've tasted - this is the worst. Very weak and if you close your eyes and think really hard about it, maybe you can almost taste cinnamon. Blech.\" \n",
        "doc = nlp(text) \n",
        "print(\"Example 3\", doc.cats) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;4mℹ Generated config template specific for your use case\u001b[0m\n",
            "- Language: en\n",
            "- Pipeline: textcat\n",
            "- Optimize for: accuracy\n",
            "- Hardware: GPU\n",
            "- Transformer: roberta-base\n",
            "\u001b[38;5;2m✔ Auto-filled config with all values\u001b[0m\n",
            "\u001b[38;5;2m✔ Saved config\u001b[0m\n",
            "config_transformer.cfg\n",
            "You can now add your data and train your pipeline:\n",
            "python -m spacy train config_transformer.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
          ]
        }
      ],
      "source": [
        "!python3 -m spacy init config config_transformer.cfg --lang \"en\" --pipeline \"textcat\" --optimize \"accuracy\" --gpu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;2m✔ Created output directory: pipeline_transformer\u001b[0m\n",
            "\u001b[38;5;4mℹ Saving to output directory: pipeline_transformer\u001b[0m\n",
            "\u001b[38;5;4mℹ Using GPU: 0\u001b[0m\n",
            "\u001b[1m\n",
            "=========================== Initializing pipeline ===========================\u001b[0m\n",
            "Some weights of RobertaModel were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
            "\u001b[1m\n",
            "============================= Training pipeline =============================\u001b[0m\n",
            "\u001b[38;5;4mℹ Pipeline: ['transformer', 'textcat']\u001b[0m\n",
            "\u001b[38;5;4mℹ Initial learn rate: 0.0\u001b[0m\n",
            "E    #       LOSS TRANS...  LOSS TEXTCAT  CATS_SCORE  SCORE \n",
            "---  ------  -------------  ------------  ----------  ------\n",
            "  0       0           0.00          0.25       43.34    0.43\n",
            "  1     200           0.89        113.61       82.90    0.83\n",
            "  2     400          15.00         78.00       85.15    0.85\n",
            "  3     600          21.33         71.08       83.78    0.84\n",
            "  4     800           5.65         69.04       78.33    0.78\n",
            "  5    1000           3.41         49.13       84.12    0.84\n",
            "  6    1200           6.17         40.11       84.10    0.84\n",
            "  7    1400           5.85         34.26       83.42    0.83\n",
            "  8    1600           8.12         48.03       76.73    0.77\n",
            "  9    1800          11.31         45.07       82.93    0.83\n",
            " 10    2000           8.14         40.29       82.51    0.83\n",
            "\u001b[38;5;2m✔ Saved pipeline to output directory\u001b[0m\n",
            "pipeline_transformer/model-last\n"
          ]
        }
      ],
      "source": [
        "!python3 -m spacy train config_transformer.cfg --paths.train \"data/train.spacy\" --paths.dev \"data/dev.spacy\" --output pipeline_transformer/ --gpu-id 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Example 1 {'positive': 0.10354223847389221, 'negative': 0.8964577913284302}\n",
            "Example 2 {'positive': 0.1040550246834755, 'negative': 0.8959449529647827}\n",
            "Example 3 {'positive': 0.103188157081604, 'negative': 0.8968119025230408}\n"
          ]
        }
      ],
      "source": [
        "import spacy  \n",
        "\n",
        "nlp = spacy.load(\"pipeline_transformer/model-best\")  \n",
        "\n",
        "text = \"Smoke Paprika My mother uses it for allot of dishes, but this particular one, doesn't compare to anything she had.  It is now being used for a decoration on the spice shelf and I will never use it and ruin a dish again. I have tried using just a little bit, thinking it was stronger than her's. And I am a decent cook. But this does not taste like the smoke paprika that I have had in the past.  Sorry I don't recommend this product at all.\" \n",
        "doc = nlp(text) \n",
        "print(\"Example 1\", doc.cats) \n",
        "\n",
        "text = \"Terrible Tasting for me The Teechino Caffeine-Free Herbal Coffee, Mediterranean Vanilla Nut tasted undrinkable to me. It lacked a deep, full-bodied flavor, which Cafix and Pero coffee-like substitute products have. I wanted to try something new, and for me, this substitute coffee drink wasn't my favorite.\" \n",
        "doc = nlp(text) \n",
        "print(\"Example 2\", doc.cats) \n",
        "\n",
        "text = \"Dishwater If I had a choice of THIS or nothing, I'd go with nothing. Of all the K-cups I've tasted - this is the worst. Very weak and if you close your eyes and think really hard about it, maybe you can almost taste cinnamon. Blech.\" \n",
        "doc = nlp(text) \n",
        "print(\"Example 3\", doc.cats) "
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}