Word2vec_cbow_exercice.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Word2vec_cbow_exercice.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm import tqdm"
      ],
      "metadata": {
        "id": "PDrUAlAZeNXo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Download Data"
      ],
      "metadata": {
        "id": "DdbHjwFfXtOp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://user.phil-fak.uni-duesseldorf.de/~samih/wp-content/uploads/2022/03/input.txt"
      ],
      "metadata": {
        "id": "aBK1gdaneZCm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Reading text file"
      ],
      "metadata": {
        "id": "01aJ8d0xXjJG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_words(text_path):\n",
        "  pass"
      ],
      "metadata": {
        "id": "Ew0xRr64tVD2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## define a function that reads a text file and returns a list of words in the text.\n",
        "text_path = '/content/input.txt'\n",
        "\n",
        "\n",
        "\n",
        "text = get_words(text_path)\n",
        "print(text)"
      ],
      "metadata": {
        "id": "x8nyd88sWned"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aL0HbhTBTyj2"
      },
      "source": [
        "# define the parameters \n",
        "\n",
        "WINDOW_SIZE = 2\n",
        "EMDEDDING_DIM = 50\n",
        "\n",
        "vocab = set(text)\n",
        "vocab_size = len(vocab)\n",
        "word_list = list(vocab)\n",
        "\n",
        "# define the mapping\n",
        "w2i = {word:idx for idx, word in enumerate(vocab)}\n",
        "i2w = {idx:word for idx, word in enumerate(vocab)}\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Preparing the data"
      ],
      "metadata": {
        "id": "swJuBDVeaQcd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y5k4PeA6FCUG"
      },
      "outputs": [],
      "source": [
        "#write a function that take take two arguments: list of words and window size.\n",
        "#it returns a list of tuples. the output should look like this. \n",
        "#[(['It', 'was', 'end', 'of'], 'the'), ... ,(['was', 'the', 'of', 'November,'], 'end')]\n",
        "\n",
        "def get_context(text, WINDOW_SIZE=2):\n",
        "   data = []\n",
        "## implimentation goes here\n",
        "       data.append((context, label))\n",
        "   return data"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## write a function \n",
        "def vectorize(context, word2idx):\n",
        "## implimentation goes here\n",
        "    return torch.tensor(idxs, dtype=torch.long)"
      ],
      "metadata": {
        "id": "at94P5MyBbsh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = get_context(text)"
      ],
      "metadata": {
        "id": "hSHYQUKwk4y2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Model(torch.nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_dim):\n",
        "        super(Model, self).__init__()\n",
        "        self.embeddings = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.linear1 = nn.Linear(embedding_dim, 100)\n",
        "        self.activation_function1 = nn.ReLU()\n",
        "        self.linear2 = nn.Linear(100, vocab_size)\n",
        "        \n",
        "\n",
        "    def forward(self, inputs):\n",
        "        embeds = sum(self.embeddings(inputs)).view(1,-1)\n",
        "        out = self.linear1(embeds)\n",
        "        out = self.activation_function1(out)\n",
        "        out = self.linear2(out)\n",
        "        return out\n",
        "\n",
        "    def get_emdedding(self, word):\n",
        "        word = torch.tensor([w2i[word]])\n",
        "        return self.embeddings(word).view(1,-1)"
      ],
      "metadata": {
        "id": "FoNe6EqSYkiH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### explain what the code is doing here"
      ],
      "metadata": {
        "id": "SphPqKDhcj5V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "model = Model(vocab_size, EMDEDDING_DIM)\n",
        "\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=0.001)\n",
        "\n"
      ],
      "metadata": {
        "id": "uVbXTw7xgNhQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in tqdm(range(500)):\n",
        "    total_loss = 0\n",
        "\n",
        "    for context, target in data:\n",
        "        context_vector = vectorize(context, w2i)  \n",
        "\n",
        "        log_probs = model(context_vector)\n",
        "\n",
        "        total_loss += loss_function(log_probs, torch.tensor([w2i[target]]))\n",
        "\n",
        "    optimizer.zero_grad()\n",
        "    total_loss.backward()\n",
        "    optimizer.step()"
      ],
      "metadata": {
        "id": "3mOQgshchMkJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(15,10))\n",
        "for w in word_list:\n",
        "    x = model.get_emdedding(w).detach().data.numpy()[0][0]\n",
        "    y = model.get_emdedding(w).detach().data.numpy()[0][1]\n",
        "    plt.scatter(x, y)\n",
        "    plt.annotate(w, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "67QkrS_glhhm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Write a function to get the model predictions"
      ],
      "metadata": {
        "id": "qPROAPmWdHrx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "7uRlohDsVN4A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "RXmsZhiCVSM5"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}