makemore_bigram_nn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "36109a35-c632-4049-8a21-24db111dbe87",
   "metadata": {},
   "source": [
    "# Nueral Network\n",
    "- bigram character level language model\n",
    "- train neural net with weights/params\n",
    "- neural network used to set probabilities of next character\n",
    "- optimize params nad lower loss with gradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "15a9c0c9-796c-4197-b8f3-3451c373ae7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3f148a25-1017-47b3-8cfc-e65c774e1962",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load the dataset\n",
    "# split into words\n",
    "words = open(\"names.txt\", \"r\").read().splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "60c6c32b-1187-4bbd-8a9d-136a6aee7fcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# look up table string to int\n",
    "chars = sorted(list(set(\"\".join(words))))\n",
    "stoi = {s:i + 1 for i,s in enumerate(chars)}\n",
    "stoi[\".\"] = 0\n",
    "\n",
    "itos = {i:s for s, i in stoi.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "38ced326-cd3e-47c2-ac72-86b31de23c0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a training set of bigrams (x, y), (input, output)\n",
    "xs, ys = [], []\n",
    "\n",
    "# just one example \"emma\"\n",
    "for w in words[:1]:\n",
    "    # create start and end tokens to specify start and ending\n",
    "    chs = [\".\"] + list(w) + [\".\"]\n",
    "    for ch1, ch2 in zip(chs, chs[1:]):\n",
    "        ix1 = stoi[ch1]\n",
    "        ix2 = stoi[ch2]\n",
    "        xs.append(ix1)\n",
    "        ys.append(ix2)\n",
    "\n",
    "xs = torch.tensor(xs)\n",
    "ys = torch.tensor(ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7b036d5d-446b-4762-814c-75c7336c3673",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0,  5, 13, 13,  1])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b1933d2d-781c-4489-8485-94239f5d8588",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 5, 13, 13,  1,  0])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2249b494-3cd9-4ee5-9828-73486ae13740",
   "metadata": {},
   "outputs": [],
   "source": [
    "# when input is 0(.), desired label is 5 (e)\n",
    "# when input is 5(e), desired label is 13 (m)\n",
    "# weights/params need to be arranged so e has a high probability given . or m has a high probabilty given e (for this training set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b429cccf-af01-4dd7-b9a8-9db8d5c772c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0],\n",
       "         [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0],\n",
       "         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0],\n",
       "         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0],\n",
       "         [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0]]),\n",
       " torch.Size([5, 27]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# neural net made up of neurons\n",
    "# neurons have weights\n",
    "# doesn't make sense to have an input neruon take on integer values\n",
    "\n",
    "# one hot encoding\n",
    "# ex. 13 -> vector of all zeros except the 13th index which is turned to 1\n",
    "# PyTorch one_hot\n",
    "\n",
    "import torch.nn.functional as F\n",
    "\n",
    "xenc = F.one_hot(xs, num_classes=27)\n",
    "xenc, xenc.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fdfc3c74-cb03-457e-a51f-d2be02ea4d40",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f932682e6b0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAACHCAYAAABK4hAcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAN2klEQVR4nO3df2hV9ePH8dfd2q4/urs6137cNufUUmpukrolkgkbTgvJ9A8r/1hDjOoqzlHJAl1CsDAIqSQjKP/xV0ImyQdDlpsE8wcTMaH21SFfr8xtKR/vdOZcu+/PH3263+9Nnd7tvXt2r88HHLj33Df3vHjzlr0899x7XMYYIwAAAAuSnA4AAAASB8UCAABYQ7EAAADWUCwAAIA1FAsAAGANxQIAAFhDsQAAANY8EsuDhUIhtbe3y+PxyOVyxfLQAABgkIwxun79unw+n5KSBj4nEdNi0d7erry8vFgeEgAAWBIIBJSbmzvgmJgWC4/HI0n631OTlPbo0D6FefnJGTYiAQCA+/hTffpZ/wr/HR9ITIvF3x9/pD2apDTP0IrFI64UG5EAAMD9/PfmHw9yGQMXbwIAAGsoFgAAwBqKBQAAsGZQxWLbtm2aNGmSRo0apdLSUp04ccJ2LgAAEIeiLhZ79+5VTU2N6urqdOrUKRUXF6uiokJdXV3DkQ8AAMSRqIvFJ598otWrV6uqqkpPPfWUtm/frjFjxujrr78ejnwAACCORFUsbt++rZaWFpWXl//fGyQlqby8XM3NzXeM7+3tVXd3d8QGAAASV1TF4sqVK+rv71dWVlbE/qysLHV0dNwxvr6+Xl6vN7zxq5sAACS2Yf1WSG1trYLBYHgLBALDeTgAAOCwqH55MyMjQ8nJyers7IzY39nZqezs7DvGu91uud3uoSUEAABxI6ozFqmpqZo1a5YaGhrC+0KhkBoaGjR37lzr4QAAQHyJ+l4hNTU1qqys1OzZs1VSUqKtW7eqp6dHVVVVw5EPAADEkaiLxYoVK/T7779r06ZN6ujo0MyZM3Xo0KE7LugEAAAPH5cxxsTqYN3d3fJ6vfr3/0we8t1NK3wz7YQCAAAD+tP0qVEHFAwGlZaWNuBY7hUCAACsifqjEBtefnKGHnGlOHHoh86P7aetvA9niAAAD4IzFgAAwBqKBQAAsIZiAQAArKFYAAAAaygWAADAGooFAACwhmIBAACsoVgAAABrKBYAAMAaigUAALCGYgEAAKyhWAAAAGsoFgAAwBqKBQAAsIZiAQAArKFYAAAAaygWAADAGooFAACw5hGnA2B4VfhmOh0BCeLH9tNW3oc1CSQ2zlgAAABrKBYAAMAaigUAALCGYgEAAKyJqljU19drzpw58ng8yszM1NKlS9Xa2jpc2QAAQJyJqlg0NTXJ7/fr2LFjOnz4sPr6+rRw4UL19PQMVz4AABBHovq66aFDhyKe79ixQ5mZmWppadH8+fOtBgMAAPFnSL9jEQwGJUnp6el3fb23t1e9vb3h593d3UM5HAAAGOEGffFmKBRSdXW15s2bp8LCwruOqa+vl9frDW95eXmDDgoAAEa+QRcLv9+vs2fPas+ePfccU1tbq2AwGN4CgcBgDwcAAOLAoD4KWbNmjQ4ePKijR48qNzf3nuPcbrfcbvegwwEAgPgSVbEwxmjt2rXav3+/GhsbVVBQMFy5AABAHIqqWPj9fu3atUsHDhyQx+NRR0eHJMnr9Wr06NHDEhAAAMSPqK6x+OKLLxQMBrVgwQLl5OSEt7179w5XPgAAEEei/igEAADgXrhXCAAAsIZiAQAArKFYAAAAaygWAADAGooFAACwhmIBAACsoVgAAABrKBYAAMAaigUAALCGYgEAAKyhWAAAAGsoFgAAwBqKBQAAsIZiAQAArKFYAAAAaygWAADAGooFAACwhmIBAACsoVgAAABrKBYAAMAaigUAALDmEacDDNaP7aetvVeFb6a19wISFf9OADwIzlgAAABrKBYAAMAaigUAALCGYgEAAKwZUrH46KOP5HK5VF1dbSkOAACIZ4MuFidPntSXX36poqIim3kAAEAcG1SxuHHjhlauXKmvvvpK48ePt50JAADEqUEVC7/frxdffFHl5eUDjuvt7VV3d3fEBgAAElfUP5C1Z88enTp1SidPnrzv2Pr6em3evHlQwQAAQPyJ6oxFIBDQunXrtHPnTo0aNeq+42traxUMBsNbIBAYdFAAADDyRXXGoqWlRV1dXXrmmWfC+/r7+3X06FF9/vnn6u3tVXJycvg1t9stt9ttLy0AABjRoioWZWVl+uWXXyL2VVVVafr06dqwYUNEqQAAAA+fqIqFx+NRYWFhxL6xY8dqwoQJd+wHAAAPH355EwAAWDPk26Y3NjZaiAEAABIBZywAAIA1Qz5jEQ1jjCTpT/VJZmjv1X09ZCHRX/40fdbeCwCARPOn/vo7+fff8YG4zIOMsuTSpUvKy8uL1eEAAIBFgUBAubm5A46JabEIhUJqb2+Xx+ORy+W657ju7m7l5eUpEAgoLS0tVvEeWsx37DDXscV8xxbzHVuxnG9jjK5fvy6fz6ekpIGvoojpRyFJSUn3bTr/X1paGoszhpjv2GGuY4v5ji3mO7ZiNd9er/eBxnHxJgAAsIZiAQAArBmRxcLtdquuro77jMQI8x07zHVsMd+xxXzH1kid75hevAkAABLbiDxjAQAA4hPFAgAAWEOxAAAA1lAsAACANRQLAABgzYgrFtu2bdOkSZM0atQolZaW6sSJE05HSkgffPCBXC5XxDZ9+nSnYyWMo0ePasmSJfL5fHK5XPr+++8jXjfGaNOmTcrJydHo0aNVXl6uc+fOORM2Adxvvl9//fU71vuiRYucCRvn6uvrNWfOHHk8HmVmZmrp0qVqbW2NGHPr1i35/X5NmDBBjz76qJYvX67Ozk6HEse3B5nvBQsW3LG+33zzTYcSj7BisXfvXtXU1Kiurk6nTp1ScXGxKioq1NXV5XS0hPT000/r8uXL4e3nn392OlLC6OnpUXFxsbZt23bX17ds2aJPP/1U27dv1/HjxzV27FhVVFTo1q1bMU6aGO4335K0aNGiiPW+e/fuGCZMHE1NTfL7/Tp27JgOHz6svr4+LVy4UD09PeEx69ev1w8//KB9+/apqalJ7e3tWrZsmYOp49eDzLckrV69OmJ9b9myxaHEkswIUlJSYvx+f/h5f3+/8fl8pr6+3sFUiamurs4UFxc7HeOhIMns378//DwUCpns7Gzz8ccfh/ddu3bNuN1us3v3bgcSJpZ/zrcxxlRWVpqXXnrJkTyJrqury0gyTU1Nxpi/1nJKSorZt29feMyvv/5qJJnm5manYiaMf863McY8//zzZt26dc6F+ocRc8bi9u3bamlpUXl5eXhfUlKSysvL1dzc7GCyxHXu3Dn5fD5NnjxZK1eu1MWLF52O9FC4cOGCOjo6Ita61+tVaWkpa30YNTY2KjMzU9OmTdNbb72lq1evOh0pIQSDQUlSenq6JKmlpUV9fX0R63v69OmaOHEi69uCf87333bu3KmMjAwVFhaqtrZWN2/edCKepBjf3XQgV65cUX9/v7KysiL2Z2Vl6bfffnMoVeIqLS3Vjh07NG3aNF2+fFmbN2/Wc889p7Nnz8rj8TgdL6F1dHRI0l3X+t+vwa5FixZp2bJlKigoUFtbm95//30tXrxYzc3NSk5Odjpe3AqFQqqurta8efNUWFgo6a/1nZqaqnHjxkWMZX0P3d3mW5Jee+015efny+fz6cyZM9qwYYNaW1v13XffOZJzxBQLxNbixYvDj4uKilRaWqr8/Hx9++23WrVqlYPJAPteeeWV8OMZM2aoqKhIU6ZMUWNjo8rKyhxMFt/8fr/Onj3L9Vkxcq/5fuONN8KPZ8yYoZycHJWVlamtrU1TpkyJdcyRc/FmRkaGkpOT77hyuLOzU9nZ2Q6leniMGzdOTz75pM6fP+90lIT393pmrTtn8uTJysjIYL0PwZo1a3Tw4EEdOXJEubm54f3Z2dm6ffu2rl27FjGe9T0095rvuyktLZUkx9b3iCkWqampmjVrlhoaGsL7QqGQGhoaNHfuXAeTPRxu3LihtrY25eTkOB0l4RUUFCg7OztirXd3d+v48eOs9Ri5dOmSrl69ynofBGOM1qxZo/379+unn35SQUFBxOuzZs1SSkpKxPpubW3VxYsXWd+DcL/5vpvTp09LkmPre0R9FFJTU6PKykrNnj1bJSUl2rp1q3p6elRVVeV0tITzzjvvaMmSJcrPz1d7e7vq6uqUnJysV1991eloCeHGjRsR/1u4cOGCTp8+rfT0dE2cOFHV1dX68MMP9cQTT6igoEAbN26Uz+fT0qVLnQsdxwaa7/T0dG3evFnLly9Xdna22tra9N5772nq1KmqqKhwMHV88vv92rVrlw4cOCCPxxO+bsLr9Wr06NHyer1atWqVampqlJ6errS0NK1du1Zz587Vs88+63D6+HO/+W5ra9OuXbv0wgsvaMKECTpz5ozWr1+v+fPnq6ioyJnQTn8t5Z8+++wzM3HiRJOammpKSkrMsWPHnI6UkFasWGFycnJMamqqefzxx82KFSvM+fPnnY6VMI4cOWIk3bFVVlYaY/76yunGjRtNVlaWcbvdpqyszLS2tjobOo4NNN83b940CxcuNI899phJSUkx+fn5ZvXq1aajo8Pp2HHpbvMsyXzzzTfhMX/88Yd5++23zfjx482YMWPMyy+/bC5fvuxc6Dh2v/m+ePGimT9/vklPTzdut9tMnTrVvPvuuyYYDDqW2fXf4AAAAEM2Yq6xAAAA8Y9iAQAArKFYAAAAaygWAADAGooFAACwhmIBAACsoVgAAABrKBYAAMAaigUAALCGYgEAAKyhWAAAAGv+A6sEjbDe9GoiAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(xenc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ea1e0da8-88fd-43b2-88a1-7622c32beb51",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# be careful\n",
    "# datatype should be a float if fed into a neural net\n",
    "xenc.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a2a02ffb-a9d7-44a2-9659-f9b413bf503c",
   "metadata": {},
   "outputs": [],
   "source": [
    "xenc = F.one_hot(xs, num_classes=27).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a7777d74-5130-4e17-8c4b-33d7fd56b01f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "          0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "          0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
       "          0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
       "          0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "          0., 0., 0., 0., 0., 0., 0., 0., 0.]]),\n",
       " torch.float32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xenc, xenc.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "58c89b84-73f2-40a7-af1c-d4341e257104",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0780],\n",
       "        [ 0.0557],\n",
       "        [ 0.4350],\n",
       "        [-1.2950],\n",
       "        [-0.6858],\n",
       "        [-0.4140],\n",
       "        [ 0.0504],\n",
       "        [-0.7037],\n",
       "        [-1.1415],\n",
       "        [-1.1591],\n",
       "        [ 0.1261],\n",
       "        [-1.7313],\n",
       "        [-0.5056],\n",
       "        [-0.6538],\n",
       "        [-0.5367],\n",
       "        [ 1.3418],\n",
       "        [-0.8485],\n",
       "        [-0.0428],\n",
       "        [-0.7017],\n",
       "        [ 1.2444],\n",
       "        [-0.2586],\n",
       "        [-0.0855],\n",
       "        [-0.5840],\n",
       "        [ 1.1810],\n",
       "        [ 1.7472],\n",
       "        [-0.5148],\n",
       "        [ 0.1656]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# construct first neuron\n",
    "# neuron will look at input vectors\n",
    "# wx + b (wx dot produt)\n",
    "\n",
    "# initialize weights\n",
    "# column vector of 27 numbers\n",
    "W = torch.randn((27, 1))\n",
    "W"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "00786658-5dd0-4a5e-b1ae-06f005dce18f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0780],\n",
       "        [-0.4140],\n",
       "        [-0.6538],\n",
       "        [-0.6538],\n",
       "        [ 0.0557]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# weights multiplied by inputs\n",
    "xenc @ W # matmul wx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fd75f9f4-9534-4697-bc75-afeac464d2ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# (5, 27) @ (27, 1) = (5, 1) \n",
    "# 27 dimensions will multiply and add\n",
    "# 5 activations of this neuron on 5 inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "075904f2-08f2-4acb-bbfd-1829eeb551e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.6129,  1.5024, -0.3730,  1.1610,  0.6176,  0.0468, -0.4312, -0.7279,\n",
       "          0.3839, -0.6634,  0.0693,  0.0421,  0.4504,  0.4992,  0.5268,  0.0693,\n",
       "         -0.4026,  0.3930, -0.4126, -0.6523,  0.7386,  2.5292, -0.6711, -0.6273,\n",
       "         -0.9378,  1.4649,  2.2418],\n",
       "        [-0.5540,  0.2061, -0.2047, -0.5337,  1.2296,  0.5108,  1.3219,  0.4500,\n",
       "         -0.8790,  1.1559, -3.0929, -0.3975, -0.5695, -0.3605,  1.0312, -1.0282,\n",
       "         -0.3140, -0.9962,  0.9886, -0.3869, -0.5897, -2.9958, -0.2369, -0.3214,\n",
       "         -1.0455,  0.2372, -2.4673],\n",
       "        [ 2.1784,  1.3450,  0.2595,  0.0613,  0.0444,  2.0564,  1.0692,  0.1808,\n",
       "          0.7143,  0.4599, -0.1578,  0.6507, -0.5619, -1.1454, -1.4254,  1.1943,\n",
       "          0.7458,  2.3220, -0.5967,  0.9672,  2.0604, -2.2235,  0.0245,  0.0815,\n",
       "          1.0706,  1.4673,  0.2673],\n",
       "        [ 2.1784,  1.3450,  0.2595,  0.0613,  0.0444,  2.0564,  1.0692,  0.1808,\n",
       "          0.7143,  0.4599, -0.1578,  0.6507, -0.5619, -1.1454, -1.4254,  1.1943,\n",
       "          0.7458,  2.3220, -0.5967,  0.9672,  2.0604, -2.2235,  0.0245,  0.0815,\n",
       "          1.0706,  1.4673,  0.2673],\n",
       "        [ 0.2368, -0.7276, -0.4971, -1.2657,  0.3818, -0.7616, -1.1682, -0.9260,\n",
       "          0.3248, -0.6620, -0.5344, -1.1574,  1.8017, -0.9882,  1.1505, -1.7646,\n",
       "         -0.3469, -0.2812,  1.9393,  0.1179, -1.1554,  0.4274, -1.0082, -0.0979,\n",
       "         -1.0662,  0.8841, -2.9304]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# instead of 1 neuron make 27 neurons\n",
    "W = torch.randn((27, 27))\n",
    "xenc @ W"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5d8eb544-498d-4650-b3a2-8f28be6f90d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 27])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# (5, 27) @ (27, 27) -> (5, 27)\n",
    "(xenc @ W).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "562643d0-35ed-4179-a96e-4f2ca7543b67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-1.1454)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for every 27 neurons what is the firing rate of those neurons on 5 examples\n",
    "\n",
    "# firing rate of 13th neuron looking at the 3rd input\n",
    "# dot product between 3rd input and 13th column of the W matrix\n",
    "(xenc @ W) [3, 13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "18a9037e-cc11-48b7-aae5-37cbd5323124",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3rd input in one hot encoding\n",
    "xenc[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "284c245a-987d-4f53-9f36-79acca10b960",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0.4992, -0.9882, -0.8694, -0.6105, -0.4751, -0.3605,  0.2041,  0.1776,\n",
       "        -1.1690,  0.5140,  0.2643,  0.1177, -0.2057, -1.1454, -0.1709,  1.6234,\n",
       "         0.8698,  0.0848, -0.6842, -1.5993,  0.4968, -0.3804, -0.5999,  0.2268,\n",
       "         0.6615, -0.8734,  0.3565])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 13th index value in each of the rows\n",
    "W[:, 13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c5187965-e494-473d-b778-5ad27e8f356b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0.0000, -0.0000, -0.0000, -0.0000, -0.0000, -0.0000,  0.0000,  0.0000,\n",
       "        -0.0000,  0.0000,  0.0000,  0.0000, -0.0000, -1.1454, -0.0000,  0.0000,\n",
       "         0.0000,  0.0000, -0.0000, -0.0000,  0.0000, -0.0000, -0.0000,  0.0000,\n",
       "         0.0000, -0.0000,  0.0000])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# each value multiplied by W and summed (wx)\n",
    "# since only 1 input value has a 1 value that is all that is taken\n",
    "xenc[3] * W[:, 13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "6cd07915-22dc-469e-938d-c8a797027e6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-1.1454)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(xenc[3] * W[:, 13]).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "b2ddba67-164e-493b-b5c0-235d47ff7056",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.6129,  1.5024, -0.3730,  1.1610,  0.6176,  0.0468, -0.4312, -0.7279,\n",
       "          0.3839, -0.6634,  0.0693,  0.0421,  0.4504,  0.4992,  0.5268,  0.0693,\n",
       "         -0.4026,  0.3930, -0.4126, -0.6523,  0.7386,  2.5292, -0.6711, -0.6273,\n",
       "         -0.9378,  1.4649,  2.2418],\n",
       "        [-0.5540,  0.2061, -0.2047, -0.5337,  1.2296,  0.5108,  1.3219,  0.4500,\n",
       "         -0.8790,  1.1559, -3.0929, -0.3975, -0.5695, -0.3605,  1.0312, -1.0282,\n",
       "         -0.3140, -0.9962,  0.9886, -0.3869, -0.5897, -2.9958, -0.2369, -0.3214,\n",
       "         -1.0455,  0.2372, -2.4673],\n",
       "        [ 2.1784,  1.3450,  0.2595,  0.0613,  0.0444,  2.0564,  1.0692,  0.1808,\n",
       "          0.7143,  0.4599, -0.1578,  0.6507, -0.5619, -1.1454, -1.4254,  1.1943,\n",
       "          0.7458,  2.3220, -0.5967,  0.9672,  2.0604, -2.2235,  0.0245,  0.0815,\n",
       "          1.0706,  1.4673,  0.2673],\n",
       "        [ 2.1784,  1.3450,  0.2595,  0.0613,  0.0444,  2.0564,  1.0692,  0.1808,\n",
       "          0.7143,  0.4599, -0.1578,  0.6507, -0.5619, -1.1454, -1.4254,  1.1943,\n",
       "          0.7458,  2.3220, -0.5967,  0.9672,  2.0604, -2.2235,  0.0245,  0.0815,\n",
       "          1.0706,  1.4673,  0.2673],\n",
       "        [ 0.2368, -0.7276, -0.4971, -1.2657,  0.3818, -0.7616, -1.1682, -0.9260,\n",
       "          0.3248, -0.6620, -0.5344, -1.1574,  1.8017, -0.9882,  1.1505, -1.7646,\n",
       "         -0.3469, -0.2812,  1.9393,  0.1179, -1.1554,  0.4274, -1.0082, -0.0979,\n",
       "         -1.0662,  0.8841, -2.9304]])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# right now we have negative and positive numbers\n",
    "(xenc @ W)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "d77872ee-3010-421f-8230-4379d50cbe6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# interpret them as log counts\n",
    "# exponentiate them\n",
    "# negative numbers turn into numbers below 1\n",
    "# positive numbers turn into more positive numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "811f4252-31eb-4264-88bb-538e3fe285dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.6129,  1.5024, -0.3730,  1.1610,  0.6176,  0.0468, -0.4312, -0.7279,\n",
       "          0.3839, -0.6634,  0.0693,  0.0421,  0.4504,  0.4992,  0.5268,  0.0693,\n",
       "         -0.4026,  0.3930, -0.4126, -0.6523,  0.7386,  2.5292, -0.6711, -0.6273,\n",
       "         -0.9378,  1.4649,  2.2418],\n",
       "        [-0.5540,  0.2061, -0.2047, -0.5337,  1.2296,  0.5108,  1.3219,  0.4500,\n",
       "         -0.8790,  1.1559, -3.0929, -0.3975, -0.5695, -0.3605,  1.0312, -1.0282,\n",
       "         -0.3140, -0.9962,  0.9886, -0.3869, -0.5897, -2.9958, -0.2369, -0.3214,\n",
       "         -1.0455,  0.2372, -2.4673],\n",
       "        [ 2.1784,  1.3450,  0.2595,  0.0613,  0.0444,  2.0564,  1.0692,  0.1808,\n",
       "          0.7143,  0.4599, -0.1578,  0.6507, -0.5619, -1.1454, -1.4254,  1.1943,\n",
       "          0.7458,  2.3220, -0.5967,  0.9672,  2.0604, -2.2235,  0.0245,  0.0815,\n",
       "          1.0706,  1.4673,  0.2673],\n",
       "        [ 2.1784,  1.3450,  0.2595,  0.0613,  0.0444,  2.0564,  1.0692,  0.1808,\n",
       "          0.7143,  0.4599, -0.1578,  0.6507, -0.5619, -1.1454, -1.4254,  1.1943,\n",
       "          0.7458,  2.3220, -0.5967,  0.9672,  2.0604, -2.2235,  0.0245,  0.0815,\n",
       "          1.0706,  1.4673,  0.2673],\n",
       "        [ 0.2368, -0.7276, -0.4971, -1.2657,  0.3818, -0.7616, -1.1682, -0.9260,\n",
       "          0.3248, -0.6620, -0.5344, -1.1574,  1.8017, -0.9882,  1.1505, -1.7646,\n",
       "         -0.3469, -0.2812,  1.9393,  0.1179, -1.1554,  0.4274, -1.0082, -0.0979,\n",
       "         -1.0662,  0.8841, -2.9304]])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# exponentiated outputs give us something we can interpet as counts which we can normalize into probability\n",
    "# sometimes called logits = log counts\n",
    "logits = xenc @ W # log counts\n",
    "logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "0be245a9-7702-4fb8-aa07-dde2d8cbf251",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0096, 0.0800, 0.0123, 0.0568, 0.0330, 0.0187, 0.0116, 0.0086, 0.0261,\n",
       "         0.0092, 0.0191, 0.0186, 0.0279, 0.0293, 0.0301, 0.0191, 0.0119, 0.0264,\n",
       "         0.0118, 0.0093, 0.0373, 0.2233, 0.0091, 0.0095, 0.0070, 0.0770, 0.1675],\n",
       "        [0.0188, 0.0401, 0.0266, 0.0191, 0.1116, 0.0544, 0.1224, 0.0512, 0.0136,\n",
       "         0.1037, 0.0015, 0.0219, 0.0185, 0.0228, 0.0915, 0.0117, 0.0238, 0.0121,\n",
       "         0.0877, 0.0222, 0.0181, 0.0016, 0.0258, 0.0237, 0.0115, 0.0414, 0.0028],\n",
       "        [0.1211, 0.0526, 0.0178, 0.0146, 0.0143, 0.1072, 0.0399, 0.0164, 0.0280,\n",
       "         0.0217, 0.0117, 0.0263, 0.0078, 0.0044, 0.0033, 0.0453, 0.0289, 0.1398,\n",
       "         0.0075, 0.0361, 0.1076, 0.0015, 0.0140, 0.0149, 0.0400, 0.0595, 0.0179],\n",
       "        [0.1211, 0.0526, 0.0178, 0.0146, 0.0143, 0.1072, 0.0399, 0.0164, 0.0280,\n",
       "         0.0217, 0.0117, 0.0263, 0.0078, 0.0044, 0.0033, 0.0453, 0.0289, 0.1398,\n",
       "         0.0075, 0.0361, 0.1076, 0.0015, 0.0140, 0.0149, 0.0400, 0.0595, 0.0179],\n",
       "        [0.0380, 0.0145, 0.0183, 0.0085, 0.0440, 0.0140, 0.0093, 0.0119, 0.0415,\n",
       "         0.0155, 0.0176, 0.0094, 0.1819, 0.0112, 0.0948, 0.0051, 0.0212, 0.0227,\n",
       "         0.2087, 0.0338, 0.0095, 0.0460, 0.0109, 0.0272, 0.0103, 0.0727, 0.0016]])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts = logits.exp() # equivalent to N in bigram (counts of each from a data sample)\n",
    "# now we can normalize\n",
    "\n",
    "probs = counts / counts.sum(1, keepdims=True)\n",
    "probs \n",
    "\n",
    "# also called the soft max\n",
    "# soft max activiation function\n",
    "# takes logits, exponentiates them and normalizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "4d1db3f2-63d0-4dd6-bc69-84dc63ece519",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.0000)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs[0].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "cce86803-c322-4d11-ab5d-cdf18a289ac2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 27])"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "19bc8d0e-20b3-4036-83cd-dab386e8c4c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for every 1 of 5 examples, we have a row that tells us the percent of time an index will be chosen as the next letter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "4ac60cc7-8e00-4854-8935-378e6208a060",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "         0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "         0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
       "         0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
       "         0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "         0., 0., 0., 0., 0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Summary\n",
    "# have sample inputs and outputs \n",
    "\n",
    "# ex.\n",
    "# input = ['.', 'e', 'm', 'm', 'a'] -> [0, 5, 13, 13, 1]\n",
    "# output = ['e', 'm', 'm', '.'] -> [5, 13, 13, 1, 0]\n",
    "\n",
    "# equivalent output value at index is what is predicted given the input at index\n",
    "xs = torch.tensor([0, 5, 13, 13, 1])\n",
    "ys = torch.tensor([5, 13, 13, 1, 0])\n",
    "\n",
    "# one hot encode inputs to make them a tensor of 5 rows and 27 values\n",
    "# 27 values will be 0 with the exception of a 1 at index = input value\n",
    "# make sure they are floats\n",
    "xenc = F.one_hot(xs, num_classes=27).float()\n",
    "xenc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "70696591-f0c1-4b87-b7b1-3fa32af03354",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1.5674, -0.2373, -0.0274, -1.1008,  0.2859, -0.0296, -1.5471,  0.6049,\n",
       "         0.0791,  0.9046, -0.4713,  0.7868, -0.3284, -0.4330,  1.3729,  2.9334,\n",
       "         1.5618, -1.6261,  0.6772, -0.8404,  0.9849, -0.1484, -1.4795,  0.4483,\n",
       "        -0.0707,  2.4968,  2.4448])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# randomly initialize 27 neuron weights, each neuron will recieve 27 inputs\n",
    "g = torch.Generator().manual_seed(2147483647)\n",
    "W = torch.randn((27, 27), generator=g)\n",
    "W[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "c44c2467-0d69-429f-89e9-5cd5d587c466",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([0.0607, 0.0100, 0.0123, 0.0042, 0.0168, 0.0123, 0.0027, 0.0232, 0.0137,\n",
       "         0.0313, 0.0079, 0.0278, 0.0091, 0.0082, 0.0500, 0.2378, 0.0603, 0.0025,\n",
       "         0.0249, 0.0055, 0.0339, 0.0109, 0.0029, 0.0198, 0.0118, 0.1537, 0.1459]),\n",
       " tensor(1.0000),\n",
       " torch.Size([5, 27]))"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = xenc @ W # predict log counts of outputs by matmuling xenc and weights\n",
    "counts = logits.exp() # exponentiate log counts, to simualate real counts\n",
    "probs = counts / counts.sum(1, keepdims=True) # normalize to make them probabilities\n",
    "probs[0], probs[0].sum(), probs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "81ace29d-6bc6-4237-b120-b926f82c03c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------\n",
      "bigram example 1: .e (index 0,5)\n",
      "input to the neural net: 0\n",
      "output probabilities for next character from the neural net:\n",
      "tensor([0.0607, 0.0100, 0.0123, 0.0042, 0.0168, 0.0123, 0.0027, 0.0232, 0.0137,\n",
      "        0.0313, 0.0079, 0.0278, 0.0091, 0.0082, 0.0500, 0.2378, 0.0603, 0.0025,\n",
      "        0.0249, 0.0055, 0.0339, 0.0109, 0.0029, 0.0198, 0.0118, 0.1537, 0.1459])\n",
      "actual next character: 5 e\n",
      "probability assigned to the next character 0.01228625513613224\n",
      "log likelihood -4.399273872375488\n",
      "negative log likelihood: 4.399273872375488\n",
      "--------\n",
      "bigram example 2: em (index 5,13)\n",
      "input to the neural net: 5\n",
      "output probabilities for next character from the neural net:\n",
      "tensor([0.0290, 0.0796, 0.0248, 0.0521, 0.1989, 0.0289, 0.0094, 0.0335, 0.0097,\n",
      "        0.0301, 0.0702, 0.0228, 0.0115, 0.0181, 0.0108, 0.0315, 0.0291, 0.0045,\n",
      "        0.0916, 0.0215, 0.0486, 0.0300, 0.0501, 0.0027, 0.0118, 0.0022, 0.0472])\n",
      "actual next character: 13 m\n",
      "probability assigned to the next character 0.018050700426101685\n",
      "log likelihood -4.014570713043213\n",
      "negative log likelihood: 4.014570713043213\n",
      "--------\n",
      "bigram example 3: mm (index 13,13)\n",
      "input to the neural net: 13\n",
      "output probabilities for next character from the neural net:\n",
      "tensor([0.0312, 0.0737, 0.0484, 0.0333, 0.0674, 0.0200, 0.0263, 0.0249, 0.1226,\n",
      "        0.0164, 0.0075, 0.0789, 0.0131, 0.0267, 0.0147, 0.0112, 0.0585, 0.0121,\n",
      "        0.0650, 0.0058, 0.0208, 0.0078, 0.0133, 0.0203, 0.1204, 0.0469, 0.0126])\n",
      "actual next character: 13 m\n",
      "probability assigned to the next character 0.026691533625125885\n",
      "log likelihood -3.623408794403076\n",
      "negative log likelihood: 3.623408794403076\n",
      "--------\n",
      "bigram example 4: ma (index 13,1)\n",
      "input to the neural net: 13\n",
      "output probabilities for next character from the neural net:\n",
      "tensor([0.0312, 0.0737, 0.0484, 0.0333, 0.0674, 0.0200, 0.0263, 0.0249, 0.1226,\n",
      "        0.0164, 0.0075, 0.0789, 0.0131, 0.0267, 0.0147, 0.0112, 0.0585, 0.0121,\n",
      "        0.0650, 0.0058, 0.0208, 0.0078, 0.0133, 0.0203, 0.1204, 0.0469, 0.0126])\n",
      "actual next character: 1 a\n",
      "probability assigned to the next character 0.07367686182260513\n",
      "log likelihood -2.6080665588378906\n",
      "negative log likelihood: 2.6080665588378906\n",
      "--------\n",
      "bigram example 5: a. (index 1,0)\n",
      "input to the neural net: 1\n",
      "output probabilities for next character from the neural net:\n",
      "tensor([0.0150, 0.0086, 0.0396, 0.0100, 0.0606, 0.0308, 0.1084, 0.0131, 0.0125,\n",
      "        0.0048, 0.1024, 0.0086, 0.0988, 0.0112, 0.0232, 0.0207, 0.0408, 0.0078,\n",
      "        0.0899, 0.0531, 0.0463, 0.0309, 0.0051, 0.0329, 0.0654, 0.0503, 0.0091])\n",
      "actual next character: 0 .\n",
      "probability assigned to the next character 0.014977526850998402\n",
      "log likelihood -4.201204299926758\n",
      "negative log likelihood: 4.201204299926758\n",
      "========\n",
      "average negative log likelihood, i.e expected loss= 3.7693049907684326\n"
     ]
    }
   ],
   "source": [
    "# calculate log likelihood\n",
    "nlls = torch.zeros(5)\n",
    "for i in range(5):\n",
    "    x = xs[i].item() # input character index\n",
    "    y = ys[i].item() # output character index\n",
    "    print(\"--------\")\n",
    "    print(f\"bigram example {i + 1}: {itos[x]}{itos[y]} (index {x},{y})\")\n",
    "    print(\"input to the neural net:\", x)\n",
    "    print(\"output probabilities for next character from the neural net:\")\n",
    "    print(probs[i])\n",
    "    print(\"actual next character:\", y, itos[y])\n",
    "    p = probs[i, y]\n",
    "    print(\"probability assigned to the next character\", p.item())\n",
    "    logp = torch.log(p)\n",
    "    print(\"log likelihood\", logp.item())\n",
    "    nll = -logp\n",
    "    print(\"negative log likelihood:\", nll.item())\n",
    "    nlls[i] = nll\n",
    "\n",
    "print(\"========\")\n",
    "print(\"average negative log likelihood, i.e expected loss=\", nlls.mean().item())\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "aab7a43e-f8ad-4580-97a7-ad18ae244919",
   "metadata": {},
   "outputs": [],
   "source": [
    "# minimize loss by tuning W based on the gradients to minimize loss\n",
    "\n",
    "# single linear layer followed by a soft max\n",
    "# negative log likelihood for loss calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "950d2b3b-781c-4123-bd9f-7dd6de1eaa2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# randomly generate neuron weights\n",
    "g = torch.Generator().manual_seed(2147483647)\n",
    "W = torch.randn((27, 27), generator=g, requires_grad=True) # must tell PyTorch we want to calculate grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "ea22d9db-258f-4654-b791-54e0ccf99b3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# forward pass\n",
    "xenc = F.one_hot(xs, num_classes=27).float()\n",
    "logits = xenc @ W # predict log counts of outputs by matmuling xenc and weights\n",
    "counts = logits.exp() # exponentiate log counts, to simualate real counts\n",
    "probs = counts / counts.sum(1, keepdims=True) # normalize to make them probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "087cbf1f-b59e-41f3-a98e-31bc076c9958",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 27])"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "4ac4b6bc-5a33-4d6c-9074-1ef0000ba63b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 0,  5, 13, 13,  1]), tensor([ 5, 13, 13,  1,  0]))"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs, ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "d0aa549a-1125-498b-b8b7-5d4a099b3f46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.0123, grad_fn=<SelectBackward0>),\n",
       " tensor(0.0181, grad_fn=<SelectBackward0>),\n",
       " tensor(0.0267, grad_fn=<SelectBackward0>),\n",
       " tensor(0.0737, grad_fn=<SelectBackward0>),\n",
       " tensor(0.0150, grad_fn=<SelectBackward0>))"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# probabilities of getting 'emma'\n",
    "probs[0, 5], probs[1, 13], probs[2, 13], probs[3, 1], probs[4, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "e71066f3-4c7c-472a-b739-6d24faa82965",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 2, 3, 4])"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.arange(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "2f40ef2b-b416-4b3f-9c96-d6475726c5f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0123, 0.0181, 0.0267, 0.0737, 0.0150], grad_fn=<IndexBackward0>)"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs[torch.arange(5), ys]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "8be361e9-00f9-4ce7-93d7-c6c6f5da7c1c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(3.7693, grad_fn=<NegBackward0>)"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# take probabilities, look at the log, take mean, take negative\n",
    "# loss vectorized\n",
    "loss =  -probs[torch.arange(5), ys].log().mean()\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "7d7a78b1-be8f-498f-8214-161dc961df61",
   "metadata": {},
   "outputs": [],
   "source": [
    "# backward pass\n",
    "# reset all gradients\n",
    "W.grad = None # set to 0\n",
    "loss.backward() # fills in gradients of all intermediates all the way back to W\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "48695781-e8e8-4a3d-a676-fb8b504c05c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 0.0121,  0.0020,  0.0025,  0.0008,  0.0034, -0.1975,  0.0005,  0.0046,\n",
       "          0.0027,  0.0063,  0.0016,  0.0056,  0.0018,  0.0016,  0.0100,  0.0476,\n",
       "          0.0121,  0.0005,  0.0050,  0.0011,  0.0068,  0.0022,  0.0006,  0.0040,\n",
       "          0.0024,  0.0307,  0.0292]),\n",
       " torch.Size([27, 27]))"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W.grad[0], W.grad.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "bb3e929d-fb45-40b5-9c43-1c00732845d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# every element of W.grad is telling us the influence of that specific weight on the loss function\n",
    "# use gradient information ot update weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "934ad45e-01b6-4a48-b8e6-85fea3d2e8b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "W.data += -0.1 * W.grad # 0.1 step size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "fe3c0088-3268-4e0a-926f-1725505ea26f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(3.7492, grad_fn=<NegBackward0>)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# forward pass\n",
    "xenc = F.one_hot(xs, num_classes=27).float()\n",
    "logits = xenc @ W # predict log counts of outputs by matmuling xenc and weights\n",
    "counts = logits.exp() # exponentiate log counts, to simualate real counts\n",
    "probs = counts / counts.sum(1, keepdims=True) # normalize to make them probabilities\n",
    "loss =  -probs[torch.arange(5), ys].log().mean()\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "30f0b828-fd65-4f4b-84b4-47b440ae630e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# a litle bit less loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "d687d335-bd5a-44a2-b0b5-94bf580535f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# all together\n",
    "# create a training set of bigrams (x, y), (input, output)\n",
    "xs, ys = [], []\n",
    "num = 0\n",
    "for w in words:\n",
    "    # create start and end tokens to specify start and ending\n",
    "    chs = [\".\"] + list(w) + [\".\"]\n",
    "    for ch1, ch2 in zip(chs, chs[1:]):\n",
    "        ix1 = stoi[ch1]\n",
    "        ix2 = stoi[ch2]\n",
    "        xs.append(ix1)\n",
    "        ys.append(ix2)\n",
    "        num += 1\n",
    "        \n",
    "xs = torch.tensor(xs)\n",
    "ys = torch.tensor(ys)\n",
    "\n",
    "g = torch.Generator().manual_seed(2147483647)\n",
    "W = torch.randn((27, 27), generator=g, requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "9a2a31fd-341a-4d03-aa28-81e755e9b086",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(3.7590, grad_fn=<NegBackward0>)\n",
      "tensor(3.3711, grad_fn=<NegBackward0>)\n",
      "tensor(3.1540, grad_fn=<NegBackward0>)\n",
      "tensor(3.0204, grad_fn=<NegBackward0>)\n",
      "tensor(2.9277, grad_fn=<NegBackward0>)\n",
      "tensor(2.8604, grad_fn=<NegBackward0>)\n",
      "tensor(2.8097, grad_fn=<NegBackward0>)\n",
      "tensor(2.7701, grad_fn=<NegBackward0>)\n",
      "tensor(2.7381, grad_fn=<NegBackward0>)\n",
      "tensor(2.7115, grad_fn=<NegBackward0>)\n",
      "tensor(2.6890, grad_fn=<NegBackward0>)\n",
      "tensor(2.6697, grad_fn=<NegBackward0>)\n",
      "tensor(2.6529, grad_fn=<NegBackward0>)\n",
      "tensor(2.6383, grad_fn=<NegBackward0>)\n",
      "tensor(2.6254, grad_fn=<NegBackward0>)\n",
      "tensor(2.6140, grad_fn=<NegBackward0>)\n",
      "tensor(2.6039, grad_fn=<NegBackward0>)\n",
      "tensor(2.5948, grad_fn=<NegBackward0>)\n",
      "tensor(2.5867, grad_fn=<NegBackward0>)\n",
      "tensor(2.5794, grad_fn=<NegBackward0>)\n",
      "tensor(2.5728, grad_fn=<NegBackward0>)\n",
      "tensor(2.5668, grad_fn=<NegBackward0>)\n",
      "tensor(2.5613, grad_fn=<NegBackward0>)\n",
      "tensor(2.5563, grad_fn=<NegBackward0>)\n",
      "tensor(2.5516, grad_fn=<NegBackward0>)\n",
      "tensor(2.5474, grad_fn=<NegBackward0>)\n",
      "tensor(2.5434, grad_fn=<NegBackward0>)\n",
      "tensor(2.5397, grad_fn=<NegBackward0>)\n",
      "tensor(2.5363, grad_fn=<NegBackward0>)\n",
      "tensor(2.5332, grad_fn=<NegBackward0>)\n",
      "tensor(2.5302, grad_fn=<NegBackward0>)\n",
      "tensor(2.5274, grad_fn=<NegBackward0>)\n",
      "tensor(2.5248, grad_fn=<NegBackward0>)\n",
      "tensor(2.5223, grad_fn=<NegBackward0>)\n",
      "tensor(2.5200, grad_fn=<NegBackward0>)\n",
      "tensor(2.5179, grad_fn=<NegBackward0>)\n",
      "tensor(2.5158, grad_fn=<NegBackward0>)\n",
      "tensor(2.5139, grad_fn=<NegBackward0>)\n",
      "tensor(2.5121, grad_fn=<NegBackward0>)\n",
      "tensor(2.5103, grad_fn=<NegBackward0>)\n",
      "tensor(2.5087, grad_fn=<NegBackward0>)\n",
      "tensor(2.5071, grad_fn=<NegBackward0>)\n",
      "tensor(2.5057, grad_fn=<NegBackward0>)\n",
      "tensor(2.5043, grad_fn=<NegBackward0>)\n",
      "tensor(2.5029, grad_fn=<NegBackward0>)\n",
      "tensor(2.5017, grad_fn=<NegBackward0>)\n",
      "tensor(2.5005, grad_fn=<NegBackward0>)\n",
      "tensor(2.4993, grad_fn=<NegBackward0>)\n",
      "tensor(2.4982, grad_fn=<NegBackward0>)\n",
      "tensor(2.4971, grad_fn=<NegBackward0>)\n",
      "tensor(2.4961, grad_fn=<NegBackward0>)\n",
      "tensor(2.4952, grad_fn=<NegBackward0>)\n",
      "tensor(2.4942, grad_fn=<NegBackward0>)\n",
      "tensor(2.4934, grad_fn=<NegBackward0>)\n",
      "tensor(2.4925, grad_fn=<NegBackward0>)\n",
      "tensor(2.4917, grad_fn=<NegBackward0>)\n",
      "tensor(2.4909, grad_fn=<NegBackward0>)\n",
      "tensor(2.4902, grad_fn=<NegBackward0>)\n",
      "tensor(2.4894, grad_fn=<NegBackward0>)\n",
      "tensor(2.4887, grad_fn=<NegBackward0>)\n",
      "tensor(2.4880, grad_fn=<NegBackward0>)\n",
      "tensor(2.4874, grad_fn=<NegBackward0>)\n",
      "tensor(2.4868, grad_fn=<NegBackward0>)\n",
      "tensor(2.4862, grad_fn=<NegBackward0>)\n",
      "tensor(2.4856, grad_fn=<NegBackward0>)\n",
      "tensor(2.4850, grad_fn=<NegBackward0>)\n",
      "tensor(2.4845, grad_fn=<NegBackward0>)\n",
      "tensor(2.4839, grad_fn=<NegBackward0>)\n",
      "tensor(2.4834, grad_fn=<NegBackward0>)\n",
      "tensor(2.4829, grad_fn=<NegBackward0>)\n",
      "tensor(2.4824, grad_fn=<NegBackward0>)\n",
      "tensor(2.4820, grad_fn=<NegBackward0>)\n",
      "tensor(2.4815, grad_fn=<NegBackward0>)\n",
      "tensor(2.4811, grad_fn=<NegBackward0>)\n",
      "tensor(2.4806, grad_fn=<NegBackward0>)\n",
      "tensor(2.4802, grad_fn=<NegBackward0>)\n",
      "tensor(2.4798, grad_fn=<NegBackward0>)\n",
      "tensor(2.4794, grad_fn=<NegBackward0>)\n",
      "tensor(2.4790, grad_fn=<NegBackward0>)\n",
      "tensor(2.4787, grad_fn=<NegBackward0>)\n",
      "tensor(2.4783, grad_fn=<NegBackward0>)\n",
      "tensor(2.4779, grad_fn=<NegBackward0>)\n",
      "tensor(2.4776, grad_fn=<NegBackward0>)\n",
      "tensor(2.4773, grad_fn=<NegBackward0>)\n",
      "tensor(2.4769, grad_fn=<NegBackward0>)\n",
      "tensor(2.4766, grad_fn=<NegBackward0>)\n",
      "tensor(2.4763, grad_fn=<NegBackward0>)\n",
      "tensor(2.4760, grad_fn=<NegBackward0>)\n",
      "tensor(2.4757, grad_fn=<NegBackward0>)\n",
      "tensor(2.4754, grad_fn=<NegBackward0>)\n",
      "tensor(2.4751, grad_fn=<NegBackward0>)\n",
      "tensor(2.4749, grad_fn=<NegBackward0>)\n",
      "tensor(2.4746, grad_fn=<NegBackward0>)\n",
      "tensor(2.4743, grad_fn=<NegBackward0>)\n",
      "tensor(2.4741, grad_fn=<NegBackward0>)\n",
      "tensor(2.4738, grad_fn=<NegBackward0>)\n",
      "tensor(2.4736, grad_fn=<NegBackward0>)\n",
      "tensor(2.4733, grad_fn=<NegBackward0>)\n",
      "tensor(2.4731, grad_fn=<NegBackward0>)\n",
      "tensor(2.4729, grad_fn=<NegBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# gradient descent\n",
    "for k in range(100):\n",
    "    # forward pass\n",
    "    xenc = F.one_hot(xs, num_classes=27).float()\n",
    "    logits = xenc @ W # predict log counts of outputs by matmuling xenc and weights\n",
    "    counts = logits.exp() # exponentiate log counts, to simualate real counts\n",
    "    probs = counts / counts.sum(1, keepdims=True) # normalize to make them probabilities\n",
    "    loss =  -probs[torch.arange(num), ys].log().mean()\n",
    "    print(loss)\n",
    "    \n",
    "    # backward pass\n",
    "    W.grad = None # set to 0\n",
    "    loss.backward() # fills in gradients of all intermediates all the way back to W    \n",
    "\n",
    "    # modify weights\n",
    "    W.data += -50 * W.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "e532ed2b-0e0e-41c0-a3de-167601fc88be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we expect to get something around 2.47 (log_likelihood of bigram with counting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "070788c6-3903-4205-997f-aab5203fa205",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0013, 0.1376, 0.0407, 0.0481, 0.0527, 0.0477, 0.0129, 0.0208, 0.0272,\n",
       "        0.0184, 0.0755, 0.0924, 0.0490, 0.0792, 0.0357, 0.0122, 0.0160, 0.0028,\n",
       "        0.0511, 0.0641, 0.0408, 0.0028, 0.0117, 0.0095, 0.0042, 0.0166, 0.0289],\n",
       "       grad_fn=<SelectBackward0>)"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make some names with probs\n",
    "probs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "4c2d6075-f939-4882-8120-d9e35c86cb16",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "m.\n",
      "dv.\n",
      "baraetla.\n",
      "az.\n",
      "arhlertriiaead.\n",
      "yra.\n",
      "fbalqao.\n",
      "meyjovianesdbeanietd.\n",
      "kv.\n",
      "etrslteceonikviiialastwohtmybaaniaemrtelwwlsxyk.\n",
      "k.\n",
      "dlsekya.\n",
      "ljagejnemflanekmynaylrla.\n",
      "jvyeb.\n",
      "dljnimhyjsru.\n",
      "oziezhksiam.\n",
      "nieadiam.\n",
      "roz.\n",
      "am.\n",
      "qlkdmralja.\n"
     ]
    }
   ],
   "source": [
    "for i in range(20):\n",
    "    out = []\n",
    "    index = 0\n",
    "    while True:\n",
    "        index = torch.multinomial(probs[index], num_samples=1, replacement=True, generator=g).item()\n",
    "        out.append(itos[index])\n",
    "        if index == 0:\n",
    "            break\n",
    "    print(\"\".join(out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cdc63e2-c90d-4d20-8e53-a54b21791738",
   "metadata": {},
   "outputs": [],
   "source": [
    "# still not great if using only 2 characters"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}