train_lr.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "from scipy.special import expit\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data/train_lr.csv\")\n",
    "valid = pd.read_csv(\"data/valid_lr.csv\")\n",
    "test = pd.read_csv(\"data/test_lr.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.array(train.iloc[:, :10000])\n",
    "y_train = np.array(train['target'])\n",
    "X_valid = np.array(valid.iloc[:, :10000])\n",
    "y_valid = np.array(valid['target'])\n",
    "X_test = np.array(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### batch gradient descent "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_BGD = []\n",
    "loss_BGD_valid = []\n",
    "c_reg = 0.01\n",
    "stepsize = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = np.zeros(10000)\n",
    "s = expit(np.dot(X_train, w))\n",
    "J = -np.dot(y_train.T, np.log(s)) - np.dot((1-y_train).T, np.log(1-s)) + c_reg * np.dot(w.T, w)\n",
    "s_v = expit(np.dot(X_valid, w))\n",
    "J_v = -np.dot(y_valid.T, np.log(s_v)) - np.dot((1-y_valid).T, np.log(1-s_v)) + c_reg * np.dot(w.T, w)\n",
    "loss_BGD.append(J)\n",
    "loss_BGD_valid.append(J_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "for epoch in range(1000):\n",
    "    w = w + stepsize * np.dot(X_train.T, (y_train - s)) - 2*stepsize*c_reg*w\n",
    "    s = expit(np.dot(X_train, w))\n",
    "    s_v = expit(np.dot(X_valid, w))\n",
    "    J = -np.dot(y_train.T, np.log(s+1e-30)) - np.dot((1-y_train).T, np.log(1-s+1e-30)) + c_reg * np.dot(w.T, w)\n",
    "    J_v = -np.dot(y_valid.T, np.log(s_v+1e-30)) - np.dot((1-y_valid).T, np.log(1-s_v+1e-30)) + c_reg * np.dot(w.T, w)\n",
    "    loss_BGD.append(J)\n",
    "    loss_BGD_valid.append(J_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEGCAYAAACJnEVTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAoQElEQVR4nO3de3wV1b338c8vCRdRhAio3ARtqYRrIBFQi2JVRNrj/YKXileq1SpHH0+R+oi2R4unrUUfFav12oOi1aIexUNRoWhbkYAICCpIQcIdBASRS5Lf88fMhk2uO8lOdrLn+3699isza9bMXrMHfmtmrTUz5u6IiEg0ZKS6ACIiUn8U9EVEIkRBX0QkQhT0RUQiREFfRCRCslJdgMq0bdvWu3btmupiiIg0KnPnzt3k7u3KW9agg37Xrl0pKChIdTFERBoVM1tZ0TI174iIRIiCvohIhCjoi4hESINu0xeR9LN3714KCwvZtWtXqovS6DVv3pxOnTrRpEmThNdR0BeRelVYWEjLli3p2rUrZpbq4jRa7s7mzZspLCzk6KOPTng9Ne+ISL3atWsXbdq0UcCvJTOjTZs21b5iStugP3flFh6ZsYy5K7ekuigiUooCfnLU5HdMy+adv322gWueLaDEnaZZGUy6dhB5XbJTXSwRkZRLyzP9eV9uoajEKXHYs7eED5ZvTnWRRKSB2Lp1K48++miN1h0+fDhbt25NboGqMGHCBHbu3Jm07aVl0D/i0IP2TZcA2S2apq4wItKgVBb0i4qKKl136tSptG7dug5KVTEF/QQsWrPtgPkZn21IUUlEJBmS2Uc3ZswYvvjiC3Jzc7n99tuZOXMmgwcP5qyzzqJHjx4AnHPOOeTl5dGzZ08ef/zxfet27dqVTZs2sWLFCnJycrjuuuvo2bMnQ4cO5dtvvy3zXevXr+fcc8+lb9++9O3bl3/84x8APPDAA/Tq1YtevXoxYcIEAL755ht++MMf0rdvX3r16sWLL77IQw89xJo1azjllFM45ZRTar3vkKZt+qW7Nt5Zsp65K7eoXV+kgbnnfz5h8ZqvK82zfddePl23nRKHDIPuR7akZfOKx6X36HAo4/6tZ4XLx48fz6JFi5g/fz4AM2fOZN68eSxatGjf0MennnqKww47jG+//ZbjjjuO888/nzZt2hywnaVLl/LCCy/wxBNPcNFFF/HKK69w+eWXH5Dn5ptv5uSTT2bKlCkUFxezY8cO5s6dy9NPP83s2bNxdwYOHMjJJ5/M8uXL6dChA2+++SYA27Zto1WrVjzwwAPMmDGDtm3bVvo7JSotz/TP69/pgMBf4vDKvMKUlUdEau7rXUWUhK/yLvFgPtkGDBhwwFj3hx56iL59+zJo0CBWrVrF0qVLy6xz9NFHk5ubC0BeXh4rVqwok+fdd9/lhhtuACAzM5NWrVrx/vvvc+6553LwwQdzyCGHcN555/Hee+/Ru3dvpk+fzs9//nPee+89WrVqlfT9hDQ908/rks3pPY7gr4vX70vTADGRhqeyM/KYuSu3cNkfP2BvUQlNsjJ4cES/pF+1H3zwwfumZ86cydtvv80///lPWrRowZAhQ8odC9+sWbN905mZmeU271TH9773PebNm8fUqVO58847OfXUU7nrrrtqtc3ypOWZPsCQYw8/YL5nh7qpNUWkbuV1yWbStYO4deixSRl+3bJlS7Zv317h8m3btpGdnU2LFi349NNP+eCDD2r8XaeeeioTJ04EoLi4mG3btjF48GBeffVVdu7cyTfffMOUKVMYPHgwa9asoUWLFlx++eXcfvvtzJs3L6HyVlfaBn115oqkj7wu2dx4yneTcobfpk0bTjzxRHr16sXtt99eZvmwYcMoKioiJyeHMWPGMGjQoBp/14MPPsiMGTPo3bs3eXl5LF68mP79+3PllVcyYMAABg4cyLXXXku/fv1YuHAhAwYMIDc3l3vuuYc777wTgFGjRjFs2LCkdeSauydlQ3UhPz/fa/oSlV9MWcik2V/um88w+PP1J6gzVyTFlixZQk5OTqqLkTbK+z3NbK6755eXP23P9M/r34mMuIZ8deaKiFQj6JtZppl9ZGZvhPNHm9lsM1tmZi+aWdMwvVk4vyxc3jVuG3eE6Z+Z2RlJ35s4eV2yyS91Vr9sffLaxUREGqPqnOnfAiyJm78f+L27fxfYAlwTpl8DbAnTfx/mw8x6ACOAnsAw4FEzy6xd8SvX7YiWB8zPWbFFD2ATkUhLKOibWSfgh8Afw3kDfgC8HGZ5FjgnnD47nCdcfmqY/2xgsrvvdvd/AcuAAUnYhwqVHq/vwPi3llSUXUQk7SV6pj8B+A+CR9kAtAG2unvsLolCoGM43RFYBRAu3xbm35dezjr7mNkoMysws4KNGzcmviflyOuSzXcOP+SANJ3ti0iUVRn0zexHwAZ3n1sP5cHdH3f3fHfPb9euXa23d/WJZd8oow5dEYmqRM70TwTOMrMVwGSCZp0HgdZmFrujtxOwOpxeDXQGCJe3AjbHp5ezTp25dOBR5Bx5YNv+RzrTF5FqOOSQoMVgzZo1XHDBBeXmGTJkCDUdYl6Z2jwKujxVBn13v8PdO7l7V4KO2Hfd/TJgBhDb+5HAa+H06+E84fJ3PbgZ4HVgRDi652igG/Bh0vakEv1LjeJZsm47z8eN4RcRSUSHDh14+eWXq86YRPUe9Cvxc+BWM1tG0Gb/ZJj+JNAmTL8VGAPg7p8ALwGLgf8FbnT34lp8f8LO69+pTNpT7y+vj68WkWRY9SG897vgby2NGTOGRx55ZN/83XffzW9/+1t27NjBqaeeSv/+/enduzevvfZamXVXrFhBr169APj2228ZMWIEOTk5nHvuuRU+e2fOnDmccMIJ9O3blwEDBrB9+3Z27drFVVddRe/evenXrx8zZswA4JNPPtl3V26fPn1YunRpmUdB11a1Hrjm7jOBmeH0csoZfePuu4ALK1j/XuDe6haytvK6ZJNzZEuWrNs/Tv+Ljd/occsiqfbWGFi3sPI8u7+G9YvAS8Ay4Ihe0OzQivMf2RvOHF/h4osvvpjRo0dz4403AvDSSy8xbdo0mjdvzpQpUzj00EPZtGkTgwYN4qyzzqrwPbQTJ06kRYsWLFmyhAULFtC/f/8yefbs2cPFF1/Miy++yHHHHcfXX3/NQQcdxIMPPoiZsXDhQj799FOGDh3K559/zmOPPcYtt9zCZZddxp49eyguLi7zKOjaSts7cksr3cTjwGN/+yI1hRGRxO3aFgR8CP7u2lZ5/ir069ePDRs2sGbNGj7++GOys7Pp3Lkz7s7YsWPp06cPp512GqtXr2b9+vUVbmfWrFn7np/fp08f+vTpUybPZ599Rvv27TnuuOMAOPTQQ8nKyuL999/ft2737t3p0qULn3/+Occffzz33Xcf999/PytXruSggw4qs83aSstHK5fnvP6deH72l8Q/aWj6Yr1cRSSlKjkj32fVh/DsWVC8BzKbwvl/hM61u8Xnwgsv5OWXX2bdunVcfPHFAEyaNImNGzcyd+5cmjRpQteuXct9pHJduvTSSxk4cCBvvvkmw4cP5w9/+APHHHNMUr8jMmf6sWfsl3bnlCouLUUktToPgJGvww9+EfytZcCHoIln8uTJvPzyy1x4YdAavW3bNg4//HCaNGnCjBkzWLlyZaXbOOmkk3j++ecBWLRoEQsWLCiT59hjj2Xt2rXMmTMHgO3bt1NUVMTgwYOZNGkSAJ9//jlffvklxx57LMuXL+eYY47h5ptv5uyzz2bBggV6tHJt/OTk75R5mYpG8og0Ap0HwODbkhLwAXr27Mn27dvp2LEj7du3B+Cyyy6joKCA3r1789xzz9G9e/dKt3HDDTewY8cOcnJyuOuuu8jLyyuTp2nTprz44ov87Gc/o2/fvpx++uns2rWLn/70p5SUlNC7d28uvvhinnnmGZo1a8ZLL71Er169yM3NZdGiRVxxxRVVPgq6utL20coVGT91CY/NOnDkznfbHczbtw1J6veISPn0aOXk0qOVqzBmeA5d27Q4IG1ZOJJHRCTdRS7oA5z43bJvlb/1xfn1XxARkXoWyaBf+umbACu/2skVT85OSXlEoqYhNys3JjX5HSMZ9PO6ZPOTk8oOg5q1dBPjp+rRyyJ1qXnz5mzevFmBv5bcnc2bN9O8efNqrReZcfqljRmew+K1XzNr6aYD0h+btZzTex6psfsidaRTp04UFhZS20enS1CBdupU9jEzlYnc6J3ShvxmBis27zwgLefIlrw1+qQ6/V4Rkbqi0TuV+N1FuWXSlqzbrmYeEUlLkQ/6eV2yGdC1bFPOY7OW66YtEUk7kQ/6AD8/M6fMaB6AsVMWavy+iKQVBX2Cs/17z+1d7rIb/7te3hIpIlIvFPRDlw48iuvLGca5bvtuznn4/RSUSEQk+RT044wZnsNJ3crerTu/cJsCv4ikBQX9Up67ZiC5nVqVSVfgF5F0oKBfjldv+n6FgV+PahCRxqzKoG9mzc3sQzP72Mw+MbN7wvRnzOxfZjY//OSG6WZmD5nZMjNbYGb947Y10syWhp+RdbZXSVBR4J+1dJMCv4g0Womc6e8GfuDufYFcYJiZDQqX3e7uueFnfph2JtAt/IwCJgKY2WHAOGAgwQvVx5lZg37Wwas3fZ/2hzYrkz5r6SY19YhIo1Rl0PfAjnC2Sfip7NkNZwPPhet9ALQ2s/bAGcB0d//K3bcA04FhtSt+3Xv4srJvwwG18YtI45RQm76ZZZrZfGADQeCOtW/cGzbh/N7MYqfEHYFVcasXhmkVpZf+rlFmVmBmBQ3hgUx5XbK5r4Ix/PMLt3H8fW/rBi4RaTQSCvruXuzuuUAnYICZ9QLuALoDxwGHAT9PRoHc/XF3z3f3/Hbt2iVjk7V26cCjeOWGEzikaWaZZWu/3s35E/+hZ/WISKNQrdE77r4VmAEMc/e1YRPObuBpgnZ6gNVA57jVOoVpFaU3Cnldsnn2moEVLn9s1nJGT/6oHkskIlJ9iYzeaWdmrcPpg4DTgU/DdnrMzIBzgEXhKq8DV4SjeAYB29x9LTANGGpm2WEH7tAwrdHI65LNKzecwJEty3buArw6f41G9ohIg5bImX57YIaZLQDmELTpvwFMMrOFwEKgLfCfYf6pwHJgGfAE8FMAd/8K+FW4jTnAL8O0RiWvSzYf/OK0cu/chWBkz+m/m1m/hRIRSVDkX6JSG6Mnf8Sr89eUu6xFkwzu/FFPLh14VD2XSkSiTi9RqSMTRvQr9yFtADv3ljB2ykKG/GaGRveISIOhoF9LY4bn8MoNJ3BYiyblLl+xeadG94hIg6GgnwR5XbKZd9dQurU7uMI8j81arrN+EUk5Bf0kmn7bEM7J7VDhcp31i0iqKegn2YQR/Sod1gnBWX/uL/+qd/CKSL1T0K8DsWGdlZ31b925l7FTFur5PSJSrxT061AiZ/3zC7fRbexUNfmISL1Q0K9jiZz17y1xHpu1XMFfROqcgn49iZ315xzZssI8seD/vV8o+ItI3dAduSkwd+UWrnt2Dl/t3FtpPt3VKyI1oTtyG5jYuP7rTzqGTKs4X+yu3u7/9y2d+YtIUuhMvwEYPfkjXp+/hpIq8jXNNK4+8WjGDM+pl3KJSONU2Zm+gn4DMn7qEp54bznFVRySDKB7+5b86pze5HVp0K8ZFpEUUNBvZK54cjazlm5KKO/BTTP58aAuOvsXkX3Upt/IPHfNwCpH+sR8s6dYI35EJGE602/g5q7cwvi3ljBv5ZYqm30ADGjXsimjTztWo35EIkrNO2li/NQlPPn+cvZW1eMbysowftSnPRNG9KvbgolIg6Kgn2aen/0lD0z/jE079iSU34CWzbO4dMBRavsXiQAF/TRV3aYfCDpxOrRuzk9P6abmH5E0paAfAeOnLuHpv/+L3YlGfyDToH0rVQAi6aZWQd/MmgOzgGZAFvCyu48zs6OByUAbYC7wY3ffY2bNgOeAPGAzcLG7rwi3dQdwDVAM3Ozu0yr7bgX96oud/c9dsaXKm73iZQBt1QEskhZqG/QNONjdd5hZE+B94BbgVuAv7j7ZzB4DPnb3iWb2U6CPu19vZiOAc939YjPrAbwADAA6AG8D33P34oq+W0G/dsZPXcKk2SvZvrvCn7hcqgBEGrekNe+YWQuCoH8D8CZwpLsXmdnxwN3ufoaZTQun/2lmWcA6oB0wBsDdfx1ua1++ir5PQT95Rk/+iDcWrKGoOqf/BBVAk6wMzux1pEYBiTQSlQX9rAQ3kEnQhPNd4BHgC2CruxeFWQqBjuF0R2AVQFghbCNoAuoIfBC32fh14r9rFDAK4KijdJaZLBNG9GPCiH7MXbmFO6cs5PP12xPq/C0BdheV8Or8Nbw6fw1NMo1uhx+iR0CINFIJBf2wCSbXzFoDU4DudVUgd38ceByCM/26+p6oyuuSzVujTwL2t//P/3JLwmP/9xY7i9du5/yJ/yDTICtTVwEijUlCQT/G3bea2QzgeKC1mWWFZ/udgNVhttVAZ6AwbN5pRdChG0uPiV9HUiCvSzZ/vv4EoGYVQLFDcdxVQGYGtDlYfQEiDVkiHbntgL1hwD8I+CtwPzASeCWuI3eBuz9qZjcCveM6cs9z94vMrCfwPPs7ct8Buqkjt+GJVQCfrN7GzkRrgFIyAFMlIJIStR290wd4Fsgk+L/8krv/0syOIRiyeRjwEXC5u+8Oh3j+CegHfAWMcPfl4bZ+AVwNFAGj3f2tyr5bQb9hiI0C2rG7mJq2txnQJNM4vGUz3RcgUsd0c5YkzfOzv+SRGUvZtGMPu6s7FCiOAZkZ0KKpHg8hkmwK+lJnYlcB3+wppqQW/5QMyMo0mmRm0KvDofz8zByNDhKpIQV9qRfxfQG7ikpqVQkAZGWAmXFQk0xdDYhUg4K+pESsEliwait7ir3G/QExBphB08wM+nRqpasBkQoo6EuDEF8J7C3xWl8JwP6+gcyMDNod0lSdxCIo6EsDNnryR0xduJYSd4pLqPXVAARPD8WCaQ0ZlShS0JdGI9YxHPQJBBVBMsSuCNRHIFGgoC+NVnzn8O7ikqRVAhBUBBkZ4B68WlL3EEi6UNCXtBK7V2DD9t0Ulzg41Xp3QFViVwUOZJgeMCeNj4K+pL34m8aCZqHkdBTHy7RYRaCOY2nYFPQlkkqPFsowqv0+gUTEOo7dgwpBdxlLqinoi8SJHzHkQEmSRg2VFrvLuMRd/QZSrxT0RapQ3lVBsoaQlicDyFSFIHVEQV+khuL7CopKSnAn6R3HpZWuENRkJNWloC+SZKU7jmMBOtmdx6WVbjJSp7KUR0FfpB7F9xmYBQG6rvoNSivdqaxhp9GkoC/SAMReSr90w3aAeq8Q4MBhp45GHKUrBX2RBq6iCqE+moxKy8o4sEIwMzLM1ITUiCjoizRy5TUZ1UencnkyAGx/haARSA2Pgr5IGquoU7muh51WprwRSLGrB70Poe7V9sXonYHngCMIjtvj7v6gmd0NXAdsDLOOdfep4Tp3ANcAxcDN7j4tTB8GPEjwkvU/uvv4yr5bQV+k9sobdhofhN1TUzHAgc85im9OKnFXB3Qt1Dbotwfau/s8M2sJzAXOAS4Cdrj7b0vl7wG8AAwAOgBvA98LF38OnA4UAnOAS9x9cUXfraAvUj/GT13Cc/9cwa6ikjKdvCUp6FcoLf7R2KWvHkDvTSgtqc07ZvYa8DBwIuUH/TsA3P3X4fw04O5w8d3ufkZ5+cqjoC/SMFR0tZCKEUiVib1Ss3TFFSsnRKOCSFrQN7OuwCygF3ArcCXwNVAA3ObuW8zsYeADd//vcJ0ngbfCTQxz92vD9B8DA939plLfMQoYBXDUUUflrVy5MuHyiUjqVDYCKRaEk/k+hGTIquDqITbdNCuTXh0ObXT9D0kJ+mZ2CPA34F53/4uZHQFsIviNfkXQBHR1bYN+PJ3pi6Sf0u9DKO+svKjYG8SVQ7zSN76VV1lkZWRwZq8jmTCiX0rLWlnQz0pwA02AV4BJ7v4XAHdfH7f8CeCNcHY10Dlu9U5hGpWki0hEXDrwqISaVsqrHEoH2vrsbygOh8hC7Dv3f3GsDMUlJbw6fw2vzV+z761sFTU1pWqYayIduQY8C3zl7qPj0tu7+9pw+t8JztpHmFlP4Hn2d+S+A3QjaG77HDiVINjPAS51908q+m6d6YtIVUq/UrOiQFsXL9ZJpvjnKkHt+h5qe6Z/IvBjYKGZzQ/TxgKXmFkuwW+7AvgJgLt/YmYvAYuBIuBGdy8OC3ITMI1gyOZTlQV8EZFE5HXJ5s/Xn5BQ3vIeoV1Rm3593/jmwN7i/bXShu17GDtlIUBSrwJ0c5aISAWen/0lD0z/jC3f7MEqaaapq7eyAQzu1pY/XTOwWuvUuk1fRCSKEu1/iBk/dQmTZq9kV1FJhSOCqjvM9cxe7Wtc/vLoTF9EJEUqGuYKqW3TFxGROpDXJZu3Rp9Ur9+ZUa/fJiIiKaWgLyISIQr6IiIRoqAvIhIhCvoiIhGioC8iEiEK+iIiEaKgLyISIQr6IiIRoqAvIhIhCvoiIhGioC8iEiEK+iIiEaKgLyISIQr6IiIRoqAvIhIhCvoiIhFSZdA3s85mNsPMFpvZJ2Z2S5h+mJlNN7Ol4d/sMN3M7CEzW2ZmC8ysf9y2Rob5l5rZyLrbLRERKU8iZ/pFwG3u3gMYBNxoZj2AMcA77t4NeCecBzgT6BZ+RgETIagkgHHAQGAAMC5WUYiISP2oMui7+1p3nxdObweWAB2Bs4Fnw2zPAueE02cDz3ngA6C1mbUHzgCmu/tX7r4FmA4MS+bOiIhI5arVpm9mXYF+wGzgCHdfGy5aBxwRTncEVsWtVhimVZRe+jtGmVmBmRVs3LixOsUTEZEqJBz0zewQ4BVgtLt/Hb/M3R3wZBTI3R9393x3z2/Xrl0yNikiIqGEgr6ZNSEI+JPc/S9h8vqw2Ybw74YwfTXQOW71TmFaRekiIlJPEhm9Y8CTwBJ3fyBu0etAbATOSOC1uPQrwlE8g4BtYTPQNGComWWHHbhDwzQREaknWQnkORH4MbDQzOaHaWOB8cBLZnYNsBK4KFw2FRgOLAN2AlcBuPtXZvYrYE6Y75fu/lUydkJERBJjQXN8w5Sfn+8FBQWpLoaISKNiZnPdPb+8ZbojV0QkQhT0RUQiREFfRCRCFPRFRCJEQV9EJEIU9EVEIkRBX0QkQhT0RUQiREFfRCRCFPRFRCJEQV9EJEIU9EVEIkRBX0QkQhT0RUQiREFfRCRCFPRFRCJEQV9EJEIU9EVEIkRBX0QkQqoM+mb2lJltMLNFcWl3m9lqM5sffobHLbvDzJaZ2WdmdkZc+rAwbZmZjUn+roiISFUSOdN/BhhWTvrv3T03/EwFMLMewAigZ7jOo2aWaWaZwCPAmUAP4JIwr4iI1KOsqjK4+ywz65rg9s4GJrv7buBfZrYMGBAuW+buywHMbHKYd3H1iywiIjVVmzb9m8xsQdj8kx2mdQRWxeUpDNMqSi/DzEaZWYGZFWzcuLEWxRMRkdJqGvQnAt8BcoG1wO+SVSB3f9zd8909v127dsnarIiIkEDzTnncfX1s2syeAN4IZ1cDneOydgrTqCRdRETqSY3O9M2sfdzsuUBsZM/rwAgza2ZmRwPdgA+BOUA3MzvazJoSdPa+XvNii4hITVR5pm9mLwBDgLZmVgiMA4aYWS7gwArgJwDu/omZvUTQQVsE3OjuxeF2bgKmAZnAU+7+SbJ3RkREKmfunuoyVCg/P98LCgpSXQwRkUbFzOa6e355y3RHrohIhCjoi4hEiIK+iEiEKOiLiESIgr6ISIQo6IuIRIiCvohIhCjoi4hEiIK+iEiEKOiLiESIgr6ISIQo6IuIRIiCvohIhCjoi4hEiIK+iEiEKOiLiESIgr6ISIQo6IuIREj6Bv3/GQ0P9oPp41JdEhGRBiM9g/5rP4O5T8OW5fD3CfDwgFSXSESkQagy6JvZU2a2wcwWxaUdZmbTzWxp+Dc7TDcze8jMlpnZAjPrH7fOyDD/UjMbWTe7E1rx/oHzmz6D//oOrPqwTr9WRKShS+RM/xlgWKm0McA77t4NeCecBzgT6BZ+RgETIagkgHHAQGAAMC5WUdSJHmeVTdu5CZ48HSZ+X8FfRCKryqDv7rOAr0olnw08G04/C5wTl/6cBz4AWptZe+AMYLq7f+XuW4DplK1Ikuf0e6DtseUvW79QwV9EIqumbfpHuPvacHodcEQ43RFYFZevMEyrKL0MMxtlZgVmVrBx48YaFg+46cOKAz/sD/7/eYQ6e0UkMmrdkevuDngSyhLb3uPunu/u+e3atavdxm76EI75QeV5inYFnb13t4ZftdUVgIiktZoG/fVhsw3h3w1h+mqgc1y+TmFaRel174op8KMHoWnLKjI6FO/dfwVwdzb8qh1M6A0Fz9RHSUVE6lxNg/7rQGwEzkjgtbj0K8JRPIOAbWEz0DRgqJllhx24Q8O0+pF/JYwthBNHQ2bzBFcqgeI9sPVLeOOW4Ergl22CimD8UWoSEpFGyYLWmUoymL0ADAHaAusJRuG8CrwEHAWsBC5y96/MzICHCTppdwJXuXtBuJ2rgbHhZu9196erKlx+fr4XFBRUf6+qUvAMzLgPvtlA7VqmMiAjA7wELAOymkH7vnDaPdBZ9waISGqY2Vx3zy93WVVBP5XqLOjHmz4OCp6G3duBkuRtNyMr+BurEAAyMiHnbDj/ieR9j4hIKQr6iSp4Bt77HexYF7TvJ69/uhQDywTiKoRY5RD7HHI4fP+2oGlKRKQaFPRratWH8PY4KJwHJXsAAy+u50JkgLG/MvCSA68eYtMZWaooRARQ0E+uWEWw5mMo2bs/CCd35GothRUEJVVXFqWXNT0Y8q4KbnATkUZJQb++xPoHir4N5uODaUkxDadSSIQFVw+xCi2jisoi0WW6IhGpcwr6DUV8pVBRUHRPQRNSqtj+fbYMMDuwkqlpxaIKSSJOQb+xiXUof7NxfxNSZQErJX0N6S6+QrK43z6soPC4yipsSquvyqkhLjODtt3hRw9ouHIDoKAfBas+hDdvhQ2fsq8Zqbr/oT2JQ1Ylwizuyi1WOVqwqNxKNNFlpa4Gy1S+lS2L+3eekRH8F4lNN8RKFIcWbWHI2BpddVYW9LOqvTVpmDoPgOvfrzpfVV65Dha/Flw5JPsfta5IIiJsovQDkw6Yjv93kPCykrLzNVlWUlL+dHnrpXLZjvXB0wAgqc2NCvpyoPOfqNubx6pzRVKfZ1mqkKShWvKagr40Ysm6IqkLyWgii+Ky4iKSeje7HCjn7KRuTkFfJKYhV0gNXfzd7CVFDbNyakzLatmmXxkFfRGpvfwrNcy1kchIdQFERKT+KOiLiESIgr6ISIQo6IuIRIiCvohIhCjoi4hESIN+9o6ZbSR4B29NtQU2Jak4jUXU9jlq+wva56iozT53cfd25S1o0EG/tsysoKKHDqWrqO1z1PYXtM9RUVf7rOYdEZEIUdAXEYmQdA/6j6e6ACkQtX2O2v6C9jkq6mSf07pNX0REDpTuZ/oiIhJHQV9EJELSMuib2TAz+8zMlpnZmFSXJ1nMrLOZzTCzxWb2iZndEqYfZmbTzWxp+Dc7TDczeyj8HRaYWf/U7kHNmFmmmX1kZm+E80eb2exwv140s6ZherNwflm4vGtKC14LZtbazF42s0/NbImZHZ/Ox9nM/j38N73IzF4ws+bpeJzN7Ckz22Bmi+LSqn1czWxkmH+pmY2sThnSLuibWSbwCHAm0AO4xMx6pLZUSVME3ObuPYBBwI3hvo0B3nH3bsA74TwEv0G38DMKmFj/RU6KW4AlcfP3A7939+8CW4BrwvRrgC1h+u/DfI3Vg8D/unt3oC/B/qflcTazjsDNQL679wIygRGk53F+BhhWKq1ax9XMDgPGAQOBAcC4WEWREHdPqw9wPDAtbv4O4I5Ul6uO9vU14HTgM6B9mNYe+Cyc/gNwSVz+ffkaywfoFP5H+AHwBmAEdylmlT7ewDTg+HA6K8xnqd6HGuxzK+BfpcuerscZ6AisAg4Lj9sbwBnpepyBrsCimh5X4BLgD3HpB+Sr6pN2Z/rs/wcUUximpZXwkrYfMBs4wt3XhovWAUeE0+nwW0wA/oP9L2FtA2x196JwPn6f9u1vuHxbmL+xORrYCDwdNmv90cwOJk2Ps7uvBn4LfAmsJThuc0n/4xxT3eNaq+OdjkE/7ZnZIcArwGh3/zp+mQdVf1qMwzWzHwEb3H1uqstSz7KA/sBEd+8HfMP+S34g7Y5zNnA2QWXXATiYsk0gkVAfxzUdg/5qoHPcfKcwLS2YWROCgD/J3f8SJq83s/bh8vbAhjC9sf8WJwJnmdkKYDJBE8+DQGszi73fOX6f9u1vuLwVsLk+C5wkhUChu88O518mqATS9TifBvzL3Te6+17gLwTHPt2Pc0x1j2utjnc6Bv05QLew578pQYfQ6ykuU1KYmQFPAkvc/YG4Ra8DsR78kQRt/bH0K8JRAIOAbXGXkQ2eu9/h7p3cvSvBcXzX3S8DZgAXhNlK72/sd7ggzN/ozobdfR2wysyODZNOBRaTpseZoFlnkJm1CP+Nx/Y3rY9znOoe12nAUDPLDq+ShoZpiUl1p0YddZQMBz4HvgB+keryJHG/vk9w6bcAmB9+hhO0Z74DLAXeBg4L8xvBSKYvgIUEoyNSvh813PchwBvh9DHAh8Ay4M9AszC9eTi/LFx+TKrLXYv9zQUKwmP9KpCdzscZuAf4FFgE/Alolo7HGXiBoN9iL8EV3TU1Oa7A1eH+LwOuqk4Z9BgGEZEIScfmHRERqYCCvohIhCjoi4hEiIK+iEiEKOiLiESIgr40Smb2azM7xczOMbM7qrluu/DpjB+Z2eBSy/4Ye0CfmY1NcpmvNLMO5X2XSH3RkE1plMzsXeCHwH3Ay+7+92qsOwI4zd2vrSLfDnc/pJrlynT34gqWzQT+j7sXVGebIsmkM31pVMzsN2a2ADgO+CdwLTDRzO4qJ29XM3s3fBb5O2Z2lJnlAv8FnG1m883soFLrzDSzfDMbDxwU5pkULrvczD4M0/4QPsYbM9thZr8zs4+B483sLjObY8Gz4R8P76i8AMgHJsW+N/Zd4TYuMbOF4Tr3x5Vnh5nda2Yfm9kHZnZEmH5hmPdjM5uV9B9a0leq71DTR5/qfggC/v8DmgB/ryTf/wAjw+mrgVfD6SuBhytYZybhnY/Ajrj0nHB7TcL5R4ErwmkHLorLe1jc9J+Afyu97fh5goeMfQm0I3jY2rvAOXHbjq3/X8Cd4fRCoGM43TrVx0SfxvPRmb40Rv2Bj4HuHPhyldKOB54Pp/9E8BiLmjoVyAPmmNn8cP6YcFkxwUPwYk4J+wwWEjwkrmcV2z4OmOnBA8eKgEnASeGyPQTPl4fgccNdw+m/A8+Y2XUELx0RSUhW1VlEGoawaeYZgqcKbgJaBMk2n+ClGt/W5dcDz7p7eZ3Guzxsxzez5gRXAfnuvsrM7iZ4VkxN7XX3WMdbMeH/WXe/3swGEvRrzDWzPHdvzE+alHqiM31pNNx9vrvnEjxMrwdBM8gZ7p5bQcD/B8HTOQEuA96r5lfuDR9lDcEDsS4ws8Nh33tNu5SzTizAb7LgvQcXxC3bDrQsZ50PgZPNrG3YT3AJ8LfKCmZm33H32e5+F8ELVzpXll8kRmf60qiYWTuC96OWmFl3d19cSfafEbx96naCwHhVNb/ucWCBmc1z98vM7E7gr2aWQfCUxBuBlfEruPtWM3uC4GmR6wge9R3zDPCYmX1L0PQUW2etmY0heJSwAW+6+2tU7jdm1i3M/w5Bc5dIlTRkU0QkQtS8IyISIQr6IiIRoqAvIhIhCvoiIhGioC8iEiEK+iIiEaKgLyISIf8fGVcURhRR3hcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = range(1001)\n",
    "plt.plot(x, loss_BGD, label = 'train cost', marker = '.')\n",
    "plt.plot(x, loss_BGD_valid, label = 'valid cost', marker = '.')\n",
    "plt.xlabel('# of iterations')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2419.5892051945298\n",
      "753.7857512398791\n"
     ]
    }
   ],
   "source": [
    "print(loss_BGD[1000])\n",
    "print(loss_BGD_valid[1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 1, ..., 1, 1, 0])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = (np.dot(X_test, w) > 0.5) + 0\n",
    "pred "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = pd.read_csv(\"data/test.csv\")\n",
    "preds_df = pd.DataFrame({\"id\": test_data[\"id\"], \"target\": pred})\n",
    "preds_df.to_csv(f\"data/output_lr_2.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### stochastic gradient descent "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_SGD = []\n",
    "loss_SGD_valid = []\n",
    "c_reg = 0.001\n",
    "stepsize = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = np.zeros(10000)\n",
    "s = expit(np.dot(X_train, w))\n",
    "s_v = expit(np.dot(X_valid, w))\n",
    "J = -np.dot(y_train.T, np.log(s)) - np.dot((1-y_train).T, np.log(1-s)) + c_reg * np.dot(w.T, w)\n",
    "J_v = -np.dot(y_valid.T, np.log(s_v)) - np.dot((1-y_valid).T, np.log(1-s_v)) + c_reg * np.dot(w.T, w)\n",
    "loss_SGD.append(J)\n",
    "loss_SGD_valid.append(J_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(17)\n",
    "for epoch in range(10000): \n",
    "    k = np.random.randint(6090)\n",
    "    w = w + stepsize * X_train[k].T * (y_train[k] - s[k]) - 2*stepsize*c_reg*w\n",
    "    s = expit(np.dot(X_train, w))\n",
    "    J = -np.dot(y_train.T, np.log(s+1e-30)) - np.dot((1-y_train).T, np.log(1-s+1e-30)) + c_reg * np.dot(w.T, w)\n",
    "    s_v = expit(np.dot(X_valid, w))\n",
    "    J_v = -np.dot(y_valid.T, np.log(s_v+1e-30)) - np.dot((1-y_valid).T, np.log(1-s_v+1e-30)) + c_reg * np.dot(w.T, w)\n",
    "    loss_SGD.append(J)\n",
    "    loss_SGD_valid.append(J_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEGCAYAAACJnEVTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAmYklEQVR4nO3de5gV1Znv8e/b+9I3bg2SDAIGnDHKTRBawMkYNSaKJCM6o4Ixo3GSOJM4MTkz4wQTH6PJJEdnMkY9iRoyGjWP8TIkRkfN4zGKJzqJl8YgoqigooKo3On7vvR7/qi1m03TV9h0Q9fv8zz17KpVq2qv1QXv2rVqVZW5OyIiEg9lA10AERHpPwr6IiIxoqAvIhIjCvoiIjGioC8iEiPJgS5Adw455BCfMGHCQBdDROSgsnz58s3uPrqzdQd00J8wYQJ1dXUDXQwRkYOKmb3V1Tp174iIxIiCvohIjCjoi4jEiIK+iEiMKOiLiMSIgr6ISIwo6IuIxMgBPU5/b7Xm8lz/2zXUVKU5ZGiaimSCinSCoeVJaqrTDClPMqIqRXkyMdBFFRHpV4My6O9oyrLkd2+Qa+v+XQFDypNUphOMHlJOTXWKEVVpaqpSjKhMM6IqRU1VmprqFMMro/SaqjTDKlMkyqyfaiIiUlqDMuh/aFgFa753GjtbcmxuaCWTa6Mlm2dnS47tTRkaWnNsrs+wozlLY2uOTQ2tbGvKsHH7TrY1ReldtRdmMKwiRU1ViuFVaUZUphhakaSmKmoohldG05DyJEMrUgyrTDKsIsWwkC+VUI+aiAycQRn0AcysPQD3VVubU9+SY1tThu3N2eizKcP2pizbmrLsaMqwrWlX+ttbm9jWlGFnN41FQZnBsMoUqUQZwytTjAxnE9FZRZqRVWmGh7OKj4yq4rCRVVSk1A0lIqUxaIP+vigrM4ZXpRhe1bcGo63NqW/NsaMpS2MmR31Ljp3NWXa2ZMNnjkyujZ0tWTK5NnY0Z9namGHd5iaeb9rOtsZMp11SFakyKlMJUokyysN8RSpBdTrJkIokQ8qTVJcnqC5PMiSdpKo8ydDy5G5nGSOqUhwypFwNiEjMKeiXUFnZ3p9dALg7Da05tjdFjcEbmxvYsK2ZnS05mjN5Mrk2WnN5WrJttOTyNLbmeGdrE42ZHI2teRpao0alO0MrklSmEoysTvMnwys4ZEg5I6vT0VSVbm8oirumhlYkSapbSmRQUNA/gJgZQ0PAHT+yiunjR/R5H9l8G42tORpac+xozlLfEn1ub8qwuSHDpvpWmjN5tjRmeH9nC6++V8+WxkyPjUV1OsHwyuisYfTQ8ugaRXmSoRVJqtJJhoXGblhFkuGVKarLC2cZ0bUNXfwWOTAo6A8yqUQZI6rSjKhKM66md9u4O02ZPFsbM9S35KhvibqidnVNRWnbmrLsaM6yuaGVDduaqW+N0luy3TcYQOhuis4ahlWkqEgnqEiWUZFKUJEqaz+7GFqxe77C57DKqBtLZxwi+0ZBXzAzqsuTVJfv3T+HfJtHDUVzjp0tWbY3ZWkoOtsobjyis4+o8fggm6clm6c5m6e+JUdTJt/jd1WnEwytSDGkIsmo6nR7Q1FoHIpHUI0II6pGVEbXZ3RfhoiCvpRAoszazy72RTbfRkNLuADeEjUU9S27LojvOguJ5rc0ZtiwvZmG1mx7nu5GT1WkykIX1K5htoXGYXg4uxhRlWLUkPIw5DZJVTpBVTq6UK5GQwYDBX05YKQSZdRUR0NX90ZhqO3OcCaxvSnL9uZoqO2O5izbQvfVjuYs9a1ZNu5o4ZX36tnelKGxF2cZ6UQZQyuS7WdFRnTfRnU6jJQKDcqwcG2jsBxdCym6X6M8SZmuccgAUdCXQaN4qO34Pm6by7fR2JpnS2N0o17hDKM5k6cpkwvdVXnqW7LhQnkecNyhMZPj3e1RA7KzOUt9aw7v5ozDLLobfHh7IxFd7K5IJahKJagqTzCkPNl+hlGd3jUktyodnYEMKU9SniyjujypYbjSJ70O+maWAOqADe7+GTObCNwNjAKWA3/j7hkzKwfuAGYBW4CF7r4u7OMy4AtAHrjE3R8pZWVE9lYyUcbwqrI+35vRmcL9GsXXMgpnH4X7NYrv39jRnOWdrU3t1zcaW/M0ZrpvOIqlk2VUp6PupyGhSyqdKGu/r2NY+3WPFNXpBJXpBOXJMspT0WdFKkE6Gd3/Ee0ramQKDU86qYvng0lfful/DVgNDAvL1wA/dPe7zexmomB+U/jc5u5/ZmaLQr6FZjYZWARMAQ4FfmtmH3X3ns+rRQ4i+3q/BkQjqlqybTS05mgK92E0hjOOxtYcDS05WnPR+p0tWZpa87Tm8iF/dE9HLu/h5r/G9jOXTL7nkVYdpRNlVIUzjvJkGZXpBFXpBJXpJJXhZsHK0OhUphPRcip6yGFlGJ1VSCtPlbUP8a1MJfRokgHQq6BvZuOATwPfA/7RzAz4BPDZkOV24EqioL8gzAMsBX4U8i8A7nb3VuBNM1sLzAb+UJKaiAwiZhYF0HQCKC/ZfrP56DlUreF5VC3ZXTf8tWbztIb7PJrCzX6NrTkaM9GNgE2ZqGGJurzy7GjK8H62jeaiUVgt2TzZfC9PUYLCgw/LLLquk06WkS76bE/rkJ5OllGRKgz73XX2smsocNTglCcTu/IV5sNTd6PQFC+9/aV/HfAvwNCwPArY7u65sLweGBvmxwLvALh7zsx2hPxjgaeL9lm8TTszuwi4COCwww7rbT1EpBdSIYgO7TnrXis0LC3ZtvbGoDmzq1FozuSjs5NMNFR3e1OW5myOtrZo29Z8G9lcG5l8G5lcG9l8G01N0ZlNJh8tR3ent9Ea7k7vbVdYMTPaz0Daz1DCWUzUxRV1lSUTRkVy97ObqnSyfZuq8t33UWhwoken2AHXsPQY9M3sM8AH7r7czE7c3wVy9yXAEoDa2tq9OJQiMpDaG5aK/vk+dyeTb9t1tlJ0FtOSy+/WABXWFRqelmx01lJomJoy0ed7O1vaz3Jy+aiB6c19JB0ly6z90SeF7q6q9K5GoSKVIFlm0d8saaQTCSrTUXfY2JpKzjxmXMn/Xr35pf8x4HQzmw9UEPXpXw+MMLNk+LU/DtgQ8m8AxgPrzSwJDCe6oFtILyjeRkRkr5gZ5clwH8U+XEfpSeE6S3M2GtFVaCQaMzlaQ3pzJk9L6AJryeZpzOTb71pvzuZpCQ1MQ2sueiRKNk8u7+TbPDrLybXRlMnR5jDzsBEDE/Td/TLgMoDwS/+f3f08M/sv4CyiETwXAPeHTR4Iy38I6x93dzezB4BfmNm1RBdyjwCeLWltRET2k+LrLCP38l6S3nB3svmoEdgf9mWc/jeAu83sX4E/AreE9FuAn4cLtVuJRuzg7i+Z2b3Ay0AOuFgjd0REdmdmpJO234bKmu/NFZB+Ultb63V1dQNdDBGRg4qZLXf32s7WaYCsiEiMKOiLiMSIgr6ISIwo6IuIxIiCvohIjCjoi4jEiIK+iEiMKOiLiMSIgr6ISIwo6IuIxIiCvohIjCjoi4jEiIK+iEiMKOiLiMSIgr6ISIwo6IuIxIiCvohIjCjoi4jEiIK+iEiMKOiLiMSIgr6ISIz0GPTNrMLMnjWzF8zsJTO7KqTfZmZvmtmKMM0I6WZmN5jZWjNbaWYzi/Z1gZmtCdMF+61WIiLSqWQv8rQCn3D3BjNLAU+Z2W/CukvdfWmH/KcBR4RpDnATMMfMRgLfBmoBB5ab2QPuvq0UFRERkZ71+EvfIw1hMRUm72aTBcAdYbungRFmNgY4FXjU3beGQP8oMG/fii8iIn3Rqz59M0uY2QrgA6LA/UxY9b3QhfNDMysPaWOBd4o2Xx/Sukrv+F0XmVmdmdVt2rSpb7UREZFu9Srou3ve3WcA44DZZjYVuAw4CjgWGAl8oxQFcvcl7l7r7rWjR48uxS5FRCTo0+gdd98OLAPmufvG0IXTCvwMmB2ybQDGF202LqR1lS4iIv2kN6N3RpvZiDBfCXwKeCX002NmBpwBrAqbPACcH0bxzAV2uPtG4BHgFDOrMbMa4JSQJiIi/aQ3o3fGALebWYKokbjX3R80s8fNbDRgwArg70P+h4H5wFqgCbgQwN23mtl3gedCvu+4+9aS1URERHpk7t0NxBlYtbW1XldXN9DFEBE5qJjZcnev7Wyd7sgVEYkRBX0RkRhR0BcRiREFfRGRGFHQFxGJEQV9EZEYUdAXEYkRBX0RkRhR0BcRiREFfRGRGFHQFxGJEQV9EZEYUdAXEYkRBX0RkRhR0BcRiREFfRGRGFHQFxGJEQV9EZEYUdAXEYkRBX0RkRhR0BcRiREFfRGRGOkx6JtZhZk9a2YvmNlLZnZVSJ9oZs+Y2Vozu8fM0iG9PCyvDesnFO3rspD+qpmdut9qJSIinerNL/1W4BPuPh2YAcwzs7nANcAP3f3PgG3AF0L+LwDbQvoPQz7MbDKwCJgCzANuNLNECesiIiI96DHoe6QhLKbC5MAngKUh/XbgjDC/ICwT1p9sZhbS73b3Vnd/E1gLzC5FJUREpHd61advZgkzWwF8ADwKvA5sd/dcyLIeGBvmxwLvAIT1O4BRxemdbFP8XReZWZ2Z1W3atKnPFRIRka71Kui7e97dZwDjiH6dH7W/CuTuS9y91t1rR48evb++RkQklvo0esfdtwPLgOOAEWaWDKvGARvC/AZgPEBYPxzYUpzeyTYiItIPejN6Z7SZjQjzlcCngNVEwf+skO0C4P4w/0BYJqx/3N09pC8Ko3smAkcAz5aoHiIi0gvJnrMwBrg9jLQpA+519wfN7GXgbjP7V+CPwC0h/y3Az81sLbCVaMQO7v6Smd0LvAzkgIvdPV/a6oiISHcs+hF+YKqtrfW6urqBLoaIyEHFzJa7e21n63RHrohIjCjoi4jESG/69EVESiabzbJ+/XpaWloGuigHvYqKCsaNG0cqler1Ngr6ItKv1q9fz9ChQ5kwYQLRzfqyN9ydLVu2sH79eiZOnNjr7dS9IyL9qqWlhVGjRing7yMzY9SoUX0+Y1LQF5F+p4BfGnvzd1TQF5FY2b59OzfeeONebTt//ny2b99e2gL14LrrrqOpqalk+1PQF5FY6S7o53K5TtMLHn74YUaMGLEfStU1BX0RkX2wePFiXn/9dWbMmMGll17KE088wfHHH8/pp5/O5MmTATjjjDOYNWsWU6ZMYcmSJe3bTpgwgc2bN7Nu3TomTZrEl770JaZMmcIpp5xCc3PzHt/1/vvvc+aZZzJ9+nSmT5/O73//ewCuvfZapk6dytSpU7nuuusAaGxs5NOf/jTTp09n6tSp3HPPPdxwww28++67nHTSSZx00kklqb9G74jIgLnqv1/i5Xd3lnSfkw8dxrf/ckqX66+++mpWrVrFihUrAHjiiSd4/vnnWbVqVfsomFtvvZWRI0fS3NzMsccey1//9V8zatSo3fazZs0a7rrrLn76059yzjnn8Mtf/pLPfe5zu+W55JJLOOGEE7jvvvvI5/M0NDSwfPlyfvazn/HMM8/g7syZM4cTTjiBN954g0MPPZSHHnoIgB07djB8+HCuvfZali1bxiGHHFKSv49+6YtI7M2ePXu3YY833HAD06dPZ+7cubzzzjusWbNmj20mTpzIjBkzAJg1axbr1q3bI8/jjz/Ol7/8ZQASiQTDhw/nqaee4swzz6S6upohQ4bwV3/1Vzz55JNMmzaNRx99lG984xs8+eSTDB8+fL/UVb/0RWTAdPeLvD9VV1e3zz/xxBP89re/5Q9/+ANVVVWceOKJnQ6LLC8vb59PJBKddu/0xUc/+lGef/55Hn74YS6//HJOPvlkrrjiin3aZ2f0S19EYmXo0KHU19d3uX7Hjh3U1NRQVVXFK6+8wtNPP73X33XyySdz0003AZDP59mxYwfHH388v/71r2lqaqKxsZH77ruP448/nnfffZeqqio+97nPcemll/L888/3qrx9paAvIrEyatQoPvaxjzF16lQuvfTSPdbPmzePXC7HpEmTWLx4MXPnzt3r77r++utZtmwZ06ZNY9asWbz88svMnDmTz3/+88yePZs5c+bwxS9+kWOOOYYXX3yR2bNnM2PGDK666iouv/xyAC666CLmzZtXsgu5erSyiPSr1atXM2nSpIEuxqDR2d9Tj1YWERFAQV9EJFYU9EVEYkRBX0QkRhT0RURiREFfRCRGFPRFRHowZMgQAN59913OOuusTvOceOKJ7I8h5vvyKOjO9Bj0zWy8mS0zs5fN7CUz+1pIv9LMNpjZijDNL9rmMjNba2avmtmpRenzQtpaM1tcslqIiPSDQw89lKVLl/brd/Z70AdywD+5+2RgLnCxmU0O637o7jPC9DBAWLcImALMA240s4SZJYAfA6cBk4Fzi/YjItIvFi9ezI9//OP25SuvvJIf/OAHNDQ0cPLJJzNz5kymTZvG/fffv8e269atY+rUqQA0NzezaNEiJk2axJlnntnls3eee+45/vzP/5zp06cze/Zs6uvraWlp4cILL2TatGkcc8wxLFu2DICXXnqp/a7co48+mjVr1uzxKOh91eMD19x9I7AxzNeb2WpgbDebLADudvdW4E0zWwvMDuvWuvsbAGZ2d8j78j6UX0QOZr9ZDO+9WNp9/sk0OO3qLlcvXLiQr3/961x88cUA3HvvvTzyyCNUVFRw3333MWzYMDZv3szcuXM5/fTTu3wl4U033URVVRWrV69m5cqVzJw5c488mUyGhQsXcs8993Dssceyc+dOKisruf766zEzXnzxRV555RVOOeUUXnvtNW6++Wa+9rWvcd5555HJZMjn83s8Cnpf9alP38wmAMcAz4SkfzCzlWZ2q5nVhLSxwDtFm60PaV2ld/yOi8yszszqNm3a1JfiiYj06JhjjuGDDz7g3Xff5YUXXqCmpobx48fj7nzzm9/k6KOP5pOf/CQbNmzg/fff73I/v/vd79qfn3/00Udz9NFH75Hn1VdfZcyYMRx77LEADBs2jGQyyVNPPdW+7VFHHcVHPvIRXnvtNY477ji+//3vc8011/DWW29RWVlZ8vr3+tHKZjYE+CXwdXffaWY3Ad8FPHz+B/C3+1ogd18CLIHo2Tv7uj8ROYB184t8fzr77LNZunQp7733HgsXLgTgzjvvZNOmTSxfvpxUKsWECRM6faTy/vTZz36WOXPm8NBDDzF//nx+8pOfcPjhh5f0O3r1S9/MUkQB/053/xWAu7/v7nl3bwN+yq4unA3A+KLNx4W0rtJFRPrVwoULufvuu1m6dClnn302ED1S+UMf+hCpVIply5bx1ltvdbuPj3/84/ziF78AYNWqVaxcuXKPPEceeSQbN27kueeeA6C+vp5cLsfxxx/PnXfeCcBrr73G22+/zZFHHskbb7zB4YcfziWXXMKCBQtYuXJl/z9a2aIOrVuA1e5+bVH6mKJsZwKrwvwDwCIzKzezicARwLPAc8ARZjbRzNJEF3sfKE01RER6b8qUKdTX1zN27FjGjIlC2XnnnUddXR3Tpk3jjjvu4Kijjup2H1/+8pdpaGhg0qRJXHHFFcyaNWuPPOl0mnvuuYevfvWrTJ8+nU996lO0tLTwla98hba2NqZNm8bChQu57bbbKC8v595772Xq1KnMmDGDVatWcf755/f4KOi+6vHRymb2F8CTwItAW0j+JnAuMIOoe2cd8Hfhoi9m9i2irp4cUXfQb0L6fOA6IAHc6u7f6+679WhlkcFHj1Yurb4+Wrk3o3eeAjq7fP1wN9t8D9gjoIdhnV1uJyIi+5fuyBURiREFfRGRGFHQF5F+dyC/pvVgsjd/RwV9EelXFRUVbNmyRYF/H7k7W7ZsoaKiok/b9frmLBGRUhg3bhzr169Hd9zvu4qKCsaNG9enbRT0RaRfpVIpJk6cONDFiC1174iIxIiCvohIjCjoi4jEiIK+iEiMKOiLiMSIgr6ISIwo6IuIxIiCvohIjCjoi4jEiIK+iEiMKOiLiMSIgr6ISIwo6IuIxIiCvohIjCjoi4jESI9B38zGm9kyM3vZzF4ys6+F9JFm9qiZrQmfNSHdzOwGM1trZivNbGbRvi4I+deY2QX7r1oiItKZ3vzSzwH/5O6TgbnAxWY2GVgMPObuRwCPhWWA04AjwnQRcBNEjQTwbWAOMBv4dqGhEBGR/tFj0Hf3je7+fJivB1YDY4EFwO0h2+3AGWF+AXCHR54GRpjZGOBU4FF33+ru24BHgXmlrIyIiHSvT336ZjYBOAZ4Bviwu28Mq94DPhzmxwLvFG22PqR1ld7xOy4yszozq9M7NEVESqvXQd/MhgC/BL7u7juL13n0WvuSvNre3Ze4e627144ePboUuxQRkaBXQd/MUkQB/053/1VIfj902xA+PwjpG4DxRZuPC2ldpYuISD/pzegdA24BVrv7tUWrHgAKI3AuAO4vSj8/jOKZC+wI3UCPAKeYWU24gHtKSBMRkX6S7EWejwF/A7xoZitC2jeBq4F7zewLwFvAOWHdw8B8YC3QBFwI4O5bzey7wHMh33fcfWspKiEiIr1jUXf8gam2ttbr6uoGuhgiIgcVM1vu7rWdrdMduSIiMaKgLyISIwr6IiIxoqAvIhIjCvoiIjGioC8iEiMK+iIiMaKgLyISIwr6IiIxoqAvIhIjCvoiIjGioC8iEiMK+iIiMaKgLyISIwr6IiIxoqAvIhIjCvoiIjGioC8iEiMK+iIiMaKgLyISIwr6IiIxoqAvIhIjPQZ9M7vVzD4ws1VFaVea2QYzWxGm+UXrLjOztWb2qpmdWpQ+L6StNbPFpa+KiIj0pDe/9G8D5nWS/kN3nxGmhwHMbDKwCJgStrnRzBJmlgB+DJwGTAbODXlFRKQfJXvK4O6/M7MJvdzfAuBud28F3jSztcDssG6tu78BYGZ3h7wv973IIiKyt/alT/8fzGxl6P6pCWljgXeK8qwPaV2l78HMLjKzOjOr27Rp0z4UT0REOtrboH8T8KfADGAj8B+lKpC7L3H3WnevHT16dKl2KyIi9KJ7pzPu/n5h3sx+CjwYFjcA44uyjgtpdJMuIiL9ZK9+6ZvZmKLFM4HCyJ4HgEVmVm5mE4EjgGeB54AjzGyimaWJLvY+sPfFFhGRvdHjL30zuws4ETjEzNYD3wZONLMZgAPrgL8DcPeXzOxeogu0OeBid8+H/fwD8AiQAG5195dKXRkREemeuftAl6FLtbW1XldXN9DFEBE5qJjZcnev7Wyd7sgVEYkRBX0RkRhR0BcRiREFfRGRGFHQFxGJEQV9EZEYUdAXEYkRBX0RkRhR0BcRiREFfRGRGFHQFxGJEQV9EZEYUdAXEYkRBX0RkRhR0BcRiREFfRGRGFHQFxGJEQV9EZEYUdAXEYkRBX0RkRhJDnQB9otMEzxzM5QloCwJVgaWgLLwmUhBWWrX+kQ6pIX5ssTu69unsrC/Qnoi2nf7+kRYlwCzgf4riIjsYZAG/UZ47KqBLUN7Q1PcGJR1aISSRQ1PqugzFX0WzxcaFrOwfdFUvJ/C/osbovZ1he8obshSe5ahvdEryle8TXFddpuKG9lQRyuL8orIAaHHoG9mtwKfAT5w96khbSRwDzABWAec4+7bzMyA64H5QBPweXd/PmxzAXB52O2/uvvtpa1KkepD4FvvQ1sumrwtmtry4PkoLZ/dtT6fDctZyGeifIV1bfkova0t7CtflJ7vsJzb9T3teYvzdfwM+2zL7ipPezlykG3eNd+WDdu1AR6+p1CvXIfvKyr3gaL9DKjQABU1Cma7N0CJ1K78u509ddh2t8au41lZx+VOGqjixnS3/KkO392h4bbOytVZelHDuFuDWtx46qxQ+ldvfunfBvwIuKMobTHwmLtfbWaLw/I3gNOAI8I0B7gJmBMaiW8DtYADy83sAXffVqqK7MYMUhX7ZdcHnbaiRmGPhqzQ6BWvC41LcaNYPOWzRY1ox32G9XhooLz7xs5DI+a+e7ny2V35ihvVQlous2cD154v1Ge3xr24ET+AGsJiu529JXZfLutwZmcdGsz2fIlO8nU8KyzOY3t+V3uejmeUpSpTJ/veI08n5dqj3J2VqRfl7rTsney7079lh/13+fc+sBvxHoO+u//OzCZ0SF4AnBjmbweeIAr6C4A73N2Bp81shJmNCXkfdfetAGb2KDAPuGvfqyDdKiuDsjSQHuiSHDja2nZv9IobrD0axnyHBqhDA9Z+xtVxXdEZX/t3FTc+uV35iqfd0ryoYeyYx4vSOsvTtmdacSPrbZDvZDvvuO8Slik2umsYetvQJuBPpsHZPyt56fa2T//D7r4xzL8HfDjMjwXeKcq3PqR1lb4HM7sIuAjgsMMO28viiXSjrAzKyoHygS5JvHTbEBXO+LpqiNp2b1T2yNdTY9XFtEeD1kmD1dbJ9/eqoe1k313Wr5O6jTx8vxyGfb6Q6+5uZl6KwoT9LQGWANTW1pZsvyIywMrKQKPEB9zeHoH3Q7cN4fODkL4BGF+Ub1xI6ypdRET60d4G/QeAC8L8BcD9RennW2QusCN0Az0CnGJmNWZWA5wS0kREpB/1ZsjmXUQXYg8xs/VEo3CuBu41sy8AbwHnhOwPEw3XXEs0ZPNCAHffambfBZ4L+b5TuKgrIiL9x6KBNgem2tpar6urG+hiiIgcVMxsubvXdrZOV1VERGJEQV9EJEYU9EVEYkRBX0QkRg7oC7lmtolodNDeOgTYXKLiHCziVue41RdU57jYlzp/xN1Hd7bigA76+8rM6rq6gj1Yxa3OcasvqM5xsb/qrO4dEZEYUdAXEYmRwR70lwx0AQZA3Ooct/qC6hwX+6XOg7pPX0REdjfYf+mLiEgRBX0RkRgZlEHfzOaZ2atmtja8w/egZWbjzWyZmb1sZi+Z2ddC+kgze9TM1oTPmpBuZnZDqPtKM5tZtK8LQv414UX1BywzS5jZH83swbA80cyeCfW6x8zSIb08LK8N6ycU7eOykP6qmZ06QFXplfBq0aVm9oqZrTaz42JwjP9X+De9yszuMrOKwXaczexWM/vAzFYVpZXsuJrZLDN7MWxzg1kvXtDr7oNqAhLA68DhRC+GfQGYPNDl2of6jAFmhvmhwGvAZODfgMUhfTFwTZifD/wGMGAu8ExIHwm8ET5rwnzNQNevm3r/I/AL4MGwfC+wKMzfDHw5zH8FuDnMLwLuCfOTw7EvByaGfxOJga5XN/W9HfhimE8DIwbzMSZ6XeqbQGXR8f38YDvOwMeBmcCqorSSHVfg2ZDXwran9Vimgf6j7Ic/8nHAI0XLlwGXDXS5Sli/+4FPAa8CY0LaGODVMP8T4Nyi/K+G9ecCPylK3y3fgTQRvVntMeATwIPhH/RmINnxGBO9jOe4MJ8M+azjcS/Od6BNwPAQAK1D+mA+xoX3Zo8Mx+1B4NTBeJyBCR2CfkmOa1j3SlH6bvm6mgZj906vX8J+sAmntMcAz7AfX05/ALgO+BegLSyPAra7ey4sF5e9vV5h/Y6Q/2Cq70RgE/Cz0KX1n2ZWzSA+xu6+AfgB8Dawkei4LWdwH+eCUh3XsWG+Y3q3BmPQH5TMbAjwS+Dr7r6zeJ1HzfygGHtrZp8BPnD35QNdln6UJOoCuMndjwEaiU772w2mYwwQ+rEXEDV4hwLVwLwBLdQAGIjjOhiD/qB7CbuZpYgC/p3u/quQPFhfTv8x4HQzWwfcTdTFcz0wwswKr/csLnt7vcL64cAWDp76QvQLbb27PxOWlxI1AoP1GAN8EnjT3Te5exb4FdGxH8zHuaBUx3VDmO+Y3q3BGPSfA44IowDSRBd9HhjgMu21cDX+FmC1u19btGpQvpze3S9z93HuPoHo2D3u7ucBy4CzQraO9S38Hc4K+T2kLwqjPiYCRxBd9DrguPt7wDtmdmRIOhl4mUF6jIO3gblmVhX+jRfqPGiPc5GSHNewbqeZzQ1/w/OL9tW1gb7IsZ8unMwnGuXyOvCtgS7PPtblL4hO/1YCK8I0n6g/8zFgDfBbYGTIb8CPQ91fBGqL9vW3RC+tXwtcONB160XdT2TX6J3Dif4zrwX+CygP6RVheW1Yf3jR9t8Kf4dX6cWohgGu6wygLhznXxON0hjUxxi4CngFWAX8nGgEzqA6zsBdRNcsskRndF8o5XEFasPf73XgR3QYDNDZpMcwiIjEyGDs3hERkS4o6IuIxIiCvohIjCjoi4jEiIK+iEiMKOjLQcnM/reZnWRmZ5jZZX3cdnR4UuMfzez4Duv+08wmh/lvlrjMnzezQzv7LpH+oiGbclAys8eBTwPfB5a6+//0YdtFwCfd/Ys95Gtw9yF9LFfC3fNdrHsC+Gd3r+vLPkVKSb/05aBiZv9uZiuBY4E/AF8EbjKzKzrJO8HMHg/PJn/MzA4zsxlEj7ZdYGYrzKyywzZPmFmtmV0NVIY8d4Z1nzOzZ0PaT8wsEdIbzOw/zOwF4Dgzu8LMnrPoOfFLwh2WZxHdSHNn4XsL3xX2cW54LvoqM7umqDwNZvY9M3vBzJ42sw+H9LND3hfM7Hcl/0PL4DXQd6xp0tTXiSjg/x8gBfxPN/n+G7ggzP8t8Osw/3ngR11s8wThTkigoSh9UthfKizfCJwf5h04pyjvyKL5nwN/2XHfxctEDxx7GxhN9PC1x4EzivZd2P7fgMvD/IvA2DA/YqCPiaaDZ9IvfTkYzSR6ccZRwOpu8h1H9CIWiILvX+zDd54MzAKeM7MVYfnwsC5P9EC8gpPCNYMXiR4YN6WHfR8LPOHRw8dywJ1EL98AyBA9ax6iRw9PCPP/A9xmZl8ienGQSK8ke84icmAIXTO3ET1NcDNQFSXbCqIXZzTvz68Hbnf3zi4at3joxzezCqKzgFp3f8fMriR6bszeyrp74cJbnvB/1t3/3szmEF3XWG5ms9x9yz58j8SEfunLQcPdV7j7DHa9MvJx4FR3n9FFwP890ZM6Ac4DnuzjV2Yteqw1RA/IOsvMPgTt7zn9SCfbFAL8ZovegXBW0bp6oldedvQscIKZHRKuE5wL/L/uCmZmf+ruz7j7FUQvYBnfXX6RAv3Sl4OKmY0Gtrl7m5kd5e4vd5P9q0Rvo7qUKDBe2MevWwKsNLPn3f08M7sc+L9mVkb01MSLgbeKN3D37Wb2U6InH75H9KjvgtuAm82smajrqbDNRjNbTPRYYQMecveeHpH772Z2RMj/GFF3l0iPNGRTRCRG1L0jIhIjCvoiIjGioC8iEiMK+iIiMaKgLyISIwr6IiIxoqAvIhIj/x8PZWqwge++CQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = range(10001)\n",
    "plt.plot(x, loss_SGD, label = 'train cost')\n",
    "plt.plot(x, loss_SGD_valid, label = 'valid cost')\n",
    "plt.xlabel('# of iterations')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3959.393876054222\n",
      "996.4602840254937\n"
     ]
    }
   ],
   "source": [
    "print(loss_SGD[10000])\n",
    "print(loss_SGD_valid[10000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "f2d0b0f03ac3a581d6b1b551bd30a19f866dd2d205f22e8dba00531575c12bcc"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 ('.venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}