exponential_regularizer_experiment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Playground for exponential regularizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import TensorDataset\n",
    "import gzip\n",
    "import pickle\n",
    "import os\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=64,\n",
    "                                          shuffle=True, num_workers=2)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=64,\n",
    "                                         shuffle=False, num_workers=2)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_stats(model_p, sparsity=1e-4):\n",
    "    params = torch.cat([param.view(-1) for param in list(model_p.parameters())])\n",
    "    params_n = params.detach().numpy()\n",
    "    N = params_n.size\n",
    "    stats = {}\n",
    "    print(\"Mean\")\n",
    "    stats['mean'] = np.mean(np.abs(params_n))\n",
    "    print(stats['mean'])\n",
    "    print(\"standard deviation\")\n",
    "    stats['std'] = np.sqrt(np.var(np.abs(params_n)))\n",
    "    print(stats['std'])\n",
    "    print(\"Sparsity\")\n",
    "    print(np.sum(np.abs(params_n)<sparsity)/N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imshow(img):\n",
    "    img = img / 2 + 0.5     # unnormalize\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAfd0lEQVR4nO2da4yc53Xf/2fuOzt73+VyuaRIimJ0lyiZFRTYTd04CRQngGygCWyghj4YYRLEaA2kHwQXqN2iH5yituEPhQu5FqIUri+JbVgojDSKYENNC8umVF0oUyJ1ockll7vc++7sXHZmTj/MCKHk5//sisudpf3+fwCxy+fs875nnnnPvDPPf8455u4QQvzqk9ptB4QQ3UHBLkRCULALkRAU7EIkBAW7EAlBwS5EQshsZ7KZPQTgywDSAP6bu38+9ve9hawPlvJhRzJpfp6UhccRHgeAVotLis1mk9oaDW6j52q2Isdr8HkR1TNmQ+xxO/HF+Os6Pxpg4I60IrKtWfioHnlgMT/SkdtSOsWNuXz4Es/ls9d0PIusI8hjBgBv8WuE2thzCYBJ5vOrNaxWNoKOXHOwm1kawH8B8NsApgD81MyedPefsTmDpTz+5PeOBW0joyV6rp5iITieMu5+tbJBbcvLK9Q2Pz9Pbe7hJ7O8VqVzrszy41W5i6g2eFB45IJbq9SD4+ls+EUWAFLkxRQAMggfDwBqNW5LZcPPWaPOX/xy4C+0fTn+mAf6wucCgEM3jwfHbzo8Qef0lnqoLZuJrGOGv4DUK+vU1lgjtg1+XW00a8Hx//jXp+ic7byNfwDA6+7+prvXAXwTwMPbOJ4QYgfZTrBPArhw1f+nOmNCiBuQ7QR76L3fL7z3NLMTZnbSzE6Wq/wtnBBiZ9lOsE8BOHDV//cDuPTuP3L3x9z9uLsf7y1saz9QCLENthPsPwVw1MwOm1kOwMcAPHl93BJCXG+u+Vbr7g0z+xSA/4W29Pa4u78Sn2VUuugp9tJZl6enguONyI716hLf/VxZ4LvxMWml1ghLIWuV8M4oADQsIilm+fKvrXIfMzm+++yp8DHXIzvnMQmtkOEfvWLv1DLp8ONupvmOdbnCd+Mvr3L/keJr/JPnzgXH//dzZ+ic0gDfjc9kc9RWiNjykTUuEsmumOXXYqUevr7L63wHf1vvq939BwB+sJ1jCCG6g75BJ0RCULALkRAU7EIkBAW7EAlBwS5EQujqt1xq1TreOHMxaJuZmaXzFhcWguP5XEQiiSTJ5JzbPPL6ZxaW3nIFLv1U1yvcVilT22CRS1QOng012Btek4hKiUYkay/jkeSUPH/cKQ9n+azXuUzpPXztF6pcevNwkhcA4JYj+4PjlyIJShem56itXOPn6ivyJJnJYZ7o5dnwc11di8ilRK5rtbh/urMLkRAU7EIkBAW7EAlBwS5EQlCwC5EQurob33KApbRvLPBda7TCbvoG30WuNPjxyqS8FACkMnxJ6vXwDnMzUlctE9kFL6S4MUV2/gEgleFJQ+lsMTheKa/SOes1vlZ5xBI/+L1iuBieV4zUVatGkjj2DvHkn+EMP+bs5fCu+8ToIJ3TB14v7Mws3yG/vMaVi6X1sKIEAIdHB4Ljk0N8B3//3v7geO78Gp2jO7sQCUHBLkRCULALkRAU7EIkBAW7EAlBwS5EQuhuuVczIB2WveYWeVIIa8kUaxlVKHDJqNXgyRjpeqTLCZHsinmeAJGONDWySAJN7JmZnuEy2uXzV4LjqUjSSqzA93ykdt1Cjd8rGiNhqezI4BidY8th3wFgLZI0dOxW3t1ltR5eyOb6Mp1z2ySX5erg0lZtjl9X9Qpf5UY1XE9uaGCYzjmwP+xjLlLXUHd2IRKCgl2IhKBgFyIhKNiFSAgKdiESgoJdiISwLenNzM4BWAXQBNBw9+PRvweQToVfX1jbIgCokKQmb0ZaCZV55tJglmcnZTL89S9LfMwYz15rNbnkkosob4VIayUnshYA9PWFZcB0ntfrW1jmctLcEpfelivc9tpUWCob6+PS2+Fbj1DbT19+mdouzfFWWcdvC3cRL2TC2YEAcOEyr0G3Z4hnHKYyfI2X5rkEO1gMXwj794Sz4QCgh6i9ke5l10Vn/+fuzldHCHFDoLfxQiSE7Qa7A/g7M3vOzE5cD4eEEDvDdt/Gv9/dL5nZHgBPmdmr7v7M1X/QeRE4AQDFXHe/nSuE+Ee2dWd390udn7MAvgfggcDfPObux939eCEb2ZESQuwo1xzsZtZrZn1v/w7gdwCcul6OCSGuL9t5Xz0O4HvWbkOTAfA/3P1vN5vU8rBM1VvgMlovyWBrtnihQSeZcgAwOhAu1gcAhUjBySYpONmKnCsd+ehSKnLb3r1cdhkY4bb1jbDUt1zmPl6e5dLb6AAvRjkfaeU0v7gYHJ+bm6Fzbh/ro7abDx+gtouXL1HbfbVwhuDgEJcAp6t7qK1yiWfmVTf4eliRy3LIhJ+bpRX+vEzsGwqOp0hbKGAbwe7ubwK491rnCyG6i6Q3IRKCgl2IhKBgFyIhKNiFSAgKdiESQtcLTmby4YytgUihvP5SWHrLpXm2WW+RZ4bVmnze3CKXO1ZJ+t36Mi+GONLLJcWBXv5au17mkpfneQ8wz4WzuXpKfH3vGOYFG2P918otLufVm+G1yhD5EgBGBngmWu/4CLX98EpY5gOAC3Phddx7E89eGxjhX/7qq3B5bXqVy3KzkXm53vD5Vlf52tdJ00SP9B3UnV2IhKBgFyIhKNiFSAgKdiESgoJdiITQ/QRzskvb08d3YosD4Z3TZoPvVp6fXaC2xQW+e14F34mtko1ki+zu78lyW0+W70y3eHk3nPzJaWqbXgsf8857b6ZzxvfwxKCFRe7jeoXvxg8PhY85sX8/nTPUz1t29Tf4Ok5OjlPb/EZ4N34ZXNHoH+BhsWeDt2Q6e4lfc97i12ohH76+W02+vktLYQWiEZmjO7sQCUHBLkRCULALkRAU7EIkBAW7EAlBwS5EQuiq9JbLZnGAyCQzczyJ4OXTl4Pj9UgiRn8vl1aG+nlSRaGX1worlsI9dzKtcJ0zABiMyGt9KW4rkYQhAMilI1V63wzXeBsZ4gk5S03ux/88+Tq1lZfXqe3Og+F6cstLF+mc4RJf+3vu+yfU9qHf/W1qa26EH1s+cuWXV5e5bWOe2gpZ0pMJQLbFW1RlSZ3C/AC/BhZXw9dcU9KbEELBLkRCULALkRAU7EIkBAW7EAlBwS5EQthUejOzxwH8PoBZd7+rMzYM4FsADgE4B+AP3Z0XAutQq9dx5q2poG1+kUsafQPhdkf33n4rnTM2OsodqYfrdwHA/CxvT7QwF5aN9g7ztkXI8EyulSaXrkolnol2S3+49Q8AzC2GJZm1JV4D7a11Lr2dn+EZgr3pmMwTnjc2yjPUzp3lbZzG9/OMsgc+wK+DdC4sXy0v81qDb7z2GrXVazx7LReRggcibcByhbBtJXKu9bWwrUGySoGt3dn/EsBD7xp7FMDT7n4UwNOd/wshbmA2DfZOv/V3v6w+DOCJzu9PAPjIdfZLCHGdudbP7OPuPg0AnZ+87aUQ4oZgx78ua2YnAJwAgGLkc4sQYme51jv7jJlNAEDn5yz7Q3d/zN2Pu/vxfDbynW4hxI5yrcH+JIBHOr8/AuD718cdIcROsRXp7RsAPghg1MymAHwWwOcBfNvMPgngPIA/2NLZUmlkSkRGO3yIThsdDBfk2zfOi/+N7eEST6XFX+P27BujtqXZ8NbE7DSXjKrGM6HWnNvO/oxnh90xPkhtv3b0aHD8x69xabOZ4VLe4XG+HqN5Lsv9+t3hterL8Xd3a2UuiWYz/FLtMS5RbTTCsmLKuRRZyHD5qpDiPsakyEaRZx3W6mH/02l+nY7uCT8vmQzP2Ns02N3948T0oc3mCiFuHPQNOiESgoJdiISgYBciISjYhUgICnYhEkJXv9JWKhXxgV9/X9BWKy/ReSnS0y29wbPG1hd49tqbl3kGVW2DSyt333VXcLynjxe3PHvmDLWtVIzaNtJc8nrhLS7Lve+eI8HxAwe5TDnz0gVqu3Mfl/nGenl/vtX5sHz1o9fP0znLVS5dXZ6Zo7aFqZ9TW2kgXMSytcaLhFaW+XpkMrznnBmX85rOMwt7+vYGx/tz/FyLJLuxuc2sNyHErwAKdiESgoJdiISgYBciISjYhUgICnYhEkJ3q0lYGq1cOINtZIj3XysVwhlDmRSXJuBcghhPcclo+tI0tS0shjOKxsbD0gkA1BtcTvr5GV7Y0Ne5rLiQCWcOAsDfngwX9Dy6nxcTevDYYWo79epb1Hb651zCvEKKWC6kuEzpxqXIp595ltpGwDP67j/2a8Fxa9TpnME8vwdWeng/OqR5FmOqwG2n3yAycY1nFQ4PhOOotqFeb0IkHgW7EAlBwS5EQlCwC5EQFOxCJISu7sbn83kcOnJL0JbN8NpkmXTYxsYBoLnBkxJ6B/jO/569k9R29uzZ4Hj54mV+riJvDbV3gu+Q16t8Nz7XilTprYVtz596k045MMrViUMHeELOXZPhHWEASKfDSsmrV3ii0fNneILP2jLfPf/xS1wx6BsM7/5PjnLfe8B3tFsV/rw0q/yas1a4DRUALCyGd90HipHd/TxpD2Y8wUd3diESgoJdiISgYBciISjYhUgICnYhEoKCXYiEsJX2T48D+H0As+5+V2fscwD+CMCVzp99xt1/sPnpHGkPSy+VNS5pOMIJEplsjs6pRiSSYqQFUamPJ5kcPByu73YmUmfu4vQ5amuQ2noAkOnjLZn2ZLgMVSItgxZzvB3Wq5evUNuZJZ5k8ltHuGT3zw6Gpa2BIq/FtrDMH/MFnhOC5y/zdUydDMty//ROLnsOF/l1lW5y6XCgh8/LNLjt6IGJ8Jw0Twyamgk/ZxsN7t9W7ux/CeChwPiX3P1Y598WAl0IsZtsGuzu/gwAnssohPilYDuf2T9lZi+Z2eNmxt9/CSFuCK412L8C4AiAYwCmAXyB/aGZnTCzk2Z2cnll7RpPJ4TYLtcU7O4+4+5Nd28B+CqAByJ/+5i7H3f34wP9vEqJEGJnuaZgN7Ortw8/CuDU9XFHCLFTbEV6+waADwIYNbMpAJ8F8EEzOwbAAZwD8MdbOdnS4iK++73vBm2VCpeTsplw3a9slmcSmfH6dAN9fN7QUETy2hOWa26/4146Z3WNf3S5eIm3LVpd4Xui5TQ/ZrEn/O6pZw+XfgYmD1Hb0jp/Xi6sccnuh6+HM8fSpAYhAAzs5c9LtR6uQwgACytcZl1rVoLjlTpfj56xUWobHuE16HIFLts261zuHVwMy2VnlvnzfAXhtefC2xaC3d0/Hhj+2mbzhBA3FvoGnRAJQcEuREJQsAuREBTsQiQEBbsQCaGrBSdbDtQa4bZMqSwvrjc4HC4QOTrKM5cyJPsLANbXwm2cAGBklBdYvOnQoeC4WaRdUJ1LV+N7w9lOAOAREWVtnWd5rZXDcs3SPC+K2T9AihcC6KnzIoq1SkRWXA7Py2T5nHKNr9XSMn/MtYjeVO0LPzdLi1x6m+MqH8orK9yPGl+r1AaX3tYvh1P6rixxSZHVWuV5crqzC5EYFOxCJAQFuxAJQcEuREJQsAuREBTsQiSErkpvY3v24E/+9F8FbamIVJaJ9IFj5PNcPylkuW29Es6SAoBsJjyvv5/3c7syN0dtZ177GbXFCl+ObfBeZJWVsKw4MsAzyvp7uQzVyoalUgBopPgxG9WwHlatRIpD1rnUVKhyyS7DXURzPezjG+dmI37wa6DQ5JUvs03+2LLga9yfCl/7+QaX8tJEwm4noobRnV2IhKBgFyIhKNiFSAgKdiESgoJdiITQ1d34dDqD/sFwUks6E/kKv5Gd3Rrfve0p8cSa/l6+013c4Dvr7uGdzr4+PqfQy2uuDQ1xP67MXKK2N8+8Sm2DxfDuc7rFs0Uqq6vUVivz5JRynl8+y6thxWBlPbLjXuD13Yb5MqLe4C2lWqR+4bl5vnO+XuY+3jLEr9ObhviO+8ZqJKGoGr7n1pt8Z72BsDLk4M+X7uxCJAQFuxAJQcEuREJQsAuREBTsQiQEBbsQCWEr7Z8OAPgrAHsBtAA85u5fNrNhAN8CcAjtFlB/6O6LsWO13FFthKWBlPPXnf7BsEQ1NL6f+x2RmhBJMMjkufzjFk7IWVmP1B6L+LERqbm2MDdPbasrfJlr9XDCyNoSr51WXeNSU5XnhGBuMXLMavix9ffz9loF4xLa6ACXUr3JbfV6WL66NMvrEM4u8NZbk338+mhE2pFFXER6MSwDDkTCk3kYu3tv5c7eAPDn7n47gAcB/JmZ3QHgUQBPu/tRAE93/i+EuEHZNNjdfdrdn+/8vgrgNIBJAA8DeKLzZ08A+MhOOSmE2D7v6TO7mR0CcB+AZwGMu/s00H5BAMDrOgshdp0tB7uZlQB8B8Cn3Z1/WPvFeSfM7KSZnVxa4p+ThBA7y5aC3cyyaAf619397QbrM2Y20bFPAAiW/nD3x9z9uLsfHyQbbUKInWfTYDczQ7sf+2l3/+JVpicBPNL5/REA37/+7gkhrhdbyXp7P4BPAHjZzF7ojH0GwOcBfNvMPgngPIA/2OxA3qyjujAVtJUGedsla5WC462IXJfN8Uy0VMTWjByzRbLeUs6lq/oql9DWIq2EKmvctrbM69qtLV8Jjs/Phsfbc7j/84tce1uvc6lsZSU87+7bDtI5E3sGqS0Vqe+GBrdVV8KPbXmJ+74xOExtvBIe8Po0/5iaj7R/KqTDkt1IiYfnxZWwJ7H2T5sGu7v/Q+QYH9psvhDixkDfoBMiISjYhUgICnYhEoKCXYiEoGAXIiF0teDk2soy/u/TPwjaciUuuxw8ekdw/MCRW+mciQNHqK1vaJLa8pHMJfdwEcVmpDVRPsXFkCxp+9M+Gc+W6+UuoroSbmvUex9/qteJjAMA56cuUFs5Ir298mp4XqvJH9f581xS3DfGH/ShSW6zveF0s5kGX/tLU1zKW3Se9ZaKPJ+1CpdSU73h56aV4cdLV8PFLc0i7bWoRQjxK4WCXYiEoGAXIiEo2IVICAp2IRKCgl2IhNBV6S2TK2D4wNGgbXGRF1F86803guMry0t0zuz5c9Q2sY/LckPjB6gt3RPOlkul+TK2GtxWcS4ZZQf3UdtEkVcvbJXDBYMakSKVGQv3DQOAfTUuyy1GJLtbb54IjqfS/P5SXuIZguN7eVbk5EG+Vvsmwr0FDz7AJbRDp2eo7f889SNqGxrlxTSPP3gPtU1fDstyC7P8ORu08HWVni3TObqzC5EQFOxCJAQFuxAJQcEuREJQsAuRELq6G58vFHHLbe8L2rLpcH03AMBG+Mv9GxW+81ir8Lpqc1Ph3X0AqKyvUlu2GK6OW4gk8QwN8R3a/pFI3b0Ut3mN10irL4UVgyXnT3V2gyfr9Dd5skuul9en6xkI+9ho8pZXvo8/ruFh3pZgcF9Y4QGAof1hVWB8ILxLDwDFSV5LrrrK/R+P1IybjdQNfHEqnLw0V+dJQ4We3uC4RZJxdGcXIiEo2IVICAp2IRKCgl2IhKBgFyIhKNiFSAibSm9mdgDAXwHYC6AF4DF3/7KZfQ7AHwF4u6/QZ9w9XGCuQ6VcxqmTPw47EqnVls+GW+cUyDgAlPp4i6c6uJx0eYG3STILvzbmcjyRJJXiPqYK3Md8RM67+SCvoTc2NBocz5AWQwBQKHEZaqXE5bBcRPocSpPnrBDxI2LLF4rU1krxxKDFVng8FWko7B6u7wYA+26+jdreOPsatZ05fZHarDcsz5YG+LWztEgeQETB3orO3gDw5+7+vJn1AXjOzJ7q2L7k7v95C8cQQuwyW+n1Ng1guvP7qpmdBsBvLUKIG5L39JndzA4BuA/As52hT5nZS2b2uJnxr4oJIXadLQe7mZUAfAfAp919BcBXABwBcAztO/8XyLwTZnbSzE6urfOvsAohdpYtBbuZZdEO9K+7+3cBwN1n3L3p7i0AXwXwQGiuuz/m7sfd/XipyDdZhBA7y6bBbmYG4GsATrv7F68avzrD4KMATl1/94QQ14ut7Ma/H8AnALxsZi90xj4D4ONmdgztzf5zAP54swO1Wg2sl8PSVjEfkU+uhGtxra3wGnTNBs8YanhE5ov4kcmElyvW9qexwf3IFUrUNh2pP3bn3bye2d33h7MKGy3+mAcGubzWO8ZbbPVFHnf7HvGLeItoYQDqG1wSjdUonL/MsxhnZy+H5ywt0DkXpy9R24VL56mtFe4O1rZtRDQxhK+RfJ5Ls2kibYI/zVvajf8Hcoiopi6EuLHQN+iESAgKdiESgoJdiISgYBciISjYhUgIXS04uVGvY/pCOPsnn+EZPlVSBDJtXMZh0g8AVBt8XmWdtzRaW+U27gg3jYyEiyECwHAfz3o79eKL1Pb03/8o7EaOZ5QVe/up7ejhQ9Q2NMCloTL5tqRFFqTR5DLl/BXekqle5hJsoxWW8xaWedrb4hK3ZXNcmk1nePZjpVKjNva4l1fCbaEAoK/EZVuG7uxCJAQFuxAJQcEuREJQsAuREBTsQiQEBbsQCaGr0pu3WqiXw33bpud4oceJiXCfr1KJF8epVMPnAYD5+XBvLSAukdTq4T5fxR6ep990nu10ZZkXIewb4IU+hgfDPecAYGwkvCYvvvwKnRPzcfnSq9SWj/TnY1mHHkkNGxsLF8sEgIHIY65WeH++xaWwrdHi97lsmstrhRyXG2uRrL0Nrvai0Qiv43qF99LL5cNSauSp1J1diKSgYBciISjYhUgICnYhEoKCXYiEoGAXIiF0VXrr6Sng3juOBG1XVnnRwxkiy03NTvGTNXkW3XqZy2tI8SXJFcKSzDqRTgDgynykUGKZ90objxxzbDTSfw1hjSdb5P3LVqtcMlqLFFFcj8ibvT3h89100z46p9TDJa/lZZ5xWK3xTLpcT1iy83rkgWX4PbAWKSA6v8rl0nJkjVmGZibHsxEvzYevnY0Gf1y6swuREBTsQiQEBbsQCUHBLkRCULALkRA23Y03swKAZwDkO3//N+7+WTM7DOCbAIYBPA/gE+4ezhTp0KjXMXMpvINeiuww7x0ZCY6vZ/kubHmN74ym0r3UtkoSdQBgYXE+OJ7N80SYrPHd23ykPl0pw3emq2t8ZzdP2gJlne/G11YjbbT4NIwN8+SUAwcOBMfrdZ7csbjC1YlaLJMkzWu/pRBej0aL+1Gp8ct4YZkn3dSaER8jKg8rVNiMtMpqkF337SbC1AD8prvfi3Z75ofM7EEAfwHgS+5+FMAigE9u4VhCiF1i02D3Nm+LnNnOPwfwmwD+pjP+BICP7IiHQojrwlb7s6c7HVxnATwF4A0AS+7+9nvUKQCTO+OiEOJ6sKVgd/emux8DsB/AAwBuD/1ZaK6ZnTCzk2Z2shz5LCSE2Fne0268uy8B+BGABwEMmtnbuw77AQSbWrv7Y+5+3N2P9+Yjuz1CiB1l02A3szEzG+z83gPgtwCcBvBDAP+i82ePAPj+TjkphNg+5rG9egBmdg/aG3BptF8cvu3u/8HMbsY/Sm//D8C/dPdIhgnQX8j4AzeF29bsmQxLNQBw6OD+4Phof0TOaEVcSfdQU6XGZa35xbBEtdGM1GKLLO/iMpcHs1ku5900eZjaikQGPPvWGTpnZZUn64wM8rZR6citwtLh56Ze5+sba5/UjNSMuzLPk2TqRLJbWuMy3/Iqt1XrXEp14z6mM/xabRGJLROZUyiEn5fXz09hnWQGbaqzu/tLAO4LjL+J9ud3IcQvAfoGnRAJQcEuREJQsAuREBTsQiQEBbsQCWFT6e26nszsCoCfd/47CmCuayfnyI93Ij/eyS+bHwfdfSxk6Gqwv+PEZifd/fiunFx+yI8E+qG38UIkBAW7EAlhN4P9sV0899XIj3ciP97Jr4wfu/aZXQjRXfQ2XoiEsCvBbmYPmdlrZva6mT26Gz50/DhnZi+b2QtmdrKL533czGbN7NRVY8Nm9pSZne38HNolPz5nZhc7a/KCmX24C34cMLMfmtlpM3vFzP51Z7yraxLxo6trYmYFM/uJmb3Y8ePfd8YPm9mznfX4lpm9twIR7t7Vf2inyr4B4GYAOQAvArij2350fDkHYHQXzvsbAO4HcOqqsf8E4NHO748C+Itd8uNzAP5Nl9djAsD9nd/7AJwBcEe31yTiR1fXBO1ys6XO71kAz6JdMObbAD7WGf+vAP70vRx3N+7sDwB43d3f9Hbp6W8CeHgX/Ng13P0ZAAvvGn4Y7boBQJcKeBI/uo67T7v7853fV9EujjKJLq9JxI+u4m2ue5HX3Qj2SQAXrvr/bhardAB/Z2bPmdmJXfLhbcbdfRpoX3QA9uyiL58ys5c6b/N3/OPE1ZjZIbTrJzyLXVyTd/kBdHlNdqLI624Ee6iKxm5JAu939/sB/C6APzOz39glP24kvgLgCNo9AqYBfKFbJzazEoDvAPi0u69067xb8KPra+LbKPLK2I1gnwJwdQ0qWqxyp3H3S52fswC+h92tvDNjZhMA0Pk5uxtOuPtM50JrAfgqurQmZpZFO8C+7u7f7Qx3fU1CfuzWmnTO/Z6LvDJ2I9h/CuBoZ2cxB+BjAJ7sthNm1mtmfW//DuB3AJyKz9pRnkS7cCewiwU83w6uDh9FF9bEzAzA1wCcdvcvXmXq6powP7q9JjtW5LVbO4zv2m38MNo7nW8A+Le75MPNaCsBLwJ4pZt+APgG2m8HN9B+p/NJACMAngZwtvNzeJf8+O8AXgbwEtrBNtEFPz6A9lvSlwC80Pn34W6vScSPrq4JgHvQLuL6EtovLP/uqmv2JwBeB/DXAPLv5bj6Bp0QCUHfoBMiISjYhUgICnYhEoKCXYiEoGAXIiEo2IVICAp2IRKCgl2IhPD/AYPHduhpJLjjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample label for this image: cat\n"
     ]
    }
   ],
   "source": [
    "for i, data in enumerate(trainloader, 0):\n",
    "    inputs, labels = data\n",
    "    imshow(inputs[0])\n",
    "    print(\"Sample label for this image: \" + str(classes[labels[0].numpy()]))\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model_error(model_t):\n",
    "    tot_num = 0.0\n",
    "    num_wrong = 0.0\n",
    "    dataiter = iter(testloader)\n",
    "    for images,labels in dataiter:\n",
    "        outputs = model_t(images)\n",
    "        for image, label, i in zip(images, labels, range(len(images))):\n",
    "    #         imshow(image)\n",
    "    #         print(model(image).detach().numpy())\n",
    "    #         expected_class = classes[outputs[i].detach().numpy().argmax()]\n",
    "            expected_class_i = int(outputs[i].detach().numpy().argmax())\n",
    "    #         print(\"Predicted class: %s\" % expected_class)\n",
    "    #         print(\"expected: \" + str(int(label)))\n",
    "            tot_num+=1\n",
    "            if int(label) != expected_class_i:\n",
    "                num_wrong+=1\n",
    "    error = num_wrong/tot_num\n",
    "    print(\"test error: \" + str(error))\n",
    "    return error\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Define the loss function to be used\n",
    "ce_loss = nn.CrossEntropyLoss(reduction='sum')\n",
    "def criterion(op, y):\n",
    "    params = list(model.parameters())\n",
    "    return ce_loss(op, y) #+ (1/len(params))*(torch.norm(torch.cat([param.view(-1) for param in params]))) #+ torch.sum((1/len(params))*torch.exp(torch.abs(torch.cat([param.view(-1) for param in params]))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-80-12ca210d8015>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0;31m# forward + backward + optimize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/playground/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    542\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/playground/lib/python3.7/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/playground/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    542\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/playground/lib/python3.7/site-packages/torch/nn/modules/pooling.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    691\u001b[0m             \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkernel_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_ratio\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    692\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturn_indices\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 693\u001b[0;31m             _random_samples=self._random_samples)\n\u001b[0m\u001b[1;32m    694\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    695\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/playground/lib/python3.7/site-packages/torch/_jit_internal.py\u001b[0m in \u001b[0;36mfn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    136\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mif_true\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 138\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mif_false\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    139\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mif_true\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mif_false\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/playground/lib/python3.7/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36m_fractional_max_pool2d\u001b[0;34m(input, kernel_size, output_size, output_ratio, return_indices, _random_samples)\u001b[0m\n\u001b[1;32m    356\u001b[0m     return fractional_max_pool2d_with_indices(input, kernel_size, output_size,\n\u001b[1;32m    357\u001b[0m                                               \u001b[0moutput_ratio\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_indices\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 358\u001b[0;31m                                               _random_samples)[0]\n\u001b[0m\u001b[1;32m    359\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    360\u001b[0m fractional_max_pool2d = boolean_dispatch(\n",
      "\u001b[0;32m~/anaconda3/envs/playground/lib/python3.7/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mfractional_max_pool2d_with_indices\u001b[0;34m(input, kernel_size, output_size, output_ratio, return_indices, _random_samples)\u001b[0m\n\u001b[1;32m    347\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0m_random_samples\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m         \u001b[0m_random_samples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 349\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfractional_max_pool2d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_random_samples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    350\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = nn.Sequential(\n",
    "    nn.Conv2d(3, 16, 3),\n",
    "    nn.FractionalMaxPool2d(2, output_ratio=(1/np.sqrt(2))),\n",
    "    nn.Conv2d(16, 32, 3),\n",
    "    nn.FractionalMaxPool2d(2, output_ratio=(1/np.sqrt(2))),\n",
    "    nn.ReLU(),\n",
    "    nn.MaxPool2d(2),\n",
    "    nn.Flatten(),\n",
    "    nn.Linear(1152, 512),\n",
    "    nn.ReLU(),\n",
    "    nn.BatchNorm1d(512),\n",
    "    nn.Linear(512, 128),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(128, 10)\n",
    ")\n",
    "alpha = 0.1\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)\n",
    "optimizer_adam = optim.Adam(list(model.parameters()), lr=0.001)\n",
    "for epoch in range(5):  # loop over the dataset multiple times\n",
    "\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(trainloader, 0):\n",
    "        # get the inputs; data is a list of [inputs, labels]\n",
    "        inputs, labels = data\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer_adam.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer_adam.step()\n",
    "\n",
    "        # print statistics\n",
    "        running_loss += loss.item()\n",
    "        if i % 2000 == 1999:    # print every 2000 mini-batches\n",
    "            print('[%d, %5d] loss: %.3f' %\n",
    "                  (epoch + 1, i + 1, running_loss / 2000))\n",
    "            running_loss = 0.0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Define the loss function to be used\n",
    "def criterion_p(op, y, model_p):\n",
    "    params = list(model_p.parameters())\n",
    "    return ce_loss(op, y) + (1/len(params))*(torch.norm(torch.cat([param.view(-1) for param in params]))) #+ torch.sum((1/len(params))*torch.exp(torch.abs(torch.cat([param.view(-1) for param in params]))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model trained on the new regularizer\n",
    "model_new = nn.Sequential(\n",
    "    nn.Conv2d(3, 16, 3),\n",
    "    nn.FractionalMaxPool2d(2, output_ratio=(1/np.sqrt(2))),\n",
    "    nn.Conv2d(16, 32, 3),\n",
    "    nn.FractionalMaxPool2d(2, output_ratio=(1/np.sqrt(2))),\n",
    "    nn.Flatten(),\n",
    "    nn.Linear(5408, 1152),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(1152, 512),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(512, 128),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(128, 10)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   300] loss: 78.161\n",
      "[1,   600] loss: 71.333\n",
      "loss for epoch 1: 20.900\n",
      "[2,   300] loss: 61.457\n",
      "[2,   600] loss: 69.346\n",
      "loss for epoch 2: 14.322\n",
      "[3,   300] loss: 32.445\n",
      "[3,   600] loss: 31.882\n",
      "loss for epoch 3: 14.453\n",
      "[4,   300] loss: 49.331\n",
      "[4,   600] loss: 47.667\n",
      "loss for epoch 4: 14.370\n",
      "[5,   300] loss: 46.724\n",
      "[5,   600] loss: 49.454\n",
      "loss for epoch 5: 18.557\n",
      "[6,   300] loss: 29.632\n",
      "[6,   600] loss: 34.976\n",
      "loss for epoch 6: 8.390\n",
      "[7,   300] loss: 29.017\n",
      "[7,   600] loss: 30.072\n",
      "loss for epoch 7: 9.639\n",
      "[8,   300] loss: 15.402\n",
      "[8,   600] loss: 15.153\n",
      "loss for epoch 8: 7.561\n",
      "[9,   300] loss: 14.011\n",
      "[9,   600] loss: 17.874\n",
      "loss for epoch 9: 8.915\n",
      "[10,   300] loss: 11.499\n",
      "[10,   600] loss: 14.864\n",
      "loss for epoch 10: 7.947\n",
      "[11,   300] loss: 10.266\n",
      "[11,   600] loss: 11.372\n",
      "loss for epoch 11: 8.366\n",
      "[12,   300] loss: 21.627\n",
      "[12,   600] loss: 17.505\n",
      "loss for epoch 12: 8.143\n",
      "[13,   300] loss: 17.817\n",
      "[13,   600] loss: 9.355\n",
      "loss for epoch 13: 11.560\n",
      "[14,   300] loss: 22.197\n",
      "[14,   600] loss: 23.183\n",
      "loss for epoch 14: 8.615\n",
      "[15,   300] loss: 10.093\n",
      "[15,   600] loss: 20.039\n",
      "loss for epoch 15: 7.780\n",
      "[16,   300] loss: 10.735\n",
      "[16,   600] loss: 19.362\n",
      "loss for epoch 16: 8.736\n",
      "[17,   300] loss: 8.202\n",
      "[17,   600] loss: 11.295\n",
      "loss for epoch 17: 9.286\n",
      "[18,   300] loss: 9.347\n",
      "[18,   600] loss: 10.475\n",
      "loss for epoch 18: 8.766\n",
      "[19,   300] loss: 13.323\n",
      "[19,   600] loss: 20.008\n",
      "loss for epoch 19: 8.481\n",
      "[20,   300] loss: 9.196\n",
      "[20,   600] loss: 9.676\n",
      "loss for epoch 20: 8.335\n"
     ]
    }
   ],
   "source": [
    "alpha = 0.1\n",
    "optimizer_new = optim.SGD(model_new.parameters(), lr=1e-4, momentum=0.9, weight_decay = 1/len(list(model_new.parameters())))\n",
    "# optimizer_old = optim.SGD(model_new.parameters(), lr=0.01, momentum=0.9, weight_decay = 1/len(list(model_new.parameters())))\n",
    "optimizer_adam_new = optim.Adam(list(model_new.parameters()), lr=3e-4)\n",
    "for epoch in range(20):  # loop over the dataset multiple times\n",
    "\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(trainloader, 0):\n",
    "        # get the inputs; data is a list of [inputs, labels]\n",
    "        inputs, labels = data\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer_adam_new.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = model_new(inputs)\n",
    "        loss = criterion_p(outputs, labels, model_new)\n",
    "        loss.backward()\n",
    "        optimizer_adam_new.step()\n",
    "\n",
    "        # print statistics\n",
    "#         running_loss += loss.item()\n",
    "        if i % 300 ==299:    # print every 2000 mini-batches\n",
    "#             print('[%d, %5d] running_loss: %.3f' %\n",
    "#                   (epoch + 1, i + 1, running_loss / 2000))\n",
    "            print('[%d, %5d] loss: %.3f' %\n",
    "                  (epoch + 1, i + 1, loss.item()))\n",
    "#             running_loss = 0.0\n",
    "    print('loss for epoch %d: %.3f' % (epoch+1, loss.item()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test error: 0.3027\n"
     ]
    }
   ],
   "source": [
    "err_n = get_model_error(model_new)\n",
    "save_model(model_new, filename=\"./cifar_20e_norm_n.pth\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### errors: \n",
    "- base - test error: 0.3309\n",
    "- norm/n resularizer - 0.3027\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save model\n",
    "def save_model(model_save, filename=\"./cifar_net.pth\"):\n",
    "    torch.save(model_save.state_dict(), filename)\n",
    "#load model\n",
    "def load_model(file, model):\n",
    "    model.load_state_dict(torch.load(file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "dataiter = iter(testloader)\n",
    "images, labels = dataiter.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAB5CAYAAAAgYXpDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO19aZAl2VXedzPz7a9e7V1d1XtPd88uzYxGIwmEEBLYIwESYQssTMCELcdEOFAYHEQYYX5gRfgHhB1gHIHlmEBCAhMIWRJIFjJGjHZgpOlZpZmeXqbX6q6u6tqr3v4yr3+cc/OcV0t39UJXP7hfREdl38yXee/Nm5nnnO8sxloLDw8PD4/eQ7DdHfDw8PDwuDH4F7iHh4dHj8K/wD08PDx6FP4F7uHh4dGj8C9wDw8Pjx6Ff4F7eHh49Chu6gVujHncGHPcGHPKGPORW9UpDw8PD49rw9yoH7gxJgRwAsCPAZgE8CyAn7XWvnrruufh4eHhsRmim/jtYwBOWWtPA4Ax5tMA3g9g0xd4sVi0AwMDN3FJDw8Pj398mJqamrXWjq5tv5kX+C4AF9T/JwG85Wo/GBgYwJNPPnkTl/Tw8PD4x4ePfvSj5zZqvxkbuNmgbZ09xhjzpDHmqDHmaK1Wu4nLeXh4eHho3MwLfBLAHvX/3QAurT3IWvuUtfZRa+2jxWLxJi7n4eHh4aFxMy/wZwEcNsYcMMZkAXwQwBdvTbc8PDw8PK6FG7aBW2s7xpgPA/h/AEIAn7DWvnK959m39AUAgLFJ2pbNULdMIN+XVqsJAOjEbTomm033xQn91iZiwTFBDAAIQtXndon2gfZlso10Xwh3TTlHnHQAAO2O9C1J2HJkIu6PWJKavE/blhIelzHS2mrRGOI4Wjf2gPvWSqStSt1ArRWnbaX7noDGhz/84XS70+msu+atwHWfz675q5sC3UatgWvUhjjj5i9Rx7t5lpNczZtqo3674z/2sY+t27fvh3lu407aNnflMgCg2ZA1c/CuQwCAgf4KACATSn+yGVp4Wd3G6zkyao116gCAcinD55C+RrwdqkW8sDAPAOjr60vbMpkMn5eOM4Gco5O0AADBBqJaYKSxViXzZhTRmszn8+m+VovO0eFnEAAK+QJfS/r2u7/9W13n371nR7pdHjlCvwvlua30lQEAK01Z19XlOe4v3e9ELYaIB1GIcmlbPuRXmHpu0weQm+JEzu/aEtXmruHGTtfnudxg7Ri+fybQ74V4g+Pot7kc9TcbSL9hadtkZf5qc8cAAF975vvrzrUZbobEhLX2ywC+fDPn8PDw8PC4MdzUC/xWoMVSlLV1aWTpM4dS2hSAvlRRxJK1lij4q2oy0th0UkMiX7iIJbyQmyJ1DpOQVIyOSBlOGk7UOVqGJJM4pC9oS++LAz6XfI0NS/F51beIJZ8goo7H7bbqSIeHJOdwEmcYbm7xCsNw0323Cjcq0ev5SOUkJSUmTmSyPAYr+5xGZCDSjpzl5iXwjVAu0r0NrDwezSq1JS0h4vNZOm+pQMdF6jJu7eTUIitk+b6rsTRjdxytq6xaJ26KokjurZPsAyXFu7nJsVaql0m11uZrCpz2aiHnDfhiGZZCnVQPAO1mk8enxsJSJa6yJhIrUnwnHKRzZeSZjkOSwIOMksDrq9S3uMr9kPM1LR3XVpJvg+dXCeVotUlLCviZqNfk3eKeEz0+pxEHgTyH1mkuPJla4+90Yj5GrmmMez/JmhkcpDHnCn18frlniVvXOelHvFrG9cKH0nt4eHj0KPwL3MPDw6NHse0mFMsmBlgxXVgmj0wsKl7SJpUmLLCZQqmhznqgiYQsq0gdKypK0g67jnOqEAAYu4ZIA2CYcLGhqIL1mHS1y3OkblVbohatrlJbaOW8fXkmsxQJVykSAVTI0TiToJXuC1JziYzdjaCdbK72a5PA31eZvK2ct8tc4Y7v0jXdLm3yoTlvtmk+Iq03x/Tb0Gx07WSDtq3hamOJ2IwVKDNWNqRrZQJpywVsHnP7FAHZrJOpJQwV4RbRfW83hQgNwCazDrVZI49kzKaibKYgx7t5UGvMkbkxmwF1vMXclSsAgLGRQTmezSVhVq4V8rXcPCtLDiI+vqlIXUewttvSthaBlX0x9zdWz0FsaMz5PunH8L4x+u3SAgCgXFtN97Ua9I6Iy/I8Jv0U2d2Xlbl31w3YztpqyvPlHB7yebkv6ZSqNeHWsfsbKJtth8ec6OXHl89GsnYLBSZ64cyAYqJJnHlWy9A3YKL0EriHh4dHj2LbJfAoZsk7lK9fwJJELlRfd8cQ8Zcw0EwN/7SjJVRHymRFetm5/24AwPLiLABgdk4klUxE0nYA+TK3OjQ9dSsBSMfOkURjc8MAgHYopEyLJYPVpfm07eI0SxJ5JVlNLQIA9u6kaw73aSnNuRbK2J1wEdv1rkoOWvK9Fe6Dt0SKT/uttAN2tewo8aXNmtDJ06cBAGM7xf0sYTJ6dEgkyDwTP8lN9PFqc5RlKTvpiOQWsvSUUQRahtuCmNZRNqOkupBdVZV2lQno3iZGaVwJu8c2mMxU66nBYy8WZQ2HjtnU4h/PQ5VdHJ977vl0V5s1gcHKm9O2XI7JfDUFqSsra6eBct8z1pH5siZt4oi8zSXwDsTVMQCt9SRUBC5rYaHSxkrMRlaKfI+ffzbd15olaXz8gbulb1fomWsambcyD2ylTkRoXo0lxxp5MCyEYcAkpn6lNIt03qjNmklbJmulRPclt7SUtkV77gMA1Ab607aEtaqY71k+ESI01fhjaQvj65envQTu4eHh0aPwL3APDw+PHsW2m1Ccnm0iSTPr1NuOjlBkwqjFam1WkUNx7NQ5ZWLgc2i/2rf86I8BAJ77278DAFxiUwoAVDsuslJUq3OTMwCAM5MX07bc4DgAYPfYAbpmTtTEFqt/mbJkfew0SO2bm5E0McVBMr9MrlJ0X0Opw2N9pOIVM6JWxm1Sg3Ww2Vr6biMS83ZEYl7d1MJkWUZFzbKPd31VSOvFJVJ1p2fJ9FToE3V4mCMOddSgI+10dOYGnV3Ti60jy+Y6q86RcZMfS79DOLKd2jLKr7rt1OdEzhFWaB6MVX7/7G+cuGjfWNb16jKZ2spFIe0Cnm8dFRlx5PIik5fzy2IaLLCfdEtZOlptulaU1WuG2mKOdO4o85GLgs4qH2fLazaJNzfr6Zl3JsFAjT3u8FiV7cKwiaNh6L5nElkLZoRMa7UV6Vv7zAnqrxEzU8LTVXX+5er5yrY5fuOCItF5PrRjRIPNoWGD50ouieZO6mP9sphK+ww986Z/RMbH120HjhhWsQ8836EixaPg+k2CXgL38PDw6FFsuwTeDOhLu1RTEVosvQyWRWyoMCkUsQSiCabUDUgRKo7krNUW0ravfonyrkwvkkQxvSrfr3MX6bhzlyTFeZgnaTwOK2lbqUJf2kyR9kV5+fLnWErMBzKW2RZFgY3v3pu2NZhcOX2aJPD5RZWTZRedd/+oaAIZdqUzyo1L5C8er/q62+T6ZM408HEDAUBL3cEGEnjMUlbC0oaOFnURblfmltO25SqNta7zX9RoNEGOyOJqXe5tucgSp+qbk+e3qmBcryaSM87lTebbkZcbugAmHPmnXAAj1hgjxRSGhubDxvru8fiYuI+Vq9nqCs3beX3NyEUui7S4p0Lz5lwGX3r55XTfG+6/HwCQaBfHmOY3r11sWROo11jDjeT8HdYAw0jI/Dbn22k2N08RHSvpPOE1bLXMyE4HLe1uyNftX+G5Gh1L9xV27KP+WCEPwa6QdmRn2lTPcG6Ty5RXBcolt8rPqx0bTtsyCfWpoTT4EmuBrRUaX1PnqClwxGtV7ks0TNqBySg3Sc530sc/DZWE3zE09yZQLrO4/mhqL4F7eHh49Cj8C9zDw8OjR7HtJpQrdVIb5ttCYn7jb74OALjviJgifuR+IgcG2V9ckycuaU2g1JGYyRLFfeHMOfIznq+TamOLQ+m+sMxk2ZCo+wWu39lSKURbTJxVBqlvlbL0ceYymUSWFxS5wSpeviCmlvMLRJ5mKqQezkxJtaTy5RUAwM6KHF9wqWsTRX6tQbWmk4GxCqlUR5dqN1SJkdy2S4+pckghSNZ/212UqLZdrLJ678jMgiK6GhyxNqVMKDMLtJ0ogqvN9pHaChG+M7Myf5MXpwAA9x0+mLbdtX839V/5xadkqouk1VYT120dJnAVajNkE17SFvNAwCa7+pKMBWw+sJwEKSzI2LN8r7Jqvk2bTGexNjtwtLFJiVMxH1WrZCqYnpbjS5UyX1Ml8uI5b63ScXnlj35lkYjQ578vZpVSjq556KDMacSmnGaN1l8hUomXmrS2YpVWOXaPWkPNx1qoKXYpXZOuWA3ep57lDJuvcqdO0umf+1a6r/NmNj2ptKyWYzSyK/JsNEDzUOZ4izAnxyclOr+xiljnZHJ9w/IOylxk88sqrcnMmDgr4ALtiypi5mxcofkNi9KWHCHf8AYnwgoU6Z7t0OREyjZor8LJbwYvgXt4eHj0KK4pgRtjPgHgJwDMWGsf4LYhAH8KYD+AswB+xlq7sNk5rtqBfpICanPyLWlniSicr6lk5y1y66lk2e1KER9O4gxDIVkaLZJgryi+aHaFvr7FASIwBkeFWKwmJEmMQEW9MeHRyohU1KiShNJYpeP3KTKkxtL2TEukYcPS0NK8krpYGqnz1z3MSr+nl2kap5ZE6t83whrGVb7Qi3UZaLlIWkGg8jK44hRdgrUjV1yQa1ca1w2+7Ru4J16eIhfLoSHSZgp5kWyaDRpzMSdtO0dJk7JKPKvWaKwlllRaDZX+kwe92pTxddI8FcqtLXVndPvWDbNLIrya92PeJexXBzkJPKek/jKTxf1MPgXsDgkAOb7HeS1wspYUNGQtpEn+uTBIa1nWWl+J9g0OiaZ4ZpK0vNMXLqdtJ049DQBYmCWJc7Uh56i1qcZKBOUWyJL9g3cfSdve9+OPAwB28Xpu5mWcjWqVfyfXrHCBdFNfwWbIhLL+XDpoR2YCklI1UnJkeYGu1Zkkt9uK0iZWLtH1W3mJdrSg94K5PJO2lSaYgKywZgl5lgrsvppdlH43mDjuzE6lbVmew84yzVVuXhwZ2nXWlgqiwSyeIeeHbEEk8L5xIl1dKiWrXAabjrxWa7iVXL8IvhUJ/JMAHl/T9hEAT1trDwN4mv/v4eHh4XEbcU0J3Fr7TWPM/jXN7wfwTt7+FICvA/jVG+nA3W94DAAw+czxtK3cT1/3x972lrStGJKduMUSsJYuDWdri63ky+jbQfWWX3z5pJx3gKS/XfvItcoqW1qGpeykOZe2tVrJumuF/MV85aWXAAAVlZC9WKIvf0nZwS5dngbQnaclZKliiN2/FhfEfrcwT9tnpsRVamKMXKSirIomWIOoIppAzNJzW9eTY9ti+hdil3TBIVritBv4FDoBXXkspgElLl8GlCvnALtitdvqXCyVFctiU3QSuOHgLKNctnIF526lyoQxsdFlM1zXN7lmpvsQ3r25CH7h7Fnut8z3yjKtu7gtmsDFi6R9LPAaqK6KPXjHMEnN5ZIE4YRcjKSlMvhFnKsn4Fw8VSWdN9xgVGGJ85eIPzkzKTxBtUW/zfezK1tJJsatxFJWZLWpcxT8cunSdNr2rW/9DQDgXuYaRgdE4qyvkmTvyp0BQPteykeyurS54p3Lytitk8YTpRKzBhMot9dVDrxbffSNAIBK9KZ0X22F7kFb5U0yOZ4bVW4wU6DrVtldUru/tjnfSEY9G3WeG+3EV2e7fG2VrlkqyFgafHyuLM/5UB+9e2L1rljltQt2ayy0VUZD7pP2+G3fQG6fG7WBj1lrpwCA/+64xvEeHh4eHrcYf+8kpjHmSWPMUWPMUZ2n2MPDw8Pj5nCjboTTxphxa+2UMWYcwMxmB1prnwLwFABMTEys0xGK/aT67zsohEqdLQp7DxxK20ZYDV88cxYA0NbRWx0yRTz2jp9K2/YefBQAcODBs2nbcy+Q2WOwTCaJSzOSCyVit6KcLibAvV2tCjm1OE9q5FA5ow+hfrCZZGRUcqG4IgWzC2ISMRyt2McuiFGoiAxWoV+/MJm2jQ6Smn14t3JlWoNP/OH/kvNzPzJKnSv3kQp46IAQt29+A7k5ubKNVpl5HClotb3E5ahRZhJHsGVzdH5NTmazZBIZHlTujK62qaoxmObYyNA5Gh05/yKTuosqdefKEqn0be06ycTjMLuCHT4kBFPGRevpwuVBl0GlC9/622d4uKqgiCOe67IWzl4moi2tXanEoUGuVF9SpG6Oj8so18KIXdwCrolZUwRkxOewKu/P5XkivtuKjS72Ofc3zhe0qtwf+X40GtLvSh+d961vejBtq3IK5Aa7zJ4/L6aR119/ncauXN7OzdHc12ty3ignZDwAlEriENDheWjH+p5xYRVF3hk2KRXGiKhcrspYrizR2I1yj21xzc+sJgMX6Tcul1IuK8/BMq/xfEa9+lyaXxWJ2eToYHDN26W6rEmXhqaoolX7dpPJNtRmvbSeK98rXbvBvTnUokxuwI/wRiXwLwJ4grefAPCFGzyPh4eHh8cNYituhH8CIixHjDGTAH4DwG8C+Iwx5kMAzgP46RvtQJgjIuDS9LG07aE3UfL5Ur980cMVIoxilgIiVQ7q9AUiGt4+eEBOXKRgj76SqiIe0bUK7LaXz6pS1vz13TUxnja9ypJHVpExy0ykHNhDGsORe+5L983Pc/GGigQEXGL3JqNIk4FBklqXWLrU+UMKRfptfUX6ffI8B1coImpMUj/Q8TUVbFSn7YwKqllhAbao2uJ77wEANCyTPUoCz7EkpKVWV5hBZ+nrHyJtIyWKlPuhc4sKlbTtIqu0rJGwNHKWA60uzohCNz9HGk+9LpJb3GRJU+VMcTk5du+h4Ki9e3an+0rpWtEk7eYS+IsnqR/Fgmg8ljW+ZkfuSz9nlXRkXUtJuVdW6R6Eaq768qRxdWIhrQ2TdiH7mplIAsNyVZIcW20hR+fnHXmpy3/R3xbnWFmpyly12L10z6i4Ig4P0uJxgUIAML9AeVSGB6gfj77x/nTfJLuKLtVlDb82SfclUOv6gKQtAQBEKhNooY+euVVVIi1ilSVWWfgiDnYJeE0myv3RcIGXSF3TbbVbKgMja9ERS9Za43HkZay0PFeqraNWZabAJGO8Pqupy52S6ShNgBl+ndEwH7sMlnwtteRcIFu3V+/1Zw/dihfKz26y693XfTUPDw8Pj1sGH4np4eHh0aPY9lwomTwRKo2GVoe5/qCKUCyWHClEqr2ul1mOSAX65FMfT9t+8l98mM6hoseyXAvQFYc4cHBXum9mngipxqqowTt3kN+4TpDf5DqFBw8RwXrXISFfl16gWoTVFVETHQnTURFodTZxDHD9vNhKVFj/IKl/HZWBPwxofJOXxLQw9gZ04Wf+2T+XPjK5V1L5VxxpUlCmJ5eaYXmZ85N0RLXPMKkWKf9Xy6poXflH24TO56p2a+I04uMzGR3hud4M4/xfG5w/pKRyTAxyPpq4JX3LhzSuxTkxAUxePAsAOMTEdxgoU5F1FddVyt2ruNwus5nOaqKQffsLoczH7j13Uf9d2tzLstZm2fQzNiYetrkRMutUF8WfOuFI0/5Bsj/kchLL0OAh1zpiQsnzcxC3ZY2FTAa6IieZrCoskaftxx4Rk8iRfRN0/pas9TOv07heP/4qAOBtbxaCc88eOv78y5Kzpx27nESb18TMqn5kuSZsYsVsWWDSuqPS9q5wJGrMRGW+X0w/YyU2aSmyTyq+q7S9cDU/6a8uRLERLD+b2oQSs6+5S9sbqGtmneFGJVpq8jtF516K2IQYcwX6rrq1/NzouqTalLpVeAncw8PDo0ex7RK44QitmpJ8GyxBZnQehDl28eF8JxkspvvGB+iLePKYRF1emjxFGzUpZXZu8iwA4OGdFP25a58wgRMzJAFVT4mUMZQj6a9vQMokvf76GbrmBEnvi8siHbX5Sz59RUlYjtxQroI1lsAN50bQ1EXJZTdMJLIya2g+WrOXsRmStkgIqQSi9pezdN5CXua0zpnkam3qx9nTZ+WaTGLuPbAvbTtzgebyS3/5dNrW5gyQec53UlTnd9Fr/RWJ6hvoJynq4YdFhRgdIanzrt00p4Fy33NSlCOaACGn6jtEOpsYp3s1sYtIaJ3hrsauZl0ayVVElwwT66M7JtK2PBPIs7Pi3lnlqGAXTtdQEZb9o7S2dilX2L5+GmdlRKTyOSa+Y5bI2qpCmXNZrCnir9V2BKVoJFmX8TJH9zhjRUPawXM/Oij3IM+E3OigsI4VdrWbO38eAHDu9bPpvp1DtP6Xpp9J2zJMXrfCzV8hkcr9EXKWxbzKj7I4Q4Ts/KrkILkyRfM72Efr/4H7RBPIsPbdVARumzUATcC79e+KnASKWHdSsC4FGKfEqWYZu3Pr6EynSM8hz1zEx+u1636TcZqRftD59IFyiYyv4tq6GbwE7uHh4dGj8C9wDw8Pjx7FtptQ0lSwSh0ZHyH1SavjX32ZfLIHOan84SFRafI5JnEi8YW+MnOWTt+UiLK9d5GfeMjnLVaEMBoZI4Jpbl7U1SUmL3Xh7R07SP2N2LzTUGSjS1JUV+p+h3/cUSdpNDlVZYe+n8NKpTZcKy9rZCw5Jnli2x3ppvHn/+ev0u2EE9QHyoe2zIRwnzJn7D9MYx4dJpPB8LhEaQ5xn/IqGdPiMTIvfe+Y1A2tW1c8gv4fKfW2wr89tFfMMG977BG6Vkl8rEushjsNtqXmtMO+zbUlMZm12Y+6oKq1DwyQ+WCak4fNqqIQBY4IHNsp81wsqhiANRhkk1mozANNLlxhlMwzP0d9Wl7mtMDK5BdyBN+5i5IwqrJM5o/+fokTcP7fTSbxjSL0ci5asCT3vWBd5KbOjUvPRKnA5kVV+X33MM1LURGKVa5231GmGVfs4gCbfI69djrdd+QIJa6CIiwvXSLf8PygmLEAvd1N2rniIokyZ6xwTMWVK2IaXFyg8554+bsAgNde+rt036FDFHOx/9C9advgCJuBlPnBpU52xT20YSJMfchV39LCJqpqPBOQUjhGkaR8vObB08jlDdjxlCTtShbHZ1X3W79LtgovgXt4eHj0KLZdAndRUv1lIZgG+mjbqJwby5YkidkF+hKO9EnXS0zAxIFIHmcvnQUAjA1K8vd9/AV37lnffU6iPy9OkaTeVxapPMNuTq+cOq967CIJ6W9TfTVXOQJuQCXg77BYOTWtEs73UZ8idlUqFkXCcvlD0BYiNK5S38Z2bJ4L5dkXvp9uFzJEKDabQrBmmYR7y1vfnLadu0iS9BxzSA/cL65mWSYga02R4jOsuTzyiBCQDY70y7K0ePigRMPezylHJ0ZE4qwU6d4mym30wmWKApxZ4GIWs1fSfVUmtxcXRQJvcUrXjHKJdLlYXKRuWxGKxQGatwcg4+vv33wunSRdU5GeoXEl6UTqjzk1acQRvokVeSibo/OPjEhkb5nXeF65ZvZzvyO+Z9q90rKrXke5d/azi2WgohcTTpsauejFpkjW/ZyAxXZEK4xZq2mpSMI6348ir81zl2X9vfo6aXfNpkR4ths0vzbUVPnmcFJrPi9jv+duigQ+dK+489ZWSBp/5XlyyX3hqBCn3/omaYDHXpW1fuTehwAAh+8WqXxgkNabI3fDrj66+d0gF7EmR10JuM76MoYuOjNWpGeSujNujq50zcaVgZQ1rFNObxVeAvfw8PDoUfgXuIeHh0ePYttNKC46bucO8cl2NfISRQaO7ybV/CibRhaNpGy1IanZ/SNCFPZX2AczL6ryfjahlDmF7R984o/SfTW+1nJdyK8a++HqzJM7OVKyMU/qXDWnr0lmnteOiz/69DSZA5ZVdObAAJ2wUiJ1OFSkU4aj48LaxbRttET7+/OioKmknACAKxeU//oQmYF27xbS7r43HKbz5+Qcr7xIRNEYq7VlVa1nhusDlipighqu0HHve/wdaVvADtX9/XTcyLD4r89z6t0z52Q+lhbJrLO8JNGnK0wWL3La3vllibDsMCGbUWl+s1wBJ1CRa/0VGtcAR24OKnNTjk1U2YKYqlbrQhKvxTD7cGvf+jJXV0lUOtRMQPOxg/3FjYpCzbLPsjPtAECeoxFDlXfWmUzSKkTKhOJ84GtVWTsuIjCnFqVlc0ptieb74lmZ73l2Ph4oyPFjnHI3n9c1ZNkkEpH5KCoK2X2F61PuGZdnro+rVS03NyfeEpUm1iW9soFuo76Fyjd8YJjSsr79nbR2Dx0Sk9y3v/F1AMCZM/JsVF/g53ZZTGwPvoGq+ezZQ+fS6ZrjDq3xWPUtYVNtVxWqtP6r+yu7XL1YTWg764f2OXeEZnqtLhKT33HKDKNNMluFl8A9PDw8ehTbLoE70q4yKBJ4J6Zu5SJxyzrChQiOPkeS1XJGItwSQ9Lc2C75kr96jNyPfuCH/1Xa9necqL9aJSmw3ZKCDjOXnWucfNNWuYZdpKLeBgOS0HcV6BxLV0Ta6YQk+Y7tECI0ZterupL4GnWSOKtMlnUSkbDaDYpE25ERSW+iTJJSsyNtayXwiydeSbeXmej6yX/yb9O2xx+n5JF//VVxN9zB5N4OrmJfUK5peY5OG+sXSayPt/PKfa/DUouTNHXOl8vHSVI6PyOudC0uzBHlJW1qXx+RvjtYImy31hNHGZWU3+WM0Lkj+vpoLJVKH+9TdRY5H830tNzvRmPz6lBFlj7bimgtsEvkQEW0miRNbUwEZEHV+UxJKiX9JZbbtNzkimm4v4pc6/D97sTS1+U5GoN+cDMsga8ukbY3dUmij8eGaCwDJYkmrrH0nChNoMNndMTpLi5QAAB3c53Mh+6TIhknTtPz8sL3xBFgLXQK5YALLgSRaNUZJvFjFb3o0rEGTOoePiKEecJut1NTn0vbFmZprCeborVNX6T6uncdJpL03vvlHDvGiFSO1Lul0+ZiEyrFbMw1Xt193LAASFdOlvX705TFPA/6FGnxFCXad0V7bhFeAvfw8PDoUWy7BO5yfwyOiITQ4a91I5BCAPkySxKcwe/8BXH+f/ubyT2ssSpfxGIfue1NXZTcFYSO61cAACAASURBVKdOUDXujqtWrbyLqmx37RsWt6+lJZJ8+ssicd59hHIzPPvSawCA54+dkX78yHsBdGdRPH2KJPRFldHQuSA26iR57xsTya3AQRtDQyL52ogkg05rczejhipt9eAbqY/veve70rbhAbJN/+BblP2aJbc+1gQqZZGKQy5S4KqmA2Jr1Un2lxbI7lphiSZRGVgO3v0AAGDHbsnYOL9AmkvfgLgWusx2xq6vGO7sqK7UFwCssk3YqhJYrlDAhSmy3TstBwDaXOxC50cpljYP5KmyttSnCjq4oJ4ZledmmYOLEs5aeMgFvAAY4PwhYUZLl7SttZQW1+eqMffRaEq/Oy2aK6MKQNgmHV9SGsnAAGkwhSzZqCMj62SAtbf+PlmTLT5HTWVbbHEG0IADSwaV5lXkLJ6TimdxheHvv/tw2nZFuX/SubQ9n+3dqm9Z3p3oB5ElU2cjbiltbPee/QCA/fv3p23PTtP97qhyb1dmFrk/JJ0fO/Zyus8FKt11l/R7bIzcGPv6hO8BB9Q1uNp9rJ69DGtcOmjHuRHqOB5rtKsijSo9fVoAQhDeQEGHa0rgxpg9xpivGWOOGWNeMcb8ErcPGWO+Yow5yX8Hr3UuDw8PD49bh62YUDoAfsVaey+AtwL4RWPMfQA+AuBpa+1hAE/z/z08PDw8bhO2UlJtCsAUb68YY44B2AXg/aBamQDwKQBfB/Cr19uBhGsM9g9JEv9qndSWWiwqhyOsXK3DE68o17QaqSrlkuTy4Fz7OHdC1L6LTO687W2UTlan6ezj9LBDE+K2dH6ezCT1pkrmXiJ1tTJKJM/DfVJ78Qqr12fPvShjqZG5YXFJrrWDq9b3W+rPvrK43u2ocBEEIyYRl0K0pFRSccIjHLznoXT7g7/wb2h8sajZx08RkZgYlUOGyc42q3PziyrpS+LywAhd6gp/JxAiamWZehJOk6p7SdWzdIU5koaQQyUmTE+fFNPWGU5h6tzwhkZkPpy6v6Sq0s/NEpFnlUkkYPc0E7i8ICqylwnTvE6lu7qWBhbk2GVxblbG8voCXdNFMQLAwCApnePjlI+jpaL22i0ywyRW+rjMZq66Mu/EHCEZsnlK1150ZpK8qu5eYPfBhlq7CRN/pTK7pap1kuUoRE34OkK4oUg7V+ndkYhtVbRjco4iZGuqhqYjAXeOy/pfi1CZENJtdU0Ynq8u9zr3G7Nun4vi7OsT805KLnYV63AmObrWyoLcxxc4JfMrLz2btg0N033cuVOI253j+/maZFYZVqbVUS5IaxRR7u5zR5n1Okxypm6E2hWRzVdWmdNsstbkcm1cF4lpjNkP4GEA3wEwxi9395LfsclvnjTGHDXGHK3VNmf+PTw8PDyuD1smMY0xZQCfA/DL1tplnbnrarDWPgXgKQCYmJhYx8KtcCKOgsrklmZmS1T5LyY/RoZIOjsRSLa0mXmSbOZC+YL1l+krec8DQkycPkuSnkuar4nFw4eJ1Dh84K607dwUSRyvvPK9tG1uloNCOOn/oHIdm3yFJPapWclBYpiIDVVA0fgecsfax1O4t08krDyXZmo2dKABSUzazWktPvBz/zLdHtxJUtFL3xcp15FBLfWVj5lUc6XDNIniSlXFWkLgtqDrs8+5RzhL5OycuAw6NzgVu4GBygD3RyTZ+TnWNlgKnJ0VwrLJ2kdHuWHGXNYuVLlQinma55xzMdQVw13yG4h0VFBZFtdikYnZSxfFHa/E5PI9qsCAy9hY5PwujbpoTQsL5G7abss4a5yrpKjcMPsrtO5LOfpbUORkxM9YrEjMTqfF51XZLV05r7T4gCoSwFpsWz15UcgkXKJcWznb4twV0jRm58Tl0mUNXFD5aJwmlesTbWktjNUSOP3VxJ5hqVXnCEklaf7rCEMAqK9SPy5flgIQly7R9lJRjsvwOnKkfEnlXylGdJwmtC9yEYmTZ+WdUq9T0ZJOTOcaGZXiHg8+SAGBhw+JxD46Smuh0i/OGLkCaQoWfH317HXSJIeKSP77IDEBwFCO088B+GNr7ee5edoYM877xwHMbPZ7Dw8PD49bj614oRgAHwdwzFr722rXFwE8wdtPAPjCre+eh4eHh8dm2IoJ5QcB/DyA7xljHDv3HwH8JoDPGGM+BOA8gJ++kQ6cPkVqy97Dkg4yH3BazJYQTRGrQUJkCOlZ5iIF99wjfrh//VdfBgDUlsRfvDhMZNOpSVIW9uwW0vPA3VRoIKfU8oN7af/ivBSFeJXrbiZMkEwuCNmzzORrIxZz0PIimWl2KILk3By1De0hc8JcTvkkJ0x6KnOJjbgWYCLq+Fov5hdePJpuv/w9uk0GYppx+SYiXXQgTY2a4WNE9Y44/axO/+nykWRVfwP2Ew8t7atkxZs0YDNTO1TqPkemKrddZDlXSbvG/slVMUG1mOQzbRWdyTacliK5Y462rK7Q8UV1H0f7qR+RMl04S8VGVObQKK2TQVVowxUkiNR8rKwSkbi6Sv3N5cT84UhAnY50YozI61xe1H1HXlrOx1FtSI8aTBAvLkh+nrl58rWuK3PNvZy2N8O+9d0FDLhep1pPTa7lOZlGH4sPd4vNU7WqnH9pkUyJWRVV6sb+9Fe/mra94y0PowuqWEHi/Ls7KgKSTSzKHR0mNe/QvlBFpr70/HMAgNUF8TcfZv/2C1PSVmEf9iw/N4mKYK6U2R9d+ednIy6EkVNxEAGbZRfIbHT2jEQ6Ly7QvD1/VOW+4biJPXskWnWCC6SMT9CzPzEm75sSp602BVWvM9g8NmEzbMUL5dvYPM3tu6/7ih4eHh4etwTbHon54imShvc+8FjaloC+fkaTdvwFX2ZCZXFRSJbhIXKhe+/jP5K2PfRGyoPwmc//WdpmOK9BP1cH3zUhLlBlJtfCjkgeQztpesYPiBS1xMn4n3+RpNypVeW+lCHCtH9ciJ2RQ9TWVQiA3faOc5GKU5dFQs0y21NXkYdVnoZOIlLDe9b4/HzrG19Jt2ucmS2bUaW4io5ElVseWs5/4ap4Z7QETv3I5xTBym54WZXFLirRWPNZGmdO5XNwqTaMyqLoyOi2KhTRYIIylVp1BBsfr0u1pSG0SuIdKNF2f4nGVC6IlJvL0PkyRu6jUe6Aa9FmUk27HUbs4hh3EXOunBzPnxJz8ixl16syzjpnYKwrH1Cn6QQZ51Yma/74sVcBAOfOnk3bXBSxVe6JE+NE2A9xRsi68vZy24sLQkDOMUlbVxquy9njPMUWl0ULCnjui5GsHZdv5fJl0XDXSuBtVUTCkeimI+dwUZ/aec6C2hzpuboqk+WKh9x9RLT1Rx56FADw3MtS5OGZZynL5iIXA4k7cg92jBMZ+fa3vz1ti/g+nz0nLsfPPEO5lB64j6K8K/3iDDHNY56eFsLerd2dY+JueODAfro+OwJUV8QN0zkEZCKR+hsb5AC6FnwuFA8PD48ehX+Be3h4ePQott2EcmKJVPTZWKXizJBKHbSUypG4GnL0d2JcbAg/9ANEQOYzolYe2EeRlT/+gQ+mbZ/9s7+ga12m804tifLWaJwCAGQhKux8nbZPnRM1Eazm2FEy0QyOiTkhrYunoh0TNjckRlR6l7xpiSMl8xmVtItTulaNSsbE5KFNtIrVrW6NjUp02lSdCJ04FrW5wnU6I9W35VkiZ1eWq9wvUTUTp/5uFB2mzCSZAt0Hm6Hru0RkABCwDaWoknu5yulxe715DJw0yWTFFpFnMrKgzBlDfaR27lE++LvHyf/W8ZTNhqjegaX1FKnIuYEKrbua5KZKceIEpUi9//770rYCm0T0dARMDSUcfTetolBdcrRmXZkp2CQYKzPJwUP7AQCjO6j/utBAhs02AyqxlCNAdZlH58P92nFKo7qqCkC4fTqGIGETUXVF5qjG/axxtGhLmbhc8Yjz00IUuhql8VXqONquCEvrNlK4KEoVJIrEEZ98qwqqXuwPvfPdvEt+4Io1HHlITLAPvInqvrqyoYGi8FzBkYMHJd4j4jndf1jSzk7sJWK4wBG9/cqE4sblCpYAYibZMSppsV1yrJBNT4Fia2N2SGgru1tiNp/LzeAlcA8PD48exbZL4McX6RvyhW9LtOND+0ga2ZkVA3+RpYDxnfSFGx8RqeSug0xGWpEapjgvySc+/Rdp23MvEinkIj27AhutI5HkHHGOrhFrYo5d8zpMiHYCRfK52VSlkRotPq/60kZMaIYsbVmVK6TDlE5Gfa1daa1We/NILdsWib2/RBLFiiJC2zFJZffc+4D8ZoKkkRmOvptR0XernBdFpz9wkqON5byliKSMe95IaTovqVJpV5ZJwq+3RCKscyEFHfWZY9fGEmsaAyr3xyhXGB+fEMnm0C5y89uREzF0lV0P59nNLszK/BVLRFqXVcTrMOe/uHRGiCuHNkvvjVXRYAJHHioR0hVriNlV8OTJE+m+lSVHJMsj5opeREp8TjgkL+BIVijXyGHWmjQ5WuMUxPW6zOmFC5Ndx6ngPlh2uay15J456bk6KxpuhvvpSth1VKRild0IO8p1USIZN5ca60r7CNklMrIqQpaf146KkO3wPLjz67JsTqDvKA3GlTdrqRwkE3s5n1HCKVsTVTSBn/Mz58U1s95yeXRUgZD+A13XX1iSa0YsUZcq+2WwLp/Qkoz50vQ8n4M6nlPpsV2AqSnL+mgsbF7mbzN4CdzDw8OjR+Ff4B4eHh49im03oayyWvHXz4v6eeJ1is58z5uERLprglT1M6cpEvIdbxZTQJ5V75WWqGef+UtKF/n8q5KQqOaiwNiEEajUnU7NCVT0mDN7xEo9a7Jpo80qnlG+xU2OaNTkTRStr99Y5MQ7WbgK2ekuxEwC6iRSHSb8sn1SxWZt6pm5S5K4Km6TKlZX6m3tAiXyGlIVwEc5zWqGq8AUVNapeugqjGg703q1uVYns8s7uCrS/fdKsqfz58k8MbcokaxNR44p8itiYrrArNOIIiwHSiW+styDy7M0luOzktTIMBFV2UFmoUJFCM4ik546TW1ZkVJrUeB71lJmCkcud9V5dP7fbH6oVCQ6OM8+9eWSkHAhj6uoojmdyeLka5QIbWleVPsljpiMlc93JssRoWo95VgfN646vYrmnGGirdYU9TzkMQz2y3pqsbmtxk7qHZUsK0nNJTofKs+H2VwG/OY3vyZj6VBVnFIk8xHzumsrM4kj0l0CL/0stdlUpZ9HRxA2mtIWpxWeODWzqn85NEDm2XJZV4RyFeL18EzXX11t3o05UCaRiJNkBWb9cW4IXeENht8fRTk+aLD5TxHU14KXwD08PDx6FNsugQ+PUH6I+QX5/E1x1Njfct1JAIjb+3iLvnSjOyWK0oT0hf3uUYnG+ouvUiRVM5EvPvhLHATrv1sxS4ZWfYade5iWAlwUZYa//EZ/LjmPgyapXC1Fnbsl5OuHliUKqzQBluK1WD6+k6TFvoqSGmvdEvjO8aF0e/L8JI9JJ8+n7TMnjqdNS+ze565eVW6KVZZ2kriL6aXjVSrhVpMktue/TdXu31mScT7A46z3izTsSDsdZdtggm2JoyM1mXruNYp2m61LZGAjQ9cv7JAxD+4kiSpXoTGFKhKzyG54uaKQ4ibcfOk7V9W4I/fARfEmHaWN8dgdiVlQkYoBa4V1lVOkOU/a4HldjIHnwaVUdflmACG7M3kl9fMlWi2Zv5UFkrgbjVX+K8Szu1N5tebbdU5Jq+qXOsLR/dXkoXP36yjtw7LUms1sTqznVSRwO+T7olJE59hJIFGup86NMuBratI44XwxWup3EamJVVG2PGrr6k6qqvdOeA9UXdco5BTOTYkcTQlNHp6uudlmjVhr1W7NmK4q893vmZaKKrV8joZ6feRC0pYmJvZhq/ASuIeHh0ePYtslcCetZlSWvE6DpKcz0yJ1NasUXPGOR6jCeWFAVY/n4gff+I5k5Kuz7batssHl2I3LSRcbVQgKlTSQfkyVbSzHkptxolCgjs+RlFFQ5bycy1FbBa6ssFTmgiCaStLrH2QXynFJDF9m/8S6CrxY++nde0QynS2zS111clYdwVnplHvYPF83y2NuKXu32F3Xu4l1JeBnnHyZ8k9cWBHJZjSg+ejSYFgqWVX29suWpL5TbBOdVDk0akXWYPZKQv2xAySh5AfElTS9DywVlcuiCRTZHh6oNWavYrtd5jw7tRVxI5y5RGuy0ZC+uXJoLg+GvsdOkwtU8FCGA80cLwJIBsiIbebaZbDNdmCdT6XZpLWzotzV3G0rVdg9VUl+tk3z3FxV1e45N8iSkjid5O3sy0bZuxO7PpjL5YYxyeZFRhJ1H1erxIMUQ30P6G+sFrMLOGqxW2yno1zruHCFVdK2ZH2U57DDNvDYaXvqXrsgJi0cW0v9bDZ0bpi463itmduUj4lVmwvi00VRuq8ZtnS/OffMoC70QtsT8BK4h4eHxz94+Be4h4eHR4/imiYUY0wewDdBNQQiAJ+11v6GMeYAgE8DGALwPICft1aFQm4RKSmkibyQVMGWIlmmV0nNef44EUHvrYlKs2LJtHBxQUwMeVahOzU5R4NVRlfDMFJRcm5fl5uYcW5IcpwNulOwZnLiErbKrlctlZLWmVO0GcGZTKocEVoeEHPJIOdSaKkUmK+xi1lGuU+9aY2WVRkUQm90jPKTTCkTSqrOqd802Uzi6iVqV734KhF2XXv4xG1Wwauzki8jyHGKXuXCdomv8aKqbH8q4vkok1pe2iNFIUYnKKfNMBdZAIAcu+a1VE8sq/m5iKuwR5pIdm2KZLyKr9bls+TSqquEO5Xa6IhaTmfrqpNr9TnL5hqdB8bt1wRhh00Gq6tcs7Spc5awC5vRLn20LrKq+MDYrgk+B0VMLi+I22aHCzRYXYGeb1qtpc0qzjzhfN6w7viMGrsrtFCrKbPeGly4IE4FJ6eoHyVV4zJi20/cVW6A5tRFWyaKWM9yrhzd5kwusU4NxPPsSEajcow4clTbqlw+FX1fnLtrErsoTUVOssmxK+eRK1hh10eOul+2VZ6leIjWxa4HxVW6393S60iJshUJvAngXdbaNwJ4CMDjxpi3AvgtAL9jrT0MYAHAh7Z+WQ8PDw+Pm8VWKvJYAM7vKcP/LIB3AXCl0D8F4D8B+Nh198CRAzpRPgebJCpvgstHcmaGvvif+MyX033veicldT9zSaS/qnPOV9+ojMvkxlJAUbkBZblQQ31FpGdHNFhFMmaYUHQSniaunKSXKMKjzi5jus0dN8BS87BKAn9ljgI5FmclA+LiOQpeOnTwADZDIS8SWY4DRjIqH0jMZJb+uHdSyYTHp3deRQroorRY2lnl8b2mpLp+Lrf2WkMS37/C2slcRSTT4T00rvEDJG0PKJfIHLslBiqfRZvXShip0mQs8UZpUIscn0rP2sXrKiRmmLArnXLlTN399HlZGwusk8jkHE12iey0ZT05iVpXRHdwZHcmq0vecRk8TQLzWsznlDtegX4zP0fX1FkGM6xRhrr6OWubHS0triHhugJXXIELpdWsctGQWlXyqaxFYFU5PieNxiK1Omm/KxgoZDdC61z1lCbFkq+Ka0rn3ipXQXcjrPgMpnBStnb17fD124rET/gdZF3JO/U8pHmNVEcM1o/FMlnd4YDBisrns/tBcsaIjNzvxROcD2q3aJvXwlar0odcD3MGwFcAvA5g0UqY3iSAXZv89kljzFFjzNGNvD48PDw8PG4MW3qBW2tja+1DAHYDeAzAvRsdtslvn7LWPmqtfbSocvt6eHh4eNwcrssP3Fq7aIz5OoC3AhgwxkQshe8GcOmqP94Ew1xJu6ES8Fc5Uiwbij+1SzPpfHm/8d2X031nuD7fYlWYjPlVUoMVF4gSq+MdVqNyqrq6U73zBZVnIXA+uqKqO5/VDpsMjPYPZZUqVhXUW+ynWlD5L1xS+aERMp20FIHb5AIG9ZxcM+HoPF2xfC3aKmKyyvks+gbkmo0qqc26YEDM6l6awVSlMjXrtfwUVqXLtUwAVdlH91uqCMe5GrXNqXwP0RhV6B7fPZq2HRil7eF+mpdARXNWWS5oKCIqYlVe16zMc5RlxNXB8wURFnI89zrK8WpINsjD4ZRNq0w5ltnf1ESjzuEi+WJtAuB1pNedW2OOVO2yYiVuPQkJHDNZ3MrIvXUV6p3pJNGEJedOaSjt143Lal9od7wzP6h+RDwW2xLieWGOzGLt1uZrsqP8wGM+rhVoAtflxdFFQLiJn6VA3QOXMjbRpg42cyUq/bIjkJ01Qx/vTGDaapM4/2xlMnNmo9TUov272cwDTbA6M4x6H7Q5rfPQ3VQ8Ytf+Pem+BtfTfP01iV0ptNlSLUHm18Q1JXBjzKgxZoC3CwB+FMAxAF8D8AE+7AkAX9j6ZT08PDw8bhZbkcDHAXzKUEKBAMBnrLVfMsa8CuDTxpj/DOAFAB+/kQ40WKrMqU9JkyWgTChSaIc/hC5BfVAQKe0sk5eBIlk6LB11FAHZ4IxrVY6E1ESNk4pKWZHSCkxsBkpqcARhoUjX1zkprnAmuUS5C0VMYAxWhGTcOURax86dRNYtVkVSWebMfatLEgU4wIn9Z6/oyMoRaLRVlfUwS2MfHJVrtss0l522yvyWuL9McCoJ3A1ZR+Sl0plm6xzRxtn62ioHSbOf+n3XgJAyg0MUPVmuyNIrF+m+5Zggbqh8Iy12O7RKeg6d+6fuB29nWJPSboSuWIEmxOxVWNoGu95F2n3UuaZpV0QeuyvsoNfTWsmaO0Bd1ZGSPPfOjS9WkY1tnodQaV5tzqcRK3fXUpM0Fyd561w1zTpL7xuUPks2iKh1/Yj0fHO/56cl/06bI0L1LVgHPXTOmRJk5ZoZlw007qpAwT/luVKnsy6Dn9IA86xhDFaE+HYl1FwBEj2nIbt85pSG6/KcdEWf8n1xkakryyqPCS/PJJI5WuJUg9GI9GPfESIqBzm6+uJrp9J9s6co42qk+pa/Sl6ZzbAVL5SXATy8QftpkD3cw8PDw2Mb4CMxPTw8PHoU257Myql4OZX0p+iIjLaojs7NM2EvZJ1gJ2F1q9NSpFPsUkpqIoq2kzRlpXy/FubJdDGvrlnhQgD9Ksqxwr7jeZB5xVWXBoCIVbxQ1WpscvIjVxBAH9epca3Bmkr6szjHYxf2Nc8Rf42rRA+GSv0aGCbzTrmk/MCbbFJSJpRO7HzDne+vSszF3/agKz0mmwVUMqaIVeIimyz6+lSEICfNL+eEjC6xb3g2J+pnizdX2W+9rghZR7TmlbqaDZ3PtKjBwRrzhL7vLSapsllFOmU2n0sXXRsoM0XGme60+YP75maoq6h4Gpmnkj3F64lkF4nsCju0WnLf62w6iesqYpJJzJIyMxX6SUXv8DjbDTlHsIGNI/WH14R2WjSeNkoqRqLKtU2Xl8Ws5yxQes2sRdhRc8x1JxMVgWtB/Q2hUujytkStKgLS2K6/AJBwsrpaJInvJJrapYNW883R0o229M2tddPlS552ks+kQj35+pqgrnBq49EjEqsR8Lvq+LPfoWvOiAk05PunC3NsZNK6FrwE7uHh4dGjMPYG3vo3iomJCfvkk0/etut5eHh4/EPARz/60eestY+ubfcSuIeHh0ePwr/APTw8PHoU/gXu4eHh0aPwL3APDw+PHsVtJTGNMVcAVAHMXuvYOxwj6O0x9Hr/gd4fQ6/3H+j9MfRS//dZa0fXNt7WFzgAGGOObsSm9hJ6fQy93n+g98fQ6/0Hen8Mvd5/wJtQPDw8PHoW/gXu4eHh0aPYjhf4U9twzVuNXh9Dr/cf6P0x9Hr/gd4fQ6/3//bbwD08PDw8bg28CcXDw8OjR3FbX+DGmMeNMceNMaeMMR+5nde+ERhj9hhjvmaMOWaMecUY80vcPmSM+Yox5iT/Hdzuvl4NXJT6BWPMl/j/B4wx3+H+/6kxJnutc2wnjDEDxpjPGmNe43vxth68B/+e19D3jTF/YozJ38n3wRjzCWPMjDHm+6ptwzk3hP/Oz/XLxphHtq/ngk3G8F94Hb1sjPkzV22M9/0aj+G4Meafbk+vrw+37QXOFX1+D8B7ANwH4GeNMffdruvfIDoAfsVaey+oDugvcp8/AuBpa+1hAE/z/+9k/BKoDJ7DbwH4He7/AoAPbUuvto7fBfCX1tp7ALwRNJaeuQfGmF0A/h2AR621D4Bq1XwQd/Z9+CSAx9e0bTbn7wFwmP89CeBjt6mP18InsX4MXwHwgLX2DQBOAPg1AODn+oMA7uff/A/TlV/2zsTtlMAfA3DKWnvaWtsC8GkA77+N179uWGunrLXP8/YK6MWxC9TvT/FhnwLwU9vTw2vDGLMbwI8D+H3+vwHwLgCf5UPu9P5XALwDXLLPWtuy1i6ih+4BIwJQMMZEAIoApnAH3wdr7TcBzK9p3mzO3w/gDy3hGVDB8/Hb09PNsdEYrLV/ZSVJ+zOQEsLvB/Bpa23TWnsGwCn0QMWx2/kC3wXggvr/JLf1BIwx+0Gl5b4DYMxaOwXQSx7Aju3r2TXx3wD8BwAuq/0wgEW1iO/0+3AQwBUAf8BmoN83xpTQQ/fAWnsRwH8FcB704l4C8Bx66z4Am895rz7b/xrA/+XtnhzD7XyBb1SxsydcYIwxZQCfA/DL1trlax1/p8AY8xMAZqy1z+nmDQ69k+9DBOARAB+z1j4MSsVwx5pLNgLbit8P4ACACQAlkNlhLe7k+3A19NqagjHm10Em0j92TRscdkePAbi9L/BJAHvU/3cDuHQbr39DMMZkQC/vP7bWfp6bp52KyH9nNvv9NuMHAbzPGHMWZLJ6F0giH2BVHrjz78MkgElr7Xf4/58FvdB75R4AwI8COGOtvWKtbQP4PIAfQG/dB2DzOe+pZ9sY8wSAnwDwc1b8qHtqDA638wX+LIDDzLxnQYTBF2/j9a8bbC/+OIBj1trfVru+COAJ3n4CwBdud9+2AmvtUPp/igAAAUpJREFUr1lrd1tr94Pm+6vW2p8D8DUAH+DD7tj+A4C19jKAC8aYu7np3QBeRY/cA8Z5AG81xhR5Tbkx9Mx9YGw2518E8AvsjfJWAEvO1HKnwRjzOIBfBfA+a21N7foigA8aY3LGmAMgQva729HH64K19rb9A/BeEPP7OoBfv53XvsH+vh2kRr0M4EX+916QHflpACf579B293ULY3kngC/x9kHQ4jwF4H8DyG13/67R94cAHOX78OcABnvtHgD4KIDXAHwfwB8ByN3J9wHAn4Ds9W2QdPqhzeYcZH74PX6uvwfytrlTx3AKZOt2z/P/VMf/Oo/hOID3bHf/t/LPR2J6eHh49Ch8JKaHh4dHj8K/wD08PDx6FP4F7uHh4dGj8C9wDw8Pjx6Ff4F7eHh49Cj8C9zDw8OjR+Ff4B4eHh49Cv8C9/Dw8OhR/H86g/sGL68EWQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# print images\n",
    "imshow(torchvision.utils.make_grid(images))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = model_new(images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAe/0lEQVR4nO2da4ycZ5Xn/6fequpb9dVtd7evsRMnIWQGJ2MyMLAsMBdlMjMKSDOzsBKKVmgyWk2kRZr9ELHSwkr7gVktID6sWJlNlgxiCVkgSjQTzQJhBsTCJHFM4lyci+NL0u6223a7711dt7MfusI64fk/3Xa7q02e/09qdfVz6nnfU+/7nnqrn3+dc8zdIYR455PbaAeEEK1BwS5EIijYhUgEBbsQiaBgFyIRFOxCJEJ+LZPN7HYAXwGQAfgf7v6F2PO7O/K+qacY3lZ8P5fsW0xSdHBbdF9kmzHxMup5zOix9+GY/2Hb5foRPY6XpdryncU2d7kKMZsWOx6N2M780q/FuCfc0oi4wXycnq9hcakedPKyg93MMgD/DcDvAxgF8JSZPeruL7I5m3qK+Ny/viG8PW/QfRULYTctxwOiUlmitlq9yvdVDL8ZAUC9EfbRI2fFcnVqy2XUBK928W2Cb7NQLAfHs8ipthz3v96oUVu1xs9Zo0GCwrgftfA1CgBYYtvDSoEb9jH2pl6p8OujXo8cx8g1nIucswq5rub5ocdCJby9b/xwLOLD5XMbgKPufszdKwAeBHDnGrYnhFhH1hLs2wC8cdHfo80xIcRVyFqCPfQ56Fc+D5rZ3WZ20MwOzi1GPpcIIdaVtQT7KIAdF/29HcCv/MPg7gfcfb+77y91rGk9UAixBtYS7E8B2Gtmu82sCOATAB69Mm4JIa40l32rdfeamd0D4P9gWXq7391fiM6BoULeX9wX+USyWtkGvmKdA1/qzucjK+SXoXhZgU9aqlSordaI+BiR3rLIKn6eTLMGX2FGjSsXsVXkRsT/irUHx+tZG58T216dHw9rcB+NqAntkXOWN27L5SPKRTVyjI3/C+vkGHtEZ8iysI8xZWJNn6vd/TEAj61lG0KI1qBv0AmRCAp2IRJBwS5EIijYhUgEBbsQidDib7k4nCVWOJd/vB6eY3Uu1TSqXPLKOiIyDngyA5O8GhHpp1goUFvNua1Rjby2yP5qtbDNIplcuYjMZxlPDPIsLK8BwGI9LLGdPs/lqfkK93Fujs/LnB+P7vbwcSwaP889nR3U1tHGJbRGjl9zuaiMFvaRXx1AlSVfRbQ33dmFSAQFuxCJoGAXIhEU7EIkgoJdiERo6Wq8eQP5Oll1zyKrxSSJoy2L5MfnY8uSkUQHkmAAgCbC1GLFwnLcj0KRr/oOXxMu3wUAM1PnqO3c+YXwvvJ8VT2HSHJKjV8ii95JbUdOng2Oe9smOqea8cSmSomv/M9NT1LbqTMXguOldv666uNT1LZzmB/HTd38OLbnY+WswtdxMXIJ14kCESu3pTu7EImgYBciERTsQiSCgl2IRFCwC5EICnYhEqHFiTAG9k19y/fxWUROqMU6cOS4LFep8YSFYqRGWr1OaoVFElMQkUKKkTpov/17v09tT//s59Q2RmS5+YiEVquXqO3k6AS1HR89RW1t/SPB8e1Du+kcb+umtkqen5dCaTO11cpzwfHzE7xzSmc/lwdH505TW5nUSgSAoW6e1tJZCCfC1KthGRUAWBMf0v1reQ43CSHeSSjYhUgEBbsQiaBgFyIRFOxCJIKCXYhEWJP0ZmYnAMwCqAOoufv+2PMblsNSLiyvTC/wjKd6rRwc7y9xea0n43JYPlKPrRGR5ZisQevqIZ5Ft7AQzsgCgB/93SPUdmaK1+s7Mxfe38lTfF8nx96gtqydy3L1rIfaunrCclihk28v386zANsiLZnac/zaOVcJtxUb2b6TzikvzlPbsWNcepucCl+nAJBt46/7ms1hW6HOpTxjdRkjmXJXQmf/iLvznEshxFWBPsYLkQhrDXYH8H0ze9rM7r4SDgkh1oe1foz/gLuPmdkWAD8ws5fc/ScXP6H5JnA3APR38yofQoj1ZU13dncfa/6eAPAwgNsCzzng7vvdfX+po8VfxRdC/JLLDnYz6zKz7jcfA/gDAM9fKceEEFeWtdxqhwA83MxIywP4X+7+D7EJtYbh7GI4w2eyyrPefvx//yk4ftP1XHL5yLsHqa0/UtyyQTLbACBH2vTkcjyjqe68bVFETcLxk8eobXKRZ4B550BwPCvxjLLcwAy1dfTx81Ipc6mpQtor9fTzc9ZT4raJ01zymrnAC052F8OXeHsHl/lev8DFpULPELVNjJ+kttLpWWob7gn70mGRTEVShJUVRQXWEOzufgzAey53vhCitUh6EyIRFOxCJIKCXYhEULALkQgKdiESobW93rI25Hv3BG0L5/n7TrUYzqCaXAhLYQCwUOG9wXqKPLOtQfpuNY3B4SzjPc/KFS7xnOXJazg3yyXAzj5eELF/cziba77B5bVBcB+zSCZapcCPY3k+LDWV57gfu4b461ogEhoATJDMNgCwQlimnJ7kxRwRKSC6OBcuYAkAWZFfB2dmeNbh+HRYwtw1yK/vHEuIi7U45CYhxDsJBbsQiaBgFyIRFOxCJIKCXYhEaOlqfHtHF274zV/JggUAjP7zy3ReqTe8Gn/b+3+bzunMeFJChawUA0Auz5NarBBema57P53TvWUHtT1z+FVqK/XxRJ5tu95NbZ4Lrz4XIivnjaXz1FapRFpsRY5VRpI4Xnj2WTqnpy3SIqmLJ8l0RerajZ0+ExyvEWUFADKygg8AAz18xX3qAk96ujDJbcfHp4PjW4eG6Zw8U5QiReh0ZxciERTsQiSCgl2IRFCwC5EICnYhEkHBLkQitFR6y2V5dPaGkx127bmezlskqsXO3dfROYNVLq1MHT9BbdVIIky9FpZdbvvQx+icnXt4R6zdv8H9ePoXXKLqL3FJZmwiXD8t77yMd1uBS16xmmZz87xN0tRkWM4bKPF9RXaFekQqG9wclmYBYKkaPp/nLoTlLgCwSMuu7kidvHzGw6lS5ok3r70xGhzf3M+TkPZuD9cUdESSZ6hFCPGOQsEuRCIo2IVIBAW7EImgYBciERTsQiTCitKbmd0P4I8BTLj7zc2xAQDfBnANgBMA/tzdeZGtN7eVyyFrC2cojZ05Quft+633Bse7enkGUjZ7itrqNS7j5CO1zo69Ec6W+2D/bjoHndupqbuLyzHteZ7J1RGpddZeJBlbkbpq27aOUNuLr71GbcUir/M3Mxs+Vrt3cIn1+htvorbJSX55lXp4i6qx0xPBcctxiaqvP9xCCwCmI7Xksohk19HJfVycDV8Hr77OszM7iuF9VWuRLEVq+f98HcDtbxu7F8Dj7r4XwOPNv4UQVzErBnuz3/rbO+fdCeCB5uMHAPBvlQghrgou93/2IXcfB4Dm7y1XziUhxHqw7gt0Zna3mR00s4PT07xmuBBifbncYD9jZiMA0PwdXgUB4O4H3H2/u+/v7e25zN0JIdbK5Qb7owDuaj6+C8AjV8YdIcR6sRrp7VsAPgxg0MxGAXwOwBcAPGRmnwbwOoA/W83OzDIU2sN393KZF0RcWgqnvRUiElRnF/8U0RVpadSW8ay3Uj7cr+nrB+6jc/7kX91DbYX509RWbOPvw7kc93H3nm3B8YnJMTqnPMez14a38MKXkzNcOlyqhM/nnut4puK113FZbvoXh6htfpa3ZJqZD/tYq3OJanEx3I4JAPr6eqmt7lwq6+3n2X61Svh8ZjneH2x0LPxhukKy/IBVBLu7f5KYfneluUKIqwd9g06IRFCwC5EICnYhEkHBLkQiKNiFSISWFpyEGSwLSxALEfmnvLAYHC9EenLNnudZXsi4ZFfAFLWN9IUzpV49wnu2jY0e5X4scDns5OgJartlONwvDwC27QoXo9w6MUTnzB/lffEG2ni2VnekH91rrx0Pjo9sDUuDADA1w79hWY1IZWfO8l51DQ/3PrNIcciFiPRmOX5d8S5rQFekUCUa4SKsRQtf9wBQOReWbd15Rqfu7EIkgoJdiERQsAuRCAp2IRJBwS5EIijYhUiE1kpvDoD07MqcSysjg2FporOdS28/OswLJfZHivLtHeDZSe1tYdmlmOdSzdmJE9TWWOLFC3dey4tYZpHX3dnTHxwfHOKFL89P8qyx6UhmWz2ibm7ZEi5elI/IpWWS/QXEs7kWyzw7rEacZOMAUF7iGZi1Gr8/bhrkBZvM+HVVtPD102aRvoMelo8LeX5N6c4uRCIo2IVIBAW7EImgYBciERTsQiRCS1fjzYBCPpxM0lvideH6usM2a/DVyhnniQfnLvCUhcFufki6iuEV1XouXCMPAE6MnaC2oX5ez2zXdbwVUpnvDk8+HW6jdWqcr9J2l8Ir+ABQKPAWTy8cfZ07Qu4jjcj9ZSmyGj83z5NC+gZ4u6YaSYQZP0MLIqOrm5+XfMYTTTo7eYJVkbXlAoBqOJGnPs/P2dCW7uB4vsDbWunOLkQiKNiFSAQFuxCJoGAXIhEU7EIkgoJdiERYTfun+wH8MYAJd7+5OfZ5AH8B4GzzaZ9198dWs8PMwlLI8JZw7bRlJ4mME0mAGNnOE0kORuSwKdtMbZ6F6+T1DvKkit4engBRaA/LJwBwTUR6K/WGE4MA4H/e/43g+ELkWM0sTlLbwiKvDViIXD3DpN1ReZLXu5sniUYA0NvDpdSXXuY1AM+cORscn4m0jOrr4y+sp6tEbZlzTbRQ4ccxWzgVHN/cxbfX2x6Oo3zk9r2aO/vXAdweGP+yu+9r/qwq0IUQG8eKwe7uPwHA3/qFEL8WrOV/9nvM7LCZ3W9m/CtYQoirgssN9q8CuBbAPgDjAL7Inmhmd5vZQTM7ODXFa7ILIdaXywp2dz/j7nV3bwD4GgDatcDdD7j7fnff39fHGw4IIdaXywp2Mxu56M+PA3j+yrgjhFgvViO9fQvAhwEMmtkogM8B+LCZ7cNyVbkTAP5yNTvL5XI0+6enn0tvtXrYzbY8zyS6fvdOajv4NJe8ZgrXUVvDZoPjQ9u4vPbikZ9T2+/8y39DbT//2T9T2/x8pE1S5VxwfOL0G3RO7D1/rspteXBpqD8XXtPd1sF9nz7LJbRaxpeFhrZwW70ezqRbjLR4Ki/yunvzkRp6tQaX86rlUWrbUghn9G0t8Sy6pVp4TuzuvWKwu/snA8P3rTRPCHF1oW/QCZEICnYhEkHBLkQiKNiFSAQFuxCJ0NKCk7lcDl2lcPZS/+AgnVezsJvlXJHOaS/1UFtfHy8o+Pobp6ntg+99d9iPOd5OqrObFzYcP8XlmKOvvEJttTpvT5Qj9QbnZ6bpnO5NI9Q2Pc1lqN4SL0Z5w/W/ERx/6tmX6JxDR45T2wc/cge1FYpcojp2NCznTUXaWsWKYpYXuby2a4hLuh1d3MeBgfC16nlegLNWCRe+dJJVCujOLkQyKNiFSAQFuxCJoGAXIhEU7EIkgoJdiERoqfTm3kCjFpY8egd4Ib/5xXAhwoU677uVZfx9bOeO7dT2ygs882p6ISyxlbp4ht2Oa6kJJ1/hxRdPjY1R2/vfT8sHYGEhLA11b91G5wxs5cU5X5/kUtniEpcci13h/ms9m3fQObd08/Ny9my4HxoAnDj5DLXNL4RlyqlpLqFt2cyLjvY6Py+7SrwQ6JYe3oOtYOFilJUq72/XRSS2HHhM6M4uRCIo2IVIBAW7EImgYBciERTsQiRCS1fjG7UqZs+PB20dkdpeS+Xwiqo1uPtmfFVycICvmr6SO0ZtE5PhVdPzGV+V7i3x2no33swTco6deJ3aqrxLEk3w2Lt3L52zdzeXDE6O8wSaF154jtrOnwsnfhTbuOrSX+KJJKMvcFVg/Byva2ckWSqLtN4a2bGH2nbxPBPs7O6gtvYcT2pZKoevn0aD1zas1sj2+GWvO7sQqaBgFyIRFOxCJIKCXYhEULALkQgKdiESYTXtn3YA+FsAwwAaAA64+1fMbADAtwFcg+UWUH/u7hdi21paWsKxo2Fpa+fed9F57bmw9Nao8ESBfDuvj9YesXV3c2mo1BOuFXbjjTfQOT/8/mPUtjDN6911bhqitqOjvK7dju3hpJzdN9xK57QV+WWwZydP8pma5Kf7xSPhhKKGcwlq9AKvrTdDkqEAoFznsu3MVFiK3DLME3JOnuf16QZ28Oak59u4H2jw1zZFZDTPcylvqbEUHK+AJ9ys5s5eA/DX7v4uAO8D8FdmdhOAewE87u57ATze/FsIcZWyYrC7+7i7H2o+ngVwBMA2AHcCeKD5tAcAfGy9nBRCrJ1L+p/dzK4BcAuAJwAMufs4sPyGAGDLlXZOCHHlWHWwm1kJwHcBfMbd+fcTf3Xe3WZ20MwOzs7yggFCiPVlVcFuZgUsB/o33f17zeEzZjbStI8ACK4aufsBd9/v7vtji19CiPVlxWA3M8NyP/Yj7v6li0yPArir+fguAI9cefeEEFeK1WS9fQDApwA8Z2ZvFvv6LIAvAHjIzD4N4HUAf7bShhaWanjmaFg22nkzr6vWQDjbzFjmDwA0ePrPzOwstU1NnaO2TQP7guN33P4ROmffe26ktoe+9zC1mXEJpbe3n9q2bQ3XcSv1cMkoq4WPLwAMDPNLZGR3ldqmO8Ly5qFneL248TmeUuYFniHYO8KzGAevC8/L8lx+rTv342UPty8DgKOnuTxYzPg2F8vl4Ph85PKuNcLXx0z9ZTpnxWB3958CYJ7+7krzhRBXB/oGnRCJoGAXIhEU7EIkgoJdiERQsAuRCC0tOFmuG16ZDmfynKvzAoBeCEsTuQovhuhEmgCAXI7bto7wb/3+i98JZ461F7jksnsXb7v0R3/6CWr7zsN/T23nTvPXPT4dLl5YLh+lc4rgGs/kIrcdPcmz9lAJy3K+mUuR/UPhIpUA0IhUUlz+zheZ1x7eZsPChSgBoBppKzZd5/tqL/Bttue59DZv4Sy7aoHvyxvh49uISLa6swuRCAp2IRJBwS5EIijYhUgEBbsQiaBgFyIRWiq9LdUNL0+F318e+SnvG7Zv12BwfLjIM5A6C5FsrWHef21kMFxUEgCu3RPOKIPzYoLjZ89T2/0Pcnnt6WdepDbW+w4AaCKg8/d1r/Pt1dv48ajnuDSUR1hirUWkoVou0istdqVGstTKlfDr9hyfk49kxGUN3tfPy1ymrIHPKzTCPmbGz1mlSvxXrzchhIJdiERQsAuRCAp2IRJBwS5EIrR0Nb4Ow1wunCzww0Ov0HmvvBZuGfWHv3UTnXPtVl6z7PixcGsiAPjQe2+mtnaSmDBb4SvMD/3DU9R26MUxaluoRVoJRVaLc4Xw+3cjUpMvZ3wVObZqXW/wBKAlssJcrfM5Zrym3RIiSSHOX1s+T1a6M36f6+zkCS1FcP/rfMEddeOhVicTa1V+Xord4ZqCuRzfj+7sQiSCgl2IRFCwC5EICnYhEkHBLkQiKNiFSIQVpTcz2wHgbwEMA2gAOODuXzGzzwP4CwBnm0/9rLs/Ft1ZPo9Ng5uDtskLXD4ZvzAVHP/Zsy/ROfXqrognXFrZPEySXQBYFpbDnjz4PJ3z9z/6ObUtNXjNNeS59JbLXfp7dH2JJ7t4RJZrROS1mOTFWigV8vySs4xLmMj4OctH5mVZeH+xJqNZ5PhmzuXBeiTZqBGRDplmNzLM5ePunrDtWFvkOHEPfkkNwF+7+yEz6wbwtJn9oGn7srv/11VsQwixwaym19s4gPHm41kzOwKAl0wVQlyVXNLnQTO7BsAtAJ5oDt1jZofN7H4z461FhRAbzqqD3cxKAL4L4DPuPgPgqwCuBbAPy3f+L5J5d5vZQTM7WFvkrZKFEOvLqoLdlqvwfxfAN939ewDg7mfcve7uDQBfAxBssO7uB9x9v7vvz3fwRhBCiPVlxWA3MwNwH4Aj7v6li8ZHLnraxwHwJWkhxIazmtX4DwD4FIDnzOyZ5thnAXzSzPZhuerVCQB/udKGzIzKJIUCl5pq5bCccPzMDJ2zNH+E2j506/XU1tE3Qm3T5bBE8uMnDtI5i84zl6o1LuO0tfHMtkakDtrCQriVUIwskpFlPOktWu+sjUheFsnKQsRmbVym7OjgtevyROqrRjLKZufnqa0ekSmXavy89PaH6ygCwPBI2FaKFN5bnA3/S+yRa2M1q/E/BRA65VFNXQhxdaFv0AmRCAp2IRJBwS5EIijYhUgEBbsQidDSgpPujkaNZFHFMoaysAxVAc92OjO3RG2HXuaFHu9Y4NLKrIfljlMX+DcD20s8u6q2wP0vL3H/OzsjUhNpexXbnuW4H7lIu6ZYBpsTGc0j95dCRG6cq/Lsu0qNS2VMlotl7MUktPlI661SH5fX+jfzlmOVWnibL73EszoLJBuxWuH+6c4uRCIo2IVIBAW7EImgYBciERTsQiSCgl2IRGip9AYHwLKGnMsdWRYu1tdwLgvVc7zA3/EJLpXd/xDP7/noh/eHtzd2NjgOAPP1WBHCiAzVzgsHZkVu6yQ9zIodXNZanOXSVSw7zCMSVYFkbGV5fs5i+8oiRSVjfewWF+YueU5sX339A9S2aYhnTJ49P0ltU+dOh8dP8p6E1+3ZHTZEJEXd2YVIBAW7EImgYBciERTsQiSCgl2IRFCwC5EILZXe8vkMm/r6grZymcth84vhTJ5ixrO/ahFZKBcpbvnjJw9T2/GxcLbc1DwvHDk5t0htJNkJANDVFcmWixQVbGsLv7Z8RK5r7+AZZVkkIy5f4Nusk/tILSJ5WcTmzn2sV/nxr1TDB7mjnUuRg5s2UdvAIJfXKpHMzaVipHgk6c/WKHD5eL4cvq7qEQlbd3YhEkHBLkQiKNiFSAQFuxCJoGAXIhFWXI03s3YAPwHQ1nz+d9z9c2a2G8CDAAYAHALwKXePrC8D3nCUySpiW+RtZ6keXm0tZHw1uMYXkeE5vrNcB18FP0ESXnKR5I5ala8wxxSDcrlMbfOR9kQ58trYKj0AdBX5qm9HJIEml4uoAu3h/XV08uNbqfBEmLOTPJGkAT4vXwgfj/6eLjpneCCsGAHA8DBPhJma53X+ZqYuUNvc9FRwvG+A7+vc2XPB8VokmWg1d/YlAB919/dguT3z7Wb2PgB/A+DL7r4XwAUAn17FtoQQG8SKwe7LvJknWGj+OICPAvhOc/wBAB9bFw+FEFeE1fZnz5odXCcA/ADAawCm3H/ZonQUwLb1cVEIcSVYVbC7e93d9wHYDuA2AO8KPS0018zuNrODZnawusBbLAsh1pdLWo139ykA/wTgfQD6zH7Z2Hs7gOB3Sd39gLvvd/f9hc6etfgqhFgDKwa7mW02s77m4w4AvwfgCIB/BPCnzafdBeCR9XJSCLF2VpMIMwLgATPLsPzm8JC7/52ZvQjgQTP7zwB+AeC+lTbUaDSwtBiWlNoyo/M6iZeNKk8yiXQtQgNcMmpEEgkapN1UrRJJ4Kjz1xVrQRSzNSKJMEx6uzDJpZ/JyHHs6eYSVW+kHlsPqYXXDi7l1RtcuspbJFmnjZ/spXJ4m+15fl5i+6otTEds3P+5qfPU1iDJOu1tXBItkzp5ZpHXRS1N3P0wgFsC48ew/P+7EOLXAH2DTohEULALkQgKdiESQcEuRCIo2IVIBItJPFd8Z2ZnAZxs/jkIIJy601rkx1uRH2/l182PXe6+OWRoabC/ZcdmB9093DxNfsgP+XHF/dDHeCESQcEuRCJsZLAf2MB9X4z8eCvy4628Y/zYsP/ZhRCtRR/jhUiEDQl2M7vdzF42s6Nmdu9G+ND044SZPWdmz5jZwRbu934zmzCz5y8aGzCzH5jZq83f/Rvkx+fN7FTzmDxjZne0wI8dZvaPZnbEzF4ws3/XHG/pMYn40dJjYmbtZvakmT3b9OM/Ncd3m9kTzePxbTPjFVdDuHtLfwBkWC5rtQdAEcCzAG5qtR9NX04AGNyA/X4IwK0Anr9o7L8AuLf5+F4Af7NBfnwewL9v8fEYAXBr83E3gFcA3NTqYxLxo6XHBIABKDUfFwA8geWCMQ8B+ERz/L8D+LeXst2NuLPfBuCoux/z5dLTDwK4cwP82DDc/ScA3l4b+U4sF+4EWlTAk/jRctx93N0PNR/PYrk4yja0+JhE/GgpvswVL/K6EcG+DcAbF/29kcUqHcD3zexpM7t7g3x4kyF3HweWLzoAWzbQl3vM7HDzY/66/ztxMWZ2DZbrJzyBDTwmb/MDaPExWY8irxsR7KFSGhslCXzA3W8F8IcA/srMPrRBflxNfBXAtVjuETAO4Iut2rGZlQB8F8Bn3H3DqpMG/Gj5MfE1FHllbESwjwLYcdHftFjleuPuY83fEwAexsZW3jljZiMA0Pw9sRFOuPuZ5oXWAPA1tOiYmFkBywH2TXf/XnO45cck5MdGHZPmvi+5yCtjI4L9KQB7myuLRQCfAPBoq50wsy4z637zMYA/APB8fNa68iiWC3cCG1jA883gavJxtOCY2HLhtPsAHHH3L11kaukxYX60+pisW5HXVq0wvm218Q4sr3S+BuA/bJAPe7CsBDwL4IVW+gHgW1j+OFjF8iedTwPYBOBxAK82fw9skB/fAPAcgMNYDraRFvjxQSx/JD0M4Jnmzx2tPiYRP1p6TAD8JpaLuB7G8hvLf7zomn0SwFEA/xtA26VsV9+gEyIR9A06IRJBwS5EIijYhUgEBbsQiaBgFyIRFOxCJIKCXYhEULALkQj/DwxILx4mBcfcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat\n"
     ]
    }
   ],
   "source": [
    "imshow(images[0])\n",
    "# print(\"predicted class: \" + str(classes[outputs[0]]))\n",
    "print(classes[outputs[0].detach().numpy().argmax()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean\n",
      "0.014815067\n",
      "standard deviation\n",
      "0.011969818\n",
      "Sparsity\n",
      "0.004412533114307208\n"
     ]
    }
   ],
   "source": [
    "# analysis of model params\n",
    "# ! pip install scipy\n",
    "print_stats(model_new, sparsity=1e-4)\n",
    "# print(\"mean: \" + np.mean([param.view(-1) for param in params]))\n",
    "model_newer = model_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean\n",
      "0.013458797\n",
      "standard deviation\n",
      "0.011134833\n",
      "Sparsity\n",
      "0.010480527728399868\n"
     ]
    }
   ],
   "source": [
    "print_stats(model_new, sparsity=1e-4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Values for model with 10 epochs of training\n",
    "- model params Conv2d -FracMaxPool - BatchNorm2d -  conv2d - FracMaxPool - linear - relu - bn - linear - relu- CELoss.   \n",
    "- exp regularizer used:  $(1/n)\\exp(|x|)$ where $n$ is the number of model parameters.  \n",
    "\n",
    "### Test error with the above regularizer: \n",
    "\n",
    "- Test error: $0.5732$\n",
    "\n",
    "### Test error without regularizer:\n",
    "- test error: $0.5625$\n",
    "\n",
    "### Test error with wight decay with $\\alpha=1$\n",
    "- test error: $0.9$ (???)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}