ULMFiT_Pretrain.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ULMFiT + Siamese Network for Sentence Vectors\n",
    "## Part Two: Pretraining\n",
    "This notebook will build a language model from lesson 10 of the Fast ai course and retrain it on the SNLI dataset. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *\n",
    "import html\n",
    "\n",
    "import json\n",
    "import html\n",
    "import re\n",
    "import pickle\n",
    "from collections import Counter\n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import sklearn\n",
    "from sklearn import model_selection\n",
    "from functools import partial\n",
    "from collections import Counter, defaultdict\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils \n",
    "import torch.optim as optim\n",
    "import torch.optim.lr_scheduler as lr_scheduler\n",
    "from torch.utils.data import dataset, dataloader\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import time\n",
    "import math\n",
    "import sys\n",
    "import data\n",
    "\n",
    "token_files = './data/tokens/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the tokens from the SNLI data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34160"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "itos = pickle.load(open(f'{token_files}itos.pkl', 'rb'))\n",
    "trn_lm = np.load(f'{token_files}trn_lm.npy')\n",
    "val_lm = np.load(f'{token_files}val_lm.npy')\n",
    "\n",
    "stoi = defaultdict(lambda:0, {v:k for k,v in enumerate(itos)})\n",
    "vocab_size = len(itos)\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_bos a very young child in a denim baseball cap eats a green apple . x_bos a boy combing a young baby 's hair . x_bos the boy was jumping on a platform . x_bos a little girl jumps while playing on the beach . x_bos two people are on the ocean . x_bos a girl is laughing because something terrible happened to someone she hates . x_bos the old woman is about to tell a story . x_bos the female is practicing while wearing no shoes . x_bos the boy is playing on the soccerfield . x_bos an african "
     ]
    }
   ],
   "source": [
    "# check to make sure that the data looks ok\n",
    "for word in trn_lm[:100]:\n",
    "    print(itos[word], end=\" \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the Wikitext LM and fix the weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#download the wikitext LM\n",
    "# ! wget -nH -r -np -P ./data/aclImdb/ http://files.fast.ai/models/wt103/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#these are the values used for the original LM\n",
    "em_sz,nh,nl = 400,1150,3\n",
    "PATH = Path('./data/aclImdb')\n",
    "PRE_PATH = PATH/'models'/'wt103'\n",
    "PRE_LM_PATH = PRE_PATH/'fwd_wt103.h5'\n",
    "wgts = torch.load(PRE_LM_PATH, map_location=lambda storage, loc: storage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the mean weight value for any new vocab\n",
    "enc_wgts = to_np(wgts['0.encoder.weight'])\n",
    "row_m = enc_wgts.mean(0)\n",
    "\n",
    "itos2 = pickle.load(Path(PRE_PATH/'itos_wt103.pkl').open('rb'))\n",
    "stoi2 = defaultdict(lambda:-1, {v:k for k,v in enumerate(itos2)})\n",
    "\n",
    "#fill in the missing values from the old vocab\n",
    "new_w = np.zeros((vocab_size, em_sz), dtype=np.float32)\n",
    "for i,w in enumerate(itos):\n",
    "    r = stoi2[w]\n",
    "    new_w[i] = enc_wgts[r] if r>=0 else row_m\n",
    "    \n",
    "#fix up the wgts with the new values\n",
    "wgts['0.encoder.weight'] = T(new_w)\n",
    "wgts['0.encoder_with_dropout.embed.weight'] = T(np.copy(new_w))\n",
    "wgts['1.decoder.weight'] = T(np.copy(new_w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_seq = 20*70\n",
    "wd = 1e-7\n",
    "bptt = 70\n",
    "batch_size = 32\n",
    "opt_fn = partial(optim.Adam, betas=(0.8, 0.99))\n",
    "\n",
    "trn_dl = LanguageModelLoader(trn_lm, batch_size, bptt)\n",
    "val_dl = LanguageModelLoader(val_lm, batch_size, bptt)\n",
    "md = LanguageModelData(PATH, 1, vocab_size, trn_dl, val_dl, bs=batch_size, bptt=bptt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the language model and load the weigths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "drops = np.array([0.25, 0.1, 0.2, 0.02, 0.15])*0.7\n",
    "learner = md.get_model(opt_fn, em_sz, nh, nl, \n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4])\n",
    "\n",
    "learner.metrics = [accuracy]\n",
    "learner.freeze_to(-1)\n",
    "\n",
    "learner.model.load_state_dict(wgts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fbe301a8ee642c9a802fa172709d799",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy   \n",
      "    0      3.417712   3.237231   0.394114  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[3.237231071355642, 0.3941137604774769]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr=1e-3\n",
    "lrs = lr\n",
    "learner.fit(lrs/2, 1, wds=wd, use_clr=(32,2), cycle_len=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "270d7781f5b44e2191bb98207fda262c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy   \n",
      "    0      3.126726   3.002443   0.417283  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "learner.lr_find(start_lr=lrs/10, end_lr=lrs*10, linear=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEOCAYAAABmVAtTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8VFX6x/HPk05CQoCEXkIVkE5AEBEQsNfVn2UV2659dy3rrrp27K66u3ax17Xg2sACKqAgvUkRkN4hlAChhJTz+2OGIWWSTGJmJuX7fr3mlVvOvfOES+aZc88955hzDhEREYCIcAcgIiJVh5KCiIj4KCmIiIiPkoKIiPgoKYiIiI+SgoiI+CgpiIiIj5KCiIj4KCmIiIiPkoKIiPhEhTuA8kpJSXFpaWnhDkNEpFqZM2fOdudcalnlql1SSEtLY/bs2eEOQ0SkWjGztYGU0+0jERHxUVIQEREfJQUREfFRUhARER8lBRER8VFSEBERn1qTFHZkZTN+8ZZwhyEiUqXVmqTw2tTVXP32HK55ezZb9xys0Dk0n7WI1HS1JincNLwjt53ciUnLMhj+5GTemb6W/PzAPuQXbtjNn96bS9d7v+G9GeuCHKmISPhYdfv2m56e7n5Lj+Y12/dx56cLmbpiB31a1+fWE48iPa0+0ZGF8+PBnDwmLcvgjZ9WM33VThJjo0hLSWDhxt1cfmwad53WmajIWpNTRaSaM7M5zrn0MsvVtqQAnttAH8/dyIPjlpC5P4fEuCiO75DKkKNSiY6M4JvFW5i0LIMDOXk0SYrjyuPSuLBfKxJionj4y194dcpqBnVI4Z/n9aBJvbhK+s1ERIJHSSEAWdm5TPk1g4lLM5i4bBvb9mYD0CgxlhOPbsxJRzehf9uGxWoR789cx12fLiI339GpSSKDOqRwUb9WtE2tWylxiYhUNiWFcnLOsWTzHnLzHN2a1yMiwkotvzIjiwlLtvLjrxnMWr2LxLgoPr7uWNJSEio9NhGR30pJIYRWZWRx3ovTqBsbxZjrBtAoUbeURKRqCTQpqKW0ErRNrcurl6WTsTebK16fRVZ2brhDEhGpECWFStKrVX2ev7g3S7fs5YrXZ7L3YE64QxIRKTclhUo0tFMj/nNhT+auy2TkqzPZfcCTGNbt2M/KjKwwRyciUrZqN/NaVXd692ZER0bwp/fmct4LP2EGy7dmYQaXH5vG30/qRFx0BBszD5AQE0X9hJhwhywi4qOG5iCZuHQb//hkIa0bxjOiSxPW7tjHW9PW0jy5Ds45Nu0+SExUBOent2Bwx0ZMX7WDLbsP8ui53UiMiw53+CJSw+jpoyrop5Xb+feEX0lNjKVvWn2WbtnLx3M3kJPniImKICcvn5H9WzPqrK78b+4GHvlqKc2T63BMmwZcP7Q99eooWYhIxQSaFHT7KISObZfCse1SCm27aXhHVm3PolfL+jz29VLenLaG1g0TeOyrpbRvVJfYqAhembKaT+Zt5PHzujPkqEbhCV5EagXVFKqQrOxcTnxqMpt2H6RVg3g+/9NAkuNjWLhhN38bs4BVGfv48NoB9GyZHO5QRaSaUT+FaqhubBSPn9eDTk0SGX1pH5LjPY3Q3VrU479X9Sc1MZbr3pnD9qzsMEcqIjWVkkIVc1yHFL6+6Xg6NUkqtL1+QgwvjezDzn2HeGDskjBFJyI1nZJCNdK1eT0uH5jGFws2qd+DiASFkkI1c9WgtsRGRfLs9yvCHYqI1EBKCtVMSt1YLunfis/mb2TB+sxyHbv7QA5j5mzg1P/8yIn/msz7MzWLnIgUpqRQDV07uB3Nkutw6WszWbRxd5nlf926l5GvzqDPAxO49aMF5ObnUyc6ktv/t5DJyzNCELGIVBd6JLWaWr9zPxeOnk7G3mxO796UmKgI4qIjuf2UTkxensH89ZncPLwjMVERXPXWbKav3MHv+7fixC6N6dWyPofy8jnjmSnsOZjDxFuHEB+jLisiNZk6r9VwLRvE8/F1x/LCpBV8OHsDsdERZO7PYdHG3cxfn0luvmPxpj088X/dmbwsg4v7t+KOUzr7jo+LiOShc7px/kvT+HjuRkb2b832rGx+//J0ujRNok1KXbq1SGJwx0ZEljHhkIjUHEGrKZhZHPADEIsn+Yxxzt1bQtnzgI+Avs65UqsBqikUl5fviDB4feoaRo1dQuemSVyQ3oL7xy6hZf141u3cz8fXDaBP6waFjnPOcfZzU8k8kMPI/q2ZumI7U1ZsJ8KM7Nx8AAZ1SOHZi3pTL15DbIhUZ1WhppANnOCcyzKzaGCKmX3lnJtesJCZJQJ/AWYEMZYa7fA3+SuPa0PHxokc3SyJ+gkxZGXn8sT45TSrF0evlvWLHWdmXH18O254by4PjvsFgJuGd+Dawe1wDsbM3cCoLxZzzvNTeeR33Wharw6tGsaH9HcTkdAKWlJwnirI4Yfpo70vf9WSB4DHgVuDFUttclyHI2MrXT+kPXsO5tKpSWKJc06f2q0J3/91MMnxMezcd4i2KQm+siP7t6ZTk0Suf3cuF4z25PK3ruzH8R1Tg/+LiEhYBPXpIzOLNLP5wDZggnNuRpH9vYCWzrmxwYyjtoqIMP5xamd+17tFiWXMjLapdWmQEEP7RnWLJY++aQ2YdOsQHj+3OwDvz9JjrCI1WVCTgnMuzznXE2gB9DOzrof3mVkE8C/gr2Wdx8yuNrPZZjY7I0OPUIZaQmwU5/dtyRUD0/h2yTbGzNkQ7pBEJEhC0k/BOZcJTAJOLrA5EegKTDKzNUB/4HMzK9YQ4pwb7ZxLd86lp6bq1kW4XDWoLe0b1eXWjxawYdf+cIcjIkEQtKRgZqlmluxdrgMMB5Ye3u+c2+2cS3HOpTnn0oDpwJllPX0k4dMsuQ5PXdADgGkrd4Q5GhEJhmDWFJoCE83sZ2AWnjaFsWY2yszODOL7ShB1bJRIg4QYpq1SUhCpiYL59NHPQC8/2+8pofyQYMUilSciwujftgGTlmWwdc9BGifFhTskEalEGvtIyu2Goe3Jzsnj5g/mU92GSRGR0ikpSLkd3awefxnWgZ9W7qDNHV9yw7tz2X8oN9xhiUglUFKQCjk/vaVvedzCzbw7Xf0XRGoCDYgnFVI/IYbPbhhIamIsf/1wAc9PWsHuAzlcdEwrmifXCXd4IlJBqilIhfVomUyz5Drcc0YX2qXW5YXJKzn7uaks27I33KGJSAUpKchv1rlpEmOuO5avbxyEARe9PJ3d+3PCHZaIVICSglSaDo0TeeWydHbuO8TNH86n70Pfcu4LP/HG1NWAZ6jugzl5YY5SREqjNgWpVN1bJNO7VTLfL90GQMbebOas3cWBnHy+X7qV1dv38fVNx5NSNzbMkYqIP0oKUulevKQPn83fxNBOqbRqkMAf35rNY1/7Rjjhme9+5f6zupZyBhEJFyUFqXSNkuK46vi2vvXRI/vw0ez1tE2ty3sz1/HmtLW0b1SX3x/TWlN9ilQxSgoSdHHRkYwckAbAvuxcxv28mbs/W8ym3Qe57eRO4Q1ORApRUpCQGtGlMV/dOIiXf1jFS5NX0jAhhk2ZB+nRsh4fzFpPvnNcO7gdQ45qFO5QRWolJQUJKTOjc9MkHji7KxszD/jmhi5o+qqdnNK1CS9c0icMEYrUblbdBjRLT093s2dryoWaYHtWNs9PXMnJXZuQuf8QnZokERcTwbAnJrM3O5cL+7Zk1FldiYnSk9Miv5WZzXHOFZvErCjVFCRsUurGcs8ZXYptn3HnMP7xv4W8P2s9PVomc1G/VmGITqR20lcwqXLiY6L41wU9ad0wnnE/b2bvwRw+mbeBrGyNxCoSbKopSJVkZlzQtyWPf72MbveNB6Bz09W8+8djaJAQE+boRGou1RSkyrp6UFsu6d+KY9s15Orj27JyWxY3vj+PA4c0VIZIsKimIFVWVGQED57dzbfeNiWB2/+3kAfGLeHhc7qxYdd+GiXGqSFapBIpKUi1cWG/VkxensHXi7aQn+94f9Z6BnVI4Y0r+qlntEglUVKQauXMHs34atEW3p+1HoAff93O8Y9PpEuzJFITY7nn9C7ERUeGOUqR6kv1bqlWhndpTEKM50P/h78NBWBj5gEmLNnKezPWMWbOhnCGJ1LtKSlItRIdGcH0fwzjh78NpVXDeN774zG0SUngif/rQZOkOJ79fgXbs7LDHaZItaUezVJjvDBpJY99vZT68dH0blWfIZ0aMX7xFp46vyepiZq/QWq3QHs0KylIjeGc49LXZvLjr9uL7Vt8/0kkxKoJTWqvQJOCbh9JjWFmjB6ZztMX9SI+pnBj89H3fsPcdbvCFJlI9aGkIDVKnZhIzuzRjLF/Po57Tu/C8gdP8e2785NF7D2YE8boRKo+JQWpkdqm1uXK49oQExXBc7/vzandmvDL5j10u288Xy3cXKz8jqxsMvaqgVpEN1mlxjute1OGdW5EXNRC/jdvI//4ZCEnd22CmfHN4i2s3r6PR7/yzCG9ZNRJxMfoz0JqL/3vl1ohLjqSpy7oSa/W9bn700V0uecburWox8zVOwuV+3LhFs7r0yJMUYqEn24fSa0yqH0KAAdy8oolBIC1O/aFOiSRKkU1BalV0lISeOr8HjROimNlRhYndGrEjqxDrN6+jycnLOOZ71fw5xM6aJA9qbXK/J9vZglmFuFd7mhmZ5pZdPBDEwmO3/VuwcD2KVw6II0W9ePp0TKZs3s15zhvLeKTeRoqQ2qvQL4O/QDEmVlz4DvgCuCNYAYlEg4Pnd2NXq2SefSrpeTm5Yc7HJGwCCQpmHNuP/A74Bnn3DlA8Yl1Raq5iAjjkmNas2t/Dq9NXc389ZnhDkkk5AJKCmY2ALgYGOfdprYIqZH6t2sIwMNfLuXs56aqs5vUOoEkhZuAO4BPnHOLzawtMLGsg8wszsxmmtkCM1tsZvf7KXOtmS00s/lmNsXMVAORsGqeXIchR6X61t+YuoZ563bpqSSpNco1IJ63wbmuc25PAGUNSHDOZXkbpqcANzrnphcok3T4XGZ2JnC9c+7k0s6rAfEk2HbtO8Segznc+tECZq05Ml7SyodP1QxvUm1V2oB4ZvaemSWZWQKwBFhmZn8r6zjnkeVdjfa+XJEyBZNLQtH9IuFQPyGG1g0TePqiXnRsXNe3/fRnpujJJKnxArl91MX74X028CXQChgZyMnNLNLM5gPbgAnOuRl+ytxgZiuBx4G/BBy5SJA1rVeH8TcP5rMbBgLwy+Y93PzBAtbv3B/myESCJ5CkEO29/XM28JlzLocAv9E75/Kccz2BFkA/M+vqp8xzzrl2wG3AXf7OY2ZXm9lsM5udkZERyFuLVJoeLZN5+qJejOzfGoAHxy0Jc0QiwRNIUngJWIPn9s4PZtYaKLNNoSDnXCYwCSitveB9PInH3/GjnXPpzrn01NRUf0VEgurMHs144Oyu9GhRj/FLtqq2IDVWmUnBOfe0c665c+5UbzvBWmBoWceZWaqZJXuX6wDDgaVFynQosHoa8Gu5ohcJsecu7k2EGW9PXxvuUESCIpCG5npm9tTh2zdm9iSeWkNZmgITzexnYBaeNoWxZjbK+6QRwJ+8j6vOB24BLqvoLyISCi3qx3PS0Y35YNZ6DhzKC3c4IpUukE5orwGLgPO96yOB1/H0cC6Rc+5noJef7fcUWL4x4EhFqojLBqTx5cItfDx3A5d42xlEaopA2hTaOefudc6t8r7uB9oGOzCRqqpfmwaexufvfiUvX09RS80SSFI4YGbHHV4xs4HAgeCFJFK1mRnXHt+WbXuz+XrRlnCHI1KpArl9dB3wppnVAwzYCVwezKBEqrrhXRpzdLMkHhi7hFO6NiFCPZ2lhgjk6aP5zrkeQHegm3Oul3NuQfBDE6m6oiMjuGJgG7bsOcgMPzO4iVRXJdYUzOyWErYD4Jx7KkgxiVQLp3VrykPjlnD9u3OYe/cI39+GSHVW2u2jxJBFIVIN1YmJZNd+z9DaV701hxb163DxMa3o0Fh/OlJ9lZgUvE8ZiUgpbhnRkacmLOfbX7YCMHfdLj68ZgBx0ZFhjkykYso1dHZVoKGzpSpxztHnwW/Zue+Q3/3T7jiBpvXqhDgqkeIqbehsESmZmfHVjYO474wu3HZyp2L77/50URiiEqk4Tasp8hs1Torj8oFt2LL7IHPW7mJfdi7TVu0AYMMudemR6qXMpGBmscC5QFrB8s65UcELS6T6aVIvjlcu89TOc/PyueKNWfz463ae+e5X/jysQxlHi1QNgdw++gw4C8gF9hV4iUgJoiIjuHZwOwCenLCc8Yu3UN3a76R2KrOh2cwWOeeKTY4TLmpolupkU+YBjn30ewD+MqwDt4zoGOaIpLaqzIbmn8ysWyXEJFLrNEuuw3VDPDWGp7/7lcz9/p9SEqkqAkkKxwFzzGyZmf1sZgu9cySISABuGt6B8/q0AOCTeRvDHI1I6QJ5+uiUoEchUoPFRkXyxP/1YMycDdz/xRK6t6hHn9YNwh2WiF+BDIi3FkgGzvC+kr3bRKQCrntnbrhDEClRINNx3gi8CzTyvt4xsz8HOzCRmmb8zccDsG1vNl8s2BTmaET8C6RN4Q/AMc65e7xTafYHrgpuWCI1T8fGiVxzvGfSwvdmrAtzNCL+BdKmYEDBGcrzvNtEpJzuOLUzGVnZfL90G845DbctVU4gNYXXgRlmdp+Z3QdMB14NalQiNVivlslk7s/h6Hu/Yeueg+EOR6SQQBqanwKuwDMN5y7gCufcv4MdmEhNNeSoRgDsP5THte/MYVOmxkeSqqPEpGBmSd6fDYA1wDvA28Ba7zYRqYCWDeLp0TIZgHnrMjn20e/5aeX2MEcl4lFaTeE97885wOwCr8PrIlJBn90wkAnep5EAHvlyaRijETmitJnXTvf+bBO6cERqjw6NE/n5vhM5/8VprNmxj0O5+cREaYoTCa9A+il8F8g2ESm/pLho/nRCe/YezGXxpt3hDkek1DaFOG/bQYqZ1TezBt5XGtAsVAGK1HR90zxNdOc8/xN5+RpeW8KrtJrCNXjaDzp5fx5+fQY8F/zQRGqHxklxtGoQD8Db09ZwKDc/vAFJrVZiUnDO/cfbnnCrc66tc66N99XDOfdsCGMUqfFeGtkHgPu+WMI/v1Gjs4RPIP0UnjGzrmZ2vpldevgViuBEaovOTZN8y98t3RbGSKS2C6Sh+V7gGe9rKPA4cGaQ4xKpdVY8dArDOzcmY0+2pu6UsAnk+bfzgGHAFufcFUAPIDaoUYnUQlGREQzumMLe7Fx+//KMcIcjtVQgSeGAcy4fyPX2ct4GtA1uWCK1U782DQGYtmoHD41bwrszNHWJhFYgo6TONrNk4GU8Tx9lATODGpVILXVUk0QSYiLZdyiPl39cDUCPFsl0bV4vzJFJbRFIQ/P1zrlM59yLwAjgMu9tJBEJgl6t6hdaf2L8sjBFIrVRaZ3Xehd9AQ2AKO+yiATBUxf0AKBOdCQAk5ZlkLE3O5whSS1iJT3lYGYTvYtxQDqwAM/kOt2BGc6540o9sVkc8AOeRukoYIxz7t4iZW4B/gjkAhnAlWXN/5yenu5mz9Z4fFI7pN0+zre85tHTwhiJVHdmNsc5l15WudI6rw11zg0F1gK9nXPpzrk+QC9gRQAxZAMnOOd6AD2Bk82sf5Ey84B051x3YAyex11FxOun20/wLT/+tadTW25evmoOEjSBPH3UyTm38PCKc24Rng/5UjmPLO9qtPflipSZ6Jzb712dDrQIKGqRWqJZch1evtTz5e75SSv57pet3P6/hfR96FvSbh/HP79ZSk6ehsWQyhNIUvjFzF4xsyFmNtjMXgZ+CeTkZhZpZvPxPMY6wTlX2sPXfwC+CuS8IrXJiC6NOa17UwD+8OZsxszZ4Nv33MSVdLhTfzZSeQJJClcAi4EbgZuAJd5tZXLO5TnneuKpAfQzs67+ypnZJXjaLf5Zwv6rzWy2mc3OyMgI5K1FapQ/HFf6tCbbNNezVJJAHkk96Jz7l3PuHO/rX865cv0PdM5lApOAk4vuM7PhwJ3Amc45vzdKnXOjvW0a6ampqeV5a5EaoXer+nx83bGFtr1+eV8ePqcb4Lm1JFIZSuy8ZmYfOufON7OFFGkLAPA2DpfIzFKBHOdcppnVAYYDjxUp0wt4CTjZOadRwERK0ad1fVY+fCorM7KIj4mkRf14nHP845OFbNurmoJUjtJ6NN/o/Xl6Bc/dFHjTzCLx1Eg+dM6NNbNRwGzn3Od4bhfVBT4yM4B1zjkNtidSgsgIo2PjRN+6mXFc+xQWb9oTxqikJiltjubN3p8VGnzFOfcznsdXi26/p8Dy8IqcW0SOOLZ9Qx7/ejtb9xykcVJcuMORaq6020d78XPbCE8HNuecS/KzT0RCLL21ZzrPxZt2KynIb1Za57VE51ySn1eiEoJI1dG5qed20qw1u8osu3r7Ptbv3F9mOam9AhklFQAza4RnyAsAnHPrghKRiJRLYlw0RzdL4qcV2/3uX7xpNwkxUaSlJHDeCz/hgLl3jwhtkFJtlJkUzOxM4EmgGZ5OaK3xdF47OrihiUigBrZP4Y2f1pCTl0905JEbAPd8toi3pnmaBfu3bcCOfYfCFaJUE4F0XnsA6A8sd861wTML29SgRiUi5dKlaRKHcvP5ZfMe/vzfefy8IZPbxvzsSwgA01ft9C3rFpKUJJCkkOOc2wFEmFmEc24iAYx9JCKh06WZp5nv7k8X8cWCTZz57FQ+mL2+xPIvTlZnN/EvkKSQaWZ18QyD/a6Z/QfPUNciUkW0SUkgMsJYsGF3qeVO6+YZQ2nDrgOhCEuqoUCSwlnAfuBm4GtgJXBGMIMSkfKJjowgKsJK3B8bFcGDZ3f1DYsxebnGEBP/Ann66GrgI+fcBuDNIMcjIhWUnesZQvu53/fmtO5NfRP0/Pj3ocTHRNKwbiwA3VvU4+cNuzmUm09MVCDfC6U2CeR/RBLwjZn9aGY3mFnjYAclIuV3eHjtIUd5Bo188ZLe3DisAy0bxPsSAsBlA9IAWL51b8hjlKqvxOk4ixU06w5cAJwLbAjXEBWajlOkZHn5jshSbiMBTFu5g4tenk5CTCSLRxUbuFhqqN88Hacf24AtwA6gUUUDE5HgKSshAPRqlRxwWal9ykwKZnadmU0CvgNSgKvKGjZbRKquuOhIRvZvzZ6Dudz16cKyD5BaJZCaQmvgJufc0c65e51zS4IdlIgEV/+2DQF4Z/qR0Wq27jnIg2OXkHb7OFZsU3tDbVXm00fOudtDEYiIhM5JRx95XsQ5x8zVO7lg9HTftumrdtK+UaK/Q6WGC3hAPBGpOaIiI7hlREeemrCcNnd8WWy/HlWtvXTlRWqptJSEEvfNW5dJ2u3jGP3DSnLy8kMYlYSbkoJILdWlafFpUepERwLw35metoaHv1xK13u/CWlcEl66fSRSS7VvVJf594wgOT6G9Tv3M2XFdo5p04ATnpxcqFx2bn5A/R+kZlBNQaQWS46PAaBlg3gu6teKtql1qR8fXazcw1/+EurQJEyUFESkkAfO7kpcdAT/OLWTb9urU1aHMSIJJSUFESnk9O7NWPrAKfyud4twhyJhoKQgIn6l1I1lzaOncXbPZqQmegbUW7JpD9e8PZt92bm0vWOcbyRWqXzrd+7n6rdCP86bGppFpEwZe7NZtHE3pz8zBYBJyzLID2wsTamgQY9PBODbJVt5cNwS3riiX6mPEVcW1RREpFTLtmYBMPqHVb5tK7Zl+ZZ3ZGWHPKba5I9vzWbNjv0MeWISc9buCvr7KSmISKmev7g3cKQPA8CiTUem/XxhkuZ7DpV/TVge9PdQUhCRUrVJSSAxNoqJy7b5tu05kONbblIvLhxh1Uq/bN4T9PdQUhCRMjWoG8O2vUduE81YvTOM0dReO/YdCvp7KCmISJlSC0znWdSaHfuYs3YXabePY9HG3SWWk+pBSUFEytQgIca3/JdhHQrtW7Eti3dnrAXwPZ0kweFvvKrKpqQgImUqeOuob1r9Qvumr9pJTOSRj5J8PasaNIM6pAT9PZQURKRM3ZrXA2Dy34ZwTJuGvu09Wni2vz9rvW/bv7/7ldc0LEZQ3Di8Q9mFfiN1XhORMt15WmdGDmhN64aezlPT7xjGim1ZJNWJ4sxnpxYq+/R3vwJwfMdU2qUmsGHXARonxWninjL8+GsGb01by+iRfcjcn1Ns//MX9yY+Jvgf2UoKIlKmuOhIOjY+Mj1nk3pxNKkXV6gTW1HDn5rMhX1b+moRax49LehxVmeXvz6LvHxHbr6j1wMTiu1vl1o3JHEodYtIhaU1jPctn+tnAL2Ct5VWbNuLc2pvKEmety2mw51f+d3fsbGSgohUcVGREax+5FTuO6ML95zRhf9c2JMhR6X6LTv8qR+4/4slIY6w5jALzSRHQUsKZhZnZjPNbIGZLTaz+/2UOd7M5ppZrpmdF6xYRCR4zIzLB7ahXp1ozurZnJdG9imx7Bs/rWHbnoMhjK7qc85VqRpUMNsUsoETnHNZZhYNTDGzr5xz0wuUWQdcDtwaxDhEJIRioyJL3d/v4e/UvgBMW7mDB8YuISICFm0M/vAVgQpaUnCe1He4FSra+3JFyqwBMLP8YMUhIlIVXfTy9DLL9G6VzNx1mdx1WucQROQR1KePzCwSmAO0B55zzs0I5vuJSNWw+pFTufadOXyzeGu4Q6nWHj+vO+0bJZZdsBIFtaHZOZfnnOsJtAD6mVnXipzHzK42s9lmNjsjI6NygxSRSmdmPHl+T4BCvZ3hSEc4KVuoEwKE6Okj51wmMAk4uYLHj3bOpTvn0lNT/T/ZICJVS93YKO44pROf3jDQt21Yp0bs2h/8kT6rus27D5RZ5prBbUMQSXHBfPoo1cySvct1gOHA0mC9n4hUPdcMbkeXZklMuW0oE24+nu+WbmPDrgOsyii501tNtPtADh8W6LMx4JHvyzzmjlNC145QUDDbFJoCb3rbFSKAD51zY81sFDDbOfe5mfUFPgHqA2eY2f3OuaODGJOIhEGL+vGF1rfsPkjbEPXQDbeJy7ZxxeuzADi6eRJHN6vat8+CVlNwzv3snOvlnOvunOvqnBvl3X6Pc+7PNfjXAAAPzklEQVRz7/Is51wL51yCc66hEoJIzfbiJZ6pPZds3sOwJyeRuf8QuXmehw+/WbyF5Vv3hjO8oDicEACyc/OZvmpHmcfcfXqXYIZUKo19JCIh0yDBM1nPg+N+AaDnKM8YP38/+Sge/3oZULPHSIo048LRZT+KOrSEXuGhoGEuRCRkmpYwn/PhhFDT5QfYcznNOxptOCgpiEjItGwQX3ahGuTwrbHDDuUG1k83IiI04xz5fe+wvbOIiB+7QjA5fahc9vrMQuvvzVwXpkgCp6QgIiE19+4Rpe73N5dAdXTO81OZuqJwo/Jn8zeFKZrAKSmISEg1SIhh3t0jfOP5DO/cmDWPnsbvejf3lcmr5vM8b959gHnrMsssd7hB+cwezYgK4y2jgvT0kYiEXP2EGP44qC1/HHSk1+4T5/Xgf3M3ArDvUC5JcdElHv/zhkw27jrAKd2aBj3Wipi8rOzheIYclUpaSgIsy+C07k15+qJeAbc5BJNqCiJSJUREGD1bJgOw288cxQB7D+ZwKDefM5+dynXvzg1leKVyzvHlws10uvsr9h/KLdYXYXDHwo+YDuqQwhtX9OPvJ3Vi1FlHM6JzYwBioiLCPpe1koKIVBnXDm4HeIaFAJi4dBsn//sHNu8+wIez19PtvvF0vOvIdJVnPjslLHEWtHPfIS59bSbXvzuXgzn5dLnnGz4t0nZwarcmhdZfvMQzEVGdmEguHZAW1qeNitLtIxGpMurHe24ZZXprCle84ekNXNJYQT9v2B2awErwzvS13PXpojLLnZ/ekts+Xuhbj46sut/Hq25kIlLrNErydG675NUZHDiUF9Ax2bmBlasMzjn+/e1ytu31TCkaSEIAz1Dih4f4AMJ+i6g0VTcyEal1GifF+pY73/N1QMfsyw5dUvhp5Q7+/e2v9HvoOyYvD2xulym3DQXghE6NgxlapVFSEJEqIz6m/He0M/cf4q8fLmDxpuDfSsop0EP5stdmllLS4/Jj03wjxMZERfDqZen8dUTHoMVXGdSmICLV2glPTgZg4cZMxt88OGjv45zj8gIjngbi+qHtCq0P69yYYZ2rdo1BNQURqVJeuTS9xH3T7xjGn4a2B6B5cp1C+5ZvLT5xz+79OeSXoyPc9FU7irVlrNiWxcbMA/zjk4UlHFWyRon+BwCsypQURKRKGd6lMT/+fahvfWD7hgDMu3sETerFcetJR7Hm0dN4oUDDrT97DubQY9R4nhi/jHs/W8SXCzezbc/BEsvvyMrmwtHT6TlqfOF4nprMwEe/Z+HG8t2e+s+FPctVvqrQ7SMRqXJaNojn21uOJyYykub165CX74o9sdOwbmyx49bv3M+gxycSGxXBB9cMAOD9WevZue8Qb05bC8CYaweQntag2LGHKxTZBXoVF3yyadHGPaXGvPC+E+l235GE0q6aziynmoKIVEntGyXSqmE8kRHm9xHOorePAAY9PhHwfLA/NG4J4OlcVtDq7fv8vt+XCzcX25abF9itp09vGEhikWE5jmqSGNCxVY2SgohUW7PuHM6bV/bzu6+kBt26sUdukKzZvo/Hvl5KTl4+936+uFjZnLySxyK6foinEXnx/Sf5hufo1twz//Irl6ZX6Q5qpdHtIxGptlITYxmcmEpyfLSvF/Rh+7Nz/R7zwNglnNKtKasysnxPLr0waWWhMsOfmsy3twym30PfFTu+QUIMc+8egXOOm4Z3LFSLefsP/XhtymoGh3E6zd+qeqYyEZEC/ntV/2LbXpmy2m/ZTbs9jc3nPP9TiedbsS2L5Vv3cshPTeHbWzyPvZoVv62VHB/DLSceVW1rCaCkICI1QMfGxe/f7y9jmIzDg+6V5Kxnp/rdHh1ZdQavCwYlBRGp9iLLOcroS5NXllnmQI7/pFKdawGBqNm/nYjUGmf1bMZ9Z3Th2d/38m3r5+fRU4BHvlparnO/cPGRPhGxVXgwu8qghmYRqRH+c6EnGcxZu9O3beaancy7ewS5+Y6+D31b4XOf0q0pn/9pINNW7sBMt49ERKqNmMjIQuv1E2JITYxlzaOn+S2f3rp+QOft3iKZawa3K7tgNaekICI1SpvUhHKVH3PdsYXWVz9yamWGU+0oKYhIjVKwc1pZOjdNAuDxc7v7tpkZNw7r4Fv313O6JlNSEJEaZ2T/1gA8fVGvUsud0MnTyez8vi359IaBzLt7BADXDTlym2hEl6o91HVlU0OziNQ4D5zdlQfO7lpqmRcu7s2JRzfxrR8eqgIKP2F012mdKz/AKkw1BRGpNS4b0Nq3fEq3piX2bzAzHvldNybcfDxRNbxfQlGqKYhIrXHX6V04mJNPfGxkmWUv6tcqBBFVPUoKIlJrREdG8Nh53csuWIvVrnqRiIiUSklBRER8lBRERMQnaEnBzOLMbKaZLTCzxWZ2v58ysWb2gZmtMLMZZpYWrHhERKRswawpZAMnOOd6AD2Bk82s6EwYfwB2OefaA/8CHgtiPCIiUoagJQXnkeVdjfa+is6CfRbwpnd5DDDMavoQhCIiVVhQ2xTMLNLM5gPbgAnOuRlFijQH1gM453KB3UDDYMYkIiIlC2pScM7lOed6Ai2AfmZWtN+5v1pB0doEZna1mc02s9kZGRnBCFVERAhR5zXnXKaZTQJOBhYV2LUBaAlsMLMooB6w08/xo4HRAGaWYWZrKznEenhqKeE8V3mOC6RsaWVK2lee7SnA9jJiCIWadu0qur+6XbvKvG6/5XyBHvdb/+ZK2x+qa9e67CKAcy4oLyAVSPYu1wF+BE4vUuYG4EXv8oXAh8GKp4xYR4f7XOU5LpCypZUpaV95tgOzw3Gtavq1q+j+6nbtKvO6heLa/da/uep07YJZU2gKvGlmkXhuU33onBtrZqO8v9jnwKvA22a2Ak8N4cIgxlOaL6rAucpzXCBlSytT0r7ybq8Katq1q+j+6nbtKjuuYF+73/o3V9r+KnXtzJt9RMrFzGY759LDHYeUn65d9RWKa6cezVJRo8MdgFSYrl31FfRrp5qCiIj4qKYgIiI+SgoiIuKjpCAiIj5KChIUZpZgZnPM7PRwxyKBM7POZvaimY0xs+vCHY8EzszONrOXzewzMzuxoudRUpBCzOw1M9tmZouKbD/ZzJZ5hzm/PYBT3QZ8GJwoxZ/KuHbOuV+cc9cC5wN6bDVEKunafeqcuwq4HLigwrHo6SMpyMyOB7KAt5xzXb3bIoHlwAg8Q5PMAi4CIoFHipziSqA7nu74ccB259zY0ERfu1XGtXPObTOzM4HbgWedc++FKv7arLKunfe4J4F3nXNzKxJLSMY+kurDOfeDn8mO+gErnHOrAMzsfeAs59wjQLHbQ2Y2FEgAugAHzOxL51x+UAOXSrl23vN8DnxuZuMAJYUQqKS/OwMeBb6qaEIAJQUJjG+Ic68NwDElFXbO3QlgZpfjqSkoIYRPua6dmQ0BfgfEAl8GNTIpS7muHfBnYDhQz8zaO+derMibKilIIAIa4rxYAefeqPxQpJzKde2cc5OAScEKRsqlvNfuaeDp3/qmamiWQBwe4vywFsCmMMUi5aNrV32F5dopKUggZgEdzKyNmcXgGc328zDHJIHRtau+wnLtlBSkEDP7LzANOMrMNpjZH5xnqtQ/Ad8Av+AZBn1xOOOU4nTtqq+qdO30SKqIiPiopiAiIj5KCiIi4qOkICIiPkoKIiLio6QgIiI+SgoiIuKjpCBBZ2ZZIXiPMwMc0rsy33OImR1bgeN6mdkr3uXLzezZyo+u/MwsrejQzX7KpJrZ16GKSUJPSUGqDe9Qwn455z53zj0ahPcsbXywIUC5kwLwD+CZCgUUZs65DGCzmQ0MdywSHEoKElJm9jczm2VmP5vZ/QW2f+qdqW2xmV1dYHuWmY0ysxnAADNbY2b3m9lcM1toZp285XzfuM3sDTN72sx+MrNVZnaed3uEmT3vfY+xZvbl4X1FYpxkZg+b2WTgRjM7w8xmmNk8M/vWzBp7hzm+FrjZzOab2SDvt+iPvb/fLH8fnGaWCHR3zi3ws6+1mX3n/bf5zsxaebe3M7Pp3nOO8lfzMs9Md+PMbIGZLTKzC7zb+3r/HRaY2UwzS/TWCH70/hvO9VfbMbNIM/tngWt1TYHdnwIX+73AUv055/TSK6gvIMv780RgNJ7RHyOAscDx3n0NvD/rAIuAht51B5xf4FxrgD97l68HXvEuX45nUhiAN4CPvO/RBc+Y9ADn4RkOOgJoAuwCzvMT7yTg+QLr9TnS+/+PwJPe5fuAWwuUew84zrvcCvjFz7mHAh8XWC8Y9xfAZd7lK4FPvctjgYu8y9ce/vcsct5zgZcLrNcDYoBVQF/vtiQ8IyPHA3HebR2A2d7lNGCRd/lq4C7vciwwG2jjXW8OLAz3/yu9gvPS0NkSSid6X/O863XxfCj9APzFzM7xbm/p3b4DyAM+LnKe/3l/zsEz9r8/nzrPPA5LzKyxd9txwEfe7VvMbGIpsX5QYLkF8IGZNcXzQbu6hGOGA108c50AkGRmic65vQXKNAUySjh+QIHf523g8QLbz/Yuvwc84efYhcATZvYYMNY596OZdQM2O+dmATjn9oCnVgE8a2Y98fz7dvRzvhOB7gVqUvXwXJPVwDagWQm/g1RzSgoSSgY84px7qdBGz8Quw4EBzrn9ZjYJz1SeAAedc3lFzpPt/ZlHyf+HswssW5GfgdhXYPkZ4Cnn3OfeWO8r4ZgIPL/DgVLOe4Ajv1tZAh6YzDm33Mz6AKcCj5jZeDy3efyd42ZgK9DDG/NBP2UMT43sGz/74vD8HlIDqU1BQukb4EozqwtgZs3NrBGeb6G7vAmhE9A/SO8/BTjX27bQGE9DcSDqARu9y5cV2L4XSCywPh7PqJYAeL+JF/UL0L6E9/kJz/DI4LlnP8W7PB3P7SEK7C/EzJoB+51z7+CpSfQGlgLNzKyvt0yit+G8Hp4aRD4wEs+cv0V9A1xnZtHeYzt6axjgqVmU+pSSVF9KChIyzrnxeG5/TDOzhcAYPB+qXwNRZvYz8ACeD8Fg+BjPxCWLgJeAGcDuAI67D/jIzH4EthfY/gVwzuGGZuAvQLq3YXYJnvv/hTjnluKZLjGx6D7v8Vd4/x1GAjd6t98E3GJmM/HcfvIXczdgppnNB+4EHnTOHQIuAJ4xswXABDzf8p8HLjOz6Xg+4Pf5Od8rwBJgrvcx1Zc4UisbCozzc4zUABo6W2oVM6vrnMsys4bATGCgc25LiGO4GdjrnHslwPLxwAHnnDOzC/E0Op8V1CBLj+cHPBPI7wpXDBI8alOQ2masmSXjaTB+INQJwesF4P/KUb4PnoZhAzLxPJkUFmaWiqd9RQmhhlJNQUREfNSmICIiPkoKIiLio6QgIiI+SgoiIuKjpCAiIj5KCiIi4vP/RFZghf5k4jkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c9be047d8fb42b2b469292b321b4252",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=15), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy   \n",
      "    0      2.888024   2.757168   0.446942  \n",
      "    1      2.853163   2.717702   0.452594  \n",
      "    2      2.780269   2.647655   0.464938  \n",
      "    3      2.716107   2.590034   0.475679  \n",
      "    4      2.672535   2.540915   0.485196  \n",
      "    5      2.637532   2.499195   0.493423  \n",
      "    6      2.600963   2.458995   0.500602  \n",
      "    7      2.53759    2.423469   0.507219  \n",
      "    8      2.495851   2.390064   0.513193  \n",
      "    9      2.474802   2.3605     0.518039  \n",
      "    10     2.432823   2.330059   0.523042  \n",
      "    11     2.3932     2.303556   0.527746  \n",
      "    12     2.350647   2.276374   0.532724  \n",
      "    13     2.328032   2.254468   0.536535  \n",
      "    14     2.320836   2.238188   0.539734  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[2.2381879509070295, 0.5397343000702536]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrs=0.004\n",
    "learner.fit(lrs, 1, wds=wd, use_clr=(20,10), cycle_len=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(learner.model, \"snli_language_model.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/0dd0df21cf404cf2bb51d0148c8b7d8b"
  },
  "gist": {
   "data": {
    "description": "fastai.text imdb example",
    "public": true
   },
   "id": "0dd0df21cf404cf2bb51d0148c8b7d8b"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "86px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}