Dataset_test_with_BlazeFace256.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import stuff\n",
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "from itertools import product as product\n",
    "import time\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Function\n",
    "import pandas as pd\n",
    "\n",
    "# import dataset\n",
    "from utils.dataset import VOCDataset, DatasetTransform, make_datapath_list, Anno_xml2list, od_collate_fn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# set up person only VOC dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "000009\n",
      "000017\n",
      "000021\n",
      "000023\n",
      "000030\n",
      "000032\n",
      "000035\n",
      "000041\n",
      "000048\n",
      "000050\n",
      "000051\n",
      "000066\n",
      "000073\n",
      "000081\n",
      "000083\n",
      "000089\n",
      "000101\n",
      "000104\n",
      "000110\n",
      "000113\n",
      "000125\n",
      "000129\n",
      "000131\n",
      "000133\n",
      "000138\n",
      "000146\n",
      "000150\n",
      "000159\n",
      "000162\n",
      "000163\n",
      "000164\n",
      "000165\n",
      "000169\n",
      "000170\n",
      "000171\n",
      "000173\n",
      "000174\n",
      "000177\n",
      "000190\n",
      "000192\n",
      "000193\n",
      "000194\n",
      "000200\n",
      "000210\n",
      "000218\n",
      "000220\n",
      "000222\n",
      "000229\n",
      "000232\n",
      "000245\n",
      "000251\n",
      "000257\n",
      "000259\n",
      "000269\n",
      "000275\n",
      "000276\n",
      "000278\n",
      "000282\n",
      "000285\n",
      "000288\n",
      "000298\n",
      "000302\n",
      "000305\n",
      "000308\n",
      "000320\n",
      "000321\n",
      "000322\n",
      "000323\n",
      "000328\n",
      "000331\n",
      "000337\n",
      "000338\n",
      "000352\n",
      "000359\n",
      "000367\n",
      "000372\n",
      "000374\n",
      "000382\n",
      "000394\n",
      "000406\n",
      "000407\n",
      "000411\n",
      "000419\n",
      "000428\n",
      "000433\n",
      "000435\n",
      "000438\n",
      "000443\n",
      "000446\n",
      "000448\n",
      "000463\n",
      "000468\n",
      "000470\n",
      "000476\n",
      "000477\n",
      "000480\n",
      "000482\n",
      "000483\n",
      "000498\n",
      "000499\n",
      "000500\n",
      "000515\n",
      "000516\n",
      "000518\n",
      "000520\n",
      "000523\n",
      "000524\n",
      "000525\n",
      "000526\n",
      "000530\n",
      "000531\n",
      "000535\n",
      "000541\n",
      "000545\n",
      "000554\n",
      "000555\n",
      "000579\n",
      "000583\n",
      "000589\n",
      "000591\n",
      "000597\n",
      "000612\n",
      "000613\n",
      "000625\n",
      "000626\n",
      "000628\n",
      "000633\n",
      "000648\n",
      "000654\n",
      "000677\n",
      "000684\n",
      "000688\n",
      "000690\n",
      "000694\n",
      "000695\n",
      "000702\n",
      "000709\n",
      "000717\n",
      "000726\n",
      "000731\n",
      "000733\n",
      "000739\n",
      "000742\n",
      "000750\n",
      "000752\n",
      "000753\n",
      "000755\n",
      "000760\n",
      "000770\n",
      "000777\n",
      "000782\n",
      "000786\n",
      "000793\n",
      "000797\n",
      "000799\n",
      "000802\n",
      "000805\n",
      "000806\n",
      "000810\n",
      "000812\n",
      "000814\n",
      "000816\n",
      "000828\n",
      "000829\n",
      "000834\n",
      "000842\n",
      "000843\n",
      "000847\n",
      "000848\n",
      "000851\n",
      "000854\n",
      "000855\n",
      "000859\n",
      "000860\n",
      "000862\n",
      "000865\n",
      "000874\n",
      "000878\n",
      "000879\n",
      "000885\n",
      "000892\n",
      "000895\n",
      "000898\n",
      "000902\n",
      "000903\n",
      "000904\n",
      "000906\n",
      "000911\n",
      "000915\n",
      "000918\n",
      "000920\n",
      "000926\n",
      "000937\n",
      "000943\n",
      "000948\n",
      "000949\n",
      "000966\n",
      "000967\n",
      "000971\n",
      "000982\n",
      "000987\n",
      "000991\n",
      "000996\n",
      "000999\n",
      "001001\n",
      "001011\n",
      "001014\n",
      "001017\n",
      "001024\n",
      "001028\n",
      "001036\n",
      "001042\n",
      "001050\n",
      "001057\n",
      "001060\n",
      "001061\n",
      "001064\n",
      "001066\n",
      "001071\n",
      "001072\n",
      "001079\n",
      "001084\n",
      "001091\n",
      "001092\n",
      "001097\n",
      "001101\n",
      "001109\n",
      "001113\n",
      "001125\n",
      "001129\n",
      "001137\n",
      "001140\n",
      "001145\n",
      "001147\n",
      "001151\n",
      "001152\n",
      "001164\n",
      "001168\n",
      "001170\n",
      "001171\n",
      "001175\n",
      "001185\n",
      "001186\n",
      "001201\n",
      "001206\n",
      "001212\n",
      "001221\n",
      "001224\n",
      "001229\n",
      "001234\n",
      "001236\n",
      "001240\n",
      "001241\n",
      "001248\n",
      "001254\n",
      "001259\n",
      "001265\n",
      "001266\n",
      "001269\n",
      "001272\n",
      "001279\n",
      "001281\n",
      "001284\n",
      "001287\n",
      "001292\n",
      "001298\n",
      "001304\n",
      "001309\n",
      "001310\n",
      "001311\n",
      "001315\n",
      "001325\n",
      "001327\n",
      "001330\n",
      "001333\n",
      "001337\n",
      "001346\n",
      "001350\n",
      "001352\n",
      "001362\n",
      "001378\n",
      "001388\n",
      "001390\n",
      "001393\n",
      "001405\n",
      "001406\n",
      "001408\n",
      "001409\n",
      "001414\n",
      "001420\n",
      "001421\n",
      "001426\n",
      "001427\n",
      "001430\n",
      "001434\n",
      "001445\n",
      "001450\n",
      "001451\n",
      "001455\n",
      "001460\n",
      "001463\n",
      "001472\n",
      "001475\n",
      "001479\n",
      "001480\n",
      "001484\n",
      "001485\n",
      "001493\n",
      "001498\n",
      "001499\n",
      "001501\n",
      "001504\n",
      "001509\n",
      "001510\n",
      "001514\n",
      "001521\n",
      "001523\n",
      "001524\n",
      "001526\n",
      "001531\n",
      "001532\n",
      "001536\n",
      "001537\n",
      "001544\n",
      "001548\n",
      "001554\n",
      "001557\n",
      "001561\n",
      "001563\n",
      "001571\n",
      "001577\n",
      "001579\n",
      "001580\n",
      "001586\n",
      "001593\n",
      "001594\n",
      "001603\n",
      "001608\n",
      "001611\n",
      "001617\n",
      "001622\n",
      "001627\n",
      "001628\n",
      "001630\n",
      "001640\n",
      "001649\n",
      "001651\n",
      "001654\n",
      "001678\n",
      "001682\n",
      "001686\n",
      "001689\n",
      "001690\n",
      "001691\n",
      "001711\n",
      "001717\n",
      "001721\n",
      "001723\n",
      "001725\n",
      "001726\n",
      "001727\n",
      "001729\n",
      "001730\n",
      "001749\n",
      "001754\n",
      "001756\n",
      "001758\n",
      "001759\n",
      "001766\n",
      "001772\n",
      "001775\n",
      "001784\n",
      "001785\n",
      "001789\n",
      "001793\n",
      "001795\n",
      "001797\n",
      "001799\n",
      "001800\n",
      "001801\n",
      "001806\n",
      "001807\n",
      "001809\n",
      "001810\n",
      "001828\n",
      "001830\n",
      "001832\n",
      "001833\n",
      "001836\n",
      "001837\n",
      "001841\n",
      "001843\n",
      "001849\n",
      "001854\n",
      "001861\n",
      "001864\n",
      "001881\n",
      "001887\n",
      "001894\n",
      "001899\n",
      "001902\n",
      "001903\n",
      "001904\n",
      "001918\n",
      "001920\n",
      "001927\n",
      "001930\n",
      "001932\n",
      "001938\n",
      "001941\n",
      "001944\n",
      "001945\n",
      "001950\n",
      "001958\n",
      "001963\n",
      "001977\n",
      "001978\n",
      "001980\n",
      "001989\n",
      "001999\n",
      "002000\n",
      "002012\n",
      "002022\n",
      "002024\n",
      "002030\n",
      "002051\n",
      "002055\n",
      "002061\n",
      "002069\n",
      "002082\n",
      "002090\n",
      "002091\n",
      "002096\n",
      "002104\n",
      "002108\n",
      "002114\n",
      "002116\n",
      "002117\n",
      "002120\n",
      "002142\n",
      "002145\n",
      "002156\n",
      "002163\n",
      "002169\n",
      "002170\n",
      "002171\n",
      "002172\n",
      "002174\n",
      "002176\n",
      "002180\n",
      "002183\n",
      "002186\n",
      "002187\n",
      "002190\n",
      "002193\n",
      "002194\n",
      "002213\n",
      "002214\n",
      "002219\n",
      "002220\n",
      "002221\n",
      "002224\n",
      "002226\n",
      "002233\n",
      "002238\n",
      "002241\n",
      "002244\n",
      "002256\n",
      "002261\n",
      "002265\n",
      "002272\n",
      "002273\n",
      "002277\n",
      "002281\n",
      "002285\n",
      "002288\n",
      "002290\n",
      "002293\n",
      "002302\n",
      "002306\n",
      "002307\n",
      "002311\n",
      "002324\n",
      "002329\n",
      "002332\n",
      "002337\n",
      "002342\n",
      "002361\n",
      "002362\n",
      "002367\n",
      "002372\n",
      "002378\n",
      "002382\n",
      "002384\n",
      "002385\n",
      "002387\n",
      "002401\n",
      "002403\n",
      "002405\n",
      "002410\n",
      "002413\n",
      "002419\n",
      "002425\n",
      "002433\n",
      "002443\n",
      "002444\n",
      "002448\n",
      "002450\n",
      "002458\n",
      "002459\n",
      "002460\n",
      "002472\n",
      "002476\n",
      "002477\n",
      "002481\n",
      "002483\n",
      "002491\n",
      "002497\n",
      "002501\n",
      "002504\n",
      "002514\n",
      "002518\n",
      "002520\n",
      "002524\n",
      "002529\n",
      "002533\n",
      "002539\n",
      "002540\n",
      "002542\n",
      "002545\n",
      "002554\n",
      "002555\n",
      "002558\n",
      "002561\n",
      "002563\n",
      "002564\n",
      "002565\n",
      "002566\n",
      "002572\n",
      "002584\n",
      "002585\n",
      "002589\n",
      "002595\n",
      "002611\n",
      "002613\n",
      "002625\n",
      "002632\n",
      "002633\n",
      "002636\n",
      "002637\n",
      "002641\n",
      "002643\n",
      "002649\n",
      "002657\n",
      "002658\n",
      "002662\n",
      "002668\n",
      "002669\n",
      "002675\n",
      "002678\n",
      "002689\n",
      "002691\n",
      "002693\n",
      "002696\n",
      "002710\n",
      "002713\n",
      "002714\n",
      "002715\n",
      "002723\n",
      "002727\n",
      "002730\n",
      "002735\n",
      "002741\n",
      "002744\n",
      "002751\n",
      "002755\n",
      "002759\n",
      "002767\n",
      "002776\n",
      "002786\n",
      "002795\n",
      "002798\n",
      "002803\n",
      "002804\n",
      "002807\n",
      "002815\n",
      "002826\n",
      "002827\n",
      "002834\n",
      "002838\n",
      "002841\n",
      "002842\n",
      "002855\n",
      "002858\n",
      "002859\n",
      "002864\n",
      "002867\n",
      "002870\n",
      "002879\n",
      "002881\n",
      "002884\n",
      "002886\n",
      "002889\n",
      "002899\n",
      "002901\n",
      "002912\n",
      "002913\n",
      "002914\n",
      "002915\n",
      "002916\n",
      "002931\n",
      "002932\n",
      "002933\n",
      "002935\n",
      "002939\n",
      "002940\n",
      "002954\n",
      "002958\n",
      "002960\n",
      "002962\n",
      "002965\n",
      "002975\n",
      "002976\n",
      "002987\n",
      "002988\n",
      "002990\n",
      "002992\n",
      "003002\n",
      "003003\n",
      "003017\n",
      "003023\n",
      "003024\n",
      "003027\n",
      "003031\n",
      "003034\n",
      "003039\n",
      "003042\n",
      "003044\n",
      "003057\n",
      "003058\n",
      "003063\n",
      "003064\n",
      "003065\n",
      "003077\n",
      "003078\n",
      "003086\n",
      "003088\n",
      "003089\n",
      "003100\n",
      "003102\n",
      "003106\n",
      "003107\n",
      "003118\n",
      "003121\n",
      "003122\n",
      "003124\n",
      "003129\n",
      "003134\n",
      "003138\n",
      "003140\n",
      "003145\n",
      "003147\n",
      "003150\n",
      "003159\n",
      "003162\n",
      "003170\n",
      "003175\n",
      "003181\n",
      "003185\n",
      "003188\n",
      "003189\n",
      "003205\n",
      "003210\n",
      "003213\n",
      "003218\n",
      "003228\n",
      "003236\n",
      "003247\n",
      "003253\n",
      "003259\n",
      "003260\n",
      "003261\n",
      "003270\n",
      "003280\n",
      "003282\n",
      "003284\n",
      "003290\n",
      "003292\n",
      "003296\n",
      "003301\n",
      "003303\n",
      "003311\n",
      "003313\n",
      "003320\n",
      "003327\n",
      "003331\n",
      "003335\n",
      "003344\n",
      "003350\n",
      "003355\n",
      "003360\n",
      "003362\n",
      "003367\n",
      "003369\n",
      "003373\n",
      "003376\n",
      "003380\n",
      "003382\n",
      "003386\n",
      "003390\n",
      "003391\n",
      "003395\n",
      "003396\n",
      "003397\n",
      "003403\n",
      "003406\n",
      "003407\n",
      "003412\n",
      "003416\n",
      "003417\n",
      "003422\n",
      "003424\n",
      "003429\n",
      "003433\n",
      "003435\n",
      "003436\n",
      "003450\n",
      "003453\n",
      "003455\n",
      "003462\n",
      "003465\n",
      "003466\n",
      "003468\n",
      "003469\n",
      "003470\n",
      "003484\n",
      "003487\n",
      "003492\n",
      "003496\n",
      "003510\n",
      "003518\n",
      "003521\n",
      "003528\n",
      "003529\n",
      "003530\n",
      "003536\n",
      "003539\n",
      "003548\n",
      "003550\n",
      "003556\n",
      "003566\n",
      "003567\n",
      "003575\n",
      "003576\n",
      "003577\n",
      "003580\n",
      "003585\n",
      "003589\n",
      "003603\n",
      "003611\n",
      "003618\n",
      "003621\n",
      "003625\n",
      "003628\n",
      "003638\n",
      "003644\n",
      "003645\n",
      "003648\n",
      "003651\n",
      "003654\n",
      "003655\n",
      "003662\n",
      "003663\n",
      "003664\n",
      "003669\n",
      "003678\n",
      "003685\n",
      "003688\n",
      "003690\n",
      "003691\n",
      "003694\n",
      "003696\n",
      "003699\n",
      "003700\n",
      "003703\n",
      "003706\n",
      "003708\n",
      "003711\n",
      "003717\n",
      "003732\n",
      "003735\n",
      "003740\n",
      "003749\n",
      "003751\n",
      "003759\n",
      "003760\n",
      "003779\n",
      "003780\n",
      "003781\n",
      "003797\n",
      "003811\n",
      "003817\n",
      "003820\n",
      "003824\n",
      "003826\n",
      "003827\n",
      "003838\n",
      "003847\n",
      "003849\n",
      "003855\n",
      "003856\n",
      "003859\n",
      "003860\n",
      "003863\n",
      "003865\n",
      "003868\n",
      "003869\n",
      "003877\n",
      "003879\n",
      "003889\n",
      "003891\n",
      "003899\n",
      "003913\n",
      "003915\n",
      "003923\n",
      "003932\n",
      "003939\n",
      "003945\n",
      "003946\n",
      "003948\n",
      "003953\n",
      "003960\n",
      "003961\n",
      "003963\n",
      "003966\n",
      "003970\n",
      "003973\n",
      "003974\n",
      "003986\n",
      "003992\n",
      "003994\n",
      "003996\n",
      "004008\n",
      "004010\n",
      "004012\n",
      "004014\n",
      "004016\n",
      "004017\n",
      "004019\n",
      "004020\n",
      "004028\n",
      "004031\n",
      "004033\n",
      "004039\n",
      "004046\n",
      "004047\n",
      "004051\n",
      "004058\n",
      "004060\n",
      "004067\n",
      "004073\n",
      "004075\n",
      "004077\n",
      "004085\n",
      "004089\n",
      "004091\n",
      "004092\n",
      "004110\n",
      "004111\n",
      "004117\n",
      "004121\n",
      "004122\n",
      "004133\n",
      "004136\n",
      "004137\n",
      "004138\n",
      "004141\n",
      "004142\n",
      "004145\n",
      "004158\n",
      "004168\n",
      "004169\n",
      "004170\n",
      "004191\n",
      "004195\n",
      "004201\n",
      "004205\n",
      "004209\n",
      "004223\n",
      "004224\n",
      "004229\n",
      "004230\n",
      "004231\n",
      "004232\n",
      "004237\n",
      "004244\n",
      "004246\n",
      "004257\n",
      "004258\n",
      "004259\n",
      "004265\n",
      "004269\n",
      "004270\n",
      "004273\n",
      "004275\n",
      "004279\n",
      "004281\n",
      "004287\n",
      "004291\n",
      "004292\n",
      "004296\n",
      "004298\n",
      "004304\n",
      "004321\n",
      "004322\n",
      "004323\n",
      "004327\n",
      "004329\n",
      "004339\n",
      "004346\n",
      "004356\n",
      "004359\n",
      "004360\n",
      "004361\n",
      "004370\n",
      "004372\n",
      "004380\n",
      "004384\n",
      "004387\n",
      "004389\n",
      "004392\n",
      "004397\n",
      "004404\n",
      "004424\n",
      "004433\n",
      "004437\n",
      "004438\n",
      "004457\n",
      "004459\n",
      "004463\n",
      "004464\n",
      "004466\n",
      "004470\n",
      "004471\n",
      "004479\n",
      "004481\n",
      "004487\n",
      "004494\n",
      "004495\n",
      "004496\n",
      "004499\n",
      "004508\n",
      "004509\n",
      "004512\n",
      "004517\n",
      "004518\n",
      "004520\n",
      "004528\n",
      "004530\n",
      "004535\n",
      "004537\n",
      "004542\n",
      "004553\n",
      "004562\n",
      "004563\n",
      "004571\n",
      "004574\n",
      "004588\n",
      "004595\n",
      "004601\n",
      "004604\n",
      "004605\n",
      "004607\n",
      "004609\n",
      "004611\n",
      "004622\n",
      "004630\n",
      "004631\n",
      "004647\n",
      "004648\n",
      "004653\n",
      "004656\n",
      "004660\n",
      "004662\n",
      "004672\n",
      "004673\n",
      "004676\n",
      "004679\n",
      "004685\n",
      "004686\n",
      "004687\n",
      "004692\n",
      "004693\n",
      "004694\n",
      "004699\n",
      "004706\n",
      "004707\n",
      "004715\n",
      "004718\n",
      "004722\n",
      "004723\n",
      "004727\n",
      "004732\n",
      "004742\n",
      "004750\n",
      "004753\n",
      "004760\n",
      "004761\n",
      "004770\n",
      "004779\n",
      "004782\n",
      "004788\n",
      "004789\n",
      "004790\n",
      "004796\n",
      "004797\n",
      "004801\n",
      "004812\n",
      "004815\n",
      "004826\n",
      "004828\n",
      "004830\n",
      "004831\n",
      "004832\n",
      "004834\n",
      "004840\n",
      "004846\n",
      "004852\n",
      "004857\n",
      "004859\n",
      "004868\n",
      "004872\n",
      "004879\n",
      "004882\n",
      "004886\n",
      "004897\n",
      "004898\n",
      "004902\n",
      "004913\n",
      "004916\n",
      "004928\n",
      "004931\n",
      "004935\n",
      "004939\n",
      "004943\n",
      "004946\n",
      "004950\n",
      "004953\n",
      "004954\n",
      "004968\n",
      "004976\n",
      "004977\n",
      "004985\n",
      "004987\n",
      "004990\n",
      "004994\n",
      "004995\n",
      "005014\n",
      "005024\n",
      "005027\n",
      "005029\n",
      "005036\n",
      "005037\n",
      "005039\n",
      "005042\n",
      "005045\n",
      "005052\n",
      "005054\n",
      "005057\n",
      "005061\n",
      "005063\n",
      "005064\n",
      "005081\n",
      "005086\n",
      "005090\n",
      "005093\n",
      "005097\n",
      "005102\n",
      "005107\n",
      "005111\n",
      "005114\n",
      "005116\n",
      "005122\n",
      "005124\n",
      "005129\n",
      "005134\n",
      "005144\n",
      "005146\n",
      "005156\n",
      "005159\n",
      "005160\n",
      "005161\n",
      "005173\n",
      "005176\n",
      "005189\n",
      "005190\n",
      "005195\n",
      "005208\n",
      "005219\n",
      "005220\n",
      "005222\n",
      "005229\n",
      "005230\n",
      "005231\n",
      "005236\n",
      "005239\n",
      "005242\n",
      "005245\n",
      "005248\n",
      "005257\n",
      "005260\n",
      "005268\n",
      "005269\n",
      "005273\n",
      "005274\n",
      "005278\n",
      "005281\n",
      "005283\n",
      "005297\n",
      "005298\n",
      "005303\n",
      "005304\n",
      "005306\n",
      "005307\n",
      "005310\n",
      "005311\n",
      "005315\n",
      "005326\n",
      "005327\n",
      "005331\n",
      "005336\n",
      "005343\n",
      "005344\n",
      "005346\n",
      "005348\n",
      "005351\n",
      "005368\n",
      "005371\n",
      "005374\n",
      "005383\n",
      "005384\n",
      "005385\n",
      "005396\n",
      "005398\n",
      "005405\n",
      "005410\n",
      "005413\n",
      "005414\n",
      "005417\n",
      "005419\n",
      "005421\n",
      "005424\n",
      "005429\n",
      "005430\n",
      "005439\n",
      "005441\n",
      "005448\n",
      "005450\n",
      "005451\n",
      "005457\n",
      "005461\n",
      "005467\n",
      "005471\n",
      "005475\n",
      "005478\n",
      "005485\n",
      "005487\n",
      "005496\n",
      "005499\n",
      "005507\n",
      "005508\n",
      "005509\n",
      "005511\n",
      "005522\n",
      "005526\n",
      "005527\n",
      "005536\n",
      "005541\n",
      "005542\n",
      "005547\n",
      "005550\n",
      "005552\n",
      "005554\n",
      "005563\n",
      "005573\n",
      "005574\n",
      "005582\n",
      "005583\n",
      "005585\n",
      "005593\n",
      "005599\n",
      "005600\n",
      "005606\n",
      "005608\n",
      "005611\n",
      "005613\n",
      "005615\n",
      "005618\n",
      "005631\n",
      "005636\n",
      "005639\n",
      "005644\n",
      "005645\n",
      "005648\n",
      "005652\n",
      "005653\n",
      "005657\n",
      "005658\n",
      "005660\n",
      "005662\n",
      "005664\n",
      "005669\n",
      "005672\n",
      "005680\n",
      "005682\n",
      "005693\n",
      "005695\n",
      "005700\n",
      "005702\n",
      "005704\n",
      "005714\n",
      "005716\n",
      "005719\n",
      "005728\n",
      "005729\n",
      "005731\n",
      "005732\n",
      "005738\n",
      "005740\n",
      "005741\n",
      "005747\n",
      "005757\n",
      "005761\n",
      "005764\n",
      "005765\n",
      "005768\n",
      "005773\n",
      "005780\n",
      "005784\n",
      "005786\n",
      "005788\n",
      "005790\n",
      "005796\n",
      "005799\n",
      "005811\n",
      "005814\n",
      "005818\n",
      "005824\n",
      "005826\n",
      "005829\n",
      "005836\n",
      "005840\n",
      "005845\n",
      "005850\n",
      "005852\n",
      "005856\n",
      "005859\n",
      "005863\n",
      "005873\n",
      "005881\n",
      "005884\n",
      "005885\n",
      "005889\n",
      "005903\n",
      "005905\n",
      "005906\n",
      "005908\n",
      "005910\n",
      "005919\n",
      "005923\n",
      "005928\n",
      "005930\n",
      "005938\n",
      "005940\n",
      "005948\n",
      "005951\n",
      "005956\n",
      "005961\n",
      "005971\n",
      "005975\n",
      "005979\n",
      "005984\n",
      "005985\n",
      "005988\n",
      "005989\n",
      "005990\n",
      "005991\n",
      "005992\n",
      "006004\n",
      "006005\n",
      "006011\n",
      "006028\n",
      "006029\n",
      "006033\n",
      "006038\n",
      "006041\n",
      "006043\n",
      "006071\n",
      "006073\n",
      "006095\n",
      "006103\n",
      "006104\n",
      "006117\n",
      "006128\n",
      "006130\n",
      "006133\n",
      "006140\n",
      "006148\n",
      "006150\n",
      "006151\n",
      "006156\n",
      "006158\n",
      "006161\n",
      "006163\n",
      "006171\n",
      "006172\n",
      "006175\n",
      "006176\n",
      "006177\n",
      "006179\n",
      "006180\n",
      "006185\n",
      "006187\n",
      "006198\n",
      "006209\n",
      "006210\n",
      "006221\n",
      "006225\n",
      "006229\n",
      "006233\n",
      "006235\n",
      "006236\n",
      "006238\n",
      "006240\n",
      "006243\n",
      "006259\n",
      "006272\n",
      "006276\n",
      "006282\n",
      "006284\n",
      "006285\n",
      "006286\n",
      "006289\n",
      "006296\n",
      "006300\n",
      "006320\n",
      "006339\n",
      "006341\n",
      "006344\n",
      "006346\n",
      "006351\n",
      "006352\n",
      "006355\n",
      "006369\n",
      "006381\n",
      "006387\n",
      "006391\n",
      "006392\n",
      "006395\n",
      "006398\n",
      "006409\n",
      "006427\n",
      "006428\n",
      "006433\n",
      "006440\n",
      "006442\n",
      "006444\n",
      "006445\n",
      "006450\n",
      "006455\n",
      "006459\n",
      "006463\n",
      "006465\n",
      "006475\n",
      "006476\n",
      "006482\n",
      "006483\n",
      "006486\n",
      "006488\n",
      "006492\n",
      "006497\n",
      "006503\n",
      "006506\n",
      "006519\n",
      "006529\n",
      "006532\n",
      "006536\n",
      "006542\n",
      "006550\n",
      "006551\n",
      "006560\n",
      "006572\n",
      "006576\n",
      "006578\n",
      "006583\n",
      "006584\n",
      "006585\n",
      "006587\n",
      "006588\n",
      "006599\n",
      "006602\n",
      "006609\n",
      "006610\n",
      "006611\n",
      "006612\n",
      "006617\n",
      "006618\n",
      "006621\n",
      "006625\n",
      "006626\n",
      "006631\n",
      "006637\n",
      "006645\n",
      "006647\n",
      "006648\n",
      "006658\n",
      "006661\n",
      "006664\n",
      "006666\n",
      "006667\n",
      "006668\n",
      "006670\n",
      "006677\n",
      "006682\n",
      "006690\n",
      "006695\n",
      "006696\n",
      "006697\n",
      "006698\n",
      "006699\n",
      "006703\n",
      "006707\n",
      "006709\n",
      "006714\n",
      "006718\n",
      "006722\n",
      "006727\n",
      "006731\n",
      "006735\n",
      "006738\n",
      "006739\n",
      "006748\n",
      "006753\n",
      "006762\n",
      "006766\n",
      "006768\n",
      "006772\n",
      "006777\n",
      "006782\n",
      "006783\n",
      "006784\n",
      "006786\n",
      "006789\n",
      "006797\n",
      "006800\n",
      "006802\n",
      "006803\n",
      "006808\n",
      "006810\n",
      "006821\n",
      "006824\n",
      "006833\n",
      "006835\n",
      "006836\n",
      "006838\n",
      "006840\n",
      "006841\n",
      "006847\n",
      "006850\n",
      "006855\n",
      "006858\n",
      "006862\n",
      "006864\n",
      "006866\n",
      "006868\n",
      "006874\n",
      "006876\n",
      "006880\n",
      "006887\n",
      "006896\n",
      "006903\n",
      "006908\n",
      "006910\n",
      "006911\n",
      "006912\n",
      "006919\n",
      "006924\n",
      "006932\n",
      "006934\n",
      "006939\n",
      "006943\n",
      "006945\n",
      "006948\n",
      "006950\n",
      "006956\n",
      "006958\n",
      "006959\n",
      "006963\n",
      "006965\n",
      "006968\n",
      "006981\n",
      "006983\n",
      "006987\n",
      "006988\n",
      "006989\n",
      "006990\n",
      "007004\n",
      "007007\n",
      "007016\n",
      "007020\n",
      "007023\n",
      "007031\n",
      "007035\n",
      "007036\n",
      "007040\n",
      "007045\n",
      "007046\n",
      "007048\n",
      "007049\n",
      "007052\n",
      "007054\n",
      "007059\n",
      "007073\n",
      "007079\n",
      "007080\n",
      "007093\n",
      "007108\n",
      "007109\n",
      "007114\n",
      "007117\n",
      "007121\n",
      "007122\n",
      "007128\n",
      "007129\n",
      "007138\n",
      "007139\n",
      "007140\n",
      "007147\n",
      "007148\n",
      "007150\n",
      "007153\n",
      "007154\n",
      "007159\n",
      "007163\n",
      "007167\n",
      "007172\n",
      "007182\n",
      "007184\n",
      "007187\n",
      "007189\n",
      "007200\n",
      "007208\n",
      "007213\n",
      "007214\n",
      "007216\n",
      "007227\n",
      "007234\n",
      "007236\n",
      "007243\n",
      "007244\n",
      "007256\n",
      "007260\n",
      "007261\n",
      "007266\n",
      "007270\n",
      "007271\n",
      "007276\n",
      "007284\n",
      "007292\n",
      "007296\n",
      "007298\n",
      "007305\n",
      "007308\n",
      "007314\n",
      "007318\n",
      "007325\n",
      "007330\n",
      "007336\n",
      "007346\n",
      "007350\n",
      "007359\n",
      "007369\n",
      "007375\n",
      "007381\n",
      "007388\n",
      "007398\n",
      "007408\n",
      "007410\n",
      "007413\n",
      "007414\n",
      "007436\n",
      "007438\n",
      "007445\n",
      "007446\n",
      "007448\n",
      "007449\n",
      "007457\n",
      "007466\n",
      "007475\n",
      "007480\n",
      "007482\n",
      "007483\n",
      "007484\n",
      "007493\n",
      "007497\n",
      "007498\n",
      "007511\n",
      "007526\n",
      "007535\n",
      "007540\n",
      "007543\n",
      "007547\n",
      "007558\n",
      "007559\n",
      "007563\n",
      "007566\n",
      "007568\n",
      "007570\n",
      "007571\n",
      "007572\n",
      "007576\n",
      "007578\n",
      "007579\n",
      "007586\n",
      "007590\n",
      "007592\n",
      "007594\n",
      "007605\n",
      "007606\n",
      "007611\n",
      "007618\n",
      "007619\n",
      "007629\n",
      "007631\n",
      "007639\n",
      "007647\n",
      "007649\n",
      "007653\n",
      "007656\n",
      "007657\n",
      "007668\n",
      "007675\n",
      "007683\n",
      "007687\n",
      "007694\n",
      "007697\n",
      "007699\n",
      "007709\n",
      "007712\n",
      "007718\n",
      "007723\n",
      "007727\n",
      "007735\n",
      "007745\n",
      "007746\n",
      "007748\n",
      "007751\n",
      "007760\n",
      "007763\n",
      "007768\n",
      "007777\n",
      "007786\n",
      "007793\n",
      "007795\n",
      "007799\n",
      "007809\n",
      "007810\n",
      "007813\n",
      "007814\n",
      "007815\n",
      "007821\n",
      "007836\n",
      "007841\n",
      "007856\n",
      "007872\n",
      "007877\n",
      "007878\n",
      "007884\n",
      "007889\n",
      "007897\n",
      "007901\n",
      "007902\n",
      "007910\n",
      "007911\n",
      "007915\n",
      "007919\n",
      "007928\n",
      "007931\n",
      "007932\n",
      "007940\n",
      "007943\n",
      "007946\n",
      "007953\n",
      "007963\n",
      "007964\n",
      "007970\n",
      "007971\n",
      "007979\n",
      "007984\n",
      "007987\n",
      "007996\n",
      "007997\n",
      "007998\n",
      "008002\n",
      "008008\n",
      "008019\n",
      "008023\n",
      "008024\n",
      "008026\n",
      "008033\n",
      "008040\n",
      "008042\n",
      "008043\n",
      "008048\n",
      "008053\n",
      "008069\n",
      "008072\n",
      "008079\n",
      "008082\n",
      "008083\n",
      "008086\n",
      "008091\n",
      "008096\n",
      "008103\n",
      "008105\n",
      "008108\n",
      "008122\n",
      "008125\n",
      "008127\n",
      "008130\n",
      "008132\n",
      "008137\n",
      "008139\n",
      "008142\n",
      "008160\n",
      "008163\n",
      "008164\n",
      "008171\n",
      "008175\n",
      "008177\n",
      "008180\n",
      "008188\n",
      "008190\n",
      "008199\n",
      "008204\n",
      "008209\n",
      "008211\n",
      "008216\n",
      "008218\n",
      "008220\n",
      "008224\n",
      "008225\n",
      "008232\n",
      "008241\n",
      "008248\n",
      "008253\n",
      "008261\n",
      "008263\n",
      "008272\n",
      "008279\n",
      "008281\n",
      "008284\n",
      "008285\n",
      "008293\n",
      "008294\n",
      "008301\n",
      "008302\n",
      "008307\n",
      "008312\n",
      "008313\n",
      "008315\n",
      "008327\n",
      "008332\n",
      "008341\n",
      "008360\n",
      "008368\n",
      "008374\n",
      "008376\n",
      "008381\n",
      "008385\n",
      "008387\n",
      "008390\n",
      "008391\n",
      "008397\n",
      "008398\n",
      "008410\n",
      "008413\n",
      "008430\n",
      "008433\n",
      "008437\n",
      "008444\n",
      "008445\n",
      "008449\n",
      "008452\n",
      "008453\n",
      "008461\n",
      "008467\n",
      "008470\n",
      "008475\n",
      "008482\n",
      "008485\n",
      "008503\n",
      "008509\n",
      "008513\n",
      "008514\n",
      "008526\n",
      "008529\n",
      "008541\n",
      "008550\n",
      "008559\n",
      "008562\n",
      "008573\n",
      "008576\n",
      "008582\n",
      "008585\n",
      "008587\n",
      "008592\n",
      "008596\n",
      "008602\n",
      "008604\n",
      "008612\n",
      "008617\n",
      "008618\n",
      "008628\n",
      "008638\n",
      "008644\n",
      "008647\n",
      "008663\n",
      "008665\n",
      "008676\n",
      "008690\n",
      "008692\n",
      "008698\n",
      "008699\n",
      "008701\n",
      "008702\n",
      "008706\n",
      "008718\n",
      "008728\n",
      "008731\n",
      "008738\n",
      "008739\n",
      "008742\n",
      "008744\n",
      "008748\n",
      "008752\n",
      "008766\n",
      "008768\n",
      "008769\n",
      "008771\n",
      "008775\n",
      "008794\n",
      "008801\n",
      "008805\n",
      "008810\n",
      "008813\n",
      "008817\n",
      "008833\n",
      "008836\n",
      "008838\n",
      "008862\n",
      "008865\n",
      "008867\n",
      "008872\n",
      "008876\n",
      "008879\n",
      "008884\n",
      "008886\n",
      "008891\n",
      "008892\n",
      "008900\n",
      "008911\n",
      "008920\n",
      "008921\n",
      "008931\n",
      "008932\n",
      "008943\n",
      "008944\n",
      "008953\n",
      "008960\n",
      "008961\n",
      "008965\n",
      "008975\n",
      "008979\n",
      "008983\n",
      "009000\n",
      "009005\n",
      "009020\n",
      "009022\n",
      "009024\n",
      "009034\n",
      "009036\n",
      "009045\n",
      "009049\n",
      "009064\n",
      "009068\n",
      "009073\n",
      "009078\n",
      "009079\n",
      "009087\n",
      "009089\n",
      "009091\n",
      "009114\n",
      "009123\n",
      "009128\n",
      "009136\n",
      "009141\n",
      "009153\n",
      "009160\n",
      "009161\n",
      "009174\n",
      "009180\n",
      "009184\n",
      "009193\n",
      "009197\n",
      "009202\n",
      "009208\n",
      "009209\n",
      "009214\n",
      "009224\n",
      "009230\n",
      "009236\n",
      "009238\n",
      "009239\n",
      "009242\n",
      "009251\n",
      "009259\n",
      "009273\n",
      "009283\n",
      "009286\n",
      "009287\n",
      "009288\n",
      "009290\n",
      "009291\n",
      "009295\n",
      "009296\n",
      "009299\n",
      "009303\n",
      "009306\n",
      "009308\n",
      "009312\n",
      "009316\n",
      "009318\n",
      "009327\n",
      "009330\n",
      "009331\n",
      "009336\n",
      "009342\n",
      "009343\n",
      "009349\n",
      "009358\n",
      "009362\n",
      "009368\n",
      "009374\n",
      "009375\n",
      "009377\n",
      "009382\n",
      "009393\n",
      "009394\n",
      "009407\n",
      "009410\n",
      "009411\n",
      "009412\n",
      "009413\n",
      "009414\n",
      "009418\n",
      "009420\n",
      "009421\n",
      "009424\n",
      "009433\n",
      "009434\n",
      "009437\n",
      "009440\n",
      "009448\n",
      "009456\n",
      "009460\n",
      "009468\n",
      "009469\n",
      "009477\n",
      "009479\n",
      "009481\n",
      "009490\n",
      "009494\n",
      "009496\n",
      "009497\n",
      "009499\n",
      "009500\n",
      "009504\n",
      "009512\n",
      "009517\n",
      "009518\n",
      "009520\n",
      "009524\n",
      "009526\n",
      "009531\n",
      "009532\n",
      "009537\n",
      "009541\n",
      "009542\n",
      "009546\n",
      "009551\n",
      "009557\n",
      "009558\n",
      "009565\n",
      "009567\n",
      "009568\n",
      "009573\n",
      "009577\n",
      "009579\n",
      "009585\n",
      "009591\n",
      "009596\n",
      "009600\n",
      "009603\n",
      "009609\n",
      "009611\n",
      "009613\n",
      "009614\n",
      "009617\n",
      "009618\n",
      "009638\n",
      "009641\n",
      "009647\n",
      "009649\n",
      "009654\n",
      "009655\n",
      "009656\n",
      "009659\n",
      "009666\n",
      "009668\n",
      "009671\n",
      "009676\n",
      "009684\n",
      "009687\n",
      "009691\n",
      "009693\n",
      "009698\n",
      "009703\n",
      "009711\n",
      "009712\n",
      "009713\n",
      "009717\n",
      "009718\n",
      "009721\n",
      "009726\n",
      "009732\n",
      "009734\n",
      "009738\n",
      "009747\n",
      "009754\n",
      "009755\n",
      "009756\n",
      "009762\n",
      "009767\n",
      "009773\n",
      "009776\n",
      "009780\n",
      "009781\n",
      "009789\n",
      "009792\n",
      "009796\n",
      "009800\n",
      "009809\n",
      "009813\n",
      "009822\n",
      "009828\n",
      "009841\n",
      "009845\n",
      "009848\n",
      "009851\n",
      "009859\n",
      "009867\n",
      "009868\n",
      "009869\n",
      "009874\n",
      "009877\n",
      "009878\n",
      "009879\n",
      "009880\n",
      "009881\n",
      "009882\n",
      "009900\n",
      "009902\n",
      "009917\n",
      "009918\n",
      "009926\n",
      "009935\n",
      "009942\n",
      "009944\n",
      "009946\n",
      "009947\n",
      "009949\n",
      "009950\n",
      "009954\n",
      "009958\n"
     ]
    }
   ],
   "source": [
    "# load files\n",
    "vocpath = os.path.join(\"..\", \"VOCdevkit\", \"VOC2007\")\n",
    "train_img_list, train_anno_list, val_img_list, val_anno_list = make_datapath_list(vocpath, cls=\"person\")\n",
    "\n",
    "# make Dataset\n",
    "voc_classes = ['person']\n",
    "color_mean = (104, 117, 123)  # (BGR)の色の平均値\n",
    "input_size = 256  # 画像のinputサイズを300×300にする\n",
    "\n",
    "## DatasetTransformを適応\n",
    "transform = DatasetTransform(input_size, color_mean)\n",
    "transform_anno = Anno_xml2list(voc_classes)\n",
    "\n",
    "train_dataset = VOCDataset(train_img_list, train_anno_list, phase = \"train\", transform=transform, transform_anno = transform_anno)\n",
    "val_dataset = VOCDataset(val_img_list, val_anno_list, phase=\"val\", transform=DatasetTransform(\n",
    "    input_size, color_mean), transform_anno=Anno_xml2list(voc_classes))\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "train_dataloader = data.DataLoader(\n",
    "    train_dataset, batch_size=batch_size, shuffle=True, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "val_dataloader = data.DataLoader(\n",
    "    val_dataset, batch_size=batch_size, shuffle=False, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "dataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 3, 256, 256])\n",
      "32\n",
      "torch.Size([1, 5])\n"
     ]
    }
   ],
   "source": [
    "# check operation\n",
    "batch_iterator = iter(dataloaders_dict[\"train\"])  # iter\n",
    "images, targets = next(batch_iterator)  # get first element\n",
    "print(images.size())  # torch.Size([4, 3, 300, 300])\n",
    "print(len(targets))\n",
    "print(targets[1].shape)  # check targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0633, 0.0000, 0.8167, 0.8119, 0.0000]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test with ssd model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BlazeFace(\n",
      "  (features): Sequential(\n",
      "    (0): Conv2d(3, 24, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "    (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (2): ReLU(inplace)\n",
      "    (3): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "    )\n",
      "    (4): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "    )\n",
      "    (5): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=24)\n",
      "        (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(24, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    )\n",
      "    (6): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "    )\n",
      "    (7): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "    )\n",
      "    (8): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=48)\n",
      "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(48, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    )\n",
      "    (9): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (10): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n",
      "BlazeFaceExtra(\n",
      "  (features): Sequential(\n",
      "    (0): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=96)\n",
      "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    )\n",
      "    (1): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (2): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from utils.blazeface import SSD256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using: cuda:0\n",
      "set weights!\n"
     ]
    }
   ],
   "source": [
    "# SSD300の設定\n",
    "ssd_cfg = {\n",
    "    'num_classes': 2,  # 背景クラスを含めた合計クラス数\n",
    "    'input_size': 256,  # 画像の入力サイズ\n",
    "    'bbox_aspect_num': [4, 6],  # 出力するDBoxのアスペクト比の種類\n",
    "    'feature_maps': [16, 8],  # 各sourceの画像サイズ\n",
    "    'steps': [8, 16],  # DBOXの大きさを決める\n",
    "    'min_sizes': [16, 32],  # DBOXの大きさを決める\n",
    "    'max_sizes': [32, 100],  # DBOXの大きさを決める\n",
    "    'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "}\n",
    "\n",
    "net = SSD256(phase=\"train\", cfg=ssd_cfg)\n",
    "\n",
    "# SSDのweightsを設定\n",
    "\n",
    "def weights_init(m):\n",
    "    if isinstance(m, nn.Conv2d):\n",
    "        init.kaiming_normal_(m.weight.data)\n",
    "        if m.bias is not None:\n",
    "            nn.init.constant_(m.bias, 0.0)\n",
    "\n",
    "# set inits for loc and conf\n",
    "net.loc.apply(weights_init)\n",
    "net.conf.apply(weights_init)\n",
    "\n",
    "# GPUが使えるか確認\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"using:\", device)\n",
    "\n",
    "print(\"set weights!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SSD256(\n",
      "  (blaze): BlazeFace(\n",
      "    (features): Sequential(\n",
      "      (0): Conv2d(3, 24, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "      (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU(inplace)\n",
      "      (3): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (4): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (5): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=24)\n",
      "          (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(24, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (6): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "          (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (7): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "          (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (8): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=48)\n",
      "          (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(48, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (9): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (10): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (extra): BlazeFaceExtra(\n",
      "    (features): Sequential(\n",
      "      (0): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (1): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (2): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (extra2): BlazeFaceExtra2(\n",
      "    (features): Sequential(\n",
      "      (0): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "          (2): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(48, 192, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (1): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(192, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=192)\n",
      "          (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(192, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "          (2): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(48, 192, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (2): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(192, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=192)\n",
      "          (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(192, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "          (2): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(48, 192, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (loc): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(96, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Conv2d(192, 24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "  )\n",
      "  (conf): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(96, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Conv2d(192, 12, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.ssd_model import MultiBoxLoss\n",
    "\n",
    "# define loss\n",
    "criterion = MultiBoxLoss(jaccard_thresh=0.5,neg_pos=3, device=device)\n",
    "\n",
    "# optim\n",
    "import torch.optim as optim\n",
    "optimizer = optim.SGD(net.parameters(), lr=1e-3, momentum=0.9, weight_decay=5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_current_lr(epoch):\n",
    "    lr = 1e-3\n",
    "    for i,lr_decay_epoch in enumerate([120,180]):\n",
    "        if epoch >= lr_decay_epoch:\n",
    "            lr *= 0.1\n",
    "    return lr\n",
    "\n",
    "def adjust_learning_rate(optimizer, epoch):\n",
    "    lr = get_current_lr(epoch)\n",
    "    print(\"lr is:\", lr)\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# モデルを学習させる関数を作成\n",
    "def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):\n",
    "\n",
    "    # GPUが使えるかを確認\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(\"used device：\", device)\n",
    "\n",
    "    # ネットワークをGPUへ\n",
    "    net.to(device)\n",
    "\n",
    "    # ネットワークがある程度固定であれば、高速化させる\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "    # イテレーションカウンタをセット\n",
    "    iteration = 1\n",
    "    epoch_train_loss = 0.0  # epochの損失和\n",
    "    epoch_val_loss = 0.0  # epochの損失和\n",
    "    logs = []\n",
    "\n",
    "    # epochのループ\n",
    "    for epoch in range(num_epochs+1):\n",
    "        \n",
    "        adjust_learning_rate(optimizer, epoch)\n",
    "        \n",
    "        # 開始時刻を保存\n",
    "        t_epoch_start = time.time()\n",
    "        t_iter_start = time.time()\n",
    "\n",
    "        print('-------------')\n",
    "        print('Epoch {}/{}'.format(epoch+1, num_epochs))\n",
    "        print('-------------')\n",
    "\n",
    "        # epochごとの訓練と検証のループ\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                net.train()  # モデルを訓練モードに\n",
    "                print('train')\n",
    "            else:\n",
    "                if((epoch+1) % 5 == 0):\n",
    "                    net.eval()   # モデルを検証モードに\n",
    "                    print('-------------')\n",
    "                    print('val')\n",
    "                else:\n",
    "                    # 検証は5回に1回だけ行う\n",
    "                    continue\n",
    "\n",
    "            # データローダーからminibatchずつ取り出すループ\n",
    "            for images, targets in dataloaders_dict[phase]:\n",
    "\n",
    "                # GPUが使えるならGPUにデータを送る\n",
    "                images = images.to(device)\n",
    "                targets = [ann.to(device)\n",
    "                           for ann in targets]  # リストの各要素のテンソルをGPUへ\n",
    "\n",
    "                # optimizerを初期化\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # 順伝搬（forward）計算\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    # 順伝搬（forward）計算\n",
    "                    outputs = net(images)\n",
    "\n",
    "                    # 損失の計算\n",
    "                    loss_l, loss_c = criterion(outputs, targets)\n",
    "                    loss = loss_l + loss_c\n",
    "\n",
    "                    # 訓練時はバックプロパゲーション\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()  # 勾配の計算\n",
    "\n",
    "                        # 勾配が大きくなりすぎると計算が不安定になるので、clipで最大でも勾配2.0に留める\n",
    "                        nn.utils.clip_grad_value_(\n",
    "                            net.parameters(), clip_value=2.0)\n",
    "\n",
    "                        optimizer.step()  # パラメータ更新\n",
    "\n",
    "                        if (iteration % 10 == 0):  # 10iterに1度、lossを表示\n",
    "                            t_iter_finish = time.time()\n",
    "                            duration = t_iter_finish - t_iter_start\n",
    "                            print('Iteration {} || Loss: {:.4f} || 10iter: {:.4f} sec.'.format(\n",
    "                                iteration, loss.item(), duration))\n",
    "                            t_iter_start = time.time()\n",
    "\n",
    "                        epoch_train_loss += loss.item()\n",
    "                        iteration += 1\n",
    "\n",
    "                    # 検証時\n",
    "                    else:\n",
    "                        epoch_val_loss += loss.item()\n",
    "\n",
    "        # epochのphaseごとのlossと正解率\n",
    "        t_epoch_finish = time.time()\n",
    "        print('-------------')\n",
    "        print('epoch {} || Epoch_TRAIN_Loss:{:.4f} ||Epoch_VAL_Loss:{:.4f}'.format(\n",
    "            epoch+1, epoch_train_loss, epoch_val_loss))\n",
    "        print('timer:  {:.4f} sec.'.format(t_epoch_finish - t_epoch_start))\n",
    "        t_epoch_start = time.time()\n",
    "\n",
    "        # ログを保存\n",
    "        log_epoch = {'epoch': epoch+1,\n",
    "                     'train_loss': epoch_train_loss, 'val_loss': epoch_val_loss}\n",
    "        logs.append(log_epoch)\n",
    "        df = pd.DataFrame(logs)\n",
    "        df.to_csv(\"log_output.csv\")\n",
    "\n",
    "        epoch_train_loss = 0.0  # epochの損失和\n",
    "        epoch_val_loss = 0.0  # epochの損失和\n",
    "\n",
    "        # ネットワークを保存する\n",
    "        if ((epoch+1) % 10 == 0):\n",
    "            torch.save(net.state_dict(), 'weights/blazeface256_' +\n",
    "                       str(epoch+1) + '.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# start training here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used device： cuda:0\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 1/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10 || Loss: 109.0224 || 10iter: 11.6945 sec.\n",
      "Iteration 20 || Loss: 51.9891 || 10iter: 5.6860 sec.\n",
      "Iteration 30 || Loss: 54.7372 || 10iter: 5.5014 sec.\n",
      "Iteration 40 || Loss: 42.4690 || 10iter: 6.0593 sec.\n",
      "Iteration 50 || Loss: 43.7510 || 10iter: 5.7763 sec.\n",
      "Iteration 60 || Loss: 46.9673 || 10iter: 4.3901 sec.\n",
      "-------------\n",
      "epoch 1 || Epoch_TRAIN_Loss:3690.0698 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.5251 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 2/200\n",
      "-------------\n",
      "train\n",
      "Iteration 70 || Loss: 42.3619 || 10iter: 8.0195 sec.\n",
      "Iteration 80 || Loss: 48.6764 || 10iter: 5.8791 sec.\n",
      "Iteration 90 || Loss: 42.2925 || 10iter: 5.4265 sec.\n",
      "Iteration 100 || Loss: 49.2742 || 10iter: 5.6672 sec.\n",
      "Iteration 110 || Loss: 35.3208 || 10iter: 5.4864 sec.\n",
      "Iteration 120 || Loss: 39.1136 || 10iter: 4.6204 sec.\n",
      "Iteration 130 || Loss: 33.9180 || 10iter: 5.3628 sec.\n",
      "-------------\n",
      "epoch 2 || Epoch_TRAIN_Loss:2603.7627 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.9282 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 3/200\n",
      "-------------\n",
      "train\n",
      "Iteration 140 || Loss: 39.0895 || 10iter: 9.3335 sec.\n",
      "Iteration 150 || Loss: 29.7569 || 10iter: 5.9040 sec.\n",
      "Iteration 160 || Loss: 38.8814 || 10iter: 5.1320 sec.\n",
      "Iteration 170 || Loss: 26.7896 || 10iter: 4.4491 sec.\n",
      "Iteration 180 || Loss: 31.3516 || 10iter: 6.4897 sec.\n",
      "Iteration 190 || Loss: 31.1963 || 10iter: 5.2496 sec.\n",
      "-------------\n",
      "epoch 3 || Epoch_TRAIN_Loss:2151.6225 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9503 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 4/200\n",
      "-------------\n",
      "train\n",
      "Iteration 200 || Loss: 23.8432 || 10iter: 5.8761 sec.\n",
      "Iteration 210 || Loss: 24.6469 || 10iter: 5.5900 sec.\n",
      "Iteration 220 || Loss: 28.2157 || 10iter: 4.3896 sec.\n",
      "Iteration 230 || Loss: 32.9187 || 10iter: 6.8242 sec.\n",
      "Iteration 240 || Loss: 24.0225 || 10iter: 5.6507 sec.\n",
      "Iteration 250 || Loss: 31.2231 || 10iter: 5.7799 sec.\n",
      "Iteration 260 || Loss: 33.7405 || 10iter: 4.6769 sec.\n",
      "-------------\n",
      "epoch 4 || Epoch_TRAIN_Loss:1922.0371 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9964 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 5/200\n",
      "-------------\n",
      "train\n",
      "Iteration 270 || Loss: 30.0378 || 10iter: 6.9496 sec.\n",
      "Iteration 280 || Loss: 31.8627 || 10iter: 4.8389 sec.\n",
      "Iteration 290 || Loss: 28.5816 || 10iter: 4.5095 sec.\n",
      "Iteration 300 || Loss: 25.4748 || 10iter: 4.7427 sec.\n",
      "Iteration 310 || Loss: 31.2024 || 10iter: 6.9317 sec.\n",
      "Iteration 320 || Loss: 31.2899 || 10iter: 5.1870 sec.\n",
      "Iteration 330 || Loss: 36.2971 || 10iter: 4.8480 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 5 || Epoch_TRAIN_Loss:1943.9876 ||Epoch_VAL_Loss:816.7870\n",
      "timer:  50.3274 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 6/200\n",
      "-------------\n",
      "train\n",
      "Iteration 340 || Loss: 25.1438 || 10iter: 11.4001 sec.\n",
      "Iteration 350 || Loss: 28.3664 || 10iter: 5.3361 sec.\n",
      "Iteration 360 || Loss: 30.4562 || 10iter: 5.9071 sec.\n",
      "Iteration 370 || Loss: 23.3022 || 10iter: 5.4677 sec.\n",
      "Iteration 380 || Loss: 25.1550 || 10iter: 4.9473 sec.\n",
      "Iteration 390 || Loss: 19.3501 || 10iter: 5.5081 sec.\n",
      "-------------\n",
      "epoch 6 || Epoch_TRAIN_Loss:1758.2050 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.1568 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 7/200\n",
      "-------------\n",
      "train\n",
      "Iteration 400 || Loss: 18.6676 || 10iter: 6.2058 sec.\n",
      "Iteration 410 || Loss: 21.7154 || 10iter: 5.9162 sec.\n",
      "Iteration 420 || Loss: 23.6463 || 10iter: 5.4929 sec.\n",
      "Iteration 430 || Loss: 22.8121 || 10iter: 4.6481 sec.\n",
      "Iteration 440 || Loss: 29.0375 || 10iter: 6.8385 sec.\n",
      "Iteration 450 || Loss: 19.9895 || 10iter: 5.4018 sec.\n",
      "Iteration 460 || Loss: 21.5373 || 10iter: 4.9616 sec.\n",
      "-------------\n",
      "epoch 7 || Epoch_TRAIN_Loss:1772.1098 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.8689 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 8/200\n",
      "-------------\n",
      "train\n",
      "Iteration 470 || Loss: 36.2643 || 10iter: 8.6768 sec.\n",
      "Iteration 480 || Loss: 18.0855 || 10iter: 4.6554 sec.\n",
      "Iteration 490 || Loss: 24.8502 || 10iter: 7.0865 sec.\n",
      "Iteration 500 || Loss: 25.6444 || 10iter: 5.7103 sec.\n",
      "Iteration 510 || Loss: 24.0727 || 10iter: 5.9817 sec.\n",
      "Iteration 520 || Loss: 35.7280 || 10iter: 5.2515 sec.\n",
      "-------------\n",
      "epoch 8 || Epoch_TRAIN_Loss:1645.8581 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.5065 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 9/200\n",
      "-------------\n",
      "train\n",
      "Iteration 530 || Loss: 25.3125 || 10iter: 3.6941 sec.\n",
      "Iteration 540 || Loss: 20.8279 || 10iter: 5.2562 sec.\n",
      "Iteration 550 || Loss: 26.6358 || 10iter: 4.2280 sec.\n",
      "Iteration 560 || Loss: 25.7371 || 10iter: 5.0811 sec.\n",
      "Iteration 570 || Loss: 18.9298 || 10iter: 6.7481 sec.\n",
      "Iteration 580 || Loss: 17.5102 || 10iter: 5.6271 sec.\n",
      "Iteration 590 || Loss: 20.6564 || 10iter: 4.5930 sec.\n",
      "-------------\n",
      "epoch 9 || Epoch_TRAIN_Loss:1690.4695 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  37.5089 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 10/200\n",
      "-------------\n",
      "train\n",
      "Iteration 600 || Loss: 26.0286 || 10iter: 7.5990 sec.\n",
      "Iteration 610 || Loss: 21.9063 || 10iter: 4.5622 sec.\n",
      "Iteration 620 || Loss: 21.7321 || 10iter: 7.0317 sec.\n",
      "Iteration 630 || Loss: 34.4750 || 10iter: 6.0925 sec.\n",
      "Iteration 640 || Loss: 19.4896 || 10iter: 5.4495 sec.\n",
      "Iteration 650 || Loss: 24.8549 || 10iter: 5.3910 sec.\n",
      "Iteration 660 || Loss: 16.7606 || 10iter: 4.2334 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 10 || Epoch_TRAIN_Loss:1525.5368 ||Epoch_VAL_Loss:627.4977\n",
      "timer:  53.5439 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 11/200\n",
      "-------------\n",
      "train\n",
      "Iteration 670 || Loss: 21.7919 || 10iter: 10.6637 sec.\n",
      "Iteration 680 || Loss: 18.3744 || 10iter: 5.1781 sec.\n",
      "Iteration 690 || Loss: 21.2952 || 10iter: 4.8924 sec.\n",
      "Iteration 700 || Loss: 25.7061 || 10iter: 5.9891 sec.\n",
      "Iteration 710 || Loss: 18.8889 || 10iter: 6.1705 sec.\n",
      "Iteration 720 || Loss: 21.7829 || 10iter: 5.0717 sec.\n",
      "-------------\n",
      "epoch 11 || Epoch_TRAIN_Loss:1536.3884 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1845 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 12/200\n",
      "-------------\n",
      "train\n",
      "Iteration 730 || Loss: 26.6570 || 10iter: 6.9586 sec.\n",
      "Iteration 740 || Loss: 25.6601 || 10iter: 4.9470 sec.\n",
      "Iteration 750 || Loss: 19.2566 || 10iter: 5.4278 sec.\n",
      "Iteration 760 || Loss: 20.6557 || 10iter: 6.4795 sec.\n",
      "Iteration 770 || Loss: 22.4029 || 10iter: 5.9907 sec.\n",
      "Iteration 780 || Loss: 28.5907 || 10iter: 5.1734 sec.\n",
      "Iteration 790 || Loss: 26.5969 || 10iter: 4.5809 sec.\n",
      "-------------\n",
      "epoch 12 || Epoch_TRAIN_Loss:1510.7264 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9447 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 13/200\n",
      "-------------\n",
      "train\n",
      "Iteration 800 || Loss: 15.8130 || 10iter: 7.1148 sec.\n",
      "Iteration 810 || Loss: 20.5318 || 10iter: 5.2175 sec.\n",
      "Iteration 820 || Loss: 25.4031 || 10iter: 4.1999 sec.\n",
      "Iteration 830 || Loss: 19.0225 || 10iter: 6.5515 sec.\n",
      "Iteration 840 || Loss: 20.3609 || 10iter: 6.2521 sec.\n",
      "Iteration 850 || Loss: 28.2737 || 10iter: 5.2959 sec.\n",
      "-------------\n",
      "epoch 13 || Epoch_TRAIN_Loss:1484.1905 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.7136 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 14/200\n",
      "-------------\n",
      "train\n",
      "Iteration 860 || Loss: 36.4610 || 10iter: 4.7526 sec.\n",
      "Iteration 870 || Loss: 15.2945 || 10iter: 5.5149 sec.\n",
      "Iteration 880 || Loss: 19.5035 || 10iter: 5.9445 sec.\n",
      "Iteration 890 || Loss: 24.9142 || 10iter: 6.1007 sec.\n",
      "Iteration 900 || Loss: 19.3462 || 10iter: 5.9568 sec.\n",
      "Iteration 910 || Loss: 17.7822 || 10iter: 5.6811 sec.\n",
      "Iteration 920 || Loss: 17.7557 || 10iter: 4.4835 sec.\n",
      "-------------\n",
      "epoch 14 || Epoch_TRAIN_Loss:1374.9734 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6467 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 15/200\n",
      "-------------\n",
      "train\n",
      "Iteration 930 || Loss: 19.7158 || 10iter: 6.0469 sec.\n",
      "Iteration 940 || Loss: 28.8496 || 10iter: 6.9475 sec.\n",
      "Iteration 950 || Loss: 20.0948 || 10iter: 6.0759 sec.\n",
      "Iteration 960 || Loss: 17.1690 || 10iter: 5.9195 sec.\n",
      "Iteration 970 || Loss: 17.4104 || 10iter: 5.9946 sec.\n",
      "Iteration 980 || Loss: 22.7455 || 10iter: 4.8602 sec.\n",
      "Iteration 990 || Loss: 19.5922 || 10iter: 4.4024 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 15 || Epoch_TRAIN_Loss:1322.8086 ||Epoch_VAL_Loss:612.4137\n",
      "timer:  54.5326 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 16/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1000 || Loss: 17.8303 || 10iter: 10.2871 sec.\n",
      "Iteration 1010 || Loss: 15.7249 || 10iter: 4.3638 sec.\n",
      "Iteration 1020 || Loss: 21.0974 || 10iter: 6.7218 sec.\n",
      "Iteration 1030 || Loss: 20.0586 || 10iter: 5.7211 sec.\n",
      "Iteration 1040 || Loss: 17.5639 || 10iter: 6.0964 sec.\n",
      "Iteration 1050 || Loss: 16.2424 || 10iter: 4.6632 sec.\n",
      "-------------\n",
      "epoch 16 || Epoch_TRAIN_Loss:1290.4380 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9425 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 17/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1060 || Loss: 19.5870 || 10iter: 5.0745 sec.\n",
      "Iteration 1070 || Loss: 17.6985 || 10iter: 5.5093 sec.\n",
      "Iteration 1080 || Loss: 15.0168 || 10iter: 4.5111 sec.\n",
      "Iteration 1090 || Loss: 19.3302 || 10iter: 5.0082 sec.\n",
      "Iteration 1100 || Loss: 16.6463 || 10iter: 6.9158 sec.\n",
      "Iteration 1110 || Loss: 21.9359 || 10iter: 5.7515 sec.\n",
      "Iteration 1120 || Loss: 18.5494 || 10iter: 4.4638 sec.\n",
      "-------------\n",
      "epoch 17 || Epoch_TRAIN_Loss:1223.7228 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.6584 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 18/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1130 || Loss: 17.0926 || 10iter: 8.8011 sec.\n",
      "Iteration 1140 || Loss: 18.1224 || 10iter: 5.0017 sec.\n",
      "Iteration 1150 || Loss: 12.5873 || 10iter: 6.7836 sec.\n",
      "Iteration 1160 || Loss: 15.5221 || 10iter: 6.1109 sec.\n",
      "Iteration 1170 || Loss: 21.5021 || 10iter: 6.0276 sec.\n",
      "Iteration 1180 || Loss: 21.7728 || 10iter: 4.6665 sec.\n",
      "-------------\n",
      "epoch 18 || Epoch_TRAIN_Loss:1210.2345 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1829 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 19/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1190 || Loss: 17.5561 || 10iter: 3.5059 sec.\n",
      "Iteration 1200 || Loss: 14.7005 || 10iter: 7.5792 sec.\n",
      "Iteration 1210 || Loss: 24.9933 || 10iter: 5.7902 sec.\n",
      "Iteration 1220 || Loss: 21.3239 || 10iter: 5.6866 sec.\n",
      "Iteration 1230 || Loss: 13.7164 || 10iter: 6.2519 sec.\n",
      "Iteration 1240 || Loss: 14.9385 || 10iter: 5.4510 sec.\n",
      "Iteration 1250 || Loss: 19.7919 || 10iter: 4.4321 sec.\n",
      "-------------\n",
      "epoch 19 || Epoch_TRAIN_Loss:1228.2475 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.7413 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 20/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1260 || Loss: 17.4029 || 10iter: 8.2660 sec.\n",
      "Iteration 1270 || Loss: 17.0330 || 10iter: 6.2391 sec.\n",
      "Iteration 1280 || Loss: 21.5109 || 10iter: 5.9115 sec.\n",
      "Iteration 1290 || Loss: 22.8055 || 10iter: 5.4010 sec.\n",
      "Iteration 1300 || Loss: 16.4833 || 10iter: 4.6940 sec.\n",
      "Iteration 1310 || Loss: 19.0399 || 10iter: 6.0326 sec.\n",
      "Iteration 1320 || Loss: 18.9292 || 10iter: 4.3136 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 20 || Epoch_TRAIN_Loss:1180.3795 ||Epoch_VAL_Loss:577.6851\n",
      "timer:  53.5443 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 21/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1330 || Loss: 13.7102 || 10iter: 8.6415 sec.\n",
      "Iteration 1340 || Loss: 18.3170 || 10iter: 4.2717 sec.\n",
      "Iteration 1350 || Loss: 14.7722 || 10iter: 5.9978 sec.\n",
      "Iteration 1360 || Loss: 14.9662 || 10iter: 6.8378 sec.\n",
      "Iteration 1370 || Loss: 13.8599 || 10iter: 5.4608 sec.\n",
      "Iteration 1380 || Loss: 19.1716 || 10iter: 4.9219 sec.\n",
      "-------------\n",
      "epoch 21 || Epoch_TRAIN_Loss:1148.5414 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.3017 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 22/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1390 || Loss: 16.9234 || 10iter: 5.2921 sec.\n",
      "Iteration 1400 || Loss: 17.4421 || 10iter: 5.3255 sec.\n",
      "Iteration 1410 || Loss: 16.3280 || 10iter: 6.3595 sec.\n",
      "Iteration 1420 || Loss: 15.5651 || 10iter: 5.9118 sec.\n",
      "Iteration 1430 || Loss: 16.3615 || 10iter: 6.0263 sec.\n",
      "Iteration 1440 || Loss: 16.9838 || 10iter: 5.2130 sec.\n",
      "Iteration 1450 || Loss: 19.2527 || 10iter: 4.2035 sec.\n",
      "-------------\n",
      "epoch 22 || Epoch_TRAIN_Loss:1160.5744 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.6291 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 23/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1460 || Loss: 17.8431 || 10iter: 9.7542 sec.\n",
      "Iteration 1470 || Loss: 18.3648 || 10iter: 6.0319 sec.\n",
      "Iteration 1480 || Loss: 14.1948 || 10iter: 5.6454 sec.\n",
      "Iteration 1490 || Loss: 20.2271 || 10iter: 6.1948 sec.\n",
      "Iteration 1500 || Loss: 19.2252 || 10iter: 4.9674 sec.\n",
      "Iteration 1510 || Loss: 16.0740 || 10iter: 4.2805 sec.\n",
      "-------------\n",
      "epoch 23 || Epoch_TRAIN_Loss:1161.3439 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.0705 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 24/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1520 || Loss: 18.0792 || 10iter: 5.2221 sec.\n",
      "Iteration 1530 || Loss: 17.2199 || 10iter: 6.4070 sec.\n",
      "Iteration 1540 || Loss: 10.3240 || 10iter: 5.6800 sec.\n",
      "Iteration 1550 || Loss: 16.1857 || 10iter: 5.7822 sec.\n",
      "Iteration 1560 || Loss: 17.6734 || 10iter: 4.5572 sec.\n",
      "Iteration 1570 || Loss: 20.2153 || 10iter: 6.5531 sec.\n",
      "Iteration 1580 || Loss: 20.8682 || 10iter: 4.7948 sec.\n",
      "-------------\n",
      "epoch 24 || Epoch_TRAIN_Loss:1200.2904 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.2920 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 25/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1590 || Loss: 19.9272 || 10iter: 7.8696 sec.\n",
      "Iteration 1600 || Loss: 15.7906 || 10iter: 5.6634 sec.\n",
      "Iteration 1610 || Loss: 14.9228 || 10iter: 4.5396 sec.\n",
      "Iteration 1620 || Loss: 14.2934 || 10iter: 5.1274 sec.\n",
      "Iteration 1630 || Loss: 13.3726 || 10iter: 4.3793 sec.\n",
      "Iteration 1640 || Loss: 15.0064 || 10iter: 5.4968 sec.\n",
      "Iteration 1650 || Loss: 22.3851 || 10iter: 4.8564 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 25 || Epoch_TRAIN_Loss:1144.3480 ||Epoch_VAL_Loss:554.1049\n",
      "timer:  51.1062 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 26/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1660 || Loss: 18.5952 || 10iter: 8.4100 sec.\n",
      "Iteration 1670 || Loss: 14.6992 || 10iter: 6.7196 sec.\n",
      "Iteration 1680 || Loss: 13.9309 || 10iter: 6.0356 sec.\n",
      "Iteration 1690 || Loss: 15.9057 || 10iter: 5.5436 sec.\n",
      "Iteration 1700 || Loss: 18.6195 || 10iter: 5.6257 sec.\n",
      "Iteration 1710 || Loss: 16.2279 || 10iter: 4.3772 sec.\n",
      "-------------\n",
      "epoch 26 || Epoch_TRAIN_Loss:1148.9851 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.5671 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 27/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1720 || Loss: 22.1748 || 10iter: 7.8933 sec.\n",
      "Iteration 1730 || Loss: 12.9746 || 10iter: 5.9903 sec.\n",
      "Iteration 1740 || Loss: 16.5674 || 10iter: 5.9641 sec.\n",
      "Iteration 1750 || Loss: 13.3709 || 10iter: 5.9520 sec.\n",
      "Iteration 1760 || Loss: 15.1574 || 10iter: 5.8700 sec.\n",
      "Iteration 1770 || Loss: 31.6266 || 10iter: 4.3211 sec.\n",
      "Iteration 1780 || Loss: 16.1337 || 10iter: 5.5366 sec.\n",
      "-------------\n",
      "epoch 27 || Epoch_TRAIN_Loss:1159.1960 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.8305 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 28/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1790 || Loss: 20.4323 || 10iter: 8.8951 sec.\n",
      "Iteration 1800 || Loss: 11.8068 || 10iter: 5.4626 sec.\n",
      "Iteration 1810 || Loss: 15.6246 || 10iter: 5.5660 sec.\n",
      "Iteration 1820 || Loss: 19.2394 || 10iter: 5.1399 sec.\n",
      "Iteration 1830 || Loss: 18.1243 || 10iter: 5.3706 sec.\n",
      "Iteration 1840 || Loss: 15.8581 || 10iter: 5.5852 sec.\n",
      "-------------\n",
      "epoch 28 || Epoch_TRAIN_Loss:1130.0660 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6646 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 29/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1850 || Loss: 18.4190 || 10iter: 6.0260 sec.\n",
      "Iteration 1860 || Loss: 14.6975 || 10iter: 5.8154 sec.\n",
      "Iteration 1870 || Loss: 15.7779 || 10iter: 5.1138 sec.\n",
      "Iteration 1880 || Loss: 20.4166 || 10iter: 4.8353 sec.\n",
      "Iteration 1890 || Loss: 17.7335 || 10iter: 4.5757 sec.\n",
      "Iteration 1900 || Loss: 16.8089 || 10iter: 3.9486 sec.\n",
      "Iteration 1910 || Loss: 17.4428 || 10iter: 5.7300 sec.\n",
      "-------------\n",
      "epoch 29 || Epoch_TRAIN_Loss:1128.8479 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.4219 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 30/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1920 || Loss: 21.8211 || 10iter: 8.5991 sec.\n",
      "Iteration 1930 || Loss: 11.9265 || 10iter: 5.9255 sec.\n",
      "Iteration 1940 || Loss: 17.1433 || 10iter: 5.3910 sec.\n",
      "Iteration 1950 || Loss: 14.9850 || 10iter: 4.4211 sec.\n",
      "Iteration 1960 || Loss: 15.2953 || 10iter: 6.6627 sec.\n",
      "Iteration 1970 || Loss: 17.2031 || 10iter: 5.6849 sec.\n",
      "Iteration 1980 || Loss: 29.6349 || 10iter: 4.4065 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 30 || Epoch_TRAIN_Loss:1140.5617 ||Epoch_VAL_Loss:541.3432\n",
      "timer:  53.1319 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 31/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1990 || Loss: 17.2704 || 10iter: 12.3712 sec.\n",
      "Iteration 2000 || Loss: 17.5364 || 10iter: 5.4827 sec.\n",
      "Iteration 2010 || Loss: 14.0423 || 10iter: 5.7198 sec.\n",
      "Iteration 2020 || Loss: 14.4911 || 10iter: 5.8152 sec.\n",
      "Iteration 2030 || Loss: 13.9099 || 10iter: 4.7442 sec.\n",
      "Iteration 2040 || Loss: 15.1452 || 10iter: 5.4785 sec.\n",
      "-------------\n",
      "epoch 31 || Epoch_TRAIN_Loss:1122.8520 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  43.0362 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 32/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2050 || Loss: 12.7132 || 10iter: 6.2744 sec.\n",
      "Iteration 2060 || Loss: 12.1778 || 10iter: 6.2543 sec.\n",
      "Iteration 2070 || Loss: 16.5430 || 10iter: 5.4818 sec.\n",
      "Iteration 2080 || Loss: 14.5878 || 10iter: 4.6627 sec.\n",
      "Iteration 2090 || Loss: 14.1997 || 10iter: 5.6523 sec.\n",
      "Iteration 2100 || Loss: 14.1387 || 10iter: 6.4177 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 2110 || Loss: 13.9025 || 10iter: 4.7387 sec.\n",
      "-------------\n",
      "epoch 32 || Epoch_TRAIN_Loss:1096.9405 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.7990 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 33/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2120 || Loss: 13.5176 || 10iter: 9.1378 sec.\n",
      "Iteration 2130 || Loss: 16.8139 || 10iter: 5.0485 sec.\n",
      "Iteration 2140 || Loss: 11.9304 || 10iter: 4.9180 sec.\n",
      "Iteration 2150 || Loss: 18.1645 || 10iter: 4.5516 sec.\n",
      "Iteration 2160 || Loss: 18.4036 || 10iter: 4.6352 sec.\n",
      "Iteration 2170 || Loss: 19.3296 || 10iter: 5.6246 sec.\n",
      "-------------\n",
      "epoch 33 || Epoch_TRAIN_Loss:1107.0035 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.1379 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 34/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2180 || Loss: 15.8720 || 10iter: 5.3103 sec.\n",
      "Iteration 2190 || Loss: 14.3596 || 10iter: 7.2093 sec.\n",
      "Iteration 2200 || Loss: 20.2769 || 10iter: 5.2394 sec.\n",
      "Iteration 2210 || Loss: 15.8526 || 10iter: 4.6394 sec.\n",
      "Iteration 2220 || Loss: 19.8023 || 10iter: 6.3191 sec.\n",
      "Iteration 2230 || Loss: 16.4650 || 10iter: 6.0116 sec.\n",
      "Iteration 2240 || Loss: 14.9615 || 10iter: 4.8040 sec.\n",
      "-------------\n",
      "epoch 34 || Epoch_TRAIN_Loss:1087.3468 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.9920 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 35/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2250 || Loss: 12.8659 || 10iter: 7.9935 sec.\n",
      "Iteration 2260 || Loss: 20.3408 || 10iter: 4.5304 sec.\n",
      "Iteration 2270 || Loss: 15.8473 || 10iter: 6.2413 sec.\n",
      "Iteration 2280 || Loss: 16.8032 || 10iter: 6.0884 sec.\n",
      "Iteration 2290 || Loss: 21.6465 || 10iter: 5.8522 sec.\n",
      "Iteration 2300 || Loss: 17.0529 || 10iter: 5.1678 sec.\n",
      "Iteration 2310 || Loss: 18.9849 || 10iter: 4.3337 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 35 || Epoch_TRAIN_Loss:1108.8510 ||Epoch_VAL_Loss:529.6819\n",
      "timer:  52.4631 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 36/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2320 || Loss: 16.7728 || 10iter: 11.0136 sec.\n",
      "Iteration 2330 || Loss: 18.0734 || 10iter: 5.4558 sec.\n",
      "Iteration 2340 || Loss: 15.9998 || 10iter: 4.9504 sec.\n",
      "Iteration 2350 || Loss: 18.9742 || 10iter: 4.5776 sec.\n",
      "Iteration 2360 || Loss: 13.8633 || 10iter: 7.0916 sec.\n",
      "Iteration 2370 || Loss: 16.2798 || 10iter: 4.8539 sec.\n",
      "-------------\n",
      "epoch 36 || Epoch_TRAIN_Loss:1076.3998 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.2348 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 37/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2380 || Loss: 22.9098 || 10iter: 6.3212 sec.\n",
      "Iteration 2390 || Loss: 12.7542 || 10iter: 6.1019 sec.\n",
      "Iteration 2400 || Loss: 20.8577 || 10iter: 4.4251 sec.\n",
      "Iteration 2410 || Loss: 14.4962 || 10iter: 4.6950 sec.\n",
      "Iteration 2420 || Loss: 15.3417 || 10iter: 4.4053 sec.\n",
      "Iteration 2430 || Loss: 19.1929 || 10iter: 5.8795 sec.\n",
      "Iteration 2440 || Loss: 19.3240 || 10iter: 4.5588 sec.\n",
      "-------------\n",
      "epoch 37 || Epoch_TRAIN_Loss:1088.6279 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  37.7571 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 38/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2450 || Loss: 18.6007 || 10iter: 10.3186 sec.\n",
      "Iteration 2460 || Loss: 13.5177 || 10iter: 6.5394 sec.\n",
      "Iteration 2470 || Loss: 17.1373 || 10iter: 4.6627 sec.\n",
      "Iteration 2480 || Loss: 16.2740 || 10iter: 6.3064 sec.\n",
      "Iteration 2490 || Loss: 22.9351 || 10iter: 6.0967 sec.\n",
      "Iteration 2500 || Loss: 14.7791 || 10iter: 5.2817 sec.\n",
      "-------------\n",
      "epoch 38 || Epoch_TRAIN_Loss:1086.4078 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  43.4741 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 39/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2510 || Loss: 12.1375 || 10iter: 4.9648 sec.\n",
      "Iteration 2520 || Loss: 14.0436 || 10iter: 5.3021 sec.\n",
      "Iteration 2530 || Loss: 16.5810 || 10iter: 5.6211 sec.\n",
      "Iteration 2540 || Loss: 19.8779 || 10iter: 6.2376 sec.\n",
      "Iteration 2550 || Loss: 15.9276 || 10iter: 5.9734 sec.\n",
      "Iteration 2560 || Loss: 13.4333 || 10iter: 5.4748 sec.\n",
      "Iteration 2570 || Loss: 13.6793 || 10iter: 4.4326 sec.\n",
      "-------------\n",
      "epoch 39 || Epoch_TRAIN_Loss:1089.7453 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2482 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 40/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2580 || Loss: 13.6810 || 10iter: 5.5091 sec.\n",
      "Iteration 2590 || Loss: 11.0514 || 10iter: 6.7181 sec.\n",
      "Iteration 2600 || Loss: 18.9660 || 10iter: 5.6684 sec.\n",
      "Iteration 2610 || Loss: 12.7942 || 10iter: 6.0204 sec.\n",
      "Iteration 2620 || Loss: 15.8144 || 10iter: 5.2506 sec.\n",
      "Iteration 2630 || Loss: 18.4350 || 10iter: 5.1659 sec.\n",
      "Iteration 2640 || Loss: 18.0616 || 10iter: 4.1015 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 40 || Epoch_TRAIN_Loss:1036.0684 ||Epoch_VAL_Loss:518.9949\n",
      "timer:  52.0728 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 41/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2650 || Loss: 15.4748 || 10iter: 10.6450 sec.\n",
      "Iteration 2660 || Loss: 16.5524 || 10iter: 4.9759 sec.\n",
      "Iteration 2670 || Loss: 17.3097 || 10iter: 4.3888 sec.\n",
      "Iteration 2680 || Loss: 20.0735 || 10iter: 5.0517 sec.\n",
      "Iteration 2690 || Loss: 15.6575 || 10iter: 4.2696 sec.\n",
      "Iteration 2700 || Loss: 18.5932 || 10iter: 4.7985 sec.\n",
      "-------------\n",
      "epoch 41 || Epoch_TRAIN_Loss:1045.9085 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  37.4504 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 42/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2710 || Loss: 17.1606 || 10iter: 7.0356 sec.\n",
      "Iteration 2720 || Loss: 17.2653 || 10iter: 6.3432 sec.\n",
      "Iteration 2730 || Loss: 12.3910 || 10iter: 5.9746 sec.\n",
      "Iteration 2740 || Loss: 18.8951 || 10iter: 5.1394 sec.\n",
      "Iteration 2750 || Loss: 21.4221 || 10iter: 4.8694 sec.\n",
      "Iteration 2760 || Loss: 14.9331 || 10iter: 6.5728 sec.\n",
      "Iteration 2770 || Loss: 18.2203 || 10iter: 4.5439 sec.\n",
      "-------------\n",
      "epoch 42 || Epoch_TRAIN_Loss:1024.2286 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.9369 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 43/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2780 || Loss: 17.3843 || 10iter: 9.2087 sec.\n",
      "Iteration 2790 || Loss: 17.4724 || 10iter: 5.2926 sec.\n",
      "Iteration 2800 || Loss: 12.4991 || 10iter: 4.4357 sec.\n",
      "Iteration 2810 || Loss: 14.4053 || 10iter: 7.1769 sec.\n",
      "Iteration 2820 || Loss: 14.8097 || 10iter: 5.7873 sec.\n",
      "Iteration 2830 || Loss: 13.8486 || 10iter: 4.8818 sec.\n",
      "-------------\n",
      "epoch 43 || Epoch_TRAIN_Loss:1078.3185 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.7826 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 44/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2840 || Loss: 17.2900 || 10iter: 4.6739 sec.\n",
      "Iteration 2850 || Loss: 16.8674 || 10iter: 4.9665 sec.\n",
      "Iteration 2860 || Loss: 18.6213 || 10iter: 6.7800 sec.\n",
      "Iteration 2870 || Loss: 16.5690 || 10iter: 6.0608 sec.\n",
      "Iteration 2880 || Loss: 20.8792 || 10iter: 5.5369 sec.\n",
      "Iteration 2890 || Loss: 15.0864 || 10iter: 5.6868 sec.\n",
      "Iteration 2900 || Loss: 15.3931 || 10iter: 4.3655 sec.\n",
      "-------------\n",
      "epoch 44 || Epoch_TRAIN_Loss:1034.4157 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1738 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 45/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2910 || Loss: 20.1342 || 10iter: 7.6191 sec.\n",
      "Iteration 2920 || Loss: 20.4938 || 10iter: 6.3800 sec.\n",
      "Iteration 2930 || Loss: 12.6574 || 10iter: 5.5243 sec.\n",
      "Iteration 2940 || Loss: 20.9581 || 10iter: 5.8239 sec.\n",
      "Iteration 2950 || Loss: 21.0009 || 10iter: 5.8123 sec.\n",
      "Iteration 2960 || Loss: 12.3621 || 10iter: 4.4935 sec.\n",
      "Iteration 2970 || Loss: 14.9244 || 10iter: 3.9463 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 45 || Epoch_TRAIN_Loss:1059.5035 ||Epoch_VAL_Loss:520.9745\n",
      "timer:  51.1710 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 46/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2980 || Loss: 13.9402 || 10iter: 10.8836 sec.\n",
      "Iteration 2990 || Loss: 17.2337 || 10iter: 5.4502 sec.\n",
      "Iteration 3000 || Loss: 13.3040 || 10iter: 5.7367 sec.\n",
      "Iteration 3010 || Loss: 18.6950 || 10iter: 4.8982 sec.\n",
      "Iteration 3020 || Loss: 12.9026 || 10iter: 6.5911 sec.\n",
      "Iteration 3030 || Loss: 21.5148 || 10iter: 4.6579 sec.\n",
      "-------------\n",
      "epoch 46 || Epoch_TRAIN_Loss:1042.3305 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.3407 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 47/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3040 || Loss: 15.1452 || 10iter: 6.9719 sec.\n",
      "Iteration 3050 || Loss: 11.2176 || 10iter: 6.1018 sec.\n",
      "Iteration 3060 || Loss: 14.8723 || 10iter: 4.4693 sec.\n",
      "Iteration 3070 || Loss: 13.5184 || 10iter: 6.3932 sec.\n",
      "Iteration 3080 || Loss: 16.7548 || 10iter: 5.8666 sec.\n",
      "Iteration 3090 || Loss: 11.3897 || 10iter: 5.9238 sec.\n",
      "Iteration 3100 || Loss: 17.2946 || 10iter: 4.4922 sec.\n",
      "-------------\n",
      "epoch 47 || Epoch_TRAIN_Loss:1027.4970 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.5258 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 48/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3110 || Loss: 16.6729 || 10iter: 7.4973 sec.\n",
      "Iteration 3120 || Loss: 14.4583 || 10iter: 5.2903 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 3130 || Loss: 8.7473 || 10iter: 6.4241 sec.\n",
      "Iteration 3140 || Loss: 19.3195 || 10iter: 5.7979 sec.\n",
      "Iteration 3150 || Loss: 14.8356 || 10iter: 5.9334 sec.\n",
      "Iteration 3160 || Loss: 16.5500 || 10iter: 4.9360 sec.\n",
      "-------------\n",
      "epoch 48 || Epoch_TRAIN_Loss:1066.0681 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.6614 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 49/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3170 || Loss: 10.7166 || 10iter: 3.3533 sec.\n",
      "Iteration 3180 || Loss: 12.4312 || 10iter: 7.7737 sec.\n",
      "Iteration 3190 || Loss: 14.7772 || 10iter: 5.3992 sec.\n",
      "Iteration 3200 || Loss: 14.9387 || 10iter: 5.9460 sec.\n",
      "Iteration 3210 || Loss: 15.3616 || 10iter: 5.9322 sec.\n",
      "Iteration 3220 || Loss: 11.4271 || 10iter: 5.6583 sec.\n",
      "Iteration 3230 || Loss: 17.5042 || 10iter: 4.1307 sec.\n",
      "-------------\n",
      "epoch 49 || Epoch_TRAIN_Loss:1018.1212 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5144 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 50/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3240 || Loss: 17.9488 || 10iter: 6.4430 sec.\n",
      "Iteration 3250 || Loss: 14.4050 || 10iter: 4.1834 sec.\n",
      "Iteration 3260 || Loss: 11.8182 || 10iter: 6.7826 sec.\n",
      "Iteration 3270 || Loss: 16.1503 || 10iter: 6.1513 sec.\n",
      "Iteration 3280 || Loss: 15.7565 || 10iter: 6.0661 sec.\n",
      "Iteration 3290 || Loss: 14.1742 || 10iter: 5.1478 sec.\n",
      "Iteration 3300 || Loss: 19.8481 || 10iter: 4.3505 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 50 || Epoch_TRAIN_Loss:1033.0824 ||Epoch_VAL_Loss:514.6496\n",
      "timer:  52.5207 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 51/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3310 || Loss: 16.5659 || 10iter: 10.7247 sec.\n",
      "Iteration 3320 || Loss: 12.7016 || 10iter: 5.0927 sec.\n",
      "Iteration 3330 || Loss: 17.0379 || 10iter: 4.4935 sec.\n",
      "Iteration 3340 || Loss: 15.1005 || 10iter: 6.9558 sec.\n",
      "Iteration 3350 || Loss: 16.4556 || 10iter: 5.6903 sec.\n",
      "Iteration 3360 || Loss: 11.3455 || 10iter: 4.7213 sec.\n",
      "-------------\n",
      "epoch 51 || Epoch_TRAIN_Loss:1024.1851 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.7639 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 52/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3370 || Loss: 12.4588 || 10iter: 6.6372 sec.\n",
      "Iteration 3380 || Loss: 11.6455 || 10iter: 4.8704 sec.\n",
      "Iteration 3390 || Loss: 17.3039 || 10iter: 5.7107 sec.\n",
      "Iteration 3400 || Loss: 13.9166 || 10iter: 6.3456 sec.\n",
      "Iteration 3410 || Loss: 14.3661 || 10iter: 5.6477 sec.\n",
      "Iteration 3420 || Loss: 15.8054 || 10iter: 5.3699 sec.\n",
      "Iteration 3430 || Loss: 15.6848 || 10iter: 4.4881 sec.\n",
      "-------------\n",
      "epoch 52 || Epoch_TRAIN_Loss:1022.6204 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3503 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 53/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3440 || Loss: 11.2649 || 10iter: 6.7061 sec.\n",
      "Iteration 3450 || Loss: 17.7232 || 10iter: 6.9787 sec.\n",
      "Iteration 3460 || Loss: 14.1251 || 10iter: 5.8491 sec.\n",
      "Iteration 3470 || Loss: 14.3032 || 10iter: 5.9474 sec.\n",
      "Iteration 3480 || Loss: 15.5099 || 10iter: 5.8106 sec.\n",
      "Iteration 3490 || Loss: 21.0352 || 10iter: 4.4186 sec.\n",
      "-------------\n",
      "epoch 53 || Epoch_TRAIN_Loss:1007.1643 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.4647 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 54/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3500 || Loss: 13.0938 || 10iter: 4.9974 sec.\n",
      "Iteration 3510 || Loss: 16.7662 || 10iter: 4.8170 sec.\n",
      "Iteration 3520 || Loss: 18.5088 || 10iter: 5.3568 sec.\n",
      "Iteration 3530 || Loss: 16.7438 || 10iter: 6.5629 sec.\n",
      "Iteration 3540 || Loss: 14.2773 || 10iter: 5.9337 sec.\n",
      "Iteration 3550 || Loss: 11.8935 || 10iter: 5.5920 sec.\n",
      "Iteration 3560 || Loss: 16.3378 || 10iter: 4.6637 sec.\n",
      "-------------\n",
      "epoch 54 || Epoch_TRAIN_Loss:1026.9428 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0519 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 55/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3570 || Loss: 15.8387 || 10iter: 7.0236 sec.\n",
      "Iteration 3580 || Loss: 11.5701 || 10iter: 7.0480 sec.\n",
      "Iteration 3590 || Loss: 11.8025 || 10iter: 5.8836 sec.\n",
      "Iteration 3600 || Loss: 11.5528 || 10iter: 6.2810 sec.\n",
      "Iteration 3610 || Loss: 15.8046 || 10iter: 5.6207 sec.\n",
      "Iteration 3620 || Loss: 11.6780 || 10iter: 4.6504 sec.\n",
      "Iteration 3630 || Loss: 13.8972 || 10iter: 4.1727 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 55 || Epoch_TRAIN_Loss:1007.8762 ||Epoch_VAL_Loss:499.7119\n",
      "timer:  55.3142 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 56/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3640 || Loss: 17.9985 || 10iter: 10.2709 sec.\n",
      "Iteration 3650 || Loss: 11.6350 || 10iter: 4.3557 sec.\n",
      "Iteration 3660 || Loss: 14.0671 || 10iter: 6.7125 sec.\n",
      "Iteration 3670 || Loss: 14.1867 || 10iter: 6.0915 sec.\n",
      "Iteration 3680 || Loss: 21.7735 || 10iter: 5.8859 sec.\n",
      "Iteration 3690 || Loss: 13.9476 || 10iter: 4.7266 sec.\n",
      "-------------\n",
      "epoch 56 || Epoch_TRAIN_Loss:1029.5409 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.0484 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 57/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3700 || Loss: 12.4418 || 10iter: 4.5203 sec.\n",
      "Iteration 3710 || Loss: 19.4634 || 10iter: 6.1728 sec.\n",
      "Iteration 3720 || Loss: 13.3959 || 10iter: 6.1213 sec.\n",
      "Iteration 3730 || Loss: 17.6485 || 10iter: 5.8327 sec.\n",
      "Iteration 3740 || Loss: 13.6238 || 10iter: 5.7168 sec.\n",
      "Iteration 3750 || Loss: 23.5306 || 10iter: 5.1860 sec.\n",
      "Iteration 3760 || Loss: 19.9493 || 10iter: 4.2067 sec.\n",
      "-------------\n",
      "epoch 57 || Epoch_TRAIN_Loss:1006.2999 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.0637 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 58/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3770 || Loss: 24.3384 || 10iter: 7.6703 sec.\n",
      "Iteration 3780 || Loss: 16.8745 || 10iter: 4.2762 sec.\n",
      "Iteration 3790 || Loss: 17.1403 || 10iter: 5.6820 sec.\n",
      "Iteration 3800 || Loss: 14.6659 || 10iter: 6.2442 sec.\n",
      "Iteration 3810 || Loss: 15.5644 || 10iter: 5.9360 sec.\n",
      "Iteration 3820 || Loss: 15.9637 || 10iter: 4.7623 sec.\n",
      "-------------\n",
      "epoch 58 || Epoch_TRAIN_Loss:1036.1489 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.8001 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 59/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3830 || Loss: 11.3857 || 10iter: 4.0861 sec.\n",
      "Iteration 3840 || Loss: 19.1324 || 10iter: 5.5881 sec.\n",
      "Iteration 3850 || Loss: 15.5791 || 10iter: 6.8715 sec.\n",
      "Iteration 3860 || Loss: 17.9680 || 10iter: 6.3342 sec.\n",
      "Iteration 3870 || Loss: 20.1824 || 10iter: 6.0805 sec.\n",
      "Iteration 3880 || Loss: 15.9785 || 10iter: 5.5940 sec.\n",
      "Iteration 3890 || Loss: 15.6084 || 10iter: 4.3465 sec.\n",
      "-------------\n",
      "epoch 59 || Epoch_TRAIN_Loss:1020.7984 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.0129 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 60/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3900 || Loss: 20.6836 || 10iter: 8.4109 sec.\n",
      "Iteration 3910 || Loss: 16.2267 || 10iter: 6.3344 sec.\n",
      "Iteration 3920 || Loss: 11.3583 || 10iter: 5.4825 sec.\n",
      "Iteration 3930 || Loss: 14.8632 || 10iter: 5.3356 sec.\n",
      "Iteration 3940 || Loss: 22.9583 || 10iter: 5.4983 sec.\n",
      "Iteration 3950 || Loss: 15.5204 || 10iter: 4.4989 sec.\n",
      "Iteration 3960 || Loss: 14.2625 || 10iter: 4.6193 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 60 || Epoch_TRAIN_Loss:1006.6983 ||Epoch_VAL_Loss:504.3472\n",
      "timer:  54.4963 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 61/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3970 || Loss: 11.8295 || 10iter: 9.5419 sec.\n",
      "Iteration 3980 || Loss: 18.2867 || 10iter: 5.3513 sec.\n",
      "Iteration 3990 || Loss: 13.3699 || 10iter: 6.5909 sec.\n",
      "Iteration 4000 || Loss: 18.5118 || 10iter: 6.2315 sec.\n",
      "Iteration 4010 || Loss: 13.2827 || 10iter: 5.8405 sec.\n",
      "Iteration 4020 || Loss: 17.1199 || 10iter: 4.7734 sec.\n",
      "-------------\n",
      "epoch 61 || Epoch_TRAIN_Loss:982.3865 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.2627 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 62/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4030 || Loss: 14.2765 || 10iter: 4.9920 sec.\n",
      "Iteration 4040 || Loss: 13.2921 || 10iter: 5.2293 sec.\n",
      "Iteration 4050 || Loss: 14.3488 || 10iter: 4.0463 sec.\n",
      "Iteration 4060 || Loss: 13.9299 || 10iter: 6.4996 sec.\n",
      "Iteration 4070 || Loss: 15.8728 || 10iter: 5.8452 sec.\n",
      "Iteration 4080 || Loss: 11.6123 || 10iter: 5.2731 sec.\n",
      "Iteration 4090 || Loss: 11.6246 || 10iter: 4.5194 sec.\n",
      "-------------\n",
      "epoch 62 || Epoch_TRAIN_Loss:997.4541 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  37.8301 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 63/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4100 || Loss: 13.0993 || 10iter: 8.3032 sec.\n",
      "Iteration 4110 || Loss: 17.5271 || 10iter: 5.9820 sec.\n",
      "Iteration 4120 || Loss: 10.6879 || 10iter: 6.7104 sec.\n",
      "Iteration 4130 || Loss: 13.5287 || 10iter: 6.1882 sec.\n",
      "Iteration 4140 || Loss: 14.6844 || 10iter: 6.1914 sec.\n",
      "Iteration 4150 || Loss: 22.0002 || 10iter: 4.8944 sec.\n",
      "-------------\n",
      "epoch 63 || Epoch_TRAIN_Loss:987.1430 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.1231 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 64/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 4160 || Loss: 18.7167 || 10iter: 4.1566 sec.\n",
      "Iteration 4170 || Loss: 14.2766 || 10iter: 7.3106 sec.\n",
      "Iteration 4180 || Loss: 13.5269 || 10iter: 5.6517 sec.\n",
      "Iteration 4190 || Loss: 17.2264 || 10iter: 5.8274 sec.\n",
      "Iteration 4200 || Loss: 14.5326 || 10iter: 5.8990 sec.\n",
      "Iteration 4210 || Loss: 11.4854 || 10iter: 5.0600 sec.\n",
      "Iteration 4220 || Loss: 11.5764 || 10iter: 4.2474 sec.\n",
      "-------------\n",
      "epoch 64 || Epoch_TRAIN_Loss:990.7665 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.8916 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 65/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4230 || Loss: 14.2379 || 10iter: 8.8906 sec.\n",
      "Iteration 4240 || Loss: 15.0938 || 10iter: 6.0149 sec.\n",
      "Iteration 4250 || Loss: 18.1892 || 10iter: 5.9513 sec.\n",
      "Iteration 4260 || Loss: 12.5868 || 10iter: 5.4881 sec.\n",
      "Iteration 4270 || Loss: 18.6650 || 10iter: 4.6883 sec.\n",
      "Iteration 4280 || Loss: 16.3001 || 10iter: 5.2455 sec.\n",
      "Iteration 4290 || Loss: 16.4084 || 10iter: 4.8040 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 65 || Epoch_TRAIN_Loss:990.8403 ||Epoch_VAL_Loss:504.2457\n",
      "timer:  54.2931 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 66/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4300 || Loss: 16.7547 || 10iter: 8.3522 sec.\n",
      "Iteration 4310 || Loss: 11.4852 || 10iter: 4.3328 sec.\n",
      "Iteration 4320 || Loss: 23.7492 || 10iter: 4.2571 sec.\n",
      "Iteration 4330 || Loss: 18.3564 || 10iter: 7.0541 sec.\n",
      "Iteration 4340 || Loss: 12.9304 || 10iter: 5.5168 sec.\n",
      "Iteration 4350 || Loss: 12.1439 || 10iter: 4.8622 sec.\n",
      "-------------\n",
      "epoch 66 || Epoch_TRAIN_Loss:999.1793 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  37.4638 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 67/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4360 || Loss: 13.1335 || 10iter: 6.8216 sec.\n",
      "Iteration 4370 || Loss: 14.2210 || 10iter: 4.7522 sec.\n",
      "Iteration 4380 || Loss: 13.3965 || 10iter: 6.6270 sec.\n",
      "Iteration 4390 || Loss: 10.5989 || 10iter: 6.2998 sec.\n",
      "Iteration 4400 || Loss: 19.3805 || 10iter: 6.3694 sec.\n",
      "Iteration 4410 || Loss: 13.4531 || 10iter: 5.5677 sec.\n",
      "Iteration 4420 || Loss: 16.0061 || 10iter: 4.4505 sec.\n",
      "-------------\n",
      "epoch 67 || Epoch_TRAIN_Loss:1008.1591 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.1884 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 68/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4430 || Loss: 12.4798 || 10iter: 7.6823 sec.\n",
      "Iteration 4440 || Loss: 13.5105 || 10iter: 6.7271 sec.\n",
      "Iteration 4450 || Loss: 11.1180 || 10iter: 5.3199 sec.\n",
      "Iteration 4460 || Loss: 13.2226 || 10iter: 5.8669 sec.\n",
      "Iteration 4470 || Loss: 11.7452 || 10iter: 5.6295 sec.\n",
      "Iteration 4480 || Loss: 16.3033 || 10iter: 4.6634 sec.\n",
      "-------------\n",
      "epoch 68 || Epoch_TRAIN_Loss:998.3814 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0716 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 69/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4490 || Loss: 17.1193 || 10iter: 6.8318 sec.\n",
      "Iteration 4500 || Loss: 19.8393 || 10iter: 6.0671 sec.\n",
      "Iteration 4510 || Loss: 15.1886 || 10iter: 5.5716 sec.\n",
      "Iteration 4520 || Loss: 17.3585 || 10iter: 5.3862 sec.\n",
      "Iteration 4530 || Loss: 11.3600 || 10iter: 5.4840 sec.\n",
      "Iteration 4540 || Loss: 14.2371 || 10iter: 4.5067 sec.\n",
      "Iteration 4550 || Loss: 12.9990 || 10iter: 5.5754 sec.\n",
      "-------------\n",
      "epoch 69 || Epoch_TRAIN_Loss:976.0073 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.7986 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 70/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4560 || Loss: 15.9286 || 10iter: 7.6131 sec.\n",
      "Iteration 4570 || Loss: 15.6294 || 10iter: 6.1468 sec.\n",
      "Iteration 4580 || Loss: 15.0040 || 10iter: 5.3200 sec.\n",
      "Iteration 4590 || Loss: 32.2140 || 10iter: 4.4694 sec.\n",
      "Iteration 4600 || Loss: 18.7015 || 10iter: 4.7742 sec.\n",
      "Iteration 4610 || Loss: 16.8723 || 10iter: 4.0044 sec.\n",
      "Iteration 4620 || Loss: 11.6509 || 10iter: 4.5418 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 70 || Epoch_TRAIN_Loss:1017.3589 ||Epoch_VAL_Loss:494.4142\n",
      "timer:  50.8085 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 71/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4630 || Loss: 14.8718 || 10iter: 9.9751 sec.\n",
      "Iteration 4640 || Loss: 14.7805 || 10iter: 4.8317 sec.\n",
      "Iteration 4650 || Loss: 21.0276 || 10iter: 6.7864 sec.\n",
      "Iteration 4660 || Loss: 18.1798 || 10iter: 6.5862 sec.\n",
      "Iteration 4670 || Loss: 13.0848 || 10iter: 6.3854 sec.\n",
      "Iteration 4680 || Loss: 15.2482 || 10iter: 4.5903 sec.\n",
      "-------------\n",
      "epoch 71 || Epoch_TRAIN_Loss:978.0705 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.1976 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 72/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4690 || Loss: 15.3392 || 10iter: 4.6368 sec.\n",
      "Iteration 4700 || Loss: 13.9817 || 10iter: 6.9372 sec.\n",
      "Iteration 4710 || Loss: 14.8214 || 10iter: 6.1017 sec.\n",
      "Iteration 4720 || Loss: 15.0243 || 10iter: 6.0317 sec.\n",
      "Iteration 4730 || Loss: 12.2511 || 10iter: 6.0745 sec.\n",
      "Iteration 4740 || Loss: 14.1386 || 10iter: 5.0429 sec.\n",
      "Iteration 4750 || Loss: 14.0562 || 10iter: 4.6783 sec.\n",
      "-------------\n",
      "epoch 72 || Epoch_TRAIN_Loss:993.6837 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.0772 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 73/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4760 || Loss: 12.5967 || 10iter: 9.5038 sec.\n",
      "Iteration 4770 || Loss: 21.4779 || 10iter: 5.8250 sec.\n",
      "Iteration 4780 || Loss: 15.1758 || 10iter: 5.6123 sec.\n",
      "Iteration 4790 || Loss: 13.2252 || 10iter: 5.6155 sec.\n",
      "Iteration 4800 || Loss: 16.6120 || 10iter: 4.5127 sec.\n",
      "Iteration 4810 || Loss: 17.2241 || 10iter: 5.8280 sec.\n",
      "-------------\n",
      "epoch 73 || Epoch_TRAIN_Loss:998.9700 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1214 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 74/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4820 || Loss: 16.8242 || 10iter: 5.5392 sec.\n",
      "Iteration 4830 || Loss: 17.6454 || 10iter: 6.4589 sec.\n",
      "Iteration 4840 || Loss: 16.3474 || 10iter: 5.1771 sec.\n",
      "Iteration 4850 || Loss: 16.1586 || 10iter: 4.5116 sec.\n",
      "Iteration 4860 || Loss: 10.1728 || 10iter: 4.9734 sec.\n",
      "Iteration 4870 || Loss: 12.8163 || 10iter: 4.0655 sec.\n",
      "Iteration 4880 || Loss: 14.8192 || 10iter: 4.8963 sec.\n",
      "-------------\n",
      "epoch 74 || Epoch_TRAIN_Loss:987.6408 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.1787 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 75/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4890 || Loss: 14.0830 || 10iter: 8.0683 sec.\n",
      "Iteration 4900 || Loss: 15.4249 || 10iter: 6.0603 sec.\n",
      "Iteration 4910 || Loss: 17.6352 || 10iter: 5.7691 sec.\n",
      "Iteration 4920 || Loss: 15.3394 || 10iter: 5.1059 sec.\n",
      "Iteration 4930 || Loss: 13.1870 || 10iter: 5.4018 sec.\n",
      "Iteration 4940 || Loss: 14.9079 || 10iter: 5.9384 sec.\n",
      "Iteration 4950 || Loss: 15.0130 || 10iter: 4.6881 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 75 || Epoch_TRAIN_Loss:972.8178 ||Epoch_VAL_Loss:488.9928\n",
      "timer:  53.8230 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 76/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4960 || Loss: 17.1723 || 10iter: 10.5933 sec.\n",
      "Iteration 4970 || Loss: 16.3492 || 10iter: 5.8907 sec.\n",
      "Iteration 4980 || Loss: 14.0426 || 10iter: 6.3034 sec.\n",
      "Iteration 4990 || Loss: 14.9772 || 10iter: 5.2219 sec.\n",
      "Iteration 5000 || Loss: 13.9097 || 10iter: 5.4034 sec.\n",
      "Iteration 5010 || Loss: 13.6050 || 10iter: 4.1536 sec.\n",
      "-------------\n",
      "epoch 76 || Epoch_TRAIN_Loss:1007.9136 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.5038 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 77/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5020 || Loss: 11.9253 || 10iter: 7.1199 sec.\n",
      "Iteration 5030 || Loss: 16.4009 || 10iter: 5.8302 sec.\n",
      "Iteration 5040 || Loss: 12.0541 || 10iter: 5.4152 sec.\n",
      "Iteration 5050 || Loss: 11.9626 || 10iter: 5.5459 sec.\n",
      "Iteration 5060 || Loss: 14.6301 || 10iter: 5.0855 sec.\n",
      "Iteration 5070 || Loss: 17.9995 || 10iter: 4.6981 sec.\n",
      "Iteration 5080 || Loss: 14.0968 || 10iter: 5.2369 sec.\n",
      "-------------\n",
      "epoch 77 || Epoch_TRAIN_Loss:976.7414 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3501 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 78/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5090 || Loss: 10.3683 || 10iter: 9.7597 sec.\n",
      "Iteration 5100 || Loss: 14.4232 || 10iter: 5.9204 sec.\n",
      "Iteration 5110 || Loss: 20.8840 || 10iter: 5.2919 sec.\n",
      "Iteration 5120 || Loss: 14.9458 || 10iter: 4.8892 sec.\n",
      "Iteration 5130 || Loss: 18.1553 || 10iter: 4.3799 sec.\n",
      "Iteration 5140 || Loss: 16.0026 || 10iter: 3.9252 sec.\n",
      "-------------\n",
      "epoch 78 || Epoch_TRAIN_Loss:1003.5671 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.9941 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 79/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5150 || Loss: 16.9236 || 10iter: 6.2516 sec.\n",
      "Iteration 5160 || Loss: 11.6032 || 10iter: 5.7965 sec.\n",
      "Iteration 5170 || Loss: 14.6878 || 10iter: 5.5012 sec.\n",
      "Iteration 5180 || Loss: 15.4998 || 10iter: 5.6558 sec.\n",
      "Iteration 5190 || Loss: 9.3984 || 10iter: 5.1798 sec.\n",
      "Iteration 5200 || Loss: 15.7360 || 10iter: 5.9901 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 5210 || Loss: 13.9013 || 10iter: 4.7455 sec.\n",
      "-------------\n",
      "epoch 79 || Epoch_TRAIN_Loss:987.7748 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.4100 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 80/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5220 || Loss: 13.0091 || 10iter: 8.6650 sec.\n",
      "Iteration 5230 || Loss: 13.8477 || 10iter: 6.5481 sec.\n",
      "Iteration 5240 || Loss: 18.4968 || 10iter: 4.8784 sec.\n",
      "Iteration 5250 || Loss: 12.2097 || 10iter: 6.1155 sec.\n",
      "Iteration 5260 || Loss: 11.8305 || 10iter: 6.2941 sec.\n",
      "Iteration 5270 || Loss: 18.1269 || 10iter: 5.4608 sec.\n",
      "Iteration 5280 || Loss: 11.0492 || 10iter: 4.4279 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 80 || Epoch_TRAIN_Loss:962.4960 ||Epoch_VAL_Loss:492.2300\n",
      "timer:  54.4483 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 81/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5290 || Loss: 13.0661 || 10iter: 11.1275 sec.\n",
      "Iteration 5300 || Loss: 11.3012 || 10iter: 5.5326 sec.\n",
      "Iteration 5310 || Loss: 12.3535 || 10iter: 5.6532 sec.\n",
      "Iteration 5320 || Loss: 17.4965 || 10iter: 4.9921 sec.\n",
      "Iteration 5330 || Loss: 14.1934 || 10iter: 5.0029 sec.\n",
      "Iteration 5340 || Loss: 14.3340 || 10iter: 5.4562 sec.\n",
      "-------------\n",
      "epoch 81 || Epoch_TRAIN_Loss:929.3852 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9065 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 82/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5350 || Loss: 16.7041 || 10iter: 7.0246 sec.\n",
      "Iteration 5360 || Loss: 13.7916 || 10iter: 5.9711 sec.\n",
      "Iteration 5370 || Loss: 10.8501 || 10iter: 5.1253 sec.\n",
      "Iteration 5380 || Loss: 12.6387 || 10iter: 4.4949 sec.\n",
      "Iteration 5390 || Loss: 14.5042 || 10iter: 5.2031 sec.\n",
      "Iteration 5400 || Loss: 13.2345 || 10iter: 4.0303 sec.\n",
      "Iteration 5410 || Loss: 17.6795 || 10iter: 5.3177 sec.\n",
      "-------------\n",
      "epoch 82 || Epoch_TRAIN_Loss:998.9397 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.6959 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 83/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5420 || Loss: 15.3235 || 10iter: 9.0655 sec.\n",
      "Iteration 5430 || Loss: 15.6885 || 10iter: 6.0145 sec.\n",
      "Iteration 5440 || Loss: 10.8087 || 10iter: 5.9995 sec.\n",
      "Iteration 5450 || Loss: 13.1291 || 10iter: 4.6773 sec.\n",
      "Iteration 5460 || Loss: 13.4269 || 10iter: 6.3202 sec.\n",
      "Iteration 5470 || Loss: 24.0452 || 10iter: 5.2826 sec.\n",
      "-------------\n",
      "epoch 83 || Epoch_TRAIN_Loss:998.8116 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.4918 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 84/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5480 || Loss: 17.7018 || 10iter: 5.0138 sec.\n",
      "Iteration 5490 || Loss: 13.3954 || 10iter: 6.9117 sec.\n",
      "Iteration 5500 || Loss: 11.6656 || 10iter: 4.8796 sec.\n",
      "Iteration 5510 || Loss: 21.3030 || 10iter: 5.9035 sec.\n",
      "Iteration 5520 || Loss: 13.6858 || 10iter: 6.5027 sec.\n",
      "Iteration 5530 || Loss: 13.6773 || 10iter: 5.0428 sec.\n",
      "Iteration 5540 || Loss: 11.7537 || 10iter: 4.5958 sec.\n",
      "-------------\n",
      "epoch 84 || Epoch_TRAIN_Loss:974.4840 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.2392 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 85/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5550 || Loss: 13.7343 || 10iter: 7.5251 sec.\n",
      "Iteration 5560 || Loss: 11.4271 || 10iter: 4.5579 sec.\n",
      "Iteration 5570 || Loss: 16.1070 || 10iter: 6.6778 sec.\n",
      "Iteration 5580 || Loss: 12.5038 || 10iter: 5.5213 sec.\n",
      "Iteration 5590 || Loss: 16.9183 || 10iter: 6.0083 sec.\n",
      "Iteration 5600 || Loss: 15.3378 || 10iter: 5.4151 sec.\n",
      "Iteration 5610 || Loss: 13.5604 || 10iter: 4.1513 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 85 || Epoch_TRAIN_Loss:969.3629 ||Epoch_VAL_Loss:483.0391\n",
      "timer:  53.3278 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 86/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5620 || Loss: 13.6105 || 10iter: 11.2711 sec.\n",
      "Iteration 5630 || Loss: 12.3925 || 10iter: 5.2334 sec.\n",
      "Iteration 5640 || Loss: 17.9386 || 10iter: 4.9456 sec.\n",
      "Iteration 5650 || Loss: 11.0786 || 10iter: 4.4996 sec.\n",
      "Iteration 5660 || Loss: 15.8776 || 10iter: 4.2088 sec.\n",
      "Iteration 5670 || Loss: 18.5076 || 10iter: 5.2186 sec.\n",
      "-------------\n",
      "epoch 86 || Epoch_TRAIN_Loss:984.6738 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.5892 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 87/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5680 || Loss: 17.6893 || 10iter: 7.0422 sec.\n",
      "Iteration 5690 || Loss: 12.4567 || 10iter: 6.2772 sec.\n",
      "Iteration 5700 || Loss: 13.6966 || 10iter: 5.9612 sec.\n",
      "Iteration 5710 || Loss: 14.3732 || 10iter: 4.4926 sec.\n",
      "Iteration 5720 || Loss: 18.2002 || 10iter: 6.0931 sec.\n",
      "Iteration 5730 || Loss: 14.6303 || 10iter: 5.6105 sec.\n",
      "Iteration 5740 || Loss: 11.9120 || 10iter: 4.6608 sec.\n",
      "-------------\n",
      "epoch 87 || Epoch_TRAIN_Loss:972.6609 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.4731 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 88/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5750 || Loss: 14.3887 || 10iter: 9.0641 sec.\n",
      "Iteration 5760 || Loss: 12.2238 || 10iter: 5.2073 sec.\n",
      "Iteration 5770 || Loss: 14.2941 || 10iter: 5.7181 sec.\n",
      "Iteration 5780 || Loss: 18.8325 || 10iter: 7.1202 sec.\n",
      "Iteration 5790 || Loss: 13.5253 || 10iter: 6.2242 sec.\n",
      "Iteration 5800 || Loss: 17.4268 || 10iter: 4.9204 sec.\n",
      "-------------\n",
      "epoch 88 || Epoch_TRAIN_Loss:986.3195 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.2286 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 89/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5810 || Loss: 12.6747 || 10iter: 3.7579 sec.\n",
      "Iteration 5820 || Loss: 19.6277 || 10iter: 5.5579 sec.\n",
      "Iteration 5830 || Loss: 14.9387 || 10iter: 6.6392 sec.\n",
      "Iteration 5840 || Loss: 11.7709 || 10iter: 5.5115 sec.\n",
      "Iteration 5850 || Loss: 17.3365 || 10iter: 5.8154 sec.\n",
      "Iteration 5860 || Loss: 14.9487 || 10iter: 5.5019 sec.\n",
      "Iteration 5870 || Loss: 15.9880 || 10iter: 4.4670 sec.\n",
      "-------------\n",
      "epoch 89 || Epoch_TRAIN_Loss:953.3715 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.3836 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 90/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5880 || Loss: 15.0473 || 10iter: 7.5732 sec.\n",
      "Iteration 5890 || Loss: 15.8875 || 10iter: 6.7382 sec.\n",
      "Iteration 5900 || Loss: 14.5802 || 10iter: 5.8709 sec.\n",
      "Iteration 5910 || Loss: 12.6661 || 10iter: 6.0976 sec.\n",
      "Iteration 5920 || Loss: 17.6094 || 10iter: 5.7224 sec.\n",
      "Iteration 5930 || Loss: 12.1811 || 10iter: 4.3184 sec.\n",
      "Iteration 5940 || Loss: 18.8277 || 10iter: 3.9480 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 90 || Epoch_TRAIN_Loss:976.9677 ||Epoch_VAL_Loss:475.7429\n",
      "timer:  52.2650 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 91/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5950 || Loss: 20.9323 || 10iter: 10.9680 sec.\n",
      "Iteration 5960 || Loss: 9.1828 || 10iter: 5.6799 sec.\n",
      "Iteration 5970 || Loss: 14.2366 || 10iter: 5.1798 sec.\n",
      "Iteration 5980 || Loss: 11.9806 || 10iter: 4.5715 sec.\n",
      "Iteration 5990 || Loss: 16.1062 || 10iter: 6.8128 sec.\n",
      "Iteration 6000 || Loss: 12.8985 || 10iter: 4.7777 sec.\n",
      "-------------\n",
      "epoch 91 || Epoch_TRAIN_Loss:954.1262 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.3212 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 92/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6010 || Loss: 16.6760 || 10iter: 6.6467 sec.\n",
      "Iteration 6020 || Loss: 15.7255 || 10iter: 6.0859 sec.\n",
      "Iteration 6030 || Loss: 13.8688 || 10iter: 4.3461 sec.\n",
      "Iteration 6040 || Loss: 14.1353 || 10iter: 6.9337 sec.\n",
      "Iteration 6050 || Loss: 13.2026 || 10iter: 6.3682 sec.\n",
      "Iteration 6060 || Loss: 15.9529 || 10iter: 5.5441 sec.\n",
      "Iteration 6070 || Loss: 15.7347 || 10iter: 4.8058 sec.\n",
      "-------------\n",
      "epoch 92 || Epoch_TRAIN_Loss:987.0139 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.1359 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 93/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6080 || Loss: 18.2559 || 10iter: 7.4871 sec.\n",
      "Iteration 6090 || Loss: 13.6344 || 10iter: 5.9289 sec.\n",
      "Iteration 6100 || Loss: 14.9522 || 10iter: 6.1536 sec.\n",
      "Iteration 6110 || Loss: 13.4468 || 10iter: 6.4406 sec.\n",
      "Iteration 6120 || Loss: 15.8941 || 10iter: 6.0158 sec.\n",
      "Iteration 6130 || Loss: 18.0777 || 10iter: 4.7282 sec.\n",
      "-------------\n",
      "epoch 93 || Epoch_TRAIN_Loss:971.0605 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4638 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 94/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6140 || Loss: 16.7608 || 10iter: 5.2858 sec.\n",
      "Iteration 6150 || Loss: 12.6784 || 10iter: 6.9478 sec.\n",
      "Iteration 6160 || Loss: 14.8260 || 10iter: 5.3623 sec.\n",
      "Iteration 6170 || Loss: 13.9354 || 10iter: 5.8409 sec.\n",
      "Iteration 6180 || Loss: 13.0245 || 10iter: 5.8497 sec.\n",
      "Iteration 6190 || Loss: 15.8238 || 10iter: 5.0743 sec.\n",
      "Iteration 6200 || Loss: 16.9637 || 10iter: 4.3355 sec.\n",
      "-------------\n",
      "epoch 94 || Epoch_TRAIN_Loss:959.0126 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.8193 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 95/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6210 || Loss: 13.9678 || 10iter: 6.0760 sec.\n",
      "Iteration 6220 || Loss: 15.1253 || 10iter: 5.3238 sec.\n",
      "Iteration 6230 || Loss: 12.7612 || 10iter: 6.3607 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 6240 || Loss: 13.7189 || 10iter: 5.5743 sec.\n",
      "Iteration 6250 || Loss: 16.9513 || 10iter: 5.6161 sec.\n",
      "Iteration 6260 || Loss: 16.5218 || 10iter: 5.1837 sec.\n",
      "Iteration 6270 || Loss: 12.3751 || 10iter: 4.1202 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 95 || Epoch_TRAIN_Loss:964.9047 ||Epoch_VAL_Loss:478.1681\n",
      "timer:  50.9588 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 96/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6280 || Loss: 10.6101 || 10iter: 10.9244 sec.\n",
      "Iteration 6290 || Loss: 13.0778 || 10iter: 5.2960 sec.\n",
      "Iteration 6300 || Loss: 17.0348 || 10iter: 4.6240 sec.\n",
      "Iteration 6310 || Loss: 13.0193 || 10iter: 6.5115 sec.\n",
      "Iteration 6320 || Loss: 15.7131 || 10iter: 6.1980 sec.\n",
      "Iteration 6330 || Loss: 14.4443 || 10iter: 4.9163 sec.\n",
      "-------------\n",
      "epoch 96 || Epoch_TRAIN_Loss:923.3120 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.7751 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 97/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6340 || Loss: 12.1546 || 10iter: 6.6679 sec.\n",
      "Iteration 6350 || Loss: 12.5369 || 10iter: 5.1325 sec.\n",
      "Iteration 6360 || Loss: 15.6752 || 10iter: 5.6698 sec.\n",
      "Iteration 6370 || Loss: 17.1729 || 10iter: 6.8995 sec.\n",
      "Iteration 6380 || Loss: 12.8216 || 10iter: 5.6816 sec.\n",
      "Iteration 6390 || Loss: 16.5939 || 10iter: 5.4506 sec.\n",
      "Iteration 6400 || Loss: 13.2461 || 10iter: 4.5020 sec.\n",
      "-------------\n",
      "epoch 97 || Epoch_TRAIN_Loss:939.6587 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.3218 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 98/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6410 || Loss: 16.5336 || 10iter: 7.0283 sec.\n",
      "Iteration 6420 || Loss: 9.8609 || 10iter: 6.7079 sec.\n",
      "Iteration 6430 || Loss: 16.3229 || 10iter: 5.7576 sec.\n",
      "Iteration 6440 || Loss: 12.3714 || 10iter: 6.0367 sec.\n",
      "Iteration 6450 || Loss: 11.3172 || 10iter: 5.4841 sec.\n",
      "Iteration 6460 || Loss: 16.8278 || 10iter: 4.5030 sec.\n",
      "-------------\n",
      "epoch 98 || Epoch_TRAIN_Loss:985.1818 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.2076 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 99/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6470 || Loss: 23.7020 || 10iter: 5.1770 sec.\n",
      "Iteration 6480 || Loss: 11.3599 || 10iter: 4.6966 sec.\n",
      "Iteration 6490 || Loss: 10.2902 || 10iter: 5.0958 sec.\n",
      "Iteration 6500 || Loss: 15.9130 || 10iter: 6.9039 sec.\n",
      "Iteration 6510 || Loss: 17.1884 || 10iter: 6.0568 sec.\n",
      "Iteration 6520 || Loss: 14.6239 || 10iter: 5.6759 sec.\n",
      "Iteration 6530 || Loss: 20.3440 || 10iter: 4.4977 sec.\n",
      "-------------\n",
      "epoch 99 || Epoch_TRAIN_Loss:981.6654 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2962 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 100/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6540 || Loss: 14.3897 || 10iter: 6.4423 sec.\n",
      "Iteration 6550 || Loss: 13.5105 || 10iter: 6.8174 sec.\n",
      "Iteration 6560 || Loss: 9.1577 || 10iter: 5.7559 sec.\n",
      "Iteration 6570 || Loss: 14.1582 || 10iter: 5.7320 sec.\n",
      "Iteration 6580 || Loss: 18.5204 || 10iter: 5.7401 sec.\n",
      "Iteration 6590 || Loss: 13.3524 || 10iter: 4.9511 sec.\n",
      "Iteration 6600 || Loss: 16.0842 || 10iter: 4.0774 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 100 || Epoch_TRAIN_Loss:984.7036 ||Epoch_VAL_Loss:479.2738\n",
      "timer:  55.5886 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 101/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6610 || Loss: 15.3480 || 10iter: 10.1778 sec.\n",
      "Iteration 6620 || Loss: 13.9617 || 10iter: 4.6350 sec.\n",
      "Iteration 6630 || Loss: 16.0838 || 10iter: 6.8762 sec.\n",
      "Iteration 6640 || Loss: 13.7185 || 10iter: 5.9280 sec.\n",
      "Iteration 6650 || Loss: 12.6900 || 10iter: 5.6440 sec.\n",
      "Iteration 6660 || Loss: 12.1379 || 10iter: 4.6762 sec.\n",
      "-------------\n",
      "epoch 101 || Epoch_TRAIN_Loss:945.0174 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.0421 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 102/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6670 || Loss: 12.7837 || 10iter: 4.7783 sec.\n",
      "Iteration 6680 || Loss: 10.1000 || 10iter: 6.9642 sec.\n",
      "Iteration 6690 || Loss: 13.3605 || 10iter: 5.5076 sec.\n",
      "Iteration 6700 || Loss: 16.4104 || 10iter: 5.8954 sec.\n",
      "Iteration 6710 || Loss: 14.5310 || 10iter: 5.8798 sec.\n",
      "Iteration 6720 || Loss: 17.7891 || 10iter: 5.4168 sec.\n",
      "Iteration 6730 || Loss: 13.1489 || 10iter: 4.2396 sec.\n",
      "-------------\n",
      "epoch 102 || Epoch_TRAIN_Loss:959.5476 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0608 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 103/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6740 || Loss: 16.1879 || 10iter: 7.9431 sec.\n",
      "Iteration 6750 || Loss: 19.3724 || 10iter: 4.5268 sec.\n",
      "Iteration 6760 || Loss: 16.0526 || 10iter: 6.7024 sec.\n",
      "Iteration 6770 || Loss: 20.3455 || 10iter: 5.7530 sec.\n",
      "Iteration 6780 || Loss: 13.2252 || 10iter: 5.6958 sec.\n",
      "Iteration 6790 || Loss: 15.0158 || 10iter: 4.9062 sec.\n",
      "-------------\n",
      "epoch 103 || Epoch_TRAIN_Loss:958.0902 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.5659 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 104/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6800 || Loss: 18.0250 || 10iter: 4.3321 sec.\n",
      "Iteration 6810 || Loss: 16.6724 || 10iter: 7.1709 sec.\n",
      "Iteration 6820 || Loss: 12.8967 || 10iter: 5.3408 sec.\n",
      "Iteration 6830 || Loss: 13.0246 || 10iter: 6.3482 sec.\n",
      "Iteration 6840 || Loss: 14.7360 || 10iter: 6.1127 sec.\n",
      "Iteration 6850 || Loss: 11.8613 || 10iter: 5.3169 sec.\n",
      "Iteration 6860 || Loss: 16.7947 || 10iter: 4.3539 sec.\n",
      "-------------\n",
      "epoch 104 || Epoch_TRAIN_Loss:965.5332 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.0111 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 105/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6870 || Loss: 17.0605 || 10iter: 9.2384 sec.\n",
      "Iteration 6880 || Loss: 14.2037 || 10iter: 6.4259 sec.\n",
      "Iteration 6890 || Loss: 13.2446 || 10iter: 5.5573 sec.\n",
      "Iteration 6900 || Loss: 15.7361 || 10iter: 5.6441 sec.\n",
      "Iteration 6910 || Loss: 17.0602 || 10iter: 4.5338 sec.\n",
      "Iteration 6920 || Loss: 13.7621 || 10iter: 6.3542 sec.\n",
      "Iteration 6930 || Loss: 13.7311 || 10iter: 4.3465 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 105 || Epoch_TRAIN_Loss:936.5213 ||Epoch_VAL_Loss:470.0446\n",
      "timer:  54.8261 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 106/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6940 || Loss: 18.6670 || 10iter: 9.1359 sec.\n",
      "Iteration 6950 || Loss: 12.8028 || 10iter: 6.2820 sec.\n",
      "Iteration 6960 || Loss: 12.8486 || 10iter: 5.7881 sec.\n",
      "Iteration 6970 || Loss: 17.9909 || 10iter: 5.6248 sec.\n",
      "Iteration 6980 || Loss: 18.1590 || 10iter: 5.4573 sec.\n",
      "Iteration 6990 || Loss: 15.4142 || 10iter: 4.3744 sec.\n",
      "-------------\n",
      "epoch 106 || Epoch_TRAIN_Loss:965.5142 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.7124 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 107/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7000 || Loss: 18.7289 || 10iter: 5.5645 sec.\n",
      "Iteration 7010 || Loss: 13.9879 || 10iter: 4.2551 sec.\n",
      "Iteration 7020 || Loss: 12.0650 || 10iter: 6.7030 sec.\n",
      "Iteration 7030 || Loss: 14.2568 || 10iter: 5.7672 sec.\n",
      "Iteration 7040 || Loss: 14.2112 || 10iter: 6.3217 sec.\n",
      "Iteration 7050 || Loss: 11.3081 || 10iter: 5.3992 sec.\n",
      "Iteration 7060 || Loss: 17.1916 || 10iter: 4.3285 sec.\n",
      "-------------\n",
      "epoch 107 || Epoch_TRAIN_Loss:936.6945 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.6127 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 108/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7070 || Loss: 11.4858 || 10iter: 7.9087 sec.\n",
      "Iteration 7080 || Loss: 16.0496 || 10iter: 6.6394 sec.\n",
      "Iteration 7090 || Loss: 11.6051 || 10iter: 5.6044 sec.\n",
      "Iteration 7100 || Loss: 16.4491 || 10iter: 5.7228 sec.\n",
      "Iteration 7110 || Loss: 11.3274 || 10iter: 5.5674 sec.\n",
      "Iteration 7120 || Loss: 11.4697 || 10iter: 4.4507 sec.\n",
      "-------------\n",
      "epoch 108 || Epoch_TRAIN_Loss:949.7767 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1208 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 109/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7130 || Loss: 12.9525 || 10iter: 7.1941 sec.\n",
      "Iteration 7140 || Loss: 13.3142 || 10iter: 6.3499 sec.\n",
      "Iteration 7150 || Loss: 14.8387 || 10iter: 6.2526 sec.\n",
      "Iteration 7160 || Loss: 14.6947 || 10iter: 5.4640 sec.\n",
      "Iteration 7170 || Loss: 19.5300 || 10iter: 5.6769 sec.\n",
      "Iteration 7180 || Loss: 11.4807 || 10iter: 4.6184 sec.\n",
      "Iteration 7190 || Loss: 18.2869 || 10iter: 5.6209 sec.\n",
      "-------------\n",
      "epoch 109 || Epoch_TRAIN_Loss:970.8179 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  43.5828 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 110/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7200 || Loss: 14.5995 || 10iter: 8.3046 sec.\n",
      "Iteration 7210 || Loss: 13.5928 || 10iter: 6.2585 sec.\n",
      "Iteration 7220 || Loss: 21.4344 || 10iter: 5.0841 sec.\n",
      "Iteration 7230 || Loss: 14.2418 || 10iter: 4.4371 sec.\n",
      "Iteration 7240 || Loss: 14.3647 || 10iter: 6.5506 sec.\n",
      "Iteration 7250 || Loss: 12.1890 || 10iter: 5.1638 sec.\n",
      "Iteration 7260 || Loss: 32.3392 || 10iter: 4.9913 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 110 || Epoch_TRAIN_Loss:982.2459 ||Epoch_VAL_Loss:471.4715\n",
      "timer:  52.5981 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 111/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 7270 || Loss: 15.7824 || 10iter: 7.8806 sec.\n",
      "Iteration 7280 || Loss: 12.5853 || 10iter: 4.4607 sec.\n",
      "Iteration 7290 || Loss: 13.3575 || 10iter: 6.7531 sec.\n",
      "Iteration 7300 || Loss: 12.9868 || 10iter: 5.7532 sec.\n",
      "Iteration 7310 || Loss: 14.6900 || 10iter: 5.9077 sec.\n",
      "Iteration 7320 || Loss: 17.9481 || 10iter: 4.6838 sec.\n",
      "-------------\n",
      "epoch 111 || Epoch_TRAIN_Loss:984.5430 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.4841 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 112/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7330 || Loss: 15.2057 || 10iter: 4.4370 sec.\n",
      "Iteration 7340 || Loss: 16.0859 || 10iter: 6.7443 sec.\n",
      "Iteration 7350 || Loss: 8.4569 || 10iter: 5.9277 sec.\n",
      "Iteration 7360 || Loss: 14.2432 || 10iter: 6.0556 sec.\n",
      "Iteration 7370 || Loss: 24.1366 || 10iter: 5.6488 sec.\n",
      "Iteration 7380 || Loss: 11.4251 || 10iter: 5.2554 sec.\n",
      "Iteration 7390 || Loss: 15.8946 || 10iter: 4.1634 sec.\n",
      "-------------\n",
      "epoch 112 || Epoch_TRAIN_Loss:947.9764 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.5240 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 113/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7400 || Loss: 13.2943 || 10iter: 10.9816 sec.\n",
      "Iteration 7410 || Loss: 12.4755 || 10iter: 6.2669 sec.\n",
      "Iteration 7420 || Loss: 16.0567 || 10iter: 6.1089 sec.\n",
      "Iteration 7430 || Loss: 16.5204 || 10iter: 5.6207 sec.\n",
      "Iteration 7440 || Loss: 11.6962 || 10iter: 4.9124 sec.\n",
      "Iteration 7450 || Loss: 17.2926 || 10iter: 5.1584 sec.\n",
      "-------------\n",
      "epoch 113 || Epoch_TRAIN_Loss:948.0815 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  43.4441 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 114/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7460 || Loss: 21.8270 || 10iter: 4.7159 sec.\n",
      "Iteration 7470 || Loss: 21.8494 || 10iter: 6.4547 sec.\n",
      "Iteration 7480 || Loss: 10.7394 || 10iter: 5.5588 sec.\n",
      "Iteration 7490 || Loss: 13.5348 || 10iter: 4.8920 sec.\n",
      "Iteration 7500 || Loss: 20.0682 || 10iter: 4.5262 sec.\n",
      "Iteration 7510 || Loss: 12.7533 || 10iter: 6.6842 sec.\n",
      "Iteration 7520 || Loss: 18.4252 || 10iter: 4.8192 sec.\n",
      "-------------\n",
      "epoch 114 || Epoch_TRAIN_Loss:965.7606 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0662 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 115/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7530 || Loss: 14.1021 || 10iter: 7.7995 sec.\n",
      "Iteration 7540 || Loss: 9.5701 || 10iter: 5.9588 sec.\n",
      "Iteration 7550 || Loss: 12.1105 || 10iter: 4.4791 sec.\n",
      "Iteration 7560 || Loss: 14.6181 || 10iter: 5.0501 sec.\n",
      "Iteration 7570 || Loss: 18.1703 || 10iter: 4.3117 sec.\n",
      "Iteration 7580 || Loss: 11.7245 || 10iter: 4.7115 sec.\n",
      "Iteration 7590 || Loss: 17.6787 || 10iter: 4.8088 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 115 || Epoch_TRAIN_Loss:927.4549 ||Epoch_VAL_Loss:468.4100\n",
      "timer:  50.2387 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 116/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7600 || Loss: 9.6902 || 10iter: 7.8163 sec.\n",
      "Iteration 7610 || Loss: 13.2329 || 10iter: 6.6317 sec.\n",
      "Iteration 7620 || Loss: 14.2033 || 10iter: 5.7609 sec.\n",
      "Iteration 7630 || Loss: 16.4161 || 10iter: 5.3677 sec.\n",
      "Iteration 7640 || Loss: 13.2963 || 10iter: 5.7478 sec.\n",
      "Iteration 7650 || Loss: 16.2765 || 10iter: 4.5254 sec.\n",
      "-------------\n",
      "epoch 116 || Epoch_TRAIN_Loss:923.5133 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.7305 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 117/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7660 || Loss: 16.3175 || 10iter: 6.9459 sec.\n",
      "Iteration 7670 || Loss: 11.9877 || 10iter: 6.4843 sec.\n",
      "Iteration 7680 || Loss: 11.8803 || 10iter: 5.1873 sec.\n",
      "Iteration 7690 || Loss: 14.4346 || 10iter: 5.9497 sec.\n",
      "Iteration 7700 || Loss: 13.5414 || 10iter: 5.9715 sec.\n",
      "Iteration 7710 || Loss: 17.3061 || 10iter: 5.2577 sec.\n",
      "Iteration 7720 || Loss: 14.7430 || 10iter: 4.7834 sec.\n",
      "-------------\n",
      "epoch 117 || Epoch_TRAIN_Loss:953.0999 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.2438 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 118/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7730 || Loss: 9.2500 || 10iter: 9.9096 sec.\n",
      "Iteration 7740 || Loss: 13.4099 || 10iter: 5.5489 sec.\n",
      "Iteration 7750 || Loss: 11.8072 || 10iter: 5.5385 sec.\n",
      "Iteration 7760 || Loss: 14.0350 || 10iter: 5.6100 sec.\n",
      "Iteration 7770 || Loss: 11.1201 || 10iter: 4.4309 sec.\n",
      "Iteration 7780 || Loss: 13.8688 || 10iter: 5.7221 sec.\n",
      "-------------\n",
      "epoch 118 || Epoch_TRAIN_Loss:940.3745 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.7788 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 119/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7790 || Loss: 15.2599 || 10iter: 5.2136 sec.\n",
      "Iteration 7800 || Loss: 11.2938 || 10iter: 6.4802 sec.\n",
      "Iteration 7810 || Loss: 18.0124 || 10iter: 5.6987 sec.\n",
      "Iteration 7820 || Loss: 13.8838 || 10iter: 4.6554 sec.\n",
      "Iteration 7830 || Loss: 10.7511 || 10iter: 5.1308 sec.\n",
      "Iteration 7840 || Loss: 19.1824 || 10iter: 4.1245 sec.\n",
      "Iteration 7850 || Loss: 15.7116 || 10iter: 4.5706 sec.\n",
      "-------------\n",
      "epoch 119 || Epoch_TRAIN_Loss:971.8737 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.7292 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 120/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7860 || Loss: 18.7349 || 10iter: 7.6686 sec.\n",
      "Iteration 7870 || Loss: 11.2684 || 10iter: 5.7461 sec.\n",
      "Iteration 7880 || Loss: 13.7486 || 10iter: 5.6841 sec.\n",
      "Iteration 7890 || Loss: 11.0297 || 10iter: 5.6105 sec.\n",
      "Iteration 7900 || Loss: 15.6891 || 10iter: 4.6371 sec.\n",
      "Iteration 7910 || Loss: 15.0965 || 10iter: 6.3361 sec.\n",
      "Iteration 7920 || Loss: 12.2637 || 10iter: 4.4864 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 120 || Epoch_TRAIN_Loss:942.6863 ||Epoch_VAL_Loss:468.1705\n",
      "timer:  53.1808 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 121/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7930 || Loss: 13.8052 || 10iter: 10.3401 sec.\n",
      "Iteration 7940 || Loss: 15.2999 || 10iter: 5.9984 sec.\n",
      "Iteration 7950 || Loss: 16.7072 || 10iter: 5.5996 sec.\n",
      "Iteration 7960 || Loss: 12.9133 || 10iter: 5.4901 sec.\n",
      "Iteration 7970 || Loss: 13.6845 || 10iter: 5.7868 sec.\n",
      "Iteration 7980 || Loss: 10.6032 || 10iter: 4.5090 sec.\n",
      "-------------\n",
      "epoch 121 || Epoch_TRAIN_Loss:950.2732 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1247 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 122/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7990 || Loss: 13.9298 || 10iter: 8.0582 sec.\n",
      "Iteration 8000 || Loss: 14.9223 || 10iter: 6.1724 sec.\n",
      "Iteration 8010 || Loss: 11.8465 || 10iter: 5.4675 sec.\n",
      "Iteration 8020 || Loss: 11.1826 || 10iter: 5.4277 sec.\n",
      "Iteration 8030 || Loss: 14.0274 || 10iter: 5.0246 sec.\n",
      "Iteration 8040 || Loss: 19.6728 || 10iter: 4.6723 sec.\n",
      "Iteration 8050 || Loss: 11.2538 || 10iter: 5.1918 sec.\n",
      "-------------\n",
      "epoch 122 || Epoch_TRAIN_Loss:907.8490 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.4758 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 123/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8060 || Loss: 14.9914 || 10iter: 9.6990 sec.\n",
      "Iteration 8070 || Loss: 13.5761 || 10iter: 5.5714 sec.\n",
      "Iteration 8080 || Loss: 15.2618 || 10iter: 5.1972 sec.\n",
      "Iteration 8090 || Loss: 11.8605 || 10iter: 4.7121 sec.\n",
      "Iteration 8100 || Loss: 16.4302 || 10iter: 4.4341 sec.\n",
      "Iteration 8110 || Loss: 12.2954 || 10iter: 3.8477 sec.\n",
      "-------------\n",
      "epoch 123 || Epoch_TRAIN_Loss:935.1732 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.6606 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 124/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8120 || Loss: 15.4937 || 10iter: 5.5033 sec.\n",
      "Iteration 8130 || Loss: 11.7546 || 10iter: 6.6334 sec.\n",
      "Iteration 8140 || Loss: 17.6447 || 10iter: 5.2484 sec.\n",
      "Iteration 8150 || Loss: 16.7278 || 10iter: 5.6761 sec.\n",
      "Iteration 8160 || Loss: 10.1220 || 10iter: 4.6551 sec.\n",
      "Iteration 8170 || Loss: 14.7795 || 10iter: 5.5680 sec.\n",
      "Iteration 8180 || Loss: 12.4753 || 10iter: 5.1576 sec.\n",
      "-------------\n",
      "epoch 124 || Epoch_TRAIN_Loss:916.3697 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.7779 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 125/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8190 || Loss: 14.0499 || 10iter: 8.1134 sec.\n",
      "Iteration 8200 || Loss: 10.9655 || 10iter: 5.3445 sec.\n",
      "Iteration 8210 || Loss: 15.6806 || 10iter: 5.2063 sec.\n",
      "Iteration 8220 || Loss: 13.0523 || 10iter: 5.1291 sec.\n",
      "Iteration 8230 || Loss: 14.1172 || 10iter: 7.1345 sec.\n",
      "Iteration 8240 || Loss: 13.0266 || 10iter: 4.9943 sec.\n",
      "Iteration 8250 || Loss: 26.7712 || 10iter: 4.4281 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 125 || Epoch_TRAIN_Loss:934.6213 ||Epoch_VAL_Loss:453.2906\n",
      "timer:  52.4234 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 126/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8260 || Loss: 11.5011 || 10iter: 11.8227 sec.\n",
      "Iteration 8270 || Loss: 12.6669 || 10iter: 5.7373 sec.\n",
      "Iteration 8280 || Loss: 11.6413 || 10iter: 5.6540 sec.\n",
      "Iteration 8290 || Loss: 15.6703 || 10iter: 5.2836 sec.\n",
      "Iteration 8300 || Loss: 15.0457 || 10iter: 5.1031 sec.\n",
      "Iteration 8310 || Loss: 16.2939 || 10iter: 5.2776 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------\n",
      "epoch 126 || Epoch_TRAIN_Loss:912.7318 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.2085 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 127/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8320 || Loss: 8.3842 || 10iter: 6.2899 sec.\n",
      "Iteration 8330 || Loss: 11.3473 || 10iter: 6.5806 sec.\n",
      "Iteration 8340 || Loss: 10.6595 || 10iter: 5.2735 sec.\n",
      "Iteration 8350 || Loss: 21.6955 || 10iter: 4.7302 sec.\n",
      "Iteration 8360 || Loss: 12.9487 || 10iter: 4.6884 sec.\n",
      "Iteration 8370 || Loss: 14.2252 || 10iter: 4.0234 sec.\n",
      "Iteration 8380 || Loss: 9.6251 || 10iter: 5.1787 sec.\n",
      "-------------\n",
      "epoch 127 || Epoch_TRAIN_Loss:891.2265 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.2336 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 128/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8390 || Loss: 15.8108 || 10iter: 8.8776 sec.\n",
      "Iteration 8400 || Loss: 12.3316 || 10iter: 5.8759 sec.\n",
      "Iteration 8410 || Loss: 8.0750 || 10iter: 5.6826 sec.\n",
      "Iteration 8420 || Loss: 11.4959 || 10iter: 4.9282 sec.\n",
      "Iteration 8430 || Loss: 12.4636 || 10iter: 6.0788 sec.\n",
      "Iteration 8440 || Loss: 11.1462 || 10iter: 5.2449 sec.\n",
      "-------------\n",
      "epoch 128 || Epoch_TRAIN_Loss:895.5529 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9619 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 129/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8450 || Loss: 16.1054 || 10iter: 5.2630 sec.\n",
      "Iteration 8460 || Loss: 13.7053 || 10iter: 6.9113 sec.\n",
      "Iteration 8470 || Loss: 11.8012 || 10iter: 4.6010 sec.\n",
      "Iteration 8480 || Loss: 12.7373 || 10iter: 5.2147 sec.\n",
      "Iteration 8490 || Loss: 15.5381 || 10iter: 6.9281 sec.\n",
      "Iteration 8500 || Loss: 13.2864 || 10iter: 5.6354 sec.\n",
      "Iteration 8510 || Loss: 10.3719 || 10iter: 4.6648 sec.\n",
      "-------------\n",
      "epoch 129 || Epoch_TRAIN_Loss:915.3317 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.5299 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 130/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8520 || Loss: 15.3779 || 10iter: 7.7391 sec.\n",
      "Iteration 8530 || Loss: 20.6143 || 10iter: 4.7733 sec.\n",
      "Iteration 8540 || Loss: 13.3054 || 10iter: 7.5215 sec.\n",
      "Iteration 8550 || Loss: 17.2792 || 10iter: 5.8232 sec.\n",
      "Iteration 8560 || Loss: 11.3048 || 10iter: 6.1319 sec.\n",
      "Iteration 8570 || Loss: 15.5831 || 10iter: 5.1807 sec.\n",
      "Iteration 8580 || Loss: 23.7557 || 10iter: 4.1689 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 130 || Epoch_TRAIN_Loss:933.9943 ||Epoch_VAL_Loss:452.9441\n",
      "timer:  53.9278 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 131/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8590 || Loss: 14.1507 || 10iter: 10.6687 sec.\n",
      "Iteration 8600 || Loss: 15.3969 || 10iter: 5.7004 sec.\n",
      "Iteration 8610 || Loss: 15.7582 || 10iter: 4.6005 sec.\n",
      "Iteration 8620 || Loss: 16.8730 || 10iter: 4.8334 sec.\n",
      "Iteration 8630 || Loss: 10.6197 || 10iter: 4.4895 sec.\n",
      "Iteration 8640 || Loss: 14.6633 || 10iter: 4.3945 sec.\n",
      "-------------\n",
      "epoch 131 || Epoch_TRAIN_Loss:942.4007 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.4953 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 132/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8650 || Loss: 14.1763 || 10iter: 6.5821 sec.\n",
      "Iteration 8660 || Loss: 10.0658 || 10iter: 6.2278 sec.\n",
      "Iteration 8670 || Loss: 17.1441 || 10iter: 5.8869 sec.\n",
      "Iteration 8680 || Loss: 12.7400 || 10iter: 5.2457 sec.\n",
      "Iteration 8690 || Loss: 14.1415 || 10iter: 4.9412 sec.\n",
      "Iteration 8700 || Loss: 11.8049 || 10iter: 6.3683 sec.\n",
      "Iteration 8710 || Loss: 17.3268 || 10iter: 4.5902 sec.\n",
      "-------------\n",
      "epoch 132 || Epoch_TRAIN_Loss:924.9322 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.2197 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 133/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8720 || Loss: 14.1800 || 10iter: 9.9275 sec.\n",
      "Iteration 8730 || Loss: 11.0566 || 10iter: 5.3696 sec.\n",
      "Iteration 8740 || Loss: 12.1690 || 10iter: 4.5391 sec.\n",
      "Iteration 8750 || Loss: 16.6111 || 10iter: 7.1483 sec.\n",
      "Iteration 8760 || Loss: 17.5903 || 10iter: 5.7590 sec.\n",
      "Iteration 8770 || Loss: 12.9027 || 10iter: 4.8613 sec.\n",
      "-------------\n",
      "epoch 133 || Epoch_TRAIN_Loss:897.0675 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.6590 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 134/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8780 || Loss: 9.7775 || 10iter: 4.9767 sec.\n",
      "Iteration 8790 || Loss: 14.1901 || 10iter: 5.4279 sec.\n",
      "Iteration 8800 || Loss: 10.7101 || 10iter: 6.4383 sec.\n",
      "Iteration 8810 || Loss: 14.0508 || 10iter: 6.6877 sec.\n",
      "Iteration 8820 || Loss: 17.5722 || 10iter: 6.5641 sec.\n",
      "Iteration 8830 || Loss: 9.8695 || 10iter: 5.5440 sec.\n",
      "Iteration 8840 || Loss: 8.8097 || 10iter: 4.3751 sec.\n",
      "-------------\n",
      "epoch 134 || Epoch_TRAIN_Loss:894.5140 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.1466 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 135/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8850 || Loss: 15.8396 || 10iter: 7.1386 sec.\n",
      "Iteration 8860 || Loss: 11.9491 || 10iter: 6.6759 sec.\n",
      "Iteration 8870 || Loss: 14.5381 || 10iter: 5.5242 sec.\n",
      "Iteration 8880 || Loss: 20.5427 || 10iter: 6.0360 sec.\n",
      "Iteration 8890 || Loss: 13.3251 || 10iter: 5.3194 sec.\n",
      "Iteration 8900 || Loss: 15.3561 || 10iter: 4.6884 sec.\n",
      "Iteration 8910 || Loss: 15.4914 || 10iter: 4.1124 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 135 || Epoch_TRAIN_Loss:907.0527 ||Epoch_VAL_Loss:448.8107\n",
      "timer:  50.1173 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 136/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8920 || Loss: 9.8894 || 10iter: 11.5302 sec.\n",
      "Iteration 8930 || Loss: 10.8600 || 10iter: 5.5708 sec.\n",
      "Iteration 8940 || Loss: 14.5206 || 10iter: 5.4867 sec.\n",
      "Iteration 8950 || Loss: 11.9545 || 10iter: 4.6746 sec.\n",
      "Iteration 8960 || Loss: 18.6129 || 10iter: 5.1377 sec.\n",
      "Iteration 8970 || Loss: 17.8533 || 10iter: 5.4575 sec.\n",
      "-------------\n",
      "epoch 136 || Epoch_TRAIN_Loss:934.6728 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.0450 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 137/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8980 || Loss: 11.5133 || 10iter: 7.1008 sec.\n",
      "Iteration 8990 || Loss: 9.9027 || 10iter: 5.7166 sec.\n",
      "Iteration 9000 || Loss: 15.5162 || 10iter: 4.7684 sec.\n",
      "Iteration 9010 || Loss: 13.0977 || 10iter: 4.6961 sec.\n",
      "Iteration 9020 || Loss: 17.5614 || 10iter: 7.2805 sec.\n",
      "Iteration 9030 || Loss: 10.8932 || 10iter: 5.3328 sec.\n",
      "Iteration 9040 || Loss: 9.6310 || 10iter: 4.9142 sec.\n",
      "-------------\n",
      "epoch 137 || Epoch_TRAIN_Loss:894.8792 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1944 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 138/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9050 || Loss: 13.2847 || 10iter: 8.8217 sec.\n",
      "Iteration 9060 || Loss: 12.4755 || 10iter: 4.4228 sec.\n",
      "Iteration 9070 || Loss: 14.2575 || 10iter: 6.8057 sec.\n",
      "Iteration 9080 || Loss: 11.4655 || 10iter: 5.9867 sec.\n",
      "Iteration 9090 || Loss: 13.3491 || 10iter: 6.3026 sec.\n",
      "Iteration 9100 || Loss: 14.2992 || 10iter: 4.9934 sec.\n",
      "-------------\n",
      "epoch 138 || Epoch_TRAIN_Loss:907.7182 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.3407 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 139/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9110 || Loss: 16.8822 || 10iter: 3.7640 sec.\n",
      "Iteration 9120 || Loss: 16.6201 || 10iter: 5.7516 sec.\n",
      "Iteration 9130 || Loss: 15.1856 || 10iter: 6.4949 sec.\n",
      "Iteration 9140 || Loss: 12.7623 || 10iter: 5.8879 sec.\n",
      "Iteration 9150 || Loss: 10.3504 || 10iter: 5.9477 sec.\n",
      "Iteration 9160 || Loss: 16.1138 || 10iter: 5.5838 sec.\n",
      "Iteration 9170 || Loss: 14.9571 || 10iter: 4.2681 sec.\n",
      "-------------\n",
      "epoch 139 || Epoch_TRAIN_Loss:892.4628 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.9052 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 140/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9180 || Loss: 10.0634 || 10iter: 7.0486 sec.\n",
      "Iteration 9190 || Loss: 14.6074 || 10iter: 4.3022 sec.\n",
      "Iteration 9200 || Loss: 16.1118 || 10iter: 6.7836 sec.\n",
      "Iteration 9210 || Loss: 13.4432 || 10iter: 5.7700 sec.\n",
      "Iteration 9220 || Loss: 13.9985 || 10iter: 6.1245 sec.\n",
      "Iteration 9230 || Loss: 15.8211 || 10iter: 5.0837 sec.\n",
      "Iteration 9240 || Loss: 12.9735 || 10iter: 4.2359 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 140 || Epoch_TRAIN_Loss:937.8549 ||Epoch_VAL_Loss:449.3141\n",
      "timer:  51.9070 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 141/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9250 || Loss: 13.0658 || 10iter: 10.4732 sec.\n",
      "Iteration 9260 || Loss: 14.4144 || 10iter: 5.2527 sec.\n",
      "Iteration 9270 || Loss: 18.9446 || 10iter: 4.9801 sec.\n",
      "Iteration 9280 || Loss: 15.6624 || 10iter: 5.8166 sec.\n",
      "Iteration 9290 || Loss: 10.6635 || 10iter: 6.3358 sec.\n",
      "Iteration 9300 || Loss: 15.9454 || 10iter: 4.5499 sec.\n",
      "-------------\n",
      "epoch 141 || Epoch_TRAIN_Loss:924.7729 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6514 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 142/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9310 || Loss: 12.5142 || 10iter: 6.1664 sec.\n",
      "Iteration 9320 || Loss: 12.3581 || 10iter: 5.7573 sec.\n",
      "Iteration 9330 || Loss: 15.3685 || 10iter: 4.4568 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 9340 || Loss: 10.2143 || 10iter: 7.2995 sec.\n",
      "Iteration 9350 || Loss: 12.1264 || 10iter: 6.1153 sec.\n",
      "Iteration 9360 || Loss: 15.4096 || 10iter: 6.0656 sec.\n",
      "Iteration 9370 || Loss: 12.5955 || 10iter: 4.7365 sec.\n",
      "-------------\n",
      "epoch 142 || Epoch_TRAIN_Loss:923.2613 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.9464 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 143/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9380 || Loss: 15.9039 || 10iter: 7.2000 sec.\n",
      "Iteration 9390 || Loss: 11.7340 || 10iter: 6.9361 sec.\n",
      "Iteration 9400 || Loss: 12.4938 || 10iter: 5.8139 sec.\n",
      "Iteration 9410 || Loss: 13.3701 || 10iter: 5.6427 sec.\n",
      "Iteration 9420 || Loss: 12.2007 || 10iter: 5.2970 sec.\n",
      "Iteration 9430 || Loss: 13.1263 || 10iter: 4.8294 sec.\n",
      "-------------\n",
      "epoch 143 || Epoch_TRAIN_Loss:899.3416 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.4516 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 144/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9440 || Loss: 12.0305 || 10iter: 4.7904 sec.\n",
      "Iteration 9450 || Loss: 13.0793 || 10iter: 4.6583 sec.\n",
      "Iteration 9460 || Loss: 10.6097 || 10iter: 4.6670 sec.\n",
      "Iteration 9470 || Loss: 13.9636 || 10iter: 6.8957 sec.\n",
      "Iteration 9480 || Loss: 12.4787 || 10iter: 5.5675 sec.\n",
      "Iteration 9490 || Loss: 16.8706 || 10iter: 5.6295 sec.\n",
      "Iteration 9500 || Loss: 16.4310 || 10iter: 4.5334 sec.\n",
      "-------------\n",
      "epoch 144 || Epoch_TRAIN_Loss:883.3896 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.9498 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 145/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9510 || Loss: 12.9037 || 10iter: 5.6359 sec.\n",
      "Iteration 9520 || Loss: 14.2345 || 10iter: 5.9616 sec.\n",
      "Iteration 9530 || Loss: 10.6169 || 10iter: 6.0187 sec.\n",
      "Iteration 9540 || Loss: 10.9065 || 10iter: 5.8784 sec.\n",
      "Iteration 9550 || Loss: 13.6863 || 10iter: 5.6874 sec.\n",
      "Iteration 9560 || Loss: 12.8965 || 10iter: 4.9099 sec.\n",
      "Iteration 9570 || Loss: 23.6590 || 10iter: 4.0263 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 145 || Epoch_TRAIN_Loss:896.8101 ||Epoch_VAL_Loss:453.6034\n",
      "timer:  51.3967 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 146/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9580 || Loss: 13.7850 || 10iter: 11.2878 sec.\n",
      "Iteration 9590 || Loss: 13.8129 || 10iter: 4.9166 sec.\n",
      "Iteration 9600 || Loss: 10.6275 || 10iter: 4.4352 sec.\n",
      "Iteration 9610 || Loss: 14.4333 || 10iter: 6.9881 sec.\n",
      "Iteration 9620 || Loss: 13.6191 || 10iter: 5.7033 sec.\n",
      "Iteration 9630 || Loss: 17.7981 || 10iter: 5.3503 sec.\n",
      "-------------\n",
      "epoch 146 || Epoch_TRAIN_Loss:924.6089 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.8348 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 147/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9640 || Loss: 15.7295 || 10iter: 6.2535 sec.\n",
      "Iteration 9650 || Loss: 11.9130 || 10iter: 4.9778 sec.\n",
      "Iteration 9660 || Loss: 15.2628 || 10iter: 6.3855 sec.\n",
      "Iteration 9670 || Loss: 17.4540 || 10iter: 5.8850 sec.\n",
      "Iteration 9680 || Loss: 11.5675 || 10iter: 5.9615 sec.\n",
      "Iteration 9690 || Loss: 10.3191 || 10iter: 5.3046 sec.\n",
      "Iteration 9700 || Loss: 15.2155 || 10iter: 4.4298 sec.\n",
      "-------------\n",
      "epoch 147 || Epoch_TRAIN_Loss:901.2196 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5149 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 148/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9710 || Loss: 13.6008 || 10iter: 7.1773 sec.\n",
      "Iteration 9720 || Loss: 11.4299 || 10iter: 4.6579 sec.\n",
      "Iteration 9730 || Loss: 12.6892 || 10iter: 4.4361 sec.\n",
      "Iteration 9740 || Loss: 12.6948 || 10iter: 6.8983 sec.\n",
      "Iteration 9750 || Loss: 13.5330 || 10iter: 5.9467 sec.\n",
      "Iteration 9760 || Loss: 10.5288 || 10iter: 4.8679 sec.\n",
      "-------------\n",
      "epoch 148 || Epoch_TRAIN_Loss:908.6395 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.0294 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 149/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9770 || Loss: 13.7474 || 10iter: 4.8985 sec.\n",
      "Iteration 9780 || Loss: 17.0925 || 10iter: 4.7125 sec.\n",
      "Iteration 9790 || Loss: 13.4918 || 10iter: 6.5473 sec.\n",
      "Iteration 9800 || Loss: 13.3543 || 10iter: 5.8082 sec.\n",
      "Iteration 9810 || Loss: 15.7533 || 10iter: 6.2444 sec.\n",
      "Iteration 9820 || Loss: 11.3133 || 10iter: 5.4495 sec.\n",
      "Iteration 9830 || Loss: 8.3350 || 10iter: 4.3821 sec.\n",
      "-------------\n",
      "epoch 149 || Epoch_TRAIN_Loss:896.1010 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1806 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 150/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9840 || Loss: 14.9328 || 10iter: 7.1487 sec.\n",
      "Iteration 9850 || Loss: 10.8317 || 10iter: 6.4007 sec.\n",
      "Iteration 9860 || Loss: 11.2239 || 10iter: 5.7310 sec.\n",
      "Iteration 9870 || Loss: 10.5478 || 10iter: 6.1639 sec.\n",
      "Iteration 9880 || Loss: 10.2742 || 10iter: 5.5218 sec.\n",
      "Iteration 9890 || Loss: 13.8498 || 10iter: 4.6859 sec.\n",
      "Iteration 9900 || Loss: 15.0083 || 10iter: 4.6294 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 150 || Epoch_TRAIN_Loss:910.4388 ||Epoch_VAL_Loss:451.7355\n",
      "timer:  55.0882 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 151/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9910 || Loss: 16.4018 || 10iter: 10.3209 sec.\n",
      "Iteration 9920 || Loss: 8.5340 || 10iter: 5.6786 sec.\n",
      "Iteration 9930 || Loss: 18.1973 || 10iter: 6.3682 sec.\n",
      "Iteration 9940 || Loss: 13.4083 || 10iter: 6.1122 sec.\n",
      "Iteration 9950 || Loss: 14.8562 || 10iter: 5.9561 sec.\n",
      "Iteration 9960 || Loss: 17.2582 || 10iter: 4.7457 sec.\n",
      "-------------\n",
      "epoch 151 || Epoch_TRAIN_Loss:929.6744 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.1036 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 152/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9970 || Loss: 15.6729 || 10iter: 5.2033 sec.\n",
      "Iteration 9980 || Loss: 10.4570 || 10iter: 4.8212 sec.\n",
      "Iteration 9990 || Loss: 9.4364 || 10iter: 4.0815 sec.\n",
      "Iteration 10000 || Loss: 17.6980 || 10iter: 6.8527 sec.\n",
      "Iteration 10010 || Loss: 14.6674 || 10iter: 6.3392 sec.\n",
      "Iteration 10020 || Loss: 17.7369 || 10iter: 5.7572 sec.\n",
      "Iteration 10030 || Loss: 12.0126 || 10iter: 4.5698 sec.\n",
      "-------------\n",
      "epoch 152 || Epoch_TRAIN_Loss:922.5782 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  38.9767 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 153/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10040 || Loss: 20.2693 || 10iter: 7.1923 sec.\n",
      "Iteration 10050 || Loss: 12.5118 || 10iter: 6.3808 sec.\n",
      "Iteration 10060 || Loss: 12.4059 || 10iter: 5.9609 sec.\n",
      "Iteration 10070 || Loss: 19.7564 || 10iter: 5.8965 sec.\n",
      "Iteration 10080 || Loss: 15.2224 || 10iter: 6.1697 sec.\n",
      "Iteration 10090 || Loss: 12.9404 || 10iter: 4.7486 sec.\n",
      "-------------\n",
      "epoch 153 || Epoch_TRAIN_Loss:892.6139 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0603 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 154/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10100 || Loss: 12.4817 || 10iter: 6.3041 sec.\n",
      "Iteration 10110 || Loss: 9.7585 || 10iter: 6.8797 sec.\n",
      "Iteration 10120 || Loss: 18.1926 || 10iter: 5.8265 sec.\n",
      "Iteration 10130 || Loss: 12.0308 || 10iter: 5.7321 sec.\n",
      "Iteration 10140 || Loss: 17.5322 || 10iter: 5.8187 sec.\n",
      "Iteration 10150 || Loss: 13.8340 || 10iter: 4.4354 sec.\n",
      "Iteration 10160 || Loss: 16.6803 || 10iter: 5.0998 sec.\n",
      "-------------\n",
      "epoch 154 || Epoch_TRAIN_Loss:916.8636 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.7423 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 155/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10170 || Loss: 11.2832 || 10iter: 7.9214 sec.\n",
      "Iteration 10180 || Loss: 16.6855 || 10iter: 6.2522 sec.\n",
      "Iteration 10190 || Loss: 15.5284 || 10iter: 5.7815 sec.\n",
      "Iteration 10200 || Loss: 12.6551 || 10iter: 5.7189 sec.\n",
      "Iteration 10210 || Loss: 15.9740 || 10iter: 5.4348 sec.\n",
      "Iteration 10220 || Loss: 13.7403 || 10iter: 5.9279 sec.\n",
      "Iteration 10230 || Loss: 12.9704 || 10iter: 4.3790 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 155 || Epoch_TRAIN_Loss:920.2844 ||Epoch_VAL_Loss:449.7269\n",
      "timer:  53.6482 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 156/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10240 || Loss: 16.1080 || 10iter: 8.1212 sec.\n",
      "Iteration 10250 || Loss: 14.1190 || 10iter: 4.1202 sec.\n",
      "Iteration 10260 || Loss: 9.7065 || 10iter: 5.2308 sec.\n",
      "Iteration 10270 || Loss: 14.2264 || 10iter: 6.6403 sec.\n",
      "Iteration 10280 || Loss: 10.1107 || 10iter: 5.4101 sec.\n",
      "Iteration 10290 || Loss: 18.5374 || 10iter: 4.6606 sec.\n",
      "-------------\n",
      "epoch 156 || Epoch_TRAIN_Loss:955.6737 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  37.4665 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 157/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10300 || Loss: 12.0086 || 10iter: 6.2474 sec.\n",
      "Iteration 10310 || Loss: 13.3836 || 10iter: 4.8018 sec.\n",
      "Iteration 10320 || Loss: 17.7201 || 10iter: 6.7756 sec.\n",
      "Iteration 10330 || Loss: 15.1572 || 10iter: 6.1022 sec.\n",
      "Iteration 10340 || Loss: 10.5441 || 10iter: 6.0320 sec.\n",
      "Iteration 10350 || Loss: 15.0902 || 10iter: 5.4671 sec.\n",
      "Iteration 10360 || Loss: 9.5163 || 10iter: 4.2872 sec.\n",
      "-------------\n",
      "epoch 157 || Epoch_TRAIN_Loss:854.4167 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9776 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 158/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 10370 || Loss: 10.9125 || 10iter: 9.2947 sec.\n",
      "Iteration 10380 || Loss: 12.3299 || 10iter: 6.2849 sec.\n",
      "Iteration 10390 || Loss: 16.0326 || 10iter: 5.8865 sec.\n",
      "Iteration 10400 || Loss: 15.3373 || 10iter: 6.1114 sec.\n",
      "Iteration 10410 || Loss: 13.4173 || 10iter: 5.7901 sec.\n",
      "Iteration 10420 || Loss: 18.7039 || 10iter: 4.3080 sec.\n",
      "-------------\n",
      "epoch 158 || Epoch_TRAIN_Loss:866.9069 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.5470 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 159/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10430 || Loss: 15.0447 || 10iter: 4.8775 sec.\n",
      "Iteration 10440 || Loss: 13.1233 || 10iter: 6.7415 sec.\n",
      "Iteration 10450 || Loss: 12.2045 || 10iter: 5.6558 sec.\n",
      "Iteration 10460 || Loss: 13.8782 || 10iter: 5.7125 sec.\n",
      "Iteration 10470 || Loss: 14.1288 || 10iter: 5.1885 sec.\n",
      "Iteration 10480 || Loss: 10.3342 || 10iter: 5.8033 sec.\n",
      "Iteration 10490 || Loss: 13.2250 || 10iter: 4.8225 sec.\n",
      "-------------\n",
      "epoch 159 || Epoch_TRAIN_Loss:909.8899 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.0883 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 160/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10500 || Loss: 11.0634 || 10iter: 7.9725 sec.\n",
      "Iteration 10510 || Loss: 12.2925 || 10iter: 5.8371 sec.\n",
      "Iteration 10520 || Loss: 15.0421 || 10iter: 5.1090 sec.\n",
      "Iteration 10530 || Loss: 12.6392 || 10iter: 4.9269 sec.\n",
      "Iteration 10540 || Loss: 13.2111 || 10iter: 4.4953 sec.\n",
      "Iteration 10550 || Loss: 11.3139 || 10iter: 3.9244 sec.\n",
      "Iteration 10560 || Loss: 14.3090 || 10iter: 5.3661 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 160 || Epoch_TRAIN_Loss:893.0910 ||Epoch_VAL_Loss:448.2201\n",
      "timer:  50.8769 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 161/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10570 || Loss: 19.1600 || 10iter: 8.7122 sec.\n",
      "Iteration 10580 || Loss: 15.8704 || 10iter: 6.0850 sec.\n",
      "Iteration 10590 || Loss: 10.3373 || 10iter: 6.1240 sec.\n",
      "Iteration 10600 || Loss: 14.0050 || 10iter: 6.2453 sec.\n",
      "Iteration 10610 || Loss: 9.8750 || 10iter: 5.6644 sec.\n",
      "Iteration 10620 || Loss: 15.5060 || 10iter: 4.6813 sec.\n",
      "-------------\n",
      "epoch 161 || Epoch_TRAIN_Loss:901.7626 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4555 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 162/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10630 || Loss: 16.8253 || 10iter: 6.1878 sec.\n",
      "Iteration 10640 || Loss: 17.6831 || 10iter: 7.1708 sec.\n",
      "Iteration 10650 || Loss: 14.8480 || 10iter: 6.0040 sec.\n",
      "Iteration 10660 || Loss: 14.9876 || 10iter: 5.6325 sec.\n",
      "Iteration 10670 || Loss: 12.0954 || 10iter: 6.0296 sec.\n",
      "Iteration 10680 || Loss: 13.4437 || 10iter: 4.6878 sec.\n",
      "Iteration 10690 || Loss: 17.8948 || 10iter: 4.5354 sec.\n",
      "-------------\n",
      "epoch 162 || Epoch_TRAIN_Loss:891.8244 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.9586 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 163/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10700 || Loss: 12.1120 || 10iter: 9.4817 sec.\n",
      "Iteration 10710 || Loss: 9.8442 || 10iter: 6.1237 sec.\n",
      "Iteration 10720 || Loss: 11.3067 || 10iter: 5.4807 sec.\n",
      "Iteration 10730 || Loss: 14.7990 || 10iter: 5.6587 sec.\n",
      "Iteration 10740 || Loss: 17.2703 || 10iter: 4.6342 sec.\n",
      "Iteration 10750 || Loss: 13.3525 || 10iter: 5.8619 sec.\n",
      "-------------\n",
      "epoch 163 || Epoch_TRAIN_Loss:909.6459 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.5811 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 164/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10760 || Loss: 14.4137 || 10iter: 4.7630 sec.\n",
      "Iteration 10770 || Loss: 15.6089 || 10iter: 6.7079 sec.\n",
      "Iteration 10780 || Loss: 12.3355 || 10iter: 5.2965 sec.\n",
      "Iteration 10790 || Loss: 17.8382 || 10iter: 4.5148 sec.\n",
      "Iteration 10800 || Loss: 11.2127 || 10iter: 4.8997 sec.\n",
      "Iteration 10810 || Loss: 14.3407 || 10iter: 4.2463 sec.\n",
      "Iteration 10820 || Loss: 14.2891 || 10iter: 5.0770 sec.\n",
      "-------------\n",
      "epoch 164 || Epoch_TRAIN_Loss:921.0212 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  37.9527 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 165/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10830 || Loss: 15.5555 || 10iter: 7.5619 sec.\n",
      "Iteration 10840 || Loss: 11.1932 || 10iter: 6.0337 sec.\n",
      "Iteration 10850 || Loss: 11.9988 || 10iter: 5.9628 sec.\n",
      "Iteration 10860 || Loss: 17.0150 || 10iter: 5.0141 sec.\n",
      "Iteration 10870 || Loss: 10.4610 || 10iter: 3.9870 sec.\n",
      "Iteration 10880 || Loss: 13.8344 || 10iter: 3.2357 sec.\n",
      "Iteration 10890 || Loss: 12.8068 || 10iter: 2.9956 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 165 || Epoch_TRAIN_Loss:888.2071 ||Epoch_VAL_Loss:447.2210\n",
      "timer:  41.5914 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 166/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10900 || Loss: 9.0986 || 10iter: 6.4950 sec.\n",
      "Iteration 10910 || Loss: 11.2941 || 10iter: 3.3385 sec.\n",
      "Iteration 10920 || Loss: 15.2934 || 10iter: 3.4065 sec.\n",
      "Iteration 10930 || Loss: 11.9637 || 10iter: 3.4829 sec.\n",
      "Iteration 10940 || Loss: 12.2463 || 10iter: 3.4601 sec.\n",
      "Iteration 10950 || Loss: 21.5050 || 10iter: 3.1212 sec.\n",
      "-------------\n",
      "epoch 166 || Epoch_TRAIN_Loss:864.9419 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.5942 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 167/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10960 || Loss: 18.1093 || 10iter: 4.1108 sec.\n",
      "Iteration 10970 || Loss: 18.9395 || 10iter: 3.8139 sec.\n",
      "Iteration 10980 || Loss: 13.3705 || 10iter: 3.5225 sec.\n",
      "Iteration 10990 || Loss: 12.6355 || 10iter: 3.4144 sec.\n",
      "Iteration 11000 || Loss: 12.4848 || 10iter: 3.4338 sec.\n",
      "Iteration 11010 || Loss: 15.8625 || 10iter: 3.3100 sec.\n",
      "Iteration 11020 || Loss: 18.4191 || 10iter: 3.1460 sec.\n",
      "-------------\n",
      "epoch 167 || Epoch_TRAIN_Loss:887.1903 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.8109 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 168/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11030 || Loss: 13.4754 || 10iter: 5.6450 sec.\n",
      "Iteration 11040 || Loss: 14.8317 || 10iter: 3.3991 sec.\n",
      "Iteration 11050 || Loss: 18.0957 || 10iter: 3.3877 sec.\n",
      "Iteration 11060 || Loss: 11.8951 || 10iter: 3.5310 sec.\n",
      "Iteration 11070 || Loss: 11.6410 || 10iter: 3.3547 sec.\n",
      "Iteration 11080 || Loss: 20.0212 || 10iter: 3.2074 sec.\n",
      "-------------\n",
      "epoch 168 || Epoch_TRAIN_Loss:927.7279 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.4329 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 169/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11090 || Loss: 11.7396 || 10iter: 2.7346 sec.\n",
      "Iteration 11100 || Loss: 11.8480 || 10iter: 4.2611 sec.\n",
      "Iteration 11110 || Loss: 12.4195 || 10iter: 3.5751 sec.\n",
      "Iteration 11120 || Loss: 14.6192 || 10iter: 3.5991 sec.\n",
      "Iteration 11130 || Loss: 13.5596 || 10iter: 3.4178 sec.\n",
      "Iteration 11140 || Loss: 11.2088 || 10iter: 3.3548 sec.\n",
      "Iteration 11150 || Loss: 12.2913 || 10iter: 3.1362 sec.\n",
      "-------------\n",
      "epoch 169 || Epoch_TRAIN_Loss:864.7938 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.7895 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 170/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11160 || Loss: 15.1636 || 10iter: 5.1151 sec.\n",
      "Iteration 11170 || Loss: 13.6271 || 10iter: 3.4353 sec.\n",
      "Iteration 11180 || Loss: 13.9042 || 10iter: 3.3998 sec.\n",
      "Iteration 11190 || Loss: 11.7710 || 10iter: 3.4468 sec.\n",
      "Iteration 11200 || Loss: 18.3810 || 10iter: 3.4037 sec.\n",
      "Iteration 11210 || Loss: 15.5047 || 10iter: 3.2527 sec.\n",
      "Iteration 11220 || Loss: 11.6615 || 10iter: 3.0207 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 170 || Epoch_TRAIN_Loss:920.8851 ||Epoch_VAL_Loss:446.9716\n",
      "timer:  31.9068 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 171/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11230 || Loss: 12.9289 || 10iter: 6.7093 sec.\n",
      "Iteration 11240 || Loss: 8.6848 || 10iter: 3.3901 sec.\n",
      "Iteration 11250 || Loss: 15.5194 || 10iter: 3.4402 sec.\n",
      "Iteration 11260 || Loss: 12.1389 || 10iter: 3.4791 sec.\n",
      "Iteration 11270 || Loss: 10.9697 || 10iter: 3.4252 sec.\n",
      "Iteration 11280 || Loss: 11.8833 || 10iter: 3.1242 sec.\n",
      "-------------\n",
      "epoch 171 || Epoch_TRAIN_Loss:877.4913 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.8441 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 172/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11290 || Loss: 16.6762 || 10iter: 4.0914 sec.\n",
      "Iteration 11300 || Loss: 13.0648 || 10iter: 3.6635 sec.\n",
      "Iteration 11310 || Loss: 13.0665 || 10iter: 3.4473 sec.\n",
      "Iteration 11320 || Loss: 20.5372 || 10iter: 3.4408 sec.\n",
      "Iteration 11330 || Loss: 12.1755 || 10iter: 3.4877 sec.\n",
      "Iteration 11340 || Loss: 19.9923 || 10iter: 3.2678 sec.\n",
      "Iteration 11350 || Loss: 11.1438 || 10iter: 3.1299 sec.\n",
      "-------------\n",
      "epoch 172 || Epoch_TRAIN_Loss:937.0917 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.5829 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 173/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11360 || Loss: 17.8899 || 10iter: 5.7612 sec.\n",
      "Iteration 11370 || Loss: 10.4533 || 10iter: 3.5361 sec.\n",
      "Iteration 11380 || Loss: 14.3989 || 10iter: 3.4140 sec.\n",
      "Iteration 11390 || Loss: 11.0375 || 10iter: 3.4516 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 11400 || Loss: 14.1533 || 10iter: 3.3881 sec.\n",
      "Iteration 11410 || Loss: 15.9351 || 10iter: 3.1915 sec.\n",
      "-------------\n",
      "epoch 173 || Epoch_TRAIN_Loss:884.7244 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.6705 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 174/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11420 || Loss: 15.8546 || 10iter: 3.0143 sec.\n",
      "Iteration 11430 || Loss: 12.6643 || 10iter: 4.0291 sec.\n",
      "Iteration 11440 || Loss: 12.7654 || 10iter: 3.3546 sec.\n",
      "Iteration 11450 || Loss: 11.8436 || 10iter: 3.4508 sec.\n",
      "Iteration 11460 || Loss: 10.7901 || 10iter: 3.4599 sec.\n",
      "Iteration 11470 || Loss: 11.5479 || 10iter: 3.3270 sec.\n",
      "Iteration 11480 || Loss: 12.6585 || 10iter: 3.1330 sec.\n",
      "-------------\n",
      "epoch 174 || Epoch_TRAIN_Loss:902.3531 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.4691 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 175/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11490 || Loss: 12.3092 || 10iter: 5.0570 sec.\n",
      "Iteration 11500 || Loss: 13.8599 || 10iter: 3.4007 sec.\n",
      "Iteration 11510 || Loss: 13.5010 || 10iter: 3.4282 sec.\n",
      "Iteration 11520 || Loss: 16.9310 || 10iter: 3.4210 sec.\n",
      "Iteration 11530 || Loss: 11.0008 || 10iter: 3.4163 sec.\n",
      "Iteration 11540 || Loss: 11.0913 || 10iter: 3.2578 sec.\n",
      "Iteration 11550 || Loss: 14.2125 || 10iter: 3.0136 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 175 || Epoch_TRAIN_Loss:893.1051 ||Epoch_VAL_Loss:447.9666\n",
      "timer:  31.9770 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 176/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11560 || Loss: 18.0471 || 10iter: 6.4220 sec.\n",
      "Iteration 11570 || Loss: 13.0207 || 10iter: 3.5111 sec.\n",
      "Iteration 11580 || Loss: 13.9472 || 10iter: 3.4541 sec.\n",
      "Iteration 11590 || Loss: 16.6900 || 10iter: 3.4001 sec.\n",
      "Iteration 11600 || Loss: 17.4832 || 10iter: 3.3902 sec.\n",
      "Iteration 11610 || Loss: 10.7548 || 10iter: 3.1392 sec.\n",
      "-------------\n",
      "epoch 176 || Epoch_TRAIN_Loss:910.6544 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.5810 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 177/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11620 || Loss: 14.8159 || 10iter: 4.1533 sec.\n",
      "Iteration 11630 || Loss: 16.2881 || 10iter: 3.6417 sec.\n",
      "Iteration 11640 || Loss: 16.6811 || 10iter: 3.3879 sec.\n",
      "Iteration 11650 || Loss: 12.9106 || 10iter: 3.4096 sec.\n",
      "Iteration 11660 || Loss: 13.4456 || 10iter: 3.3866 sec.\n",
      "Iteration 11670 || Loss: 10.8550 || 10iter: 3.3049 sec.\n",
      "Iteration 11680 || Loss: 12.8873 || 10iter: 3.1185 sec.\n",
      "-------------\n",
      "epoch 177 || Epoch_TRAIN_Loss:893.0366 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.4837 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 178/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11690 || Loss: 12.4140 || 10iter: 5.7000 sec.\n",
      "Iteration 11700 || Loss: 13.1015 || 10iter: 3.4012 sec.\n",
      "Iteration 11710 || Loss: 15.0979 || 10iter: 3.4773 sec.\n",
      "Iteration 11720 || Loss: 15.0959 || 10iter: 3.4744 sec.\n",
      "Iteration 11730 || Loss: 10.7485 || 10iter: 3.4081 sec.\n",
      "Iteration 11740 || Loss: 16.2038 || 10iter: 3.1986 sec.\n",
      "-------------\n",
      "epoch 178 || Epoch_TRAIN_Loss:892.5611 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.5700 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 179/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11750 || Loss: 15.3928 || 10iter: 2.7199 sec.\n",
      "Iteration 11760 || Loss: 13.1330 || 10iter: 4.2955 sec.\n",
      "Iteration 11770 || Loss: 12.7775 || 10iter: 3.3885 sec.\n",
      "Iteration 11780 || Loss: 10.6746 || 10iter: 3.4409 sec.\n",
      "Iteration 11790 || Loss: 15.3470 || 10iter: 3.4135 sec.\n",
      "Iteration 11800 || Loss: 16.4969 || 10iter: 3.3653 sec.\n",
      "Iteration 11810 || Loss: 14.0286 || 10iter: 3.1282 sec.\n",
      "-------------\n",
      "epoch 179 || Epoch_TRAIN_Loss:925.7152 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.4525 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 180/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11820 || Loss: 10.9537 || 10iter: 5.1764 sec.\n",
      "Iteration 11830 || Loss: 12.0183 || 10iter: 3.3515 sec.\n",
      "Iteration 11840 || Loss: 14.2800 || 10iter: 3.4320 sec.\n",
      "Iteration 11850 || Loss: 14.5207 || 10iter: 3.4681 sec.\n",
      "Iteration 11860 || Loss: 12.6548 || 10iter: 3.5403 sec.\n",
      "Iteration 11870 || Loss: 13.4517 || 10iter: 3.2649 sec.\n",
      "Iteration 11880 || Loss: 16.5880 || 10iter: 3.0099 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 180 || Epoch_TRAIN_Loss:898.1246 ||Epoch_VAL_Loss:446.4465\n",
      "timer:  31.9773 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 181/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11890 || Loss: 10.6537 || 10iter: 6.6164 sec.\n",
      "Iteration 11900 || Loss: 16.0970 || 10iter: 3.3900 sec.\n",
      "Iteration 11910 || Loss: 12.6114 || 10iter: 3.4442 sec.\n",
      "Iteration 11920 || Loss: 13.5055 || 10iter: 3.3667 sec.\n",
      "Iteration 11930 || Loss: 13.3226 || 10iter: 3.3989 sec.\n",
      "Iteration 11940 || Loss: 14.8956 || 10iter: 3.1289 sec.\n",
      "-------------\n",
      "epoch 181 || Epoch_TRAIN_Loss:888.0084 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.6029 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 182/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11950 || Loss: 14.5149 || 10iter: 4.1153 sec.\n",
      "Iteration 11960 || Loss: 10.9595 || 10iter: 3.6611 sec.\n",
      "Iteration 11970 || Loss: 9.3715 || 10iter: 3.3956 sec.\n",
      "Iteration 11980 || Loss: 15.3344 || 10iter: 3.4426 sec.\n",
      "Iteration 11990 || Loss: 14.0598 || 10iter: 3.7125 sec.\n",
      "Iteration 12000 || Loss: 13.6695 || 10iter: 3.4796 sec.\n",
      "Iteration 12010 || Loss: 13.5227 || 10iter: 3.1354 sec.\n",
      "-------------\n",
      "epoch 182 || Epoch_TRAIN_Loss:894.0152 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  26.0135 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 183/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12020 || Loss: 10.5449 || 10iter: 6.1897 sec.\n",
      "Iteration 12030 || Loss: 10.9134 || 10iter: 3.4785 sec.\n",
      "Iteration 12040 || Loss: 13.9742 || 10iter: 3.4433 sec.\n",
      "Iteration 12050 || Loss: 11.8764 || 10iter: 3.4611 sec.\n",
      "Iteration 12060 || Loss: 15.0170 || 10iter: 3.3795 sec.\n",
      "Iteration 12070 || Loss: 15.1527 || 10iter: 3.1939 sec.\n",
      "-------------\n",
      "epoch 183 || Epoch_TRAIN_Loss:875.1050 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  26.0564 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 184/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12080 || Loss: 17.6413 || 10iter: 3.2172 sec.\n",
      "Iteration 12090 || Loss: 12.8826 || 10iter: 4.0011 sec.\n",
      "Iteration 12100 || Loss: 14.4827 || 10iter: 3.3472 sec.\n",
      "Iteration 12110 || Loss: 12.4152 || 10iter: 3.4482 sec.\n",
      "Iteration 12120 || Loss: 15.3032 || 10iter: 3.4426 sec.\n",
      "Iteration 12130 || Loss: 12.3657 || 10iter: 3.3433 sec.\n",
      "Iteration 12140 || Loss: 14.8759 || 10iter: 3.1261 sec.\n",
      "-------------\n",
      "epoch 184 || Epoch_TRAIN_Loss:877.9550 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.6305 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 185/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12150 || Loss: 18.7328 || 10iter: 5.1061 sec.\n",
      "Iteration 12160 || Loss: 11.1043 || 10iter: 3.4801 sec.\n",
      "Iteration 12170 || Loss: 15.8659 || 10iter: 3.4212 sec.\n",
      "Iteration 12180 || Loss: 11.7815 || 10iter: 3.4400 sec.\n",
      "Iteration 12190 || Loss: 17.0666 || 10iter: 3.4293 sec.\n",
      "Iteration 12200 || Loss: 15.6774 || 10iter: 3.2254 sec.\n",
      "Iteration 12210 || Loss: 13.6903 || 10iter: 2.9898 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 185 || Epoch_TRAIN_Loss:895.5526 ||Epoch_VAL_Loss:442.7623\n",
      "timer:  31.7144 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 186/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12220 || Loss: 13.7094 || 10iter: 6.5779 sec.\n",
      "Iteration 12230 || Loss: 12.2338 || 10iter: 3.3243 sec.\n",
      "Iteration 12240 || Loss: 9.8190 || 10iter: 3.4650 sec.\n",
      "Iteration 12250 || Loss: 12.7480 || 10iter: 3.4666 sec.\n",
      "Iteration 12260 || Loss: 16.9761 || 10iter: 3.4128 sec.\n",
      "Iteration 12270 || Loss: 15.3624 || 10iter: 3.1296 sec.\n",
      "-------------\n",
      "epoch 186 || Epoch_TRAIN_Loss:917.4943 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.6300 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 187/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12280 || Loss: 15.1618 || 10iter: 4.3092 sec.\n",
      "Iteration 12290 || Loss: 14.1666 || 10iter: 3.4056 sec.\n",
      "Iteration 12300 || Loss: 17.5480 || 10iter: 3.4128 sec.\n",
      "Iteration 12310 || Loss: 12.3649 || 10iter: 3.4287 sec.\n",
      "Iteration 12320 || Loss: 13.6804 || 10iter: 3.4017 sec.\n",
      "Iteration 12330 || Loss: 10.9900 || 10iter: 3.2820 sec.\n",
      "Iteration 12340 || Loss: 14.3069 || 10iter: 3.1240 sec.\n",
      "-------------\n",
      "epoch 187 || Epoch_TRAIN_Loss:887.2123 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.4473 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 188/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12350 || Loss: 14.7847 || 10iter: 5.6405 sec.\n",
      "Iteration 12360 || Loss: 12.2131 || 10iter: 3.4704 sec.\n",
      "Iteration 12370 || Loss: 10.7095 || 10iter: 3.4561 sec.\n",
      "Iteration 12380 || Loss: 11.9595 || 10iter: 3.4821 sec.\n",
      "Iteration 12390 || Loss: 15.8935 || 10iter: 3.4221 sec.\n",
      "Iteration 12400 || Loss: 13.0019 || 10iter: 3.1867 sec.\n",
      "-------------\n",
      "epoch 188 || Epoch_TRAIN_Loss:891.6883 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.5734 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 189/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 12410 || Loss: 11.2111 || 10iter: 3.2243 sec.\n",
      "Iteration 12420 || Loss: 12.0982 || 10iter: 3.7587 sec.\n",
      "Iteration 12430 || Loss: 13.8475 || 10iter: 3.4758 sec.\n",
      "Iteration 12440 || Loss: 14.8300 || 10iter: 3.6678 sec.\n",
      "Iteration 12450 || Loss: 17.4080 || 10iter: 3.4875 sec.\n",
      "Iteration 12460 || Loss: 12.6967 || 10iter: 3.3884 sec.\n",
      "Iteration 12470 || Loss: 16.5735 || 10iter: 3.1351 sec.\n",
      "-------------\n",
      "epoch 189 || Epoch_TRAIN_Loss:912.0688 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.8486 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 190/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12480 || Loss: 10.3492 || 10iter: 4.9027 sec.\n",
      "Iteration 12490 || Loss: 15.6157 || 10iter: 3.6552 sec.\n",
      "Iteration 12500 || Loss: 12.0285 || 10iter: 3.3948 sec.\n",
      "Iteration 12510 || Loss: 13.4263 || 10iter: 3.4436 sec.\n",
      "Iteration 12520 || Loss: 11.4418 || 10iter: 3.4434 sec.\n",
      "Iteration 12530 || Loss: 12.8234 || 10iter: 3.2017 sec.\n",
      "Iteration 12540 || Loss: 15.9670 || 10iter: 2.9974 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 190 || Epoch_TRAIN_Loss:866.7902 ||Epoch_VAL_Loss:442.9090\n",
      "timer:  31.7587 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 191/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12550 || Loss: 11.0685 || 10iter: 6.6810 sec.\n",
      "Iteration 12560 || Loss: 12.7172 || 10iter: 3.3983 sec.\n",
      "Iteration 12570 || Loss: 13.2059 || 10iter: 3.4652 sec.\n",
      "Iteration 12580 || Loss: 13.0226 || 10iter: 3.5150 sec.\n",
      "Iteration 12590 || Loss: 14.9268 || 10iter: 3.3800 sec.\n",
      "Iteration 12600 || Loss: 14.5347 || 10iter: 3.1305 sec.\n",
      "-------------\n",
      "epoch 191 || Epoch_TRAIN_Loss:877.1783 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.8338 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 192/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12610 || Loss: 10.6129 || 10iter: 3.9090 sec.\n",
      "Iteration 12620 || Loss: 11.7802 || 10iter: 3.8077 sec.\n",
      "Iteration 12630 || Loss: 13.4024 || 10iter: 3.3976 sec.\n",
      "Iteration 12640 || Loss: 10.7620 || 10iter: 3.4106 sec.\n",
      "Iteration 12650 || Loss: 12.0469 || 10iter: 3.5165 sec.\n",
      "Iteration 12660 || Loss: 11.7558 || 10iter: 3.3343 sec.\n",
      "Iteration 12670 || Loss: 15.1205 || 10iter: 3.1322 sec.\n",
      "-------------\n",
      "epoch 192 || Epoch_TRAIN_Loss:888.5574 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.5617 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 193/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12680 || Loss: 11.2932 || 10iter: 5.4325 sec.\n",
      "Iteration 12690 || Loss: 12.9133 || 10iter: 3.4300 sec.\n",
      "Iteration 12700 || Loss: 12.2464 || 10iter: 3.4801 sec.\n",
      "Iteration 12710 || Loss: 12.1508 || 10iter: 3.4163 sec.\n",
      "Iteration 12720 || Loss: 10.5767 || 10iter: 3.4030 sec.\n",
      "Iteration 12730 || Loss: 15.2359 || 10iter: 3.2090 sec.\n",
      "-------------\n",
      "epoch 193 || Epoch_TRAIN_Loss:885.5716 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.3031 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 194/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12740 || Loss: 14.3615 || 10iter: 3.5158 sec.\n",
      "Iteration 12750 || Loss: 12.4768 || 10iter: 3.5774 sec.\n",
      "Iteration 12760 || Loss: 11.2761 || 10iter: 3.4187 sec.\n",
      "Iteration 12770 || Loss: 12.4797 || 10iter: 3.4309 sec.\n",
      "Iteration 12780 || Loss: 12.4816 || 10iter: 3.5115 sec.\n",
      "Iteration 12790 || Loss: 13.7752 || 10iter: 3.3178 sec.\n",
      "Iteration 12800 || Loss: 14.1416 || 10iter: 3.1175 sec.\n",
      "-------------\n",
      "epoch 194 || Epoch_TRAIN_Loss:887.1300 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.5910 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 195/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12810 || Loss: 15.1212 || 10iter: 5.0196 sec.\n",
      "Iteration 12820 || Loss: 15.7840 || 10iter: 3.4594 sec.\n",
      "Iteration 12830 || Loss: 13.7378 || 10iter: 3.4304 sec.\n",
      "Iteration 12840 || Loss: 14.7485 || 10iter: 3.4051 sec.\n",
      "Iteration 12850 || Loss: 15.1331 || 10iter: 3.4560 sec.\n",
      "Iteration 12860 || Loss: 10.9125 || 10iter: 3.1956 sec.\n",
      "Iteration 12870 || Loss: 17.4466 || 10iter: 2.9908 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 195 || Epoch_TRAIN_Loss:922.4720 ||Epoch_VAL_Loss:443.5161\n",
      "timer:  31.6785 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 196/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12880 || Loss: 11.0609 || 10iter: 6.6987 sec.\n",
      "Iteration 12890 || Loss: 12.3313 || 10iter: 3.3586 sec.\n",
      "Iteration 12900 || Loss: 12.5390 || 10iter: 3.5130 sec.\n",
      "Iteration 12910 || Loss: 14.1387 || 10iter: 3.4958 sec.\n",
      "Iteration 12920 || Loss: 11.1224 || 10iter: 3.3900 sec.\n",
      "Iteration 12930 || Loss: 16.6331 || 10iter: 3.1409 sec.\n",
      "-------------\n",
      "epoch 196 || Epoch_TRAIN_Loss:912.7228 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.8746 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 197/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12940 || Loss: 12.6669 || 10iter: 4.3055 sec.\n",
      "Iteration 12950 || Loss: 12.9950 || 10iter: 3.6136 sec.\n",
      "Iteration 12960 || Loss: 10.6797 || 10iter: 3.3956 sec.\n",
      "Iteration 12970 || Loss: 12.8368 || 10iter: 3.4337 sec.\n",
      "Iteration 12980 || Loss: 12.3469 || 10iter: 3.4961 sec.\n",
      "Iteration 12990 || Loss: 11.0352 || 10iter: 3.3012 sec.\n",
      "Iteration 13000 || Loss: 13.8408 || 10iter: 3.1289 sec.\n",
      "-------------\n",
      "epoch 197 || Epoch_TRAIN_Loss:874.5465 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.7555 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 198/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13010 || Loss: 15.5403 || 10iter: 5.7277 sec.\n",
      "Iteration 13020 || Loss: 14.7894 || 10iter: 3.5112 sec.\n",
      "Iteration 13030 || Loss: 18.2963 || 10iter: 3.5025 sec.\n",
      "Iteration 13040 || Loss: 11.0973 || 10iter: 3.4326 sec.\n",
      "Iteration 13050 || Loss: 13.0559 || 10iter: 3.5127 sec.\n",
      "Iteration 13060 || Loss: 11.9797 || 10iter: 3.2188 sec.\n",
      "-------------\n",
      "epoch 198 || Epoch_TRAIN_Loss:873.7311 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.8204 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 199/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13070 || Loss: 10.7961 || 10iter: 3.0601 sec.\n",
      "Iteration 13080 || Loss: 13.6552 || 10iter: 4.1939 sec.\n",
      "Iteration 13090 || Loss: 13.6754 || 10iter: 3.3773 sec.\n",
      "Iteration 13100 || Loss: 12.0770 || 10iter: 3.4867 sec.\n",
      "Iteration 13110 || Loss: 13.9706 || 10iter: 3.4315 sec.\n",
      "Iteration 13120 || Loss: 18.8023 || 10iter: 3.3344 sec.\n",
      "Iteration 13130 || Loss: 14.7221 || 10iter: 3.1320 sec.\n",
      "-------------\n",
      "epoch 199 || Epoch_TRAIN_Loss:859.8809 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.7345 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 200/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13140 || Loss: 16.4445 || 10iter: 5.1552 sec.\n",
      "Iteration 13150 || Loss: 15.4783 || 10iter: 3.3639 sec.\n",
      "Iteration 13160 || Loss: 11.9480 || 10iter: 3.4820 sec.\n",
      "Iteration 13170 || Loss: 11.9657 || 10iter: 3.4716 sec.\n",
      "Iteration 13180 || Loss: 17.4440 || 10iter: 3.4538 sec.\n",
      "Iteration 13190 || Loss: 11.0828 || 10iter: 3.2714 sec.\n",
      "Iteration 13200 || Loss: 13.2597 || 10iter: 3.0164 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 200 || Epoch_TRAIN_Loss:906.3419 ||Epoch_VAL_Loss:443.3210\n",
      "timer:  32.0838 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 201/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13210 || Loss: 12.7132 || 10iter: 6.4709 sec.\n",
      "Iteration 13220 || Loss: 10.6353 || 10iter: 3.3203 sec.\n",
      "Iteration 13230 || Loss: 15.2054 || 10iter: 3.4721 sec.\n",
      "Iteration 13240 || Loss: 13.1060 || 10iter: 3.5125 sec.\n",
      "Iteration 13250 || Loss: 9.8960 || 10iter: 3.4448 sec.\n",
      "Iteration 13260 || Loss: 13.4711 || 10iter: 3.1350 sec.\n",
      "-------------\n",
      "epoch 201 || Epoch_TRAIN_Loss:915.0661 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  25.6379 sec.\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 200\n",
    "train_model(net, dataloaders_dict, criterion, optimizer, num_epochs=num_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That's all :)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}