Dataset_test_with_BlazeFace128-VOC-allclasses.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import stuff\n",
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "from itertools import product as product\n",
    "import time\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Function\n",
    "import pandas as pd\n",
    "\n",
    "# import dataset\n",
    "from utils.dataset import VOCDataset, DatasetTransform, make_datapath_list, Anno_xml2list, od_collate_fn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# set up person only VOC dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "000005\n",
      "000007\n",
      "000009\n",
      "000012\n",
      "000016\n",
      "000017\n",
      "000019\n",
      "000020\n",
      "000021\n",
      "000023\n",
      "000024\n",
      "000026\n",
      "000030\n",
      "000032\n",
      "000033\n",
      "000034\n",
      "000035\n",
      "000036\n",
      "000039\n",
      "000041\n",
      "000042\n",
      "000044\n",
      "000046\n",
      "000047\n",
      "000048\n",
      "000050\n",
      "000051\n",
      "000052\n",
      "000060\n",
      "000061\n",
      "000063\n",
      "000064\n",
      "000065\n",
      "000066\n",
      "000072\n",
      "000073\n",
      "000077\n",
      "000078\n",
      "000081\n",
      "000083\n",
      "000089\n",
      "000091\n",
      "000093\n",
      "000095\n",
      "000099\n",
      "000101\n",
      "000102\n",
      "000104\n",
      "000107\n",
      "000109\n",
      "000110\n",
      "000112\n",
      "000113\n",
      "000117\n",
      "000118\n",
      "000120\n",
      "000121\n",
      "000122\n",
      "000123\n",
      "000125\n",
      "000129\n",
      "000130\n",
      "000131\n",
      "000132\n",
      "000133\n",
      "000134\n",
      "000138\n",
      "000140\n",
      "000141\n",
      "000142\n",
      "000143\n",
      "000146\n",
      "000147\n",
      "000150\n",
      "000153\n",
      "000154\n",
      "000156\n",
      "000158\n",
      "000159\n",
      "000161\n",
      "000162\n",
      "000163\n",
      "000164\n",
      "000165\n",
      "000169\n",
      "000170\n",
      "000171\n",
      "000173\n",
      "000174\n",
      "000177\n",
      "000180\n",
      "000184\n",
      "000187\n",
      "000189\n",
      "000190\n",
      "000192\n",
      "000193\n",
      "000194\n",
      "000198\n",
      "000200\n",
      "000203\n",
      "000207\n",
      "000208\n",
      "000209\n",
      "000210\n",
      "000211\n",
      "000214\n",
      "000215\n",
      "000218\n",
      "000219\n",
      "000220\n",
      "000221\n",
      "000222\n",
      "000224\n",
      "000225\n",
      "000228\n",
      "000229\n",
      "000232\n",
      "000233\n",
      "000235\n",
      "000236\n",
      "000241\n",
      "000242\n",
      "000244\n",
      "000245\n",
      "000246\n",
      "000249\n",
      "000250\n",
      "000251\n",
      "000256\n",
      "000257\n",
      "000259\n",
      "000262\n",
      "000263\n",
      "000266\n",
      "000268\n",
      "000269\n",
      "000270\n",
      "000275\n",
      "000276\n",
      "000278\n",
      "000282\n",
      "000285\n",
      "000288\n",
      "000289\n",
      "000294\n",
      "000296\n",
      "000298\n",
      "000302\n",
      "000303\n",
      "000304\n",
      "000305\n",
      "000306\n",
      "000307\n",
      "000308\n",
      "000311\n",
      "000312\n",
      "000317\n",
      "000318\n",
      "000320\n",
      "000321\n",
      "000322\n",
      "000323\n",
      "000325\n",
      "000328\n",
      "000329\n",
      "000331\n",
      "000332\n",
      "000334\n",
      "000336\n",
      "000337\n",
      "000338\n",
      "000340\n",
      "000343\n",
      "000344\n",
      "000347\n",
      "000349\n",
      "000352\n",
      "000354\n",
      "000355\n",
      "000359\n",
      "000363\n",
      "000367\n",
      "000370\n",
      "000372\n",
      "000373\n",
      "000374\n",
      "000379\n",
      "000380\n",
      "000381\n",
      "000382\n",
      "000387\n",
      "000391\n",
      "000394\n",
      "000395\n",
      "000396\n",
      "000400\n",
      "000403\n",
      "000404\n",
      "000406\n",
      "000407\n",
      "000408\n",
      "000411\n",
      "000416\n",
      "000417\n",
      "000419\n",
      "000420\n",
      "000424\n",
      "000427\n",
      "000428\n",
      "000430\n",
      "000431\n",
      "000433\n",
      "000435\n",
      "000438\n",
      "000439\n",
      "000443\n",
      "000446\n",
      "000448\n",
      "000450\n",
      "000454\n",
      "000459\n",
      "000460\n",
      "000461\n",
      "000462\n",
      "000463\n",
      "000464\n",
      "000468\n",
      "000469\n",
      "000470\n",
      "000474\n",
      "000476\n",
      "000477\n",
      "000480\n",
      "000482\n",
      "000483\n",
      "000484\n",
      "000486\n",
      "000489\n",
      "000491\n",
      "000492\n",
      "000494\n",
      "000496\n",
      "000498\n",
      "000499\n",
      "000500\n",
      "000501\n",
      "000503\n",
      "000508\n",
      "000509\n",
      "000513\n",
      "000514\n",
      "000515\n",
      "000516\n",
      "000518\n",
      "000519\n",
      "000520\n",
      "000522\n",
      "000523\n",
      "000524\n",
      "000525\n",
      "000526\n",
      "000528\n",
      "000530\n",
      "000531\n",
      "000535\n",
      "000537\n",
      "000540\n",
      "000541\n",
      "000543\n",
      "000544\n",
      "000545\n",
      "000549\n",
      "000550\n",
      "000552\n",
      "000554\n",
      "000555\n",
      "000559\n",
      "000563\n",
      "000564\n",
      "000565\n",
      "000577\n",
      "000579\n",
      "000581\n",
      "000582\n",
      "000583\n",
      "000588\n",
      "000589\n",
      "000590\n",
      "000591\n",
      "000592\n",
      "000597\n",
      "000598\n",
      "000599\n",
      "000601\n",
      "000605\n",
      "000608\n",
      "000609\n",
      "000610\n",
      "000612\n",
      "000613\n",
      "000619\n",
      "000620\n",
      "000622\n",
      "000625\n",
      "000626\n",
      "000628\n",
      "000632\n",
      "000633\n",
      "000635\n",
      "000637\n",
      "000645\n",
      "000647\n",
      "000648\n",
      "000653\n",
      "000654\n",
      "000656\n",
      "000657\n",
      "000660\n",
      "000661\n",
      "000663\n",
      "000667\n",
      "000671\n",
      "000672\n",
      "000675\n",
      "000676\n",
      "000677\n",
      "000680\n",
      "000682\n",
      "000684\n",
      "000685\n",
      "000686\n",
      "000688\n",
      "000689\n",
      "000690\n",
      "000694\n",
      "000695\n",
      "000699\n",
      "000700\n",
      "000702\n",
      "000705\n",
      "000707\n",
      "000709\n",
      "000710\n",
      "000711\n",
      "000712\n",
      "000713\n",
      "000714\n",
      "000717\n",
      "000720\n",
      "000726\n",
      "000728\n",
      "000729\n",
      "000730\n",
      "000731\n",
      "000733\n",
      "000738\n",
      "000739\n",
      "000740\n",
      "000742\n",
      "000746\n",
      "000748\n",
      "000750\n",
      "000752\n",
      "000753\n",
      "000754\n",
      "000755\n",
      "000756\n",
      "000760\n",
      "000761\n",
      "000763\n",
      "000764\n",
      "000767\n",
      "000768\n",
      "000770\n",
      "000771\n",
      "000772\n",
      "000774\n",
      "000776\n",
      "000777\n",
      "000780\n",
      "000782\n",
      "000786\n",
      "000787\n",
      "000791\n",
      "000793\n",
      "000794\n",
      "000796\n",
      "000797\n",
      "000799\n",
      "000800\n",
      "000802\n",
      "000804\n",
      "000805\n",
      "000806\n",
      "000808\n",
      "000810\n",
      "000812\n",
      "000814\n",
      "000815\n",
      "000816\n",
      "000818\n",
      "000820\n",
      "000822\n",
      "000823\n",
      "000826\n",
      "000827\n",
      "000828\n",
      "000829\n",
      "000830\n",
      "000831\n",
      "000832\n",
      "000834\n",
      "000842\n",
      "000843\n",
      "000845\n",
      "000847\n",
      "000848\n",
      "000849\n",
      "000850\n",
      "000851\n",
      "000854\n",
      "000855\n",
      "000857\n",
      "000859\n",
      "000860\n",
      "000862\n",
      "000863\n",
      "000865\n",
      "000867\n",
      "000868\n",
      "000871\n",
      "000872\n",
      "000874\n",
      "000876\n",
      "000878\n",
      "000879\n",
      "000880\n",
      "000882\n",
      "000885\n",
      "000887\n",
      "000888\n",
      "000889\n",
      "000892\n",
      "000895\n",
      "000896\n",
      "000898\n",
      "000899\n",
      "000900\n",
      "000902\n",
      "000903\n",
      "000904\n",
      "000906\n",
      "000908\n",
      "000911\n",
      "000912\n",
      "000915\n",
      "000917\n",
      "000918\n",
      "000919\n",
      "000920\n",
      "000921\n",
      "000923\n",
      "000926\n",
      "000929\n",
      "000931\n",
      "000934\n",
      "000935\n",
      "000936\n",
      "000937\n",
      "000943\n",
      "000946\n",
      "000947\n",
      "000948\n",
      "000949\n",
      "000950\n",
      "000951\n",
      "000954\n",
      "000958\n",
      "000962\n",
      "000964\n",
      "000965\n",
      "000966\n",
      "000967\n",
      "000971\n",
      "000972\n",
      "000973\n",
      "000977\n",
      "000980\n",
      "000982\n",
      "000987\n",
      "000989\n",
      "000991\n",
      "000993\n",
      "000996\n",
      "000997\n",
      "000999\n",
      "001001\n",
      "001002\n",
      "001004\n",
      "001008\n",
      "001009\n",
      "001010\n",
      "001011\n",
      "001012\n",
      "001014\n",
      "001015\n",
      "001017\n",
      "001018\n",
      "001024\n",
      "001027\n",
      "001028\n",
      "001036\n",
      "001041\n",
      "001042\n",
      "001043\n",
      "001045\n",
      "001050\n",
      "001052\n",
      "001053\n",
      "001056\n",
      "001057\n",
      "001060\n",
      "001061\n",
      "001062\n",
      "001064\n",
      "001066\n",
      "001068\n",
      "001069\n",
      "001071\n",
      "001072\n",
      "001073\n",
      "001074\n",
      "001077\n",
      "001078\n",
      "001079\n",
      "001082\n",
      "001083\n",
      "001084\n",
      "001091\n",
      "001092\n",
      "001093\n",
      "001097\n",
      "001101\n",
      "001102\n",
      "001104\n",
      "001106\n",
      "001107\n",
      "001109\n",
      "001110\n",
      "001112\n",
      "001113\n",
      "001119\n",
      "001121\n",
      "001124\n",
      "001125\n",
      "001127\n",
      "001129\n",
      "001130\n",
      "001136\n",
      "001137\n",
      "001140\n",
      "001142\n",
      "001143\n",
      "001144\n",
      "001145\n",
      "001147\n",
      "001148\n",
      "001149\n",
      "001151\n",
      "001152\n",
      "001154\n",
      "001156\n",
      "001158\n",
      "001160\n",
      "001161\n",
      "001164\n",
      "001166\n",
      "001168\n",
      "001170\n",
      "001171\n",
      "001172\n",
      "001174\n",
      "001175\n",
      "001176\n",
      "001182\n",
      "001184\n",
      "001185\n",
      "001186\n",
      "001187\n",
      "001191\n",
      "001192\n",
      "001194\n",
      "001199\n",
      "001200\n",
      "001201\n",
      "001203\n",
      "001204\n",
      "001205\n",
      "001206\n",
      "001207\n",
      "001209\n",
      "001211\n",
      "001212\n",
      "001214\n",
      "001215\n",
      "001221\n",
      "001224\n",
      "001225\n",
      "001226\n",
      "001229\n",
      "001230\n",
      "001231\n",
      "001233\n",
      "001234\n",
      "001236\n",
      "001237\n",
      "001239\n",
      "001240\n",
      "001241\n",
      "001247\n",
      "001248\n",
      "001250\n",
      "001254\n",
      "001258\n",
      "001259\n",
      "001260\n",
      "001263\n",
      "001265\n",
      "001266\n",
      "001268\n",
      "001269\n",
      "001270\n",
      "001272\n",
      "001273\n",
      "001274\n",
      "001277\n",
      "001279\n",
      "001281\n",
      "001284\n",
      "001286\n",
      "001287\n",
      "001288\n",
      "001289\n",
      "001290\n",
      "001292\n",
      "001293\n",
      "001294\n",
      "001298\n",
      "001299\n",
      "001304\n",
      "001309\n",
      "001310\n",
      "001311\n",
      "001312\n",
      "001314\n",
      "001315\n",
      "001316\n",
      "001323\n",
      "001324\n",
      "001325\n",
      "001326\n",
      "001327\n",
      "001330\n",
      "001332\n",
      "001333\n",
      "001334\n",
      "001337\n",
      "001341\n",
      "001343\n",
      "001345\n",
      "001346\n",
      "001348\n",
      "001350\n",
      "001352\n",
      "001360\n",
      "001361\n",
      "001362\n",
      "001364\n",
      "001365\n",
      "001371\n",
      "001375\n",
      "001378\n",
      "001383\n",
      "001384\n",
      "001385\n",
      "001386\n",
      "001387\n",
      "001388\n",
      "001390\n",
      "001393\n",
      "001395\n",
      "001397\n",
      "001400\n",
      "001402\n",
      "001404\n",
      "001405\n",
      "001406\n",
      "001408\n",
      "001409\n",
      "001413\n",
      "001414\n",
      "001418\n",
      "001420\n",
      "001421\n",
      "001426\n",
      "001427\n",
      "001430\n",
      "001432\n",
      "001434\n",
      "001436\n",
      "001439\n",
      "001441\n",
      "001442\n",
      "001443\n",
      "001444\n",
      "001445\n",
      "001450\n",
      "001451\n",
      "001453\n",
      "001455\n",
      "001457\n",
      "001460\n",
      "001463\n",
      "001464\n",
      "001465\n",
      "001466\n",
      "001467\n",
      "001468\n",
      "001470\n",
      "001472\n",
      "001475\n",
      "001479\n",
      "001480\n",
      "001481\n",
      "001483\n",
      "001484\n",
      "001485\n",
      "001486\n",
      "001488\n",
      "001490\n",
      "001492\n",
      "001493\n",
      "001494\n",
      "001497\n",
      "001498\n",
      "001499\n",
      "001501\n",
      "001504\n",
      "001509\n",
      "001510\n",
      "001512\n",
      "001514\n",
      "001515\n",
      "001517\n",
      "001521\n",
      "001522\n",
      "001523\n",
      "001524\n",
      "001526\n",
      "001528\n",
      "001529\n",
      "001531\n",
      "001532\n",
      "001536\n",
      "001537\n",
      "001539\n",
      "001541\n",
      "001543\n",
      "001544\n",
      "001545\n",
      "001548\n",
      "001553\n",
      "001554\n",
      "001555\n",
      "001556\n",
      "001557\n",
      "001559\n",
      "001561\n",
      "001563\n",
      "001565\n",
      "001571\n",
      "001576\n",
      "001577\n",
      "001579\n",
      "001580\n",
      "001582\n",
      "001586\n",
      "001588\n",
      "001590\n",
      "001593\n",
      "001594\n",
      "001595\n",
      "001597\n",
      "001598\n",
      "001603\n",
      "001604\n",
      "001607\n",
      "001608\n",
      "001610\n",
      "001611\n",
      "001612\n",
      "001614\n",
      "001617\n",
      "001618\n",
      "001622\n",
      "001627\n",
      "001628\n",
      "001630\n",
      "001632\n",
      "001633\n",
      "001636\n",
      "001638\n",
      "001640\n",
      "001642\n",
      "001643\n",
      "001647\n",
      "001649\n",
      "001650\n",
      "001651\n",
      "001653\n",
      "001654\n",
      "001661\n",
      "001662\n",
      "001669\n",
      "001673\n",
      "001675\n",
      "001676\n",
      "001677\n",
      "001678\n",
      "001680\n",
      "001682\n",
      "001683\n",
      "001684\n",
      "001685\n",
      "001686\n",
      "001688\n",
      "001689\n",
      "001690\n",
      "001691\n",
      "001693\n",
      "001699\n",
      "001707\n",
      "001708\n",
      "001711\n",
      "001713\n",
      "001714\n",
      "001717\n",
      "001718\n",
      "001721\n",
      "001723\n",
      "001724\n",
      "001725\n",
      "001726\n",
      "001727\n",
      "001729\n",
      "001730\n",
      "001732\n",
      "001733\n",
      "001734\n",
      "001738\n",
      "001739\n",
      "001741\n",
      "001746\n",
      "001747\n",
      "001749\n",
      "001750\n",
      "001752\n",
      "001754\n",
      "001755\n",
      "001756\n",
      "001758\n",
      "001759\n",
      "001761\n",
      "001765\n",
      "001766\n",
      "001768\n",
      "001771\n",
      "001772\n",
      "001775\n",
      "001777\n",
      "001778\n",
      "001780\n",
      "001782\n",
      "001784\n",
      "001785\n",
      "001787\n",
      "001789\n",
      "001793\n",
      "001795\n",
      "001797\n",
      "001799\n",
      "001800\n",
      "001801\n",
      "001806\n",
      "001807\n",
      "001809\n",
      "001810\n",
      "001816\n",
      "001818\n",
      "001821\n",
      "001825\n",
      "001827\n",
      "001828\n",
      "001830\n",
      "001832\n",
      "001833\n",
      "001834\n",
      "001836\n",
      "001837\n",
      "001840\n",
      "001841\n",
      "001842\n",
      "001843\n",
      "001845\n",
      "001847\n",
      "001849\n",
      "001853\n",
      "001854\n",
      "001855\n",
      "001858\n",
      "001860\n",
      "001861\n",
      "001862\n",
      "001864\n",
      "001870\n",
      "001872\n",
      "001875\n",
      "001877\n",
      "001878\n",
      "001881\n",
      "001882\n",
      "001887\n",
      "001888\n",
      "001892\n",
      "001894\n",
      "001896\n",
      "001898\n",
      "001899\n",
      "001901\n",
      "001902\n",
      "001903\n",
      "001904\n",
      "001906\n",
      "001907\n",
      "001911\n",
      "001915\n",
      "001918\n",
      "001920\n",
      "001922\n",
      "001927\n",
      "001928\n",
      "001930\n",
      "001931\n",
      "001932\n",
      "001933\n",
      "001934\n",
      "001936\n",
      "001937\n",
      "001938\n",
      "001940\n",
      "001941\n",
      "001944\n",
      "001945\n",
      "001948\n",
      "001950\n",
      "001952\n",
      "001954\n",
      "001958\n",
      "001960\n",
      "001962\n",
      "001963\n",
      "001964\n",
      "001970\n",
      "001971\n",
      "001972\n",
      "001976\n",
      "001977\n",
      "001978\n",
      "001980\n",
      "001981\n",
      "001982\n",
      "001985\n",
      "001989\n",
      "001995\n",
      "001999\n",
      "002000\n",
      "002001\n",
      "002002\n",
      "002004\n",
      "002006\n",
      "002011\n",
      "002012\n",
      "002015\n",
      "002019\n",
      "002020\n",
      "002021\n",
      "002022\n",
      "002023\n",
      "002024\n",
      "002025\n",
      "002027\n",
      "002030\n",
      "002034\n",
      "002036\n",
      "002037\n",
      "002039\n",
      "002042\n",
      "002043\n",
      "002045\n",
      "002047\n",
      "002049\n",
      "002051\n",
      "002054\n",
      "002055\n",
      "002056\n",
      "002058\n",
      "002061\n",
      "002063\n",
      "002064\n",
      "002067\n",
      "002068\n",
      "002069\n",
      "002070\n",
      "002082\n",
      "002083\n",
      "002086\n",
      "002088\n",
      "002090\n",
      "002091\n",
      "002094\n",
      "002095\n",
      "002096\n",
      "002098\n",
      "002099\n",
      "002101\n",
      "002102\n",
      "002104\n",
      "002108\n",
      "002109\n",
      "002112\n",
      "002114\n",
      "002116\n",
      "002117\n",
      "002120\n",
      "002124\n",
      "002125\n",
      "002126\n",
      "002129\n",
      "002132\n",
      "002134\n",
      "002135\n",
      "002136\n",
      "002139\n",
      "002140\n",
      "002142\n",
      "002145\n",
      "002146\n",
      "002151\n",
      "002152\n",
      "002153\n",
      "002155\n",
      "002156\n",
      "002158\n",
      "002163\n",
      "002165\n",
      "002166\n",
      "002169\n",
      "002170\n",
      "002171\n",
      "002172\n",
      "002174\n",
      "002176\n",
      "002178\n",
      "002179\n",
      "002180\n",
      "002181\n",
      "002182\n",
      "002183\n",
      "002184\n",
      "002186\n",
      "002187\n",
      "002190\n",
      "002191\n",
      "002192\n",
      "002193\n",
      "002194\n",
      "002196\n",
      "002197\n",
      "002199\n",
      "002201\n",
      "002202\n",
      "002208\n",
      "002209\n",
      "002212\n",
      "002213\n",
      "002214\n",
      "002215\n",
      "002218\n",
      "002219\n",
      "002220\n",
      "002221\n",
      "002224\n",
      "002226\n",
      "002228\n",
      "002233\n",
      "002234\n",
      "002237\n",
      "002238\n",
      "002241\n",
      "002244\n",
      "002247\n",
      "002248\n",
      "002249\n",
      "002251\n",
      "002253\n",
      "002255\n",
      "002256\n",
      "002257\n",
      "002259\n",
      "002260\n",
      "002261\n",
      "002263\n",
      "002265\n",
      "002266\n",
      "002267\n",
      "002268\n",
      "002270\n",
      "002272\n",
      "002273\n",
      "002276\n",
      "002277\n",
      "002278\n",
      "002279\n",
      "002280\n",
      "002281\n",
      "002284\n",
      "002285\n",
      "002287\n",
      "002288\n",
      "002290\n",
      "002291\n",
      "002293\n",
      "002300\n",
      "002302\n",
      "002305\n",
      "002306\n",
      "002307\n",
      "002308\n",
      "002310\n",
      "002311\n",
      "002315\n",
      "002318\n",
      "002320\n",
      "002321\n",
      "002323\n",
      "002324\n",
      "002328\n",
      "002329\n",
      "002330\n",
      "002332\n",
      "002333\n",
      "002334\n",
      "002335\n",
      "002337\n",
      "002340\n",
      "002342\n",
      "002343\n",
      "002345\n",
      "002347\n",
      "002348\n",
      "002350\n",
      "002352\n",
      "002354\n",
      "002355\n",
      "002359\n",
      "002361\n",
      "002362\n",
      "002364\n",
      "002366\n",
      "002367\n",
      "002368\n",
      "002369\n",
      "002371\n",
      "002372\n",
      "002373\n",
      "002374\n",
      "002375\n",
      "002376\n",
      "002377\n",
      "002378\n",
      "002382\n",
      "002384\n",
      "002385\n",
      "002387\n",
      "002391\n",
      "002392\n",
      "002393\n",
      "002401\n",
      "002403\n",
      "002404\n",
      "002405\n",
      "002407\n",
      "002410\n",
      "002411\n",
      "002413\n",
      "002415\n",
      "002417\n",
      "002419\n",
      "002420\n",
      "002423\n",
      "002425\n",
      "002427\n",
      "002433\n",
      "002435\n",
      "002436\n",
      "002437\n",
      "002439\n",
      "002441\n",
      "002442\n",
      "002443\n",
      "002444\n",
      "002445\n",
      "002448\n",
      "002450\n",
      "002452\n",
      "002454\n",
      "002456\n",
      "002458\n",
      "002459\n",
      "002460\n",
      "002461\n",
      "002462\n",
      "002465\n",
      "002466\n",
      "002468\n",
      "002470\n",
      "002471\n",
      "002472\n",
      "002476\n",
      "002477\n",
      "002478\n",
      "002479\n",
      "002480\n",
      "002481\n",
      "002483\n",
      "002490\n",
      "002491\n",
      "002492\n",
      "002493\n",
      "002494\n",
      "002496\n",
      "002497\n",
      "002500\n",
      "002501\n",
      "002502\n",
      "002504\n",
      "002505\n",
      "002508\n",
      "002512\n",
      "002513\n",
      "002514\n",
      "002518\n",
      "002519\n",
      "002520\n",
      "002523\n",
      "002524\n",
      "002525\n",
      "002529\n",
      "002533\n",
      "002534\n",
      "002537\n",
      "002539\n",
      "002540\n",
      "002542\n",
      "002544\n",
      "002545\n",
      "002546\n",
      "002547\n",
      "002549\n",
      "002554\n",
      "002555\n",
      "002558\n",
      "002559\n",
      "002561\n",
      "002563\n",
      "002564\n",
      "002565\n",
      "002566\n",
      "002567\n",
      "002569\n",
      "002571\n",
      "002572\n",
      "002578\n",
      "002579\n",
      "002584\n",
      "002585\n",
      "002586\n",
      "002589\n",
      "002590\n",
      "002593\n",
      "002594\n",
      "002595\n",
      "002598\n",
      "002599\n",
      "002600\n",
      "002603\n",
      "002605\n",
      "002606\n",
      "002609\n",
      "002611\n",
      "002613\n",
      "002615\n",
      "002618\n",
      "002621\n",
      "002625\n",
      "002627\n",
      "002632\n",
      "002633\n",
      "002634\n",
      "002635\n",
      "002636\n",
      "002637\n",
      "002641\n",
      "002643\n",
      "002645\n",
      "002646\n",
      "002647\n",
      "002648\n",
      "002649\n",
      "002653\n",
      "002657\n",
      "002658\n",
      "002659\n",
      "002662\n",
      "002664\n",
      "002666\n",
      "002667\n",
      "002668\n",
      "002669\n",
      "002670\n",
      "002675\n",
      "002677\n",
      "002678\n",
      "002680\n",
      "002682\n",
      "002683\n",
      "002684\n",
      "002689\n",
      "002690\n",
      "002691\n",
      "002693\n",
      "002695\n",
      "002696\n",
      "002697\n",
      "002699\n",
      "002702\n",
      "002704\n",
      "002706\n",
      "002709\n",
      "002710\n",
      "002713\n",
      "002714\n",
      "002715\n",
      "002717\n",
      "002718\n",
      "002721\n",
      "002722\n",
      "002723\n",
      "002727\n",
      "002730\n",
      "002732\n",
      "002734\n",
      "002735\n",
      "002737\n",
      "002738\n",
      "002741\n",
      "002744\n",
      "002745\n",
      "002747\n",
      "002749\n",
      "002751\n",
      "002755\n",
      "002757\n",
      "002759\n",
      "002760\n",
      "002762\n",
      "002763\n",
      "002765\n",
      "002766\n",
      "002767\n",
      "002772\n",
      "002774\n",
      "002775\n",
      "002776\n",
      "002778\n",
      "002779\n",
      "002782\n",
      "002783\n",
      "002784\n",
      "002785\n",
      "002786\n",
      "002791\n",
      "002794\n",
      "002795\n",
      "002796\n",
      "002798\n",
      "002800\n",
      "002801\n",
      "002803\n",
      "002804\n",
      "002807\n",
      "002810\n",
      "002812\n",
      "002815\n",
      "002816\n",
      "002817\n",
      "002820\n",
      "002826\n",
      "002827\n",
      "002833\n",
      "002834\n",
      "002835\n",
      "002836\n",
      "002838\n",
      "002841\n",
      "002842\n",
      "002844\n",
      "002845\n",
      "002847\n",
      "002848\n",
      "002854\n",
      "002855\n",
      "002858\n",
      "002859\n",
      "002864\n",
      "002866\n",
      "002867\n",
      "002868\n",
      "002869\n",
      "002870\n",
      "002873\n",
      "002875\n",
      "002879\n",
      "002880\n",
      "002881\n",
      "002884\n",
      "002886\n",
      "002889\n",
      "002891\n",
      "002893\n",
      "002896\n",
      "002899\n",
      "002901\n",
      "002906\n",
      "002910\n",
      "002912\n",
      "002913\n",
      "002914\n",
      "002915\n",
      "002916\n",
      "002917\n",
      "002919\n",
      "002924\n",
      "002931\n",
      "002932\n",
      "002933\n",
      "002934\n",
      "002935\n",
      "002937\n",
      "002938\n",
      "002939\n",
      "002940\n",
      "002941\n",
      "002942\n",
      "002943\n",
      "002944\n",
      "002946\n",
      "002947\n",
      "002952\n",
      "002953\n",
      "002954\n",
      "002956\n",
      "002957\n",
      "002958\n",
      "002960\n",
      "002962\n",
      "002963\n",
      "002965\n",
      "002966\n",
      "002967\n",
      "002969\n",
      "002975\n",
      "002976\n",
      "002977\n",
      "002978\n",
      "002984\n",
      "002986\n",
      "002987\n",
      "002988\n",
      "002989\n",
      "002990\n",
      "002992\n",
      "002994\n",
      "002995\n",
      "003000\n",
      "003002\n",
      "003003\n",
      "003004\n",
      "003005\n",
      "003007\n",
      "003008\n",
      "003009\n",
      "003011\n",
      "003013\n",
      "003015\n",
      "003017\n",
      "003021\n",
      "003023\n",
      "003024\n",
      "003027\n",
      "003028\n",
      "003031\n",
      "003032\n",
      "003034\n",
      "003038\n",
      "003039\n",
      "003042\n",
      "003044\n",
      "003045\n",
      "003047\n",
      "003051\n",
      "003053\n",
      "003054\n",
      "003056\n",
      "003057\n",
      "003058\n",
      "003061\n",
      "003063\n",
      "003064\n",
      "003065\n",
      "003066\n",
      "003072\n",
      "003074\n",
      "003077\n",
      "003078\n",
      "003082\n",
      "003083\n",
      "003085\n",
      "003086\n",
      "003088\n",
      "003089\n",
      "003090\n",
      "003092\n",
      "003093\n",
      "003094\n",
      "003098\n",
      "003100\n",
      "003102\n",
      "003103\n",
      "003105\n",
      "003106\n",
      "003107\n",
      "003108\n",
      "003110\n",
      "003112\n",
      "003116\n",
      "003117\n",
      "003118\n",
      "003120\n",
      "003121\n",
      "003122\n",
      "003124\n",
      "003126\n",
      "003127\n",
      "003129\n",
      "003133\n",
      "003134\n",
      "003135\n",
      "003137\n",
      "003138\n",
      "003140\n",
      "003142\n",
      "003145\n",
      "003146\n",
      "003147\n",
      "003149\n",
      "003150\n",
      "003154\n",
      "003155\n",
      "003157\n",
      "003159\n",
      "003161\n",
      "003162\n",
      "003163\n",
      "003164\n",
      "003165\n",
      "003169\n",
      "003170\n",
      "003175\n",
      "003176\n",
      "003177\n",
      "003178\n",
      "003181\n",
      "003183\n",
      "003184\n",
      "003185\n",
      "003186\n",
      "003188\n",
      "003189\n",
      "003194\n",
      "003195\n",
      "003199\n",
      "003200\n",
      "003202\n",
      "003204\n",
      "003205\n",
      "003207\n",
      "003210\n",
      "003211\n",
      "003213\n",
      "003214\n",
      "003216\n",
      "003218\n",
      "003219\n",
      "003223\n",
      "003228\n",
      "003229\n",
      "003231\n",
      "003233\n",
      "003236\n",
      "003239\n",
      "003240\n",
      "003242\n",
      "003243\n",
      "003244\n",
      "003247\n",
      "003250\n",
      "003253\n",
      "003254\n",
      "003255\n",
      "003256\n",
      "003258\n",
      "003259\n",
      "003260\n",
      "003261\n",
      "003262\n",
      "003269\n",
      "003270\n",
      "003271\n",
      "003272\n",
      "003273\n",
      "003274\n",
      "003279\n",
      "003280\n",
      "003282\n",
      "003284\n",
      "003285\n",
      "003290\n",
      "003292\n",
      "003293\n",
      "003294\n",
      "003296\n",
      "003299\n",
      "003300\n",
      "003301\n",
      "003303\n",
      "003307\n",
      "003308\n",
      "003311\n",
      "003313\n",
      "003316\n",
      "003320\n",
      "003325\n",
      "003327\n",
      "003330\n",
      "003331\n",
      "003335\n",
      "003336\n",
      "003337\n",
      "003338\n",
      "003339\n",
      "003343\n",
      "003344\n",
      "003349\n",
      "003350\n",
      "003351\n",
      "003354\n",
      "003355\n",
      "003356\n",
      "003359\n",
      "003360\n",
      "003362\n",
      "003363\n",
      "003365\n",
      "003367\n",
      "003369\n",
      "003370\n",
      "003373\n",
      "003374\n",
      "003376\n",
      "003377\n",
      "003379\n",
      "003380\n",
      "003382\n",
      "003386\n",
      "003390\n",
      "003391\n",
      "003392\n",
      "003395\n",
      "003396\n",
      "003397\n",
      "003398\n",
      "003401\n",
      "003403\n",
      "003404\n",
      "003406\n",
      "003407\n",
      "003408\n",
      "003410\n",
      "003412\n",
      "003413\n",
      "003415\n",
      "003416\n",
      "003417\n",
      "003419\n",
      "003420\n",
      "003421\n",
      "003422\n",
      "003424\n",
      "003425\n",
      "003429\n",
      "003430\n",
      "003433\n",
      "003435\n",
      "003436\n",
      "003439\n",
      "003441\n",
      "003443\n",
      "003444\n",
      "003449\n",
      "003450\n",
      "003451\n",
      "003452\n",
      "003453\n",
      "003455\n",
      "003458\n",
      "003461\n",
      "003462\n",
      "003464\n",
      "003465\n",
      "003466\n",
      "003468\n",
      "003469\n",
      "003470\n",
      "003477\n",
      "003484\n",
      "003487\n",
      "003489\n",
      "003491\n",
      "003492\n",
      "003493\n",
      "003496\n",
      "003497\n",
      "003499\n",
      "003500\n",
      "003506\n",
      "003508\n",
      "003509\n",
      "003510\n",
      "003511\n",
      "003516\n",
      "003518\n",
      "003519\n",
      "003521\n",
      "003522\n",
      "003524\n",
      "003525\n",
      "003528\n",
      "003529\n",
      "003530\n",
      "003536\n",
      "003537\n",
      "003539\n",
      "003546\n",
      "003548\n",
      "003549\n",
      "003550\n",
      "003551\n",
      "003554\n",
      "003555\n",
      "003556\n",
      "003564\n",
      "003565\n",
      "003566\n",
      "003567\n",
      "003575\n",
      "003576\n",
      "003577\n",
      "003580\n",
      "003585\n",
      "003586\n",
      "003587\n",
      "003588\n",
      "003589\n",
      "003593\n",
      "003594\n",
      "003596\n",
      "003597\n",
      "003599\n",
      "003603\n",
      "003604\n",
      "003605\n",
      "003606\n",
      "003608\n",
      "003609\n",
      "003611\n",
      "003614\n",
      "003618\n",
      "003620\n",
      "003621\n",
      "003622\n",
      "003623\n",
      "003625\n",
      "003627\n",
      "003628\n",
      "003629\n",
      "003632\n",
      "003634\n",
      "003635\n",
      "003636\n",
      "003638\n",
      "003639\n",
      "003640\n",
      "003642\n",
      "003644\n",
      "003645\n",
      "003646\n",
      "003648\n",
      "003651\n",
      "003654\n",
      "003655\n",
      "003656\n",
      "003657\n",
      "003658\n",
      "003660\n",
      "003662\n",
      "003663\n",
      "003664\n",
      "003667\n",
      "003669\n",
      "003671\n",
      "003673\n",
      "003674\n",
      "003675\n",
      "003678\n",
      "003679\n",
      "003681\n",
      "003684\n",
      "003685\n",
      "003688\n",
      "003690\n",
      "003691\n",
      "003694\n",
      "003695\n",
      "003696\n",
      "003698\n",
      "003699\n",
      "003700\n",
      "003703\n",
      "003704\n",
      "003705\n",
      "003706\n",
      "003708\n",
      "003709\n",
      "003711\n",
      "003713\n",
      "003714\n",
      "003717\n",
      "003721\n",
      "003722\n",
      "003727\n",
      "003729\n",
      "003732\n",
      "003735\n",
      "003740\n",
      "003743\n",
      "003748\n",
      "003749\n",
      "003750\n",
      "003751\n",
      "003752\n",
      "003753\n",
      "003754\n",
      "003758\n",
      "003759\n",
      "003760\n",
      "003763\n",
      "003767\n",
      "003772\n",
      "003773\n",
      "003774\n",
      "003779\n",
      "003780\n",
      "003781\n",
      "003783\n",
      "003784\n",
      "003786\n",
      "003788\n",
      "003790\n",
      "003791\n",
      "003792\n",
      "003793\n",
      "003796\n",
      "003797\n",
      "003798\n",
      "003803\n",
      "003806\n",
      "003807\n",
      "003808\n",
      "003809\n",
      "003811\n",
      "003814\n",
      "003817\n",
      "003818\n",
      "003820\n",
      "003821\n",
      "003824\n",
      "003826\n",
      "003827\n",
      "003828\n",
      "003830\n",
      "003834\n",
      "003835\n",
      "003837\n",
      "003838\n",
      "003844\n",
      "003845\n",
      "003846\n",
      "003847\n",
      "003848\n",
      "003849\n",
      "003855\n",
      "003856\n",
      "003857\n",
      "003859\n",
      "003860\n",
      "003861\n",
      "003863\n",
      "003865\n",
      "003866\n",
      "003868\n",
      "003869\n",
      "003871\n",
      "003872\n",
      "003874\n",
      "003876\n",
      "003877\n",
      "003879\n",
      "003885\n",
      "003886\n",
      "003887\n",
      "003889\n",
      "003890\n",
      "003891\n",
      "003895\n",
      "003898\n",
      "003899\n",
      "003905\n",
      "003907\n",
      "003911\n",
      "003912\n",
      "003913\n",
      "003915\n",
      "003918\n",
      "003919\n",
      "003921\n",
      "003923\n",
      "003924\n",
      "003926\n",
      "003932\n",
      "003935\n",
      "003936\n",
      "003937\n",
      "003939\n",
      "003941\n",
      "003945\n",
      "003946\n",
      "003947\n",
      "003948\n",
      "003949\n",
      "003953\n",
      "003954\n",
      "003956\n",
      "003957\n",
      "003960\n",
      "003961\n",
      "003963\n",
      "003965\n",
      "003966\n",
      "003969\n",
      "003970\n",
      "003971\n",
      "003973\n",
      "003974\n",
      "003979\n",
      "003983\n",
      "003984\n",
      "003986\n",
      "003987\n",
      "003988\n",
      "003990\n",
      "003991\n",
      "003992\n",
      "003993\n",
      "003994\n",
      "003996\n",
      "003997\n",
      "003998\n",
      "004003\n",
      "004005\n",
      "004008\n",
      "004009\n",
      "004010\n",
      "004011\n",
      "004012\n",
      "004013\n",
      "004014\n",
      "004015\n",
      "004016\n",
      "004017\n",
      "004019\n",
      "004020\n",
      "004023\n",
      "004025\n",
      "004028\n",
      "004031\n",
      "004033\n",
      "004034\n",
      "004035\n",
      "004037\n",
      "004039\n",
      "004046\n",
      "004047\n",
      "004051\n",
      "004052\n",
      "004057\n",
      "004058\n",
      "004060\n",
      "004066\n",
      "004067\n",
      "004069\n",
      "004073\n",
      "004075\n",
      "004076\n",
      "004077\n",
      "004082\n",
      "004085\n",
      "004087\n",
      "004089\n",
      "004091\n",
      "004092\n",
      "004093\n",
      "004095\n",
      "004100\n",
      "004102\n",
      "004105\n",
      "004106\n",
      "004108\n",
      "004110\n",
      "004111\n",
      "004113\n",
      "004117\n",
      "004120\n",
      "004121\n",
      "004122\n",
      "004129\n",
      "004131\n",
      "004133\n",
      "004135\n",
      "004136\n",
      "004137\n",
      "004138\n",
      "004140\n",
      "004141\n",
      "004142\n",
      "004143\n",
      "004145\n",
      "004146\n",
      "004148\n",
      "004149\n",
      "004150\n",
      "004152\n",
      "004158\n",
      "004163\n",
      "004164\n",
      "004168\n",
      "004169\n",
      "004170\n",
      "004171\n",
      "004174\n",
      "004178\n",
      "004185\n",
      "004186\n",
      "004189\n",
      "004190\n",
      "004191\n",
      "004192\n",
      "004193\n",
      "004194\n",
      "004195\n",
      "004196\n",
      "004200\n",
      "004201\n",
      "004203\n",
      "004204\n",
      "004205\n",
      "004209\n",
      "004212\n",
      "004215\n",
      "004220\n",
      "004221\n",
      "004223\n",
      "004224\n",
      "004228\n",
      "004229\n",
      "004230\n",
      "004231\n",
      "004232\n",
      "004237\n",
      "004239\n",
      "004241\n",
      "004242\n",
      "004244\n",
      "004246\n",
      "004247\n",
      "004253\n",
      "004255\n",
      "004256\n",
      "004257\n",
      "004258\n",
      "004259\n",
      "004263\n",
      "004264\n",
      "004265\n",
      "004269\n",
      "004270\n",
      "004271\n",
      "004272\n",
      "004273\n",
      "004274\n",
      "004275\n",
      "004279\n",
      "004280\n",
      "004281\n",
      "004283\n",
      "004284\n",
      "004286\n",
      "004287\n",
      "004291\n",
      "004292\n",
      "004293\n",
      "004295\n",
      "004296\n",
      "004298\n",
      "004300\n",
      "004303\n",
      "004304\n",
      "004307\n",
      "004310\n",
      "004312\n",
      "004315\n",
      "004318\n",
      "004321\n",
      "004322\n",
      "004323\n",
      "004325\n",
      "004326\n",
      "004327\n",
      "004329\n",
      "004331\n",
      "004333\n",
      "004338\n",
      "004339\n",
      "004341\n",
      "004345\n",
      "004346\n",
      "004347\n",
      "004349\n",
      "004351\n",
      "004352\n",
      "004354\n",
      "004356\n",
      "004359\n",
      "004360\n",
      "004361\n",
      "004364\n",
      "004365\n",
      "004367\n",
      "004368\n",
      "004369\n",
      "004370\n",
      "004371\n",
      "004372\n",
      "004376\n",
      "004379\n",
      "004380\n",
      "004384\n",
      "004386\n",
      "004387\n",
      "004389\n",
      "004390\n",
      "004391\n",
      "004392\n",
      "004396\n",
      "004397\n",
      "004404\n",
      "004405\n",
      "004409\n",
      "004411\n",
      "004421\n",
      "004423\n",
      "004424\n",
      "004429\n",
      "004430\n",
      "004432\n",
      "004433\n",
      "004434\n",
      "004436\n",
      "004437\n",
      "004438\n",
      "004439\n",
      "004441\n",
      "004446\n",
      "004450\n",
      "004452\n",
      "004455\n",
      "004457\n",
      "004459\n",
      "004463\n",
      "004464\n",
      "004466\n",
      "004468\n",
      "004470\n",
      "004471\n",
      "004474\n",
      "004479\n",
      "004481\n",
      "004484\n",
      "004487\n",
      "004488\n",
      "004490\n",
      "004493\n",
      "004494\n",
      "004495\n",
      "004496\n",
      "004498\n",
      "004499\n",
      "004500\n",
      "004502\n",
      "004507\n",
      "004508\n",
      "004509\n",
      "004510\n",
      "004512\n",
      "004514\n",
      "004517\n",
      "004518\n",
      "004519\n",
      "004520\n",
      "004524\n",
      "004526\n",
      "004527\n",
      "004528\n",
      "004530\n",
      "004532\n",
      "004535\n",
      "004537\n",
      "004539\n",
      "004540\n",
      "004542\n",
      "004544\n",
      "004548\n",
      "004549\n",
      "004551\n",
      "004552\n",
      "004553\n",
      "004555\n",
      "004558\n",
      "004562\n",
      "004563\n",
      "004565\n",
      "004566\n",
      "004570\n",
      "004571\n",
      "004574\n",
      "004576\n",
      "004579\n",
      "004581\n",
      "004584\n",
      "004585\n",
      "004587\n",
      "004588\n",
      "004591\n",
      "004592\n",
      "004595\n",
      "004597\n",
      "004600\n",
      "004601\n",
      "004604\n",
      "004605\n",
      "004606\n",
      "004607\n",
      "004609\n",
      "004611\n",
      "004612\n",
      "004618\n",
      "004622\n",
      "004623\n",
      "004625\n",
      "004626\n",
      "004627\n",
      "004628\n",
      "004630\n",
      "004631\n",
      "004632\n",
      "004634\n",
      "004636\n",
      "004643\n",
      "004644\n",
      "004647\n",
      "004648\n",
      "004649\n",
      "004651\n",
      "004652\n",
      "004653\n",
      "004654\n",
      "004655\n",
      "004656\n",
      "004660\n",
      "004662\n",
      "004671\n",
      "004672\n",
      "004673\n",
      "004674\n",
      "004675\n",
      "004676\n",
      "004679\n",
      "004682\n",
      "004683\n",
      "004685\n",
      "004686\n",
      "004687\n",
      "004689\n",
      "004691\n",
      "004692\n",
      "004693\n",
      "004694\n",
      "004699\n",
      "004701\n",
      "004702\n",
      "004705\n",
      "004706\n",
      "004707\n",
      "004708\n",
      "004710\n",
      "004714\n",
      "004715\n",
      "004718\n",
      "004719\n",
      "004722\n",
      "004723\n",
      "004727\n",
      "004732\n",
      "004735\n",
      "004737\n",
      "004742\n",
      "004743\n",
      "004746\n",
      "004747\n",
      "004748\n",
      "004750\n",
      "004753\n",
      "004754\n",
      "004760\n",
      "004761\n",
      "004768\n",
      "004770\n",
      "004773\n",
      "004776\n",
      "004777\n",
      "004779\n",
      "004782\n",
      "004783\n",
      "004785\n",
      "004786\n",
      "004788\n",
      "004789\n",
      "004790\n",
      "004792\n",
      "004793\n",
      "004794\n",
      "004796\n",
      "004797\n",
      "004799\n",
      "004801\n",
      "004805\n",
      "004808\n",
      "004812\n",
      "004814\n",
      "004815\n",
      "004816\n",
      "004818\n",
      "004823\n",
      "004825\n",
      "004826\n",
      "004828\n",
      "004830\n",
      "004831\n",
      "004832\n",
      "004834\n",
      "004836\n",
      "004837\n",
      "004839\n",
      "004840\n",
      "004841\n",
      "004842\n",
      "004846\n",
      "004848\n",
      "004849\n",
      "004850\n",
      "004852\n",
      "004856\n",
      "004857\n",
      "004859\n",
      "004863\n",
      "004866\n",
      "004867\n",
      "004868\n",
      "004869\n",
      "004872\n",
      "004873\n",
      "004876\n",
      "004878\n",
      "004879\n",
      "004882\n",
      "004885\n",
      "004886\n",
      "004890\n",
      "004895\n",
      "004896\n",
      "004897\n",
      "004898\n",
      "004902\n",
      "004903\n",
      "004905\n",
      "004907\n",
      "004910\n",
      "004911\n",
      "004912\n",
      "004913\n",
      "004916\n",
      "004926\n",
      "004928\n",
      "004929\n",
      "004931\n",
      "004935\n",
      "004936\n",
      "004938\n",
      "004939\n",
      "004943\n",
      "004946\n",
      "004948\n",
      "004950\n",
      "004951\n",
      "004953\n",
      "004954\n",
      "004955\n",
      "004956\n",
      "004958\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "004960\n",
      "004961\n",
      "004962\n",
      "004963\n",
      "004966\n",
      "004967\n",
      "004968\n",
      "004972\n",
      "004973\n",
      "004974\n",
      "004976\n",
      "004977\n",
      "004982\n",
      "004983\n",
      "004984\n",
      "004985\n",
      "004986\n",
      "004987\n",
      "004990\n",
      "004991\n",
      "004992\n",
      "004994\n",
      "004995\n",
      "004997\n",
      "004998\n",
      "004999\n",
      "005001\n",
      "005003\n",
      "005004\n",
      "005006\n",
      "005007\n",
      "005014\n",
      "005016\n",
      "005018\n",
      "005020\n",
      "005023\n",
      "005024\n",
      "005026\n",
      "005027\n",
      "005028\n",
      "005029\n",
      "005032\n",
      "005033\n",
      "005036\n",
      "005037\n",
      "005039\n",
      "005042\n",
      "005045\n",
      "005047\n",
      "005052\n",
      "005054\n",
      "005055\n",
      "005056\n",
      "005057\n",
      "005058\n",
      "005061\n",
      "005062\n",
      "005063\n",
      "005064\n",
      "005065\n",
      "005067\n",
      "005068\n",
      "005071\n",
      "005072\n",
      "005073\n",
      "005077\n",
      "005078\n",
      "005079\n",
      "005081\n",
      "005084\n",
      "005085\n",
      "005086\n",
      "005090\n",
      "005093\n",
      "005094\n",
      "005097\n",
      "005101\n",
      "005102\n",
      "005104\n",
      "005107\n",
      "005108\n",
      "005110\n",
      "005111\n",
      "005114\n",
      "005116\n",
      "005121\n",
      "005122\n",
      "005124\n",
      "005128\n",
      "005129\n",
      "005130\n",
      "005131\n",
      "005134\n",
      "005135\n",
      "005136\n",
      "005138\n",
      "005143\n",
      "005144\n",
      "005145\n",
      "005146\n",
      "005150\n",
      "005153\n",
      "005156\n",
      "005159\n",
      "005160\n",
      "005161\n",
      "005168\n",
      "005169\n",
      "005171\n",
      "005173\n",
      "005175\n",
      "005176\n",
      "005177\n",
      "005179\n",
      "005181\n",
      "005183\n",
      "005185\n",
      "005186\n",
      "005189\n",
      "005190\n",
      "005191\n",
      "005195\n",
      "005199\n",
      "005202\n",
      "005203\n",
      "005208\n",
      "005209\n",
      "005210\n",
      "005212\n",
      "005214\n",
      "005215\n",
      "005217\n",
      "005219\n",
      "005220\n",
      "005222\n",
      "005223\n",
      "005224\n",
      "005229\n",
      "005230\n",
      "005231\n",
      "005236\n",
      "005239\n",
      "005242\n",
      "005244\n",
      "005245\n",
      "005246\n",
      "005248\n",
      "005253\n",
      "005254\n",
      "005257\n",
      "005258\n",
      "005259\n",
      "005260\n",
      "005262\n",
      "005263\n",
      "005264\n",
      "005267\n",
      "005268\n",
      "005269\n",
      "005270\n",
      "005273\n",
      "005274\n",
      "005278\n",
      "005281\n",
      "005283\n",
      "005285\n",
      "005288\n",
      "005290\n",
      "005292\n",
      "005293\n",
      "005297\n",
      "005298\n",
      "005303\n",
      "005304\n",
      "005305\n",
      "005306\n",
      "005307\n",
      "005310\n",
      "005311\n",
      "005312\n",
      "005314\n",
      "005315\n",
      "005318\n",
      "005319\n",
      "005320\n",
      "005325\n",
      "005326\n",
      "005327\n",
      "005328\n",
      "005331\n",
      "005336\n",
      "005337\n",
      "005338\n",
      "005340\n",
      "005343\n",
      "005344\n",
      "005345\n",
      "005346\n",
      "005348\n",
      "005349\n",
      "005350\n",
      "005351\n",
      "005352\n",
      "005355\n",
      "005358\n",
      "005360\n",
      "005363\n",
      "005365\n",
      "005367\n",
      "005368\n",
      "005369\n",
      "005370\n",
      "005371\n",
      "005373\n",
      "005374\n",
      "005378\n",
      "005379\n",
      "005380\n",
      "005383\n",
      "005384\n",
      "005385\n",
      "005387\n",
      "005388\n",
      "005389\n",
      "005391\n",
      "005393\n",
      "005395\n",
      "005396\n",
      "005397\n",
      "005398\n",
      "005404\n",
      "005405\n",
      "005406\n",
      "005407\n",
      "005408\n",
      "005410\n",
      "005413\n",
      "005414\n",
      "005416\n",
      "005417\n",
      "005418\n",
      "005419\n",
      "005420\n",
      "005421\n",
      "005423\n",
      "005424\n",
      "005429\n",
      "005430\n",
      "005431\n",
      "005433\n",
      "005434\n",
      "005436\n",
      "005438\n",
      "005439\n",
      "005440\n",
      "005441\n",
      "005445\n",
      "005448\n",
      "005450\n",
      "005451\n",
      "005453\n",
      "005454\n",
      "005455\n",
      "005457\n",
      "005461\n",
      "005465\n",
      "005467\n",
      "005469\n",
      "005470\n",
      "005471\n",
      "005475\n",
      "005478\n",
      "005481\n",
      "005483\n",
      "005485\n",
      "005486\n",
      "005487\n",
      "005489\n",
      "005496\n",
      "005497\n",
      "005499\n",
      "005507\n",
      "005508\n",
      "005509\n",
      "005510\n",
      "005511\n",
      "005514\n",
      "005515\n",
      "005517\n",
      "005518\n",
      "005519\n",
      "005521\n",
      "005522\n",
      "005524\n",
      "005526\n",
      "005527\n",
      "005530\n",
      "005531\n",
      "005535\n",
      "005536\n",
      "005539\n",
      "005541\n",
      "005542\n",
      "005544\n",
      "005547\n",
      "005549\n",
      "005550\n",
      "005552\n",
      "005554\n",
      "005559\n",
      "005563\n",
      "005566\n",
      "005568\n",
      "005573\n",
      "005574\n",
      "005576\n",
      "005577\n",
      "005579\n",
      "005582\n",
      "005583\n",
      "005584\n",
      "005585\n",
      "005586\n",
      "005588\n",
      "005590\n",
      "005591\n",
      "005592\n",
      "005593\n",
      "005599\n",
      "005600\n",
      "005601\n",
      "005603\n",
      "005605\n",
      "005606\n",
      "005608\n",
      "005609\n",
      "005611\n",
      "005613\n",
      "005614\n",
      "005615\n",
      "005618\n",
      "005620\n",
      "005624\n",
      "005625\n",
      "005629\n",
      "005630\n",
      "005631\n",
      "005636\n",
      "005637\n",
      "005639\n",
      "005640\n",
      "005641\n",
      "005644\n",
      "005645\n",
      "005647\n",
      "005648\n",
      "005652\n",
      "005653\n",
      "005654\n",
      "005655\n",
      "005657\n",
      "005658\n",
      "005660\n",
      "005662\n",
      "005664\n",
      "005668\n",
      "005669\n",
      "005672\n",
      "005674\n",
      "005676\n",
      "005679\n",
      "005680\n",
      "005682\n",
      "005685\n",
      "005686\n",
      "005687\n",
      "005693\n",
      "005695\n",
      "005696\n",
      "005697\n",
      "005699\n",
      "005700\n",
      "005701\n",
      "005702\n",
      "005704\n",
      "005705\n",
      "005710\n",
      "005713\n",
      "005714\n",
      "005715\n",
      "005716\n",
      "005718\n",
      "005719\n",
      "005723\n",
      "005728\n",
      "005729\n",
      "005730\n",
      "005731\n",
      "005732\n",
      "005735\n",
      "005736\n",
      "005738\n",
      "005740\n",
      "005741\n",
      "005742\n",
      "005743\n",
      "005747\n",
      "005749\n",
      "005752\n",
      "005755\n",
      "005756\n",
      "005757\n",
      "005760\n",
      "005761\n",
      "005762\n",
      "005764\n",
      "005765\n",
      "005768\n",
      "005769\n",
      "005773\n",
      "005779\n",
      "005780\n",
      "005781\n",
      "005782\n",
      "005783\n",
      "005784\n",
      "005786\n",
      "005788\n",
      "005789\n",
      "005790\n",
      "005791\n",
      "005794\n",
      "005796\n",
      "005799\n",
      "005803\n",
      "005805\n",
      "005806\n",
      "005811\n",
      "005812\n",
      "005813\n",
      "005814\n",
      "005815\n",
      "005817\n",
      "005818\n",
      "005819\n",
      "005821\n",
      "005824\n",
      "005825\n",
      "005826\n",
      "005828\n",
      "005829\n",
      "005830\n",
      "005831\n",
      "005836\n",
      "005838\n",
      "005839\n",
      "005840\n",
      "005841\n",
      "005843\n",
      "005845\n",
      "005850\n",
      "005851\n",
      "005852\n",
      "005853\n",
      "005854\n",
      "005856\n",
      "005859\n",
      "005860\n",
      "005861\n",
      "005863\n",
      "005864\n",
      "005867\n",
      "005868\n",
      "005873\n",
      "005874\n",
      "005875\n",
      "005877\n",
      "005878\n",
      "005879\n",
      "005881\n",
      "005884\n",
      "005885\n",
      "005888\n",
      "005889\n",
      "005893\n",
      "005894\n",
      "005895\n",
      "005897\n",
      "005899\n",
      "005901\n",
      "005903\n",
      "005905\n",
      "005906\n",
      "005908\n",
      "005909\n",
      "005910\n",
      "005911\n",
      "005912\n",
      "005914\n",
      "005917\n",
      "005918\n",
      "005919\n",
      "005920\n",
      "005923\n",
      "005928\n",
      "005930\n",
      "005938\n",
      "005940\n",
      "005947\n",
      "005948\n",
      "005951\n",
      "005952\n",
      "005954\n",
      "005956\n",
      "005960\n",
      "005961\n",
      "005963\n",
      "005964\n",
      "005968\n",
      "005970\n",
      "005971\n",
      "005975\n",
      "005979\n",
      "005980\n",
      "005981\n",
      "005983\n",
      "005984\n",
      "005985\n",
      "005988\n",
      "005989\n",
      "005990\n",
      "005991\n",
      "005992\n",
      "005995\n",
      "005996\n",
      "005998\n",
      "006000\n",
      "006001\n",
      "006004\n",
      "006005\n",
      "006009\n",
      "006011\n",
      "006012\n",
      "006018\n",
      "006020\n",
      "006023\n",
      "006025\n",
      "006026\n",
      "006027\n",
      "006028\n",
      "006029\n",
      "006030\n",
      "006033\n",
      "006035\n",
      "006038\n",
      "006041\n",
      "006042\n",
      "006043\n",
      "006045\n",
      "006046\n",
      "006055\n",
      "006058\n",
      "006061\n",
      "006062\n",
      "006065\n",
      "006066\n",
      "006067\n",
      "006069\n",
      "006070\n",
      "006071\n",
      "006073\n",
      "006074\n",
      "006078\n",
      "006079\n",
      "006084\n",
      "006088\n",
      "006089\n",
      "006091\n",
      "006095\n",
      "006096\n",
      "006097\n",
      "006098\n",
      "006100\n",
      "006103\n",
      "006104\n",
      "006105\n",
      "006107\n",
      "006108\n",
      "006111\n",
      "006117\n",
      "006120\n",
      "006123\n",
      "006124\n",
      "006125\n",
      "006128\n",
      "006129\n",
      "006130\n",
      "006131\n",
      "006133\n",
      "006134\n",
      "006135\n",
      "006136\n",
      "006139\n",
      "006140\n",
      "006141\n",
      "006146\n",
      "006148\n",
      "006150\n",
      "006151\n",
      "006153\n",
      "006156\n",
      "006158\n",
      "006159\n",
      "006161\n",
      "006162\n",
      "006163\n",
      "006166\n",
      "006170\n",
      "006171\n",
      "006172\n",
      "006174\n",
      "006175\n",
      "006176\n",
      "006177\n",
      "006179\n",
      "006180\n",
      "006181\n",
      "006183\n",
      "006184\n",
      "006185\n",
      "006187\n",
      "006188\n",
      "006189\n",
      "006190\n",
      "006196\n",
      "006198\n",
      "006201\n",
      "006202\n",
      "006203\n",
      "006206\n",
      "006208\n",
      "006209\n",
      "006210\n",
      "006212\n",
      "006214\n",
      "006215\n",
      "006216\n",
      "006218\n",
      "006219\n",
      "006220\n",
      "006221\n",
      "006222\n",
      "006223\n",
      "006224\n",
      "006225\n",
      "006229\n",
      "006230\n",
      "006233\n",
      "006234\n",
      "006235\n",
      "006236\n",
      "006238\n",
      "006240\n",
      "006241\n",
      "006243\n",
      "006247\n",
      "006249\n",
      "006250\n",
      "006251\n",
      "006252\n",
      "006254\n",
      "006258\n",
      "006259\n",
      "006260\n",
      "006261\n",
      "006262\n",
      "006264\n",
      "006267\n",
      "006269\n",
      "006270\n",
      "006272\n",
      "006275\n",
      "006276\n",
      "006277\n",
      "006279\n",
      "006281\n",
      "006282\n",
      "006284\n",
      "006285\n",
      "006286\n",
      "006289\n",
      "006290\n",
      "006291\n",
      "006295\n",
      "006296\n",
      "006299\n",
      "006300\n",
      "006301\n",
      "006304\n",
      "006305\n",
      "006306\n",
      "006309\n",
      "006314\n",
      "006318\n",
      "006319\n",
      "006320\n",
      "006321\n",
      "006323\n",
      "006325\n",
      "006329\n",
      "006330\n",
      "006335\n",
      "006337\n",
      "006338\n",
      "006339\n",
      "006341\n",
      "006344\n",
      "006346\n",
      "006348\n",
      "006349\n",
      "006350\n",
      "006351\n",
      "006352\n",
      "006353\n",
      "006355\n",
      "006357\n",
      "006362\n",
      "006363\n",
      "006366\n",
      "006367\n",
      "006369\n",
      "006371\n",
      "006374\n",
      "006375\n",
      "006377\n",
      "006381\n",
      "006382\n",
      "006385\n",
      "006387\n",
      "006391\n",
      "006392\n",
      "006395\n",
      "006396\n",
      "006398\n",
      "006400\n",
      "006404\n",
      "006409\n",
      "006411\n",
      "006417\n",
      "006418\n",
      "006419\n",
      "006421\n",
      "006424\n",
      "006425\n",
      "006427\n",
      "006428\n",
      "006429\n",
      "006430\n",
      "006433\n",
      "006434\n",
      "006436\n",
      "006437\n",
      "006438\n",
      "006440\n",
      "006442\n",
      "006443\n",
      "006444\n",
      "006445\n",
      "006447\n",
      "006448\n",
      "006449\n",
      "006450\n",
      "006455\n",
      "006456\n",
      "006458\n",
      "006459\n",
      "006462\n",
      "006463\n",
      "006465\n",
      "006466\n",
      "006468\n",
      "006470\n",
      "006472\n",
      "006473\n",
      "006474\n",
      "006475\n",
      "006476\n",
      "006480\n",
      "006482\n",
      "006483\n",
      "006484\n",
      "006486\n",
      "006488\n",
      "006492\n",
      "006495\n",
      "006497\n",
      "006499\n",
      "006501\n",
      "006503\n",
      "006506\n",
      "006507\n",
      "006509\n",
      "006512\n",
      "006515\n",
      "006519\n",
      "006520\n",
      "006523\n",
      "006524\n",
      "006529\n",
      "006530\n",
      "006532\n",
      "006534\n",
      "006536\n",
      "006538\n",
      "006542\n",
      "006543\n",
      "006547\n",
      "006548\n",
      "006549\n",
      "006550\n",
      "006551\n",
      "006553\n",
      "006556\n",
      "006560\n",
      "006562\n",
      "006564\n",
      "006565\n",
      "006569\n",
      "006570\n",
      "006572\n",
      "006575\n",
      "006576\n",
      "006578\n",
      "006583\n",
      "006584\n",
      "006585\n",
      "006587\n",
      "006588\n",
      "006593\n",
      "006595\n",
      "006597\n",
      "006599\n",
      "006602\n",
      "006603\n",
      "006605\n",
      "006606\n",
      "006609\n",
      "006610\n",
      "006611\n",
      "006612\n",
      "006617\n",
      "006618\n",
      "006619\n",
      "006621\n",
      "006622\n",
      "006625\n",
      "006626\n",
      "006627\n",
      "006628\n",
      "006631\n",
      "006632\n",
      "006635\n",
      "006636\n",
      "006637\n",
      "006638\n",
      "006643\n",
      "006645\n",
      "006647\n",
      "006648\n",
      "006652\n",
      "006654\n",
      "006657\n",
      "006658\n",
      "006660\n",
      "006661\n",
      "006664\n",
      "006666\n",
      "006667\n",
      "006668\n",
      "006670\n",
      "006671\n",
      "006673\n",
      "006674\n",
      "006677\n",
      "006678\n",
      "006679\n",
      "006681\n",
      "006682\n",
      "006684\n",
      "006687\n",
      "006689\n",
      "006690\n",
      "006694\n",
      "006695\n",
      "006696\n",
      "006697\n",
      "006698\n",
      "006699\n",
      "006702\n",
      "006703\n",
      "006704\n",
      "006706\n",
      "006707\n",
      "006708\n",
      "006709\n",
      "006714\n",
      "006718\n",
      "006719\n",
      "006722\n",
      "006725\n",
      "006726\n",
      "006727\n",
      "006730\n",
      "006731\n",
      "006734\n",
      "006735\n",
      "006736\n",
      "006738\n",
      "006739\n",
      "006740\n",
      "006747\n",
      "006748\n",
      "006751\n",
      "006753\n",
      "006755\n",
      "006759\n",
      "006760\n",
      "006761\n",
      "006762\n",
      "006765\n",
      "006766\n",
      "006768\n",
      "006769\n",
      "006772\n",
      "006773\n",
      "006777\n",
      "006781\n",
      "006782\n",
      "006783\n",
      "006784\n",
      "006786\n",
      "006789\n",
      "006794\n",
      "006797\n",
      "006799\n",
      "006800\n",
      "006802\n",
      "006803\n",
      "006805\n",
      "006806\n",
      "006808\n",
      "006810\n",
      "006813\n",
      "006814\n",
      "006819\n",
      "006821\n",
      "006822\n",
      "006824\n",
      "006825\n",
      "006827\n",
      "006828\n",
      "006829\n",
      "006833\n",
      "006835\n",
      "006836\n",
      "006838\n",
      "006839\n",
      "006840\n",
      "006841\n",
      "006842\n",
      "006844\n",
      "006845\n",
      "006847\n",
      "006848\n",
      "006849\n",
      "006850\n",
      "006852\n",
      "006855\n",
      "006858\n",
      "006859\n",
      "006860\n",
      "006862\n",
      "006864\n",
      "006865\n",
      "006866\n",
      "006867\n",
      "006868\n",
      "006869\n",
      "006874\n",
      "006876\n",
      "006878\n",
      "006880\n",
      "006883\n",
      "006884\n",
      "006886\n",
      "006887\n",
      "006892\n",
      "006893\n",
      "006896\n",
      "006899\n",
      "006900\n",
      "006903\n",
      "006908\n",
      "006909\n",
      "006910\n",
      "006911\n",
      "006912\n",
      "006914\n",
      "006916\n",
      "006917\n",
      "006918\n",
      "006919\n",
      "006922\n",
      "006924\n",
      "006930\n",
      "006931\n",
      "006932\n",
      "006933\n",
      "006934\n",
      "006935\n",
      "006939\n",
      "006940\n",
      "006943\n",
      "006944\n",
      "006945\n",
      "006947\n",
      "006948\n",
      "006949\n",
      "006950\n",
      "006952\n",
      "006953\n",
      "006956\n",
      "006958\n",
      "006959\n",
      "006962\n",
      "006963\n",
      "006965\n",
      "006966\n",
      "006968\n",
      "006971\n",
      "006972\n",
      "006976\n",
      "006981\n",
      "006983\n",
      "006987\n",
      "006988\n",
      "006989\n",
      "006990\n",
      "006994\n",
      "006995\n",
      "007002\n",
      "007003\n",
      "007004\n",
      "007006\n",
      "007007\n",
      "007008\n",
      "007009\n",
      "007011\n",
      "007016\n",
      "007018\n",
      "007020\n",
      "007021\n",
      "007022\n",
      "007023\n",
      "007025\n",
      "007029\n",
      "007031\n",
      "007033\n",
      "007035\n",
      "007036\n",
      "007038\n",
      "007039\n",
      "007040\n",
      "007042\n",
      "007045\n",
      "007046\n",
      "007048\n",
      "007049\n",
      "007050\n",
      "007052\n",
      "007054\n",
      "007056\n",
      "007058\n",
      "007059\n",
      "007062\n",
      "007064\n",
      "007065\n",
      "007068\n",
      "007070\n",
      "007071\n",
      "007072\n",
      "007073\n",
      "007074\n",
      "007075\n",
      "007077\n",
      "007078\n",
      "007079\n",
      "007080\n",
      "007084\n",
      "007086\n",
      "007088\n",
      "007089\n",
      "007090\n",
      "007092\n",
      "007093\n",
      "007095\n",
      "007097\n",
      "007100\n",
      "007101\n",
      "007104\n",
      "007105\n",
      "007108\n",
      "007109\n",
      "007113\n",
      "007114\n",
      "007117\n",
      "007121\n",
      "007122\n",
      "007123\n",
      "007125\n",
      "007128\n",
      "007129\n",
      "007130\n",
      "007132\n",
      "007133\n",
      "007138\n",
      "007139\n",
      "007140\n",
      "007141\n",
      "007144\n",
      "007146\n",
      "007147\n",
      "007148\n",
      "007149\n",
      "007150\n",
      "007152\n",
      "007153\n",
      "007154\n",
      "007159\n",
      "007162\n",
      "007163\n",
      "007165\n",
      "007166\n",
      "007167\n",
      "007168\n",
      "007172\n",
      "007174\n",
      "007177\n",
      "007180\n",
      "007182\n",
      "007184\n",
      "007185\n",
      "007187\n",
      "007189\n",
      "007191\n",
      "007193\n",
      "007194\n",
      "007197\n",
      "007200\n",
      "007204\n",
      "007205\n",
      "007208\n",
      "007210\n",
      "007211\n",
      "007212\n",
      "007213\n",
      "007214\n",
      "007215\n",
      "007216\n",
      "007217\n",
      "007219\n",
      "007222\n",
      "007223\n",
      "007224\n",
      "007227\n",
      "007230\n",
      "007234\n",
      "007236\n",
      "007241\n",
      "007243\n",
      "007244\n",
      "007245\n",
      "007247\n",
      "007249\n",
      "007250\n",
      "007256\n",
      "007258\n",
      "007259\n",
      "007260\n",
      "007261\n",
      "007263\n",
      "007266\n",
      "007270\n",
      "007271\n",
      "007274\n",
      "007275\n",
      "007276\n",
      "007279\n",
      "007280\n",
      "007283\n",
      "007284\n",
      "007285\n",
      "007289\n",
      "007292\n",
      "007294\n",
      "007295\n",
      "007296\n",
      "007297\n",
      "007298\n",
      "007299\n",
      "007300\n",
      "007302\n",
      "007305\n",
      "007308\n",
      "007311\n",
      "007314\n",
      "007318\n",
      "007322\n",
      "007323\n",
      "007325\n",
      "007327\n",
      "007329\n",
      "007330\n",
      "007334\n",
      "007336\n",
      "007343\n",
      "007344\n",
      "007346\n",
      "007350\n",
      "007351\n",
      "007356\n",
      "007359\n",
      "007361\n",
      "007363\n",
      "007365\n",
      "007369\n",
      "007370\n",
      "007372\n",
      "007373\n",
      "007374\n",
      "007375\n",
      "007376\n",
      "007381\n",
      "007383\n",
      "007385\n",
      "007388\n",
      "007389\n",
      "007390\n",
      "007394\n",
      "007396\n",
      "007398\n",
      "007408\n",
      "007410\n",
      "007411\n",
      "007413\n",
      "007414\n",
      "007416\n",
      "007417\n",
      "007419\n",
      "007421\n",
      "007422\n",
      "007424\n",
      "007425\n",
      "007427\n",
      "007431\n",
      "007432\n",
      "007433\n",
      "007435\n",
      "007436\n",
      "007437\n",
      "007438\n",
      "007439\n",
      "007443\n",
      "007445\n",
      "007446\n",
      "007448\n",
      "007449\n",
      "007451\n",
      "007454\n",
      "007457\n",
      "007458\n",
      "007460\n",
      "007461\n",
      "007465\n",
      "007466\n",
      "007467\n",
      "007468\n",
      "007470\n",
      "007474\n",
      "007475\n",
      "007477\n",
      "007479\n",
      "007480\n",
      "007481\n",
      "007482\n",
      "007483\n",
      "007484\n",
      "007486\n",
      "007489\n",
      "007490\n",
      "007491\n",
      "007493\n",
      "007497\n",
      "007498\n",
      "007503\n",
      "007506\n",
      "007511\n",
      "007513\n",
      "007517\n",
      "007519\n",
      "007521\n",
      "007523\n",
      "007524\n",
      "007525\n",
      "007526\n",
      "007527\n",
      "007528\n",
      "007530\n",
      "007533\n",
      "007535\n",
      "007536\n",
      "007537\n",
      "007538\n",
      "007540\n",
      "007543\n",
      "007544\n",
      "007546\n",
      "007547\n",
      "007551\n",
      "007555\n",
      "007558\n",
      "007559\n",
      "007563\n",
      "007565\n",
      "007566\n",
      "007568\n",
      "007570\n",
      "007571\n",
      "007572\n",
      "007575\n",
      "007576\n",
      "007578\n",
      "007579\n",
      "007585\n",
      "007586\n",
      "007590\n",
      "007592\n",
      "007594\n",
      "007600\n",
      "007601\n",
      "007603\n",
      "007605\n",
      "007606\n",
      "007611\n",
      "007612\n",
      "007614\n",
      "007615\n",
      "007618\n",
      "007619\n",
      "007621\n",
      "007622\n",
      "007624\n",
      "007626\n",
      "007629\n",
      "007631\n",
      "007633\n",
      "007637\n",
      "007639\n",
      "007640\n",
      "007642\n",
      "007647\n",
      "007649\n",
      "007650\n",
      "007653\n",
      "007654\n",
      "007655\n",
      "007656\n",
      "007657\n",
      "007662\n",
      "007663\n",
      "007664\n",
      "007666\n",
      "007667\n",
      "007668\n",
      "007670\n",
      "007671\n",
      "007672\n",
      "007673\n",
      "007675\n",
      "007677\n",
      "007678\n",
      "007679\n",
      "007680\n",
      "007682\n",
      "007683\n",
      "007685\n",
      "007687\n",
      "007688\n",
      "007691\n",
      "007692\n",
      "007694\n",
      "007696\n",
      "007697\n",
      "007699\n",
      "007702\n",
      "007704\n",
      "007705\n",
      "007709\n",
      "007712\n",
      "007713\n",
      "007715\n",
      "007718\n",
      "007720\n",
      "007721\n",
      "007723\n",
      "007724\n",
      "007727\n",
      "007729\n",
      "007731\n",
      "007732\n",
      "007735\n",
      "007736\n",
      "007740\n",
      "007742\n",
      "007743\n",
      "007745\n",
      "007746\n",
      "007748\n",
      "007749\n",
      "007751\n",
      "007753\n",
      "007754\n",
      "007758\n",
      "007760\n",
      "007762\n",
      "007763\n",
      "007765\n",
      "007767\n",
      "007768\n",
      "007772\n",
      "007773\n",
      "007775\n",
      "007776\n",
      "007777\n",
      "007779\n",
      "007781\n",
      "007786\n",
      "007790\n",
      "007791\n",
      "007793\n",
      "007795\n",
      "007798\n",
      "007799\n",
      "007803\n",
      "007809\n",
      "007810\n",
      "007812\n",
      "007813\n",
      "007814\n",
      "007815\n",
      "007819\n",
      "007820\n",
      "007821\n",
      "007824\n",
      "007826\n",
      "007831\n",
      "007833\n",
      "007834\n",
      "007836\n",
      "007838\n",
      "007840\n",
      "007841\n",
      "007843\n",
      "007845\n",
      "007847\n",
      "007853\n",
      "007854\n",
      "007855\n",
      "007856\n",
      "007857\n",
      "007859\n",
      "007863\n",
      "007864\n",
      "007865\n",
      "007868\n",
      "007869\n",
      "007872\n",
      "007873\n",
      "007876\n",
      "007877\n",
      "007878\n",
      "007883\n",
      "007884\n",
      "007885\n",
      "007886\n",
      "007889\n",
      "007890\n",
      "007897\n",
      "007898\n",
      "007899\n",
      "007900\n",
      "007901\n",
      "007902\n",
      "007905\n",
      "007908\n",
      "007909\n",
      "007910\n",
      "007911\n",
      "007914\n",
      "007915\n",
      "007916\n",
      "007919\n",
      "007920\n",
      "007921\n",
      "007923\n",
      "007924\n",
      "007925\n",
      "007926\n",
      "007928\n",
      "007931\n",
      "007932\n",
      "007933\n",
      "007935\n",
      "007939\n",
      "007940\n",
      "007943\n",
      "007946\n",
      "007947\n",
      "007950\n",
      "007953\n",
      "007954\n",
      "007956\n",
      "007958\n",
      "007959\n",
      "007963\n",
      "007964\n",
      "007968\n",
      "007970\n",
      "007971\n",
      "007974\n",
      "007976\n",
      "007979\n",
      "007980\n",
      "007984\n",
      "007987\n",
      "007991\n",
      "007996\n",
      "007997\n",
      "007998\n",
      "007999\n",
      "008001\n",
      "008002\n",
      "008004\n",
      "008005\n",
      "008008\n",
      "008009\n",
      "008012\n",
      "008017\n",
      "008019\n",
      "008023\n",
      "008024\n",
      "008026\n",
      "008029\n",
      "008031\n",
      "008032\n",
      "008033\n",
      "008036\n",
      "008037\n",
      "008040\n",
      "008042\n",
      "008043\n",
      "008044\n",
      "008048\n",
      "008049\n",
      "008051\n",
      "008053\n",
      "008057\n",
      "008060\n",
      "008061\n",
      "008062\n",
      "008063\n",
      "008064\n",
      "008067\n",
      "008068\n",
      "008069\n",
      "008072\n",
      "008075\n",
      "008076\n",
      "008079\n",
      "008082\n",
      "008083\n",
      "008084\n",
      "008085\n",
      "008086\n",
      "008087\n",
      "008091\n",
      "008093\n",
      "008095\n",
      "008096\n",
      "008098\n",
      "008100\n",
      "008101\n",
      "008103\n",
      "008105\n",
      "008106\n",
      "008107\n",
      "008108\n",
      "008112\n",
      "008115\n",
      "008116\n",
      "008117\n",
      "008121\n",
      "008122\n",
      "008125\n",
      "008127\n",
      "008130\n",
      "008132\n",
      "008137\n",
      "008138\n",
      "008139\n",
      "008140\n",
      "008141\n",
      "008142\n",
      "008144\n",
      "008150\n",
      "008151\n",
      "008159\n",
      "008160\n",
      "008163\n",
      "008164\n",
      "008166\n",
      "008168\n",
      "008169\n",
      "008171\n",
      "008173\n",
      "008174\n",
      "008175\n",
      "008177\n",
      "008180\n",
      "008186\n",
      "008188\n",
      "008189\n",
      "008190\n",
      "008191\n",
      "008197\n",
      "008199\n",
      "008200\n",
      "008202\n",
      "008203\n",
      "008204\n",
      "008208\n",
      "008209\n",
      "008211\n",
      "008213\n",
      "008216\n",
      "008218\n",
      "008220\n",
      "008222\n",
      "008223\n",
      "008224\n",
      "008225\n",
      "008226\n",
      "008229\n",
      "008232\n",
      "008235\n",
      "008236\n",
      "008241\n",
      "008244\n",
      "008248\n",
      "008250\n",
      "008251\n",
      "008252\n",
      "008253\n",
      "008254\n",
      "008258\n",
      "008260\n",
      "008261\n",
      "008262\n",
      "008263\n",
      "008268\n",
      "008269\n",
      "008272\n",
      "008275\n",
      "008279\n",
      "008280\n",
      "008281\n",
      "008282\n",
      "008284\n",
      "008285\n",
      "008292\n",
      "008293\n",
      "008294\n",
      "008295\n",
      "008296\n",
      "008297\n",
      "008299\n",
      "008300\n",
      "008301\n",
      "008302\n",
      "008306\n",
      "008307\n",
      "008310\n",
      "008311\n",
      "008312\n",
      "008313\n",
      "008315\n",
      "008316\n",
      "008317\n",
      "008318\n",
      "008319\n",
      "008320\n",
      "008322\n",
      "008323\n",
      "008326\n",
      "008327\n",
      "008329\n",
      "008332\n",
      "008335\n",
      "008336\n",
      "008338\n",
      "008341\n",
      "008342\n",
      "008345\n",
      "008346\n",
      "008349\n",
      "008351\n",
      "008355\n",
      "008359\n",
      "008360\n",
      "008364\n",
      "008365\n",
      "008368\n",
      "008370\n",
      "008372\n",
      "008374\n",
      "008376\n",
      "008381\n",
      "008384\n",
      "008385\n",
      "008386\n",
      "008387\n",
      "008388\n",
      "008390\n",
      "008391\n",
      "008397\n",
      "008398\n",
      "008403\n",
      "008409\n",
      "008410\n",
      "008413\n",
      "008415\n",
      "008416\n",
      "008422\n",
      "008423\n",
      "008424\n",
      "008425\n",
      "008426\n",
      "008427\n",
      "008429\n",
      "008430\n",
      "008433\n",
      "008434\n",
      "008437\n",
      "008438\n",
      "008442\n",
      "008443\n",
      "008444\n",
      "008445\n",
      "008449\n",
      "008450\n",
      "008452\n",
      "008453\n",
      "008454\n",
      "008456\n",
      "008461\n",
      "008462\n",
      "008465\n",
      "008466\n",
      "008467\n",
      "008468\n",
      "008470\n",
      "008472\n",
      "008475\n",
      "008477\n",
      "008478\n",
      "008482\n",
      "008483\n",
      "008484\n",
      "008485\n",
      "008492\n",
      "008494\n",
      "008495\n",
      "008498\n",
      "008499\n",
      "008502\n",
      "008503\n",
      "008506\n",
      "008509\n",
      "008512\n",
      "008513\n",
      "008514\n",
      "008517\n",
      "008518\n",
      "008519\n",
      "008521\n",
      "008522\n",
      "008523\n",
      "008524\n",
      "008526\n",
      "008529\n",
      "008530\n",
      "008533\n",
      "008534\n",
      "008535\n",
      "008536\n",
      "008541\n",
      "008542\n",
      "008549\n",
      "008550\n",
      "008553\n",
      "008556\n",
      "008557\n",
      "008558\n",
      "008559\n",
      "008562\n",
      "008564\n",
      "008568\n",
      "008572\n",
      "008573\n",
      "008576\n",
      "008581\n",
      "008582\n",
      "008584\n",
      "008585\n",
      "008586\n",
      "008587\n",
      "008588\n",
      "008592\n",
      "008595\n",
      "008596\n",
      "008601\n",
      "008602\n",
      "008604\n",
      "008606\n",
      "008607\n",
      "008608\n",
      "008610\n",
      "008612\n",
      "008615\n",
      "008617\n",
      "008618\n",
      "008620\n",
      "008621\n",
      "008624\n",
      "008628\n",
      "008633\n",
      "008635\n",
      "008636\n",
      "008638\n",
      "008639\n",
      "008644\n",
      "008645\n",
      "008647\n",
      "008653\n",
      "008654\n",
      "008655\n",
      "008663\n",
      "008665\n",
      "008667\n",
      "008670\n",
      "008676\n",
      "008680\n",
      "008683\n",
      "008687\n",
      "008688\n",
      "008690\n",
      "008691\n",
      "008692\n",
      "008695\n",
      "008698\n",
      "008699\n",
      "008701\n",
      "008702\n",
      "008706\n",
      "008709\n",
      "008710\n",
      "008713\n",
      "008716\n",
      "008717\n",
      "008718\n",
      "008720\n",
      "008722\n",
      "008723\n",
      "008725\n",
      "008727\n",
      "008728\n",
      "008730\n",
      "008731\n",
      "008732\n",
      "008733\n",
      "008738\n",
      "008739\n",
      "008741\n",
      "008742\n",
      "008744\n",
      "008747\n",
      "008748\n",
      "008749\n",
      "008750\n",
      "008752\n",
      "008753\n",
      "008755\n",
      "008756\n",
      "008757\n",
      "008759\n",
      "008760\n",
      "008764\n",
      "008766\n",
      "008768\n",
      "008769\n",
      "008770\n",
      "008771\n",
      "008772\n",
      "008773\n",
      "008775\n",
      "008776\n",
      "008783\n",
      "008784\n",
      "008790\n",
      "008793\n",
      "008794\n",
      "008796\n",
      "008799\n",
      "008801\n",
      "008805\n",
      "008806\n",
      "008809\n",
      "008810\n",
      "008811\n",
      "008813\n",
      "008814\n",
      "008815\n",
      "008817\n",
      "008819\n",
      "008822\n",
      "008823\n",
      "008826\n",
      "008831\n",
      "008833\n",
      "008835\n",
      "008836\n",
      "008837\n",
      "008838\n",
      "008840\n",
      "008841\n",
      "008843\n",
      "008847\n",
      "008848\n",
      "008849\n",
      "008854\n",
      "008856\n",
      "008858\n",
      "008859\n",
      "008862\n",
      "008865\n",
      "008867\n",
      "008871\n",
      "008872\n",
      "008873\n",
      "008874\n",
      "008876\n",
      "008878\n",
      "008879\n",
      "008880\n",
      "008883\n",
      "008884\n",
      "008885\n",
      "008886\n",
      "008888\n",
      "008890\n",
      "008891\n",
      "008892\n",
      "008900\n",
      "008905\n",
      "008909\n",
      "008911\n",
      "008913\n",
      "008914\n",
      "008917\n",
      "008919\n",
      "008920\n",
      "008921\n",
      "008923\n",
      "008926\n",
      "008927\n",
      "008929\n",
      "008930\n",
      "008931\n",
      "008932\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "008933\n",
      "008936\n",
      "008939\n",
      "008940\n",
      "008942\n",
      "008943\n",
      "008944\n",
      "008948\n",
      "008951\n",
      "008953\n",
      "008955\n",
      "008958\n",
      "008960\n",
      "008961\n",
      "008962\n",
      "008965\n",
      "008966\n",
      "008967\n",
      "008968\n",
      "008969\n",
      "008970\n",
      "008971\n",
      "008973\n",
      "008975\n",
      "008976\n",
      "008978\n",
      "008979\n",
      "008980\n",
      "008982\n",
      "008983\n",
      "008985\n",
      "008987\n",
      "008988\n",
      "008989\n",
      "008995\n",
      "008997\n",
      "008999\n",
      "009000\n",
      "009002\n",
      "009004\n",
      "009005\n",
      "009006\n",
      "009007\n",
      "009015\n",
      "009016\n",
      "009018\n",
      "009019\n",
      "009020\n",
      "009022\n",
      "009024\n",
      "009027\n",
      "009029\n",
      "009032\n",
      "009034\n",
      "009035\n",
      "009036\n",
      "009037\n",
      "009039\n",
      "009042\n",
      "009045\n",
      "009048\n",
      "009049\n",
      "009051\n",
      "009053\n",
      "009058\n",
      "009059\n",
      "009060\n",
      "009063\n",
      "009064\n",
      "009066\n",
      "009068\n",
      "009072\n",
      "009073\n",
      "009078\n",
      "009079\n",
      "009080\n",
      "009085\n",
      "009086\n",
      "009087\n",
      "009089\n",
      "009091\n",
      "009094\n",
      "009098\n",
      "009099\n",
      "009100\n",
      "009105\n",
      "009106\n",
      "009108\n",
      "009112\n",
      "009113\n",
      "009114\n",
      "009116\n",
      "009117\n",
      "009121\n",
      "009123\n",
      "009126\n",
      "009128\n",
      "009129\n",
      "009131\n",
      "009133\n",
      "009136\n",
      "009138\n",
      "009141\n",
      "009144\n",
      "009147\n",
      "009148\n",
      "009150\n",
      "009151\n",
      "009153\n",
      "009155\n",
      "009157\n",
      "009159\n",
      "009160\n",
      "009161\n",
      "009162\n",
      "009163\n",
      "009166\n",
      "009168\n",
      "009173\n",
      "009174\n",
      "009175\n",
      "009177\n",
      "009178\n",
      "009179\n",
      "009180\n",
      "009181\n",
      "009184\n",
      "009185\n",
      "009186\n",
      "009187\n",
      "009189\n",
      "009191\n",
      "009192\n",
      "009193\n",
      "009194\n",
      "009195\n",
      "009196\n",
      "009197\n",
      "009200\n",
      "009202\n",
      "009205\n",
      "009208\n",
      "009209\n",
      "009212\n",
      "009213\n",
      "009214\n",
      "009215\n",
      "009218\n",
      "009221\n",
      "009224\n",
      "009227\n",
      "009230\n",
      "009236\n",
      "009238\n",
      "009239\n",
      "009242\n",
      "009244\n",
      "009245\n",
      "009246\n",
      "009247\n",
      "009249\n",
      "009250\n",
      "009251\n",
      "009252\n",
      "009254\n",
      "009255\n",
      "009259\n",
      "009268\n",
      "009269\n",
      "009270\n",
      "009271\n",
      "009272\n",
      "009273\n",
      "009278\n",
      "009279\n",
      "009281\n",
      "009282\n",
      "009283\n",
      "009285\n",
      "009286\n",
      "009287\n",
      "009288\n",
      "009289\n",
      "009290\n",
      "009291\n",
      "009295\n",
      "009296\n",
      "009299\n",
      "009303\n",
      "009306\n",
      "009307\n",
      "009308\n",
      "009309\n",
      "009312\n",
      "009315\n",
      "009316\n",
      "009318\n",
      "009323\n",
      "009324\n",
      "009325\n",
      "009326\n",
      "009327\n",
      "009330\n",
      "009331\n",
      "009333\n",
      "009334\n",
      "009336\n",
      "009337\n",
      "009339\n",
      "009342\n",
      "009343\n",
      "009347\n",
      "009348\n",
      "009349\n",
      "009350\n",
      "009351\n",
      "009354\n",
      "009358\n",
      "009359\n",
      "009362\n",
      "009365\n",
      "009368\n",
      "009371\n",
      "009373\n",
      "009374\n",
      "009375\n",
      "009377\n",
      "009378\n",
      "009382\n",
      "009386\n",
      "009388\n",
      "009389\n",
      "009392\n",
      "009393\n",
      "009394\n",
      "009398\n",
      "009401\n",
      "009405\n",
      "009406\n",
      "009407\n",
      "009408\n",
      "009409\n",
      "009410\n",
      "009411\n",
      "009412\n",
      "009413\n",
      "009414\n",
      "009417\n",
      "009418\n",
      "009419\n",
      "009420\n",
      "009421\n",
      "009422\n",
      "009424\n",
      "009429\n",
      "009432\n",
      "009433\n",
      "009434\n",
      "009437\n",
      "009438\n",
      "009439\n",
      "009440\n",
      "009443\n",
      "009445\n",
      "009446\n",
      "009448\n",
      "009454\n",
      "009455\n",
      "009456\n",
      "009457\n",
      "009458\n",
      "009459\n",
      "009460\n",
      "009461\n",
      "009463\n",
      "009464\n",
      "009465\n",
      "009466\n",
      "009468\n",
      "009469\n",
      "009470\n",
      "009472\n",
      "009476\n",
      "009477\n",
      "009479\n",
      "009480\n",
      "009481\n",
      "009484\n",
      "009488\n",
      "009490\n",
      "009491\n",
      "009494\n",
      "009496\n",
      "009497\n",
      "009499\n",
      "009500\n",
      "009502\n",
      "009504\n",
      "009507\n",
      "009508\n",
      "009512\n",
      "009515\n",
      "009516\n",
      "009517\n",
      "009518\n",
      "009519\n",
      "009520\n",
      "009523\n",
      "009524\n",
      "009526\n",
      "009527\n",
      "009528\n",
      "009531\n",
      "009532\n",
      "009533\n",
      "009537\n",
      "009540\n",
      "009541\n",
      "009542\n",
      "009543\n",
      "009545\n",
      "009546\n",
      "009549\n",
      "009550\n",
      "009551\n",
      "009557\n",
      "009558\n",
      "009560\n",
      "009562\n",
      "009565\n",
      "009566\n",
      "009567\n",
      "009568\n",
      "009571\n",
      "009573\n",
      "009576\n",
      "009577\n",
      "009579\n",
      "009580\n",
      "009584\n",
      "009585\n",
      "009586\n",
      "009587\n",
      "009588\n",
      "009591\n",
      "009596\n",
      "009597\n",
      "009598\n",
      "009600\n",
      "009603\n",
      "009605\n",
      "009609\n",
      "009611\n",
      "009613\n",
      "009614\n",
      "009615\n",
      "009617\n",
      "009618\n",
      "009619\n",
      "009620\n",
      "009621\n",
      "009623\n",
      "009627\n",
      "009629\n",
      "009634\n",
      "009636\n",
      "009637\n",
      "009638\n",
      "009641\n",
      "009644\n",
      "009647\n",
      "009649\n",
      "009650\n",
      "009654\n",
      "009655\n",
      "009656\n",
      "009658\n",
      "009659\n",
      "009664\n",
      "009666\n",
      "009667\n",
      "009668\n",
      "009670\n",
      "009671\n",
      "009676\n",
      "009678\n",
      "009679\n",
      "009681\n",
      "009684\n",
      "009685\n",
      "009686\n",
      "009687\n",
      "009691\n",
      "009692\n",
      "009693\n",
      "009695\n",
      "009698\n",
      "009699\n",
      "009700\n",
      "009702\n",
      "009703\n",
      "009706\n",
      "009707\n",
      "009709\n",
      "009710\n",
      "009711\n",
      "009712\n",
      "009713\n",
      "009717\n",
      "009718\n",
      "009719\n",
      "009721\n",
      "009724\n",
      "009726\n",
      "009729\n",
      "009732\n",
      "009733\n",
      "009734\n",
      "009735\n",
      "009737\n",
      "009738\n",
      "009743\n",
      "009745\n",
      "009746\n",
      "009747\n",
      "009748\n",
      "009749\n",
      "009754\n",
      "009755\n",
      "009756\n",
      "009758\n",
      "009761\n",
      "009762\n",
      "009763\n",
      "009764\n",
      "009767\n",
      "009772\n",
      "009773\n",
      "009774\n",
      "009776\n",
      "009778\n",
      "009780\n",
      "009781\n",
      "009785\n",
      "009789\n",
      "009790\n",
      "009792\n",
      "009794\n",
      "009796\n",
      "009797\n",
      "009800\n",
      "009801\n",
      "009805\n",
      "009807\n",
      "009808\n",
      "009809\n",
      "009810\n",
      "009813\n",
      "009816\n",
      "009819\n",
      "009822\n",
      "009823\n",
      "009825\n",
      "009828\n",
      "009830\n",
      "009831\n",
      "009832\n",
      "009833\n",
      "009834\n",
      "009836\n",
      "009839\n",
      "009841\n",
      "009842\n",
      "009845\n",
      "009848\n",
      "009851\n",
      "009852\n",
      "009855\n",
      "009858\n",
      "009859\n",
      "009860\n",
      "009862\n",
      "009863\n",
      "009865\n",
      "009867\n",
      "009868\n",
      "009869\n",
      "009870\n",
      "009872\n",
      "009874\n",
      "009877\n",
      "009878\n",
      "009879\n",
      "009880\n",
      "009881\n",
      "009882\n",
      "009884\n",
      "009886\n",
      "009887\n",
      "009894\n",
      "009896\n",
      "009897\n",
      "009898\n",
      "009900\n",
      "009902\n",
      "009904\n",
      "009905\n",
      "009908\n",
      "009911\n",
      "009913\n",
      "009917\n",
      "009918\n",
      "009920\n",
      "009923\n",
      "009926\n",
      "009932\n",
      "009935\n",
      "009938\n",
      "009939\n",
      "009940\n",
      "009942\n",
      "009944\n",
      "009945\n",
      "009946\n",
      "009947\n",
      "009949\n",
      "009950\n",
      "009954\n",
      "009955\n",
      "009958\n",
      "009959\n",
      "009961\n"
     ]
    }
   ],
   "source": [
    "# load files\n",
    "vocpath = os.path.join(\"..\", \"VOCdevkit\", \"VOC2007\")\n",
    "train_img_list, train_anno_list, val_img_list, val_anno_list = make_datapath_list(vocpath)\n",
    "\n",
    "# make Dataset\n",
    "voc_classes = ['aeroplane', 'bicycle', 'bird', 'boat',\n",
    "               'bottle', 'bus', 'car', 'cat', 'chair',\n",
    "               'cow', 'diningtable', 'dog', 'horse',\n",
    "               'motorbike', 'person', 'pottedplant',\n",
    "               'sheep', 'sofa', 'train', 'tvmonitor']\n",
    "color_mean = (104, 117, 123)  # (BGR)の色の平均値\n",
    "input_size = 128  # 画像のinputサイズを300×300にする\n",
    "\n",
    "## DatasetTransformを適応\n",
    "transform = DatasetTransform(input_size, color_mean)\n",
    "transform_anno = Anno_xml2list(voc_classes)\n",
    "\n",
    "train_dataset = VOCDataset(train_img_list, train_anno_list, phase = \"train\", transform=transform, transform_anno = transform_anno)\n",
    "val_dataset = VOCDataset(val_img_list, val_anno_list, phase=\"val\", transform=DatasetTransform(\n",
    "    input_size, color_mean), transform_anno=Anno_xml2list(voc_classes))\n",
    "\n",
    "batch_size = 64\n",
    "\n",
    "train_dataloader = data.DataLoader(\n",
    "    train_dataset, batch_size=batch_size, shuffle=True, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "val_dataloader = data.DataLoader(\n",
    "    val_dataset, batch_size=batch_size, shuffle=False, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "dataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 3, 128, 128])\n",
      "64\n",
      "torch.Size([1, 5])\n"
     ]
    }
   ],
   "source": [
    "# check operation\n",
    "batch_iterator = iter(dataloaders_dict[\"train\"])  # iter\n",
    "images, targets = next(batch_iterator)  # get first element\n",
    "print(images.size())  # torch.Size([4, 3, 300, 300])\n",
    "print(len(targets))\n",
    "print(targets[1].shape)  # check targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.6791, 0.1970, 0.8268, 0.2903, 9.0000]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test with ssd model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BlazeFace(\n",
      "  (features): Sequential(\n",
      "    (0): Conv2d(3, 24, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "    (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (2): ReLU(inplace)\n",
      "    (3): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "    )\n",
      "    (4): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "    )\n",
      "    (5): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=24)\n",
      "        (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(24, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    )\n",
      "    (6): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "    )\n",
      "    (7): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "    )\n",
      "    (8): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=48)\n",
      "        (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(48, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    )\n",
      "    (9): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (10): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n",
      "BlazeFaceExtra(\n",
      "  (features): Sequential(\n",
      "    (0): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=96)\n",
      "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    )\n",
      "    (1): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (2): BlazeBlock(\n",
      "      (conv1): Sequential(\n",
      "        (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "        (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (act): ReLU(inplace)\n",
      "      (conv2): Sequential(\n",
      "        (0): ReLU(inplace)\n",
      "        (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "        (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "        (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from utils.blazeface import SSD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using: cuda:0\n",
      "set weights!\n"
     ]
    }
   ],
   "source": [
    "# SSD300の設定\n",
    "ssd_cfg = {\n",
    "    'num_classes': 21,  # 背景クラスを含めた合計クラス数\n",
    "    'input_size': 128,  # 画像の入力サイズ\n",
    "    'bbox_aspect_num': [4, 6],  # 出力するDBoxのアスペクト比の種類\n",
    "    'feature_maps': [16, 8],  # 各sourceの画像サイズ\n",
    "    'steps': [4, 8],  # DBOXの大きさを決める\n",
    "    'min_sizes': [30, 60],  # DBOXの大きさを決める\n",
    "    'max_sizes': [60, 128],  # DBOXの大きさを決める\n",
    "    'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "}\n",
    "\n",
    "net = SSD(phase=\"train\", cfg=ssd_cfg)\n",
    "\n",
    "# SSDのweightsを設定\n",
    "\n",
    "def weights_init(m):\n",
    "    if isinstance(m, nn.Conv2d):\n",
    "        init.kaiming_normal_(m.weight.data)\n",
    "        if m.bias is not None:\n",
    "            nn.init.constant_(m.bias, 0.0)\n",
    "\n",
    "# set inits for loc and conf\n",
    "net.loc.apply(weights_init)\n",
    "net.conf.apply(weights_init)\n",
    "\n",
    "# GPUが使えるか確認\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"using:\", device)\n",
    "\n",
    "print(\"set weights!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SSD(\n",
      "  (blaze): BlazeFace(\n",
      "    (features): Sequential(\n",
      "      (0): Conv2d(3, 24, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "      (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU(inplace)\n",
      "      (3): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (4): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (5): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(24, 24, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=24)\n",
      "          (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(24, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (6): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "          (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (7): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=48)\n",
      "          (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "      )\n",
      "      (8): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(48, 48, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=48)\n",
      "          (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(48, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (9): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (10): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (extra): BlazeFaceExtra(\n",
      "    (features): Sequential(\n",
      "      (0): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (1): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (2): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=96)\n",
      "          (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace)\n",
      "          (1): Conv2d(24, 24, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=24)\n",
      "          (2): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(24, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (loc): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(96, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Conv2d(96, 24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "  )\n",
      "  (conf): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(96, 84, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Conv2d(96, 126, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.ssd_model import MultiBoxLoss\n",
    "\n",
    "# define loss\n",
    "criterion = MultiBoxLoss(jaccard_thresh=0.5,neg_pos=2, device=device)\n",
    "\n",
    "# optim\n",
    "import torch.optim as optim\n",
    "optimizer = optim.SGD(net.parameters(), lr=1e-3, momentum=0.9, weight_decay=5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_current_lr(epoch):\n",
    "    lr = 1e-3\n",
    "    for i,lr_decay_epoch in enumerate([120,180]):\n",
    "        if epoch >= lr_decay_epoch:\n",
    "            lr *= 0.1\n",
    "    return lr\n",
    "\n",
    "def adjust_learning_rate(optimizer, epoch):\n",
    "    lr = get_current_lr(epoch)\n",
    "    print(\"lr is:\", lr)\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# モデルを学習させる関数を作成\n",
    "def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):\n",
    "\n",
    "    # GPUが使えるかを確認\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(\"used device：\", device)\n",
    "\n",
    "    # ネットワークをGPUへ\n",
    "    net.to(device)\n",
    "\n",
    "    # ネットワークがある程度固定であれば、高速化させる\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "    # イテレーションカウンタをセット\n",
    "    iteration = 1\n",
    "    epoch_train_loss = 0.0  # epochの損失和\n",
    "    epoch_val_loss = 0.0  # epochの損失和\n",
    "    logs = []\n",
    "\n",
    "    # epochのループ\n",
    "    for epoch in range(num_epochs+1):\n",
    "        \n",
    "        adjust_learning_rate(optimizer, epoch)\n",
    "        \n",
    "        # 開始時刻を保存\n",
    "        t_epoch_start = time.time()\n",
    "        t_iter_start = time.time()\n",
    "\n",
    "        print('-------------')\n",
    "        print('Epoch {}/{}'.format(epoch+1, num_epochs))\n",
    "        print('-------------')\n",
    "\n",
    "        # epochごとの訓練と検証のループ\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                net.train()  # モデルを訓練モードに\n",
    "                print('train')\n",
    "            else:\n",
    "                if((epoch+1) % 5 == 0):\n",
    "                    net.eval()   # モデルを検証モードに\n",
    "                    print('-------------')\n",
    "                    print('val')\n",
    "                else:\n",
    "                    # 検証は5回に1回だけ行う\n",
    "                    continue\n",
    "\n",
    "            # データローダーからminibatchずつ取り出すループ\n",
    "            for images, targets in dataloaders_dict[phase]:\n",
    "\n",
    "                # GPUが使えるならGPUにデータを送る\n",
    "                images = images.to(device)\n",
    "                targets = [ann.to(device)\n",
    "                           for ann in targets]  # リストの各要素のテンソルをGPUへ\n",
    "\n",
    "                # optimizerを初期化\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # 順伝搬（forward）計算\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    # 順伝搬（forward）計算\n",
    "                    outputs = net(images)\n",
    "\n",
    "                    # 損失の計算\n",
    "                    loss_l, loss_c = criterion(outputs, targets)\n",
    "                    loss = loss_l + loss_c\n",
    "\n",
    "                    # 訓練時はバックプロパゲーション\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()  # 勾配の計算\n",
    "\n",
    "                        # 勾配が大きくなりすぎると計算が不安定になるので、clipで最大でも勾配2.0に留める\n",
    "                        nn.utils.clip_grad_value_(\n",
    "                            net.parameters(), clip_value=2.0)\n",
    "\n",
    "                        optimizer.step()  # パラメータ更新\n",
    "\n",
    "                        if (iteration % 10 == 0):  # 10iterに1度、lossを表示\n",
    "                            t_iter_finish = time.time()\n",
    "                            duration = t_iter_finish - t_iter_start\n",
    "                            print('Iteration {} || Loss: {:.4f} || 10iter: {:.4f} sec.'.format(\n",
    "                                iteration, loss.item(), duration))\n",
    "                            t_iter_start = time.time()\n",
    "\n",
    "                        epoch_train_loss += loss.item()\n",
    "                        iteration += 1\n",
    "\n",
    "                    # 検証時\n",
    "                    else:\n",
    "                        epoch_val_loss += loss.item()\n",
    "\n",
    "        # epochのphaseごとのlossと正解率\n",
    "        t_epoch_finish = time.time()\n",
    "        print('-------------')\n",
    "        print('epoch {} || Epoch_TRAIN_Loss:{:.4f} ||Epoch_VAL_Loss:{:.4f}'.format(\n",
    "            epoch+1, epoch_train_loss, epoch_val_loss))\n",
    "        print('timer:  {:.4f} sec.'.format(t_epoch_finish - t_epoch_start))\n",
    "        t_epoch_start = time.time()\n",
    "\n",
    "        # ログを保存\n",
    "        log_epoch = {'epoch': epoch+1,\n",
    "                     'train_loss': epoch_train_loss, 'val_loss': epoch_val_loss}\n",
    "        logs.append(log_epoch)\n",
    "        df = pd.DataFrame(logs)\n",
    "        df.to_csv(\"log_output.csv\")\n",
    "\n",
    "        epoch_train_loss = 0.0  # epochの損失和\n",
    "        epoch_val_loss = 0.0  # epochの損失和\n",
    "\n",
    "        # ネットワークを保存する\n",
    "        if ((epoch+1) % 10 == 0):\n",
    "            torch.save(net.state_dict(), 'weights/blazeface128VOC_' +\n",
    "                       str(epoch+1) + '.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# start training here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used device： cuda:0\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 1/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10 || Loss: 29.6023 || 10iter: 10.9506 sec.\n",
      "Iteration 20 || Loss: 23.3059 || 10iter: 4.6900 sec.\n",
      "Iteration 30 || Loss: 18.4569 || 10iter: 4.3560 sec.\n",
      "Iteration 40 || Loss: 18.7418 || 10iter: 4.4052 sec.\n",
      "Iteration 50 || Loss: 15.1519 || 10iter: 4.7336 sec.\n",
      "Iteration 60 || Loss: 15.2802 || 10iter: 4.8466 sec.\n",
      "Iteration 70 || Loss: 13.5241 || 10iter: 3.3809 sec.\n",
      "-------------\n",
      "epoch 1 || Epoch_TRAIN_Loss:1651.0105 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3107 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 2/200\n",
      "-------------\n",
      "train\n",
      "Iteration 80 || Loss: 15.9082 || 10iter: 4.1762 sec.\n",
      "Iteration 90 || Loss: 16.3893 || 10iter: 6.9769 sec.\n",
      "Iteration 100 || Loss: 14.1174 || 10iter: 5.0676 sec.\n",
      "Iteration 110 || Loss: 13.2159 || 10iter: 4.4170 sec.\n",
      "Iteration 120 || Loss: 12.5102 || 10iter: 4.8756 sec.\n",
      "Iteration 130 || Loss: 13.7482 || 10iter: 4.5590 sec.\n",
      "Iteration 140 || Loss: 12.0504 || 10iter: 4.7415 sec.\n",
      "Iteration 150 || Loss: 12.8908 || 10iter: 3.2477 sec.\n",
      "-------------\n",
      "epoch 2 || Epoch_TRAIN_Loss:1097.6198 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5114 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 3/200\n",
      "-------------\n",
      "train\n",
      "Iteration 160 || Loss: 11.4969 || 10iter: 5.5071 sec.\n",
      "Iteration 170 || Loss: 11.0109 || 10iter: 6.1675 sec.\n",
      "Iteration 180 || Loss: 13.1074 || 10iter: 4.7741 sec.\n",
      "Iteration 190 || Loss: 11.2927 || 10iter: 4.6654 sec.\n",
      "Iteration 200 || Loss: 12.1686 || 10iter: 4.4176 sec.\n",
      "Iteration 210 || Loss: 10.4912 || 10iter: 4.9738 sec.\n",
      "Iteration 220 || Loss: 13.0646 || 10iter: 4.7208 sec.\n",
      "Iteration 230 || Loss: 11.0671 || 10iter: 2.9610 sec.\n",
      "-------------\n",
      "epoch 3 || Epoch_TRAIN_Loss:946.7882 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2671 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 4/200\n",
      "-------------\n",
      "train\n",
      "Iteration 240 || Loss: 11.7840 || 10iter: 5.5654 sec.\n",
      "Iteration 250 || Loss: 10.9628 || 10iter: 6.5506 sec.\n",
      "Iteration 260 || Loss: 11.3041 || 10iter: 4.6590 sec.\n",
      "Iteration 270 || Loss: 11.5175 || 10iter: 5.0349 sec.\n",
      "Iteration 280 || Loss: 10.9121 || 10iter: 4.8406 sec.\n",
      "Iteration 290 || Loss: 11.1461 || 10iter: 5.0038 sec.\n",
      "Iteration 300 || Loss: 10.6519 || 10iter: 4.7458 sec.\n",
      "Iteration 310 || Loss: 10.0756 || 10iter: 2.8216 sec.\n",
      "-------------\n",
      "epoch 4 || Epoch_TRAIN_Loss:867.7431 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1017 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 5/200\n",
      "-------------\n",
      "train\n",
      "Iteration 320 || Loss: 10.4117 || 10iter: 6.1443 sec.\n",
      "Iteration 330 || Loss: 10.9794 || 10iter: 6.0983 sec.\n",
      "Iteration 340 || Loss: 10.7972 || 10iter: 4.8044 sec.\n",
      "Iteration 350 || Loss: 10.7618 || 10iter: 4.5918 sec.\n",
      "Iteration 360 || Loss: 9.9607 || 10iter: 4.6148 sec.\n",
      "Iteration 370 || Loss: 9.7488 || 10iter: 4.6393 sec.\n",
      "Iteration 380 || Loss: 10.4887 || 10iter: 4.6562 sec.\n",
      "Iteration 390 || Loss: 10.2088 || 10iter: 2.7082 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 5 || Epoch_TRAIN_Loss:814.7554 ||Epoch_VAL_Loss:396.0065\n",
      "timer:  49.5697 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 6/200\n",
      "-------------\n",
      "train\n",
      "Iteration 400 || Loss: 10.3091 || 10iter: 7.3069 sec.\n",
      "Iteration 410 || Loss: 10.0162 || 10iter: 5.8669 sec.\n",
      "Iteration 420 || Loss: 10.0392 || 10iter: 4.9238 sec.\n",
      "Iteration 430 || Loss: 8.6826 || 10iter: 4.4265 sec.\n",
      "Iteration 440 || Loss: 8.5996 || 10iter: 4.6507 sec.\n",
      "Iteration 450 || Loss: 9.3381 || 10iter: 4.6038 sec.\n",
      "Iteration 460 || Loss: 10.8486 || 10iter: 4.5622 sec.\n",
      "Iteration 470 || Loss: 9.5540 || 10iter: 2.7084 sec.\n",
      "-------------\n",
      "epoch 6 || Epoch_TRAIN_Loss:773.4461 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4443 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 7/200\n",
      "-------------\n",
      "train\n",
      "Iteration 480 || Loss: 10.2458 || 10iter: 7.9192 sec.\n",
      "Iteration 490 || Loss: 9.5136 || 10iter: 5.8924 sec.\n",
      "Iteration 500 || Loss: 10.1731 || 10iter: 4.6456 sec.\n",
      "Iteration 510 || Loss: 9.4933 || 10iter: 4.9775 sec.\n",
      "Iteration 520 || Loss: 9.7825 || 10iter: 4.4482 sec.\n",
      "Iteration 530 || Loss: 8.8590 || 10iter: 4.5930 sec.\n",
      "Iteration 540 || Loss: 9.4224 || 10iter: 4.3309 sec.\n",
      "Iteration 550 || Loss: 9.5265 || 10iter: 2.6559 sec.\n",
      "-------------\n",
      "epoch 7 || Epoch_TRAIN_Loss:748.6080 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6254 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 8/200\n",
      "-------------\n",
      "train\n",
      "Iteration 560 || Loss: 9.5676 || 10iter: 8.3686 sec.\n",
      "Iteration 570 || Loss: 9.3105 || 10iter: 5.5680 sec.\n",
      "Iteration 580 || Loss: 9.5285 || 10iter: 4.8304 sec.\n",
      "Iteration 590 || Loss: 9.5836 || 10iter: 4.6714 sec.\n",
      "Iteration 600 || Loss: 9.6097 || 10iter: 4.9568 sec.\n",
      "Iteration 610 || Loss: 8.7795 || 10iter: 4.5716 sec.\n",
      "Iteration 620 || Loss: 9.3114 || 10iter: 4.7372 sec.\n",
      "Iteration 630 || Loss: 9.1287 || 10iter: 2.6912 sec.\n",
      "-------------\n",
      "epoch 8 || Epoch_TRAIN_Loss:738.4797 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.2089 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 9/200\n",
      "-------------\n",
      "train\n",
      "Iteration 640 || Loss: 9.4821 || 10iter: 9.2461 sec.\n",
      "Iteration 650 || Loss: 8.9120 || 10iter: 5.8165 sec.\n",
      "Iteration 660 || Loss: 9.5889 || 10iter: 4.6161 sec.\n",
      "Iteration 670 || Loss: 8.5669 || 10iter: 4.5588 sec.\n",
      "Iteration 680 || Loss: 9.6852 || 10iter: 4.6073 sec.\n",
      "Iteration 690 || Loss: 8.9279 || 10iter: 4.8739 sec.\n",
      "Iteration 700 || Loss: 8.9252 || 10iter: 3.6269 sec.\n",
      "Iteration 710 || Loss: 9.3587 || 10iter: 2.6434 sec.\n",
      "-------------\n",
      "epoch 9 || Epoch_TRAIN_Loss:729.8965 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5613 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 10/200\n",
      "-------------\n",
      "train\n",
      "Iteration 720 || Loss: 9.7791 || 10iter: 9.1630 sec.\n",
      "Iteration 730 || Loss: 8.5802 || 10iter: 5.8264 sec.\n",
      "Iteration 740 || Loss: 9.0055 || 10iter: 4.6624 sec.\n",
      "Iteration 750 || Loss: 9.1444 || 10iter: 4.7533 sec.\n",
      "Iteration 760 || Loss: 8.9717 || 10iter: 4.6254 sec.\n",
      "Iteration 770 || Loss: 9.1759 || 10iter: 4.6417 sec.\n",
      "Iteration 780 || Loss: 8.4333 || 10iter: 3.6995 sec.\n",
      "Iteration 790 || Loss: 10.4306 || 10iter: 2.5272 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 10 || Epoch_TRAIN_Loss:713.6229 ||Epoch_VAL_Loss:348.6567\n",
      "timer:  50.0183 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 11/200\n",
      "-------------\n",
      "train\n",
      "Iteration 800 || Loss: 9.6739 || 10iter: 10.2425 sec.\n",
      "Iteration 810 || Loss: 8.9333 || 10iter: 5.0999 sec.\n",
      "Iteration 820 || Loss: 8.8946 || 10iter: 4.4515 sec.\n",
      "Iteration 830 || Loss: 9.3950 || 10iter: 4.8372 sec.\n",
      "Iteration 840 || Loss: 9.3361 || 10iter: 4.6819 sec.\n",
      "Iteration 850 || Loss: 9.2128 || 10iter: 4.9031 sec.\n",
      "Iteration 860 || Loss: 9.0475 || 10iter: 3.5118 sec.\n",
      "-------------\n",
      "epoch 11 || Epoch_TRAIN_Loss:696.6674 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4001 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 12/200\n",
      "-------------\n",
      "train\n",
      "Iteration 870 || Loss: 9.2240 || 10iter: 4.7450 sec.\n",
      "Iteration 880 || Loss: 9.3062 || 10iter: 6.6024 sec.\n",
      "Iteration 890 || Loss: 8.5865 || 10iter: 4.6062 sec.\n",
      "Iteration 900 || Loss: 8.2258 || 10iter: 4.5018 sec.\n",
      "Iteration 910 || Loss: 9.3607 || 10iter: 4.5725 sec.\n",
      "Iteration 920 || Loss: 7.9690 || 10iter: 4.4732 sec.\n",
      "Iteration 930 || Loss: 7.9636 || 10iter: 4.9432 sec.\n",
      "Iteration 940 || Loss: 8.7244 || 10iter: 3.3327 sec.\n",
      "-------------\n",
      "epoch 12 || Epoch_TRAIN_Loss:688.5135 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2672 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 13/200\n",
      "-------------\n",
      "train\n",
      "Iteration 950 || Loss: 9.0866 || 10iter: 5.5380 sec.\n",
      "Iteration 960 || Loss: 9.2487 || 10iter: 7.0321 sec.\n",
      "Iteration 970 || Loss: 9.4090 || 10iter: 4.8139 sec.\n",
      "Iteration 980 || Loss: 8.8948 || 10iter: 4.7681 sec.\n",
      "Iteration 990 || Loss: 8.3171 || 10iter: 4.6602 sec.\n",
      "Iteration 1000 || Loss: 8.4866 || 10iter: 4.5135 sec.\n",
      "Iteration 1010 || Loss: 8.5152 || 10iter: 4.9007 sec.\n",
      "Iteration 1020 || Loss: 8.0458 || 10iter: 3.0129 sec.\n",
      "-------------\n",
      "epoch 13 || Epoch_TRAIN_Loss:682.5245 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.3966 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 14/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1030 || Loss: 9.1267 || 10iter: 6.5621 sec.\n",
      "Iteration 1040 || Loss: 8.9034 || 10iter: 5.7444 sec.\n",
      "Iteration 1050 || Loss: 8.5557 || 10iter: 4.8903 sec.\n",
      "Iteration 1060 || Loss: 8.7428 || 10iter: 4.6779 sec.\n",
      "Iteration 1070 || Loss: 8.2205 || 10iter: 4.7013 sec.\n",
      "Iteration 1080 || Loss: 8.4990 || 10iter: 4.3473 sec.\n",
      "Iteration 1090 || Loss: 8.5393 || 10iter: 4.9167 sec.\n",
      "Iteration 1100 || Loss: 7.7955 || 10iter: 2.8976 sec.\n",
      "-------------\n",
      "epoch 14 || Epoch_TRAIN_Loss:681.7006 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5732 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 15/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1110 || Loss: 8.9387 || 10iter: 6.7569 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1120 || Loss: 8.4006 || 10iter: 6.0264 sec.\n",
      "Iteration 1130 || Loss: 9.0219 || 10iter: 4.5365 sec.\n",
      "Iteration 1140 || Loss: 8.2331 || 10iter: 4.6713 sec.\n",
      "Iteration 1150 || Loss: 8.6595 || 10iter: 4.6912 sec.\n",
      "Iteration 1160 || Loss: 8.4130 || 10iter: 4.4023 sec.\n",
      "Iteration 1170 || Loss: 8.1138 || 10iter: 5.0531 sec.\n",
      "Iteration 1180 || Loss: 8.1990 || 10iter: 2.7085 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 15 || Epoch_TRAIN_Loss:676.4399 ||Epoch_VAL_Loss:335.2492\n",
      "timer:  50.3796 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 16/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1190 || Loss: 8.7035 || 10iter: 7.3222 sec.\n",
      "Iteration 1200 || Loss: 8.7335 || 10iter: 5.4954 sec.\n",
      "Iteration 1210 || Loss: 8.1417 || 10iter: 5.1213 sec.\n",
      "Iteration 1220 || Loss: 8.8950 || 10iter: 4.3846 sec.\n",
      "Iteration 1230 || Loss: 7.9730 || 10iter: 4.6609 sec.\n",
      "Iteration 1240 || Loss: 8.8896 || 10iter: 4.5830 sec.\n",
      "Iteration 1250 || Loss: 8.3721 || 10iter: 4.2469 sec.\n",
      "Iteration 1260 || Loss: 8.0381 || 10iter: 2.6585 sec.\n",
      "-------------\n",
      "epoch 16 || Epoch_TRAIN_Loss:662.4377 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.8564 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 17/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1270 || Loss: 8.0678 || 10iter: 8.2324 sec.\n",
      "Iteration 1280 || Loss: 8.8131 || 10iter: 6.0197 sec.\n",
      "Iteration 1290 || Loss: 8.4521 || 10iter: 5.2626 sec.\n",
      "Iteration 1300 || Loss: 7.8528 || 10iter: 4.7741 sec.\n",
      "Iteration 1310 || Loss: 8.1045 || 10iter: 4.7841 sec.\n",
      "Iteration 1320 || Loss: 7.9920 || 10iter: 4.6032 sec.\n",
      "Iteration 1330 || Loss: 7.8755 || 10iter: 4.1178 sec.\n",
      "Iteration 1340 || Loss: 8.9018 || 10iter: 2.6665 sec.\n",
      "-------------\n",
      "epoch 17 || Epoch_TRAIN_Loss:656.1724 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.5381 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 18/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1350 || Loss: 8.3710 || 10iter: 8.4177 sec.\n",
      "Iteration 1360 || Loss: 8.2334 || 10iter: 5.5354 sec.\n",
      "Iteration 1370 || Loss: 8.1039 || 10iter: 4.6836 sec.\n",
      "Iteration 1380 || Loss: 8.3008 || 10iter: 4.8102 sec.\n",
      "Iteration 1390 || Loss: 7.8026 || 10iter: 4.8772 sec.\n",
      "Iteration 1400 || Loss: 8.2411 || 10iter: 4.4774 sec.\n",
      "Iteration 1410 || Loss: 8.3573 || 10iter: 4.2914 sec.\n",
      "Iteration 1420 || Loss: 7.6083 || 10iter: 2.6784 sec.\n",
      "-------------\n",
      "epoch 18 || Epoch_TRAIN_Loss:659.1587 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5653 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 19/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1430 || Loss: 8.0996 || 10iter: 8.6839 sec.\n",
      "Iteration 1440 || Loss: 8.4889 || 10iter: 5.9263 sec.\n",
      "Iteration 1450 || Loss: 9.1760 || 10iter: 4.4697 sec.\n",
      "Iteration 1460 || Loss: 8.6031 || 10iter: 4.6120 sec.\n",
      "Iteration 1470 || Loss: 8.4419 || 10iter: 5.1086 sec.\n",
      "Iteration 1480 || Loss: 8.6216 || 10iter: 4.4444 sec.\n",
      "Iteration 1490 || Loss: 8.3998 || 10iter: 3.7963 sec.\n",
      "Iteration 1500 || Loss: 8.3943 || 10iter: 2.6340 sec.\n",
      "-------------\n",
      "epoch 19 || Epoch_TRAIN_Loss:657.9213 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2303 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 20/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1510 || Loss: 7.4985 || 10iter: 10.0098 sec.\n",
      "Iteration 1520 || Loss: 8.0166 || 10iter: 5.3238 sec.\n",
      "Iteration 1530 || Loss: 8.8459 || 10iter: 4.8016 sec.\n",
      "Iteration 1540 || Loss: 8.2161 || 10iter: 4.3843 sec.\n",
      "Iteration 1550 || Loss: 7.5829 || 10iter: 4.7152 sec.\n",
      "Iteration 1560 || Loss: 8.6627 || 10iter: 4.8237 sec.\n",
      "Iteration 1570 || Loss: 7.7170 || 10iter: 3.7539 sec.\n",
      "Iteration 1580 || Loss: 7.2484 || 10iter: 2.5225 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 20 || Epoch_TRAIN_Loss:647.5922 ||Epoch_VAL_Loss:325.6890\n",
      "timer:  50.3728 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 21/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1590 || Loss: 7.6469 || 10iter: 11.0125 sec.\n",
      "Iteration 1600 || Loss: 7.7881 || 10iter: 4.4525 sec.\n",
      "Iteration 1610 || Loss: 8.8743 || 10iter: 4.9891 sec.\n",
      "Iteration 1620 || Loss: 8.7278 || 10iter: 5.0080 sec.\n",
      "Iteration 1630 || Loss: 7.8691 || 10iter: 4.5129 sec.\n",
      "Iteration 1640 || Loss: 7.7571 || 10iter: 4.8990 sec.\n",
      "Iteration 1650 || Loss: 8.6897 || 10iter: 3.4501 sec.\n",
      "-------------\n",
      "epoch 21 || Epoch_TRAIN_Loss:656.8855 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9993 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 22/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1660 || Loss: 7.4586 || 10iter: 4.1819 sec.\n",
      "Iteration 1670 || Loss: 8.2894 || 10iter: 7.1665 sec.\n",
      "Iteration 1680 || Loss: 8.3504 || 10iter: 4.8792 sec.\n",
      "Iteration 1690 || Loss: 7.4266 || 10iter: 4.6536 sec.\n",
      "Iteration 1700 || Loss: 8.5980 || 10iter: 4.6916 sec.\n",
      "Iteration 1710 || Loss: 8.1503 || 10iter: 4.7401 sec.\n",
      "Iteration 1720 || Loss: 7.6339 || 10iter: 4.5375 sec.\n",
      "Iteration 1730 || Loss: 9.1599 || 10iter: 3.3621 sec.\n",
      "-------------\n",
      "epoch 22 || Epoch_TRAIN_Loss:647.2146 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6664 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 23/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1740 || Loss: 8.2857 || 10iter: 5.3361 sec.\n",
      "Iteration 1750 || Loss: 8.4311 || 10iter: 6.0744 sec.\n",
      "Iteration 1760 || Loss: 8.3167 || 10iter: 4.6469 sec.\n",
      "Iteration 1770 || Loss: 7.6699 || 10iter: 4.7173 sec.\n",
      "Iteration 1780 || Loss: 8.4314 || 10iter: 4.6182 sec.\n",
      "Iteration 1790 || Loss: 7.5438 || 10iter: 4.5761 sec.\n",
      "Iteration 1800 || Loss: 8.1301 || 10iter: 4.5727 sec.\n",
      "Iteration 1810 || Loss: 7.9373 || 10iter: 3.0280 sec.\n",
      "-------------\n",
      "epoch 23 || Epoch_TRAIN_Loss:644.2420 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.7369 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 24/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1820 || Loss: 8.6455 || 10iter: 5.9561 sec.\n",
      "Iteration 1830 || Loss: 8.2995 || 10iter: 5.9892 sec.\n",
      "Iteration 1840 || Loss: 7.9355 || 10iter: 4.8513 sec.\n",
      "Iteration 1850 || Loss: 8.6172 || 10iter: 4.6046 sec.\n",
      "Iteration 1860 || Loss: 8.2024 || 10iter: 4.7278 sec.\n",
      "Iteration 1870 || Loss: 8.1768 || 10iter: 4.2655 sec.\n",
      "Iteration 1880 || Loss: 8.2628 || 10iter: 4.9856 sec.\n",
      "Iteration 1890 || Loss: 7.9635 || 10iter: 2.8852 sec.\n",
      "-------------\n",
      "epoch 24 || Epoch_TRAIN_Loss:643.9270 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1928 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 25/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1900 || Loss: 8.3501 || 10iter: 6.7796 sec.\n",
      "Iteration 1910 || Loss: 7.9343 || 10iter: 5.7972 sec.\n",
      "Iteration 1920 || Loss: 7.5297 || 10iter: 4.9754 sec.\n",
      "Iteration 1930 || Loss: 8.3862 || 10iter: 4.2203 sec.\n",
      "Iteration 1940 || Loss: 8.4835 || 10iter: 4.9714 sec.\n",
      "Iteration 1950 || Loss: 8.4573 || 10iter: 4.5844 sec.\n",
      "Iteration 1960 || Loss: 7.7589 || 10iter: 5.1950 sec.\n",
      "Iteration 1970 || Loss: 8.7675 || 10iter: 2.8335 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 25 || Epoch_TRAIN_Loss:639.6546 ||Epoch_VAL_Loss:321.0395\n",
      "timer:  50.1242 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 26/200\n",
      "-------------\n",
      "train\n",
      "Iteration 1980 || Loss: 7.9853 || 10iter: 7.4597 sec.\n",
      "Iteration 1990 || Loss: 8.2616 || 10iter: 5.7697 sec.\n",
      "Iteration 2000 || Loss: 8.9364 || 10iter: 4.7357 sec.\n",
      "Iteration 2010 || Loss: 7.9715 || 10iter: 4.7472 sec.\n",
      "Iteration 2020 || Loss: 8.5392 || 10iter: 4.5052 sec.\n",
      "Iteration 2030 || Loss: 7.8313 || 10iter: 4.4188 sec.\n",
      "Iteration 2040 || Loss: 9.0743 || 10iter: 4.8557 sec.\n",
      "Iteration 2050 || Loss: 8.3235 || 10iter: 2.6732 sec.\n",
      "-------------\n",
      "epoch 26 || Epoch_TRAIN_Loss:642.7465 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5765 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 27/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2060 || Loss: 8.2598 || 10iter: 7.7407 sec.\n",
      "Iteration 2070 || Loss: 7.7269 || 10iter: 5.6722 sec.\n",
      "Iteration 2080 || Loss: 8.3142 || 10iter: 4.8428 sec.\n",
      "Iteration 2090 || Loss: 8.1624 || 10iter: 4.6810 sec.\n",
      "Iteration 2100 || Loss: 7.0627 || 10iter: 4.6123 sec.\n",
      "Iteration 2110 || Loss: 7.9537 || 10iter: 4.9511 sec.\n",
      "Iteration 2120 || Loss: 8.3111 || 10iter: 4.3827 sec.\n",
      "Iteration 2130 || Loss: 8.6000 || 10iter: 2.6451 sec.\n",
      "-------------\n",
      "epoch 27 || Epoch_TRAIN_Loss:638.7381 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5609 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 28/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2140 || Loss: 8.1342 || 10iter: 8.5574 sec.\n",
      "Iteration 2150 || Loss: 7.7221 || 10iter: 5.5261 sec.\n",
      "Iteration 2160 || Loss: 8.0942 || 10iter: 4.1978 sec.\n",
      "Iteration 2170 || Loss: 8.6867 || 10iter: 4.9389 sec.\n",
      "Iteration 2180 || Loss: 7.1775 || 10iter: 4.2661 sec.\n",
      "Iteration 2190 || Loss: 8.2143 || 10iter: 4.6508 sec.\n",
      "Iteration 2200 || Loss: 8.4733 || 10iter: 4.1788 sec.\n",
      "Iteration 2210 || Loss: 7.6557 || 10iter: 2.6289 sec.\n",
      "-------------\n",
      "epoch 28 || Epoch_TRAIN_Loss:634.2949 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.7542 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 29/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 2220 || Loss: 8.2445 || 10iter: 8.7083 sec.\n",
      "Iteration 2230 || Loss: 8.0514 || 10iter: 5.8645 sec.\n",
      "Iteration 2240 || Loss: 7.8717 || 10iter: 4.2027 sec.\n",
      "Iteration 2250 || Loss: 6.9328 || 10iter: 4.7247 sec.\n",
      "Iteration 2260 || Loss: 8.0714 || 10iter: 4.4041 sec.\n",
      "Iteration 2270 || Loss: 8.1670 || 10iter: 4.8034 sec.\n",
      "Iteration 2280 || Loss: 7.1901 || 10iter: 3.9027 sec.\n",
      "Iteration 2290 || Loss: 7.4076 || 10iter: 2.6778 sec.\n",
      "-------------\n",
      "epoch 29 || Epoch_TRAIN_Loss:632.6935 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.8111 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 30/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2300 || Loss: 7.8879 || 10iter: 11.2488 sec.\n",
      "Iteration 2310 || Loss: 8.2625 || 10iter: 5.1958 sec.\n",
      "Iteration 2320 || Loss: 7.9251 || 10iter: 4.2402 sec.\n",
      "Iteration 2330 || Loss: 7.4470 || 10iter: 4.7375 sec.\n",
      "Iteration 2340 || Loss: 8.2425 || 10iter: 4.5130 sec.\n",
      "Iteration 2350 || Loss: 7.5599 || 10iter: 4.7223 sec.\n",
      "Iteration 2360 || Loss: 8.4366 || 10iter: 3.8192 sec.\n",
      "Iteration 2370 || Loss: 7.7690 || 10iter: 2.5412 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 30 || Epoch_TRAIN_Loss:629.2460 ||Epoch_VAL_Loss:319.9825\n",
      "timer:  51.2200 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 31/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2380 || Loss: 8.1775 || 10iter: 10.8457 sec.\n",
      "Iteration 2390 || Loss: 8.0362 || 10iter: 5.0868 sec.\n",
      "Iteration 2400 || Loss: 7.7730 || 10iter: 4.4869 sec.\n",
      "Iteration 2410 || Loss: 7.8330 || 10iter: 4.5551 sec.\n",
      "Iteration 2420 || Loss: 7.2634 || 10iter: 4.5098 sec.\n",
      "Iteration 2430 || Loss: 7.6382 || 10iter: 5.0714 sec.\n",
      "Iteration 2440 || Loss: 7.7608 || 10iter: 3.4208 sec.\n",
      "-------------\n",
      "epoch 31 || Epoch_TRAIN_Loss:631.3884 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6196 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 32/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2450 || Loss: 7.9470 || 10iter: 4.5116 sec.\n",
      "Iteration 2460 || Loss: 7.2252 || 10iter: 6.6247 sec.\n",
      "Iteration 2470 || Loss: 8.2420 || 10iter: 4.6850 sec.\n",
      "Iteration 2480 || Loss: 8.3505 || 10iter: 4.7956 sec.\n",
      "Iteration 2490 || Loss: 7.5859 || 10iter: 4.8246 sec.\n",
      "Iteration 2500 || Loss: 8.1850 || 10iter: 4.4118 sec.\n",
      "Iteration 2510 || Loss: 7.4202 || 10iter: 4.8003 sec.\n",
      "Iteration 2520 || Loss: 8.0267 || 10iter: 3.2623 sec.\n",
      "-------------\n",
      "epoch 32 || Epoch_TRAIN_Loss:627.9564 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3237 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 33/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2530 || Loss: 7.8649 || 10iter: 5.0674 sec.\n",
      "Iteration 2540 || Loss: 7.6001 || 10iter: 6.7209 sec.\n",
      "Iteration 2550 || Loss: 7.6772 || 10iter: 4.7190 sec.\n",
      "Iteration 2560 || Loss: 8.4777 || 10iter: 4.8249 sec.\n",
      "Iteration 2570 || Loss: 7.5722 || 10iter: 4.4855 sec.\n",
      "Iteration 2580 || Loss: 7.5188 || 10iter: 4.5283 sec.\n",
      "Iteration 2590 || Loss: 7.8215 || 10iter: 4.8115 sec.\n",
      "Iteration 2600 || Loss: 7.9524 || 10iter: 3.0116 sec.\n",
      "-------------\n",
      "epoch 33 || Epoch_TRAIN_Loss:628.5034 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3384 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 34/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2610 || Loss: 8.0198 || 10iter: 6.2545 sec.\n",
      "Iteration 2620 || Loss: 8.0075 || 10iter: 6.0385 sec.\n",
      "Iteration 2630 || Loss: 8.0038 || 10iter: 5.5282 sec.\n",
      "Iteration 2640 || Loss: 7.3396 || 10iter: 4.8127 sec.\n",
      "Iteration 2650 || Loss: 8.2796 || 10iter: 4.4968 sec.\n",
      "Iteration 2660 || Loss: 7.9620 || 10iter: 4.7905 sec.\n",
      "Iteration 2670 || Loss: 7.9005 || 10iter: 4.6384 sec.\n",
      "Iteration 2680 || Loss: 7.4796 || 10iter: 2.7911 sec.\n",
      "-------------\n",
      "epoch 34 || Epoch_TRAIN_Loss:627.2395 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.2361 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 35/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2690 || Loss: 8.3611 || 10iter: 6.5049 sec.\n",
      "Iteration 2700 || Loss: 8.6093 || 10iter: 5.4580 sec.\n",
      "Iteration 2710 || Loss: 7.7945 || 10iter: 5.0006 sec.\n",
      "Iteration 2720 || Loss: 8.2689 || 10iter: 4.5171 sec.\n",
      "Iteration 2730 || Loss: 7.9635 || 10iter: 4.7335 sec.\n",
      "Iteration 2740 || Loss: 8.3260 || 10iter: 4.5911 sec.\n",
      "Iteration 2750 || Loss: 8.3258 || 10iter: 4.5385 sec.\n",
      "Iteration 2760 || Loss: 7.4004 || 10iter: 2.7426 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 35 || Epoch_TRAIN_Loss:625.9183 ||Epoch_VAL_Loss:315.5078\n",
      "timer:  49.0886 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 36/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2770 || Loss: 7.4979 || 10iter: 7.2012 sec.\n",
      "Iteration 2780 || Loss: 8.1386 || 10iter: 5.9185 sec.\n",
      "Iteration 2790 || Loss: 7.5024 || 10iter: 4.9848 sec.\n",
      "Iteration 2800 || Loss: 8.6388 || 10iter: 4.6391 sec.\n",
      "Iteration 2810 || Loss: 7.9884 || 10iter: 4.6377 sec.\n",
      "Iteration 2820 || Loss: 7.6112 || 10iter: 4.2019 sec.\n",
      "Iteration 2830 || Loss: 7.3334 || 10iter: 4.8435 sec.\n",
      "Iteration 2840 || Loss: 7.8521 || 10iter: 2.7144 sec.\n",
      "-------------\n",
      "epoch 36 || Epoch_TRAIN_Loss:622.1281 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5435 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 37/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2850 || Loss: 7.7191 || 10iter: 7.8451 sec.\n",
      "Iteration 2860 || Loss: 7.6610 || 10iter: 5.4707 sec.\n",
      "Iteration 2870 || Loss: 7.7082 || 10iter: 4.8826 sec.\n",
      "Iteration 2880 || Loss: 7.9534 || 10iter: 4.4247 sec.\n",
      "Iteration 2890 || Loss: 7.9615 || 10iter: 4.5629 sec.\n",
      "Iteration 2900 || Loss: 7.9196 || 10iter: 4.5761 sec.\n",
      "Iteration 2910 || Loss: 7.5209 || 10iter: 4.5799 sec.\n",
      "Iteration 2920 || Loss: 8.0235 || 10iter: 2.6635 sec.\n",
      "-------------\n",
      "epoch 37 || Epoch_TRAIN_Loss:624.7532 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0965 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 38/200\n",
      "-------------\n",
      "train\n",
      "Iteration 2930 || Loss: 8.0988 || 10iter: 8.6393 sec.\n",
      "Iteration 2940 || Loss: 7.3061 || 10iter: 5.3830 sec.\n",
      "Iteration 2950 || Loss: 7.1311 || 10iter: 4.5627 sec.\n",
      "Iteration 2960 || Loss: 8.0818 || 10iter: 5.4075 sec.\n",
      "Iteration 2970 || Loss: 7.8694 || 10iter: 5.0811 sec.\n",
      "Iteration 2980 || Loss: 7.8832 || 10iter: 4.6554 sec.\n",
      "Iteration 2990 || Loss: 8.7554 || 10iter: 4.2068 sec.\n",
      "Iteration 3000 || Loss: 7.8165 || 10iter: 2.6864 sec.\n",
      "-------------\n",
      "epoch 38 || Epoch_TRAIN_Loss:621.6146 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.4073 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 39/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3010 || Loss: 8.2641 || 10iter: 8.6442 sec.\n",
      "Iteration 3020 || Loss: 7.1236 || 10iter: 5.9127 sec.\n",
      "Iteration 3030 || Loss: 7.6639 || 10iter: 4.4369 sec.\n",
      "Iteration 3040 || Loss: 7.3787 || 10iter: 4.3927 sec.\n",
      "Iteration 3050 || Loss: 8.4828 || 10iter: 4.8009 sec.\n",
      "Iteration 3060 || Loss: 8.1681 || 10iter: 4.6274 sec.\n",
      "Iteration 3070 || Loss: 7.6358 || 10iter: 3.7929 sec.\n",
      "Iteration 3080 || Loss: 8.1800 || 10iter: 2.6611 sec.\n",
      "-------------\n",
      "epoch 39 || Epoch_TRAIN_Loss:618.3329 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.8129 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 40/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3090 || Loss: 7.8292 || 10iter: 9.8497 sec.\n",
      "Iteration 3100 || Loss: 7.2254 || 10iter: 5.4849 sec.\n",
      "Iteration 3110 || Loss: 8.1887 || 10iter: 4.3125 sec.\n",
      "Iteration 3120 || Loss: 7.9821 || 10iter: 4.6910 sec.\n",
      "Iteration 3130 || Loss: 8.3199 || 10iter: 4.6262 sec.\n",
      "Iteration 3140 || Loss: 7.7103 || 10iter: 4.6945 sec.\n",
      "Iteration 3150 || Loss: 8.7376 || 10iter: 3.7842 sec.\n",
      "Iteration 3160 || Loss: 8.4243 || 10iter: 2.5302 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 40 || Epoch_TRAIN_Loss:621.5411 ||Epoch_VAL_Loss:313.1447\n",
      "timer:  49.6854 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 41/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3170 || Loss: 8.0614 || 10iter: 10.5289 sec.\n",
      "Iteration 3180 || Loss: 7.8143 || 10iter: 5.0397 sec.\n",
      "Iteration 3190 || Loss: 8.0506 || 10iter: 4.4663 sec.\n",
      "Iteration 3200 || Loss: 8.3507 || 10iter: 4.7380 sec.\n",
      "Iteration 3210 || Loss: 7.8184 || 10iter: 4.7891 sec.\n",
      "Iteration 3220 || Loss: 8.6507 || 10iter: 4.7658 sec.\n",
      "Iteration 3230 || Loss: 7.5659 || 10iter: 3.4976 sec.\n",
      "-------------\n",
      "epoch 41 || Epoch_TRAIN_Loss:620.3961 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4406 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 42/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3240 || Loss: 7.7822 || 10iter: 4.5212 sec.\n",
      "Iteration 3250 || Loss: 7.8683 || 10iter: 6.6811 sec.\n",
      "Iteration 3260 || Loss: 7.9965 || 10iter: 4.7023 sec.\n",
      "Iteration 3270 || Loss: 8.2631 || 10iter: 4.6523 sec.\n",
      "Iteration 3280 || Loss: 7.9024 || 10iter: 4.6757 sec.\n",
      "Iteration 3290 || Loss: 7.8484 || 10iter: 5.0353 sec.\n",
      "Iteration 3300 || Loss: 7.8087 || 10iter: 5.3019 sec.\n",
      "Iteration 3310 || Loss: 7.9395 || 10iter: 3.1978 sec.\n",
      "-------------\n",
      "epoch 42 || Epoch_TRAIN_Loss:618.7313 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1681 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 43/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 3320 || Loss: 7.5642 || 10iter: 5.2141 sec.\n",
      "Iteration 3330 || Loss: 7.6851 || 10iter: 6.2053 sec.\n",
      "Iteration 3340 || Loss: 7.3296 || 10iter: 4.8349 sec.\n",
      "Iteration 3350 || Loss: 7.7371 || 10iter: 4.7184 sec.\n",
      "Iteration 3360 || Loss: 7.2479 || 10iter: 4.6146 sec.\n",
      "Iteration 3370 || Loss: 7.4304 || 10iter: 4.4780 sec.\n",
      "Iteration 3380 || Loss: 7.4144 || 10iter: 4.7501 sec.\n",
      "Iteration 3390 || Loss: 7.7933 || 10iter: 3.1247 sec.\n",
      "-------------\n",
      "epoch 43 || Epoch_TRAIN_Loss:619.9213 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1257 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 44/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3400 || Loss: 8.3615 || 10iter: 6.1860 sec.\n",
      "Iteration 3410 || Loss: 7.3907 || 10iter: 5.8462 sec.\n",
      "Iteration 3420 || Loss: 8.0827 || 10iter: 4.7511 sec.\n",
      "Iteration 3430 || Loss: 7.6008 || 10iter: 4.8242 sec.\n",
      "Iteration 3440 || Loss: 7.7430 || 10iter: 4.6976 sec.\n",
      "Iteration 3450 || Loss: 7.5046 || 10iter: 4.5491 sec.\n",
      "Iteration 3460 || Loss: 7.7838 || 10iter: 4.8837 sec.\n",
      "Iteration 3470 || Loss: 7.6497 || 10iter: 2.8096 sec.\n",
      "-------------\n",
      "epoch 44 || Epoch_TRAIN_Loss:617.1250 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4449 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 45/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3480 || Loss: 7.6317 || 10iter: 6.3780 sec.\n",
      "Iteration 3490 || Loss: 7.1457 || 10iter: 5.9108 sec.\n",
      "Iteration 3500 || Loss: 7.5645 || 10iter: 5.1248 sec.\n",
      "Iteration 3510 || Loss: 7.9433 || 10iter: 4.5715 sec.\n",
      "Iteration 3520 || Loss: 7.4792 || 10iter: 4.8529 sec.\n",
      "Iteration 3530 || Loss: 7.4261 || 10iter: 4.7610 sec.\n",
      "Iteration 3540 || Loss: 7.9399 || 10iter: 4.5421 sec.\n",
      "Iteration 3550 || Loss: 7.4232 || 10iter: 2.7069 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 45 || Epoch_TRAIN_Loss:614.3904 ||Epoch_VAL_Loss:311.1203\n",
      "timer:  49.8402 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 46/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3560 || Loss: 7.9366 || 10iter: 7.1910 sec.\n",
      "Iteration 3570 || Loss: 8.0600 || 10iter: 6.1456 sec.\n",
      "Iteration 3580 || Loss: 7.9264 || 10iter: 4.5560 sec.\n",
      "Iteration 3590 || Loss: 7.7362 || 10iter: 4.5548 sec.\n",
      "Iteration 3600 || Loss: 7.6520 || 10iter: 4.5505 sec.\n",
      "Iteration 3610 || Loss: 7.7950 || 10iter: 4.5338 sec.\n",
      "Iteration 3620 || Loss: 6.9130 || 10iter: 4.4025 sec.\n",
      "Iteration 3630 || Loss: 7.5054 || 10iter: 2.7428 sec.\n",
      "-------------\n",
      "epoch 46 || Epoch_TRAIN_Loss:617.4177 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0525 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 47/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3640 || Loss: 7.3203 || 10iter: 7.7013 sec.\n",
      "Iteration 3650 || Loss: 7.6666 || 10iter: 5.3917 sec.\n",
      "Iteration 3660 || Loss: 8.4936 || 10iter: 4.9063 sec.\n",
      "Iteration 3670 || Loss: 7.6901 || 10iter: 4.6238 sec.\n",
      "Iteration 3680 || Loss: 8.2897 || 10iter: 4.4831 sec.\n",
      "Iteration 3690 || Loss: 7.8342 || 10iter: 5.0612 sec.\n",
      "Iteration 3700 || Loss: 7.3256 || 10iter: 4.1349 sec.\n",
      "Iteration 3710 || Loss: 7.6150 || 10iter: 2.6797 sec.\n",
      "-------------\n",
      "epoch 47 || Epoch_TRAIN_Loss:613.4642 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0743 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 48/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3720 || Loss: 9.3944 || 10iter: 8.5525 sec.\n",
      "Iteration 3730 || Loss: 7.3090 || 10iter: 5.9852 sec.\n",
      "Iteration 3740 || Loss: 7.6674 || 10iter: 4.3586 sec.\n",
      "Iteration 3750 || Loss: 7.8799 || 10iter: 4.6328 sec.\n",
      "Iteration 3760 || Loss: 8.1651 || 10iter: 4.8940 sec.\n",
      "Iteration 3770 || Loss: 7.6349 || 10iter: 4.7488 sec.\n",
      "Iteration 3780 || Loss: 8.3539 || 10iter: 3.9127 sec.\n",
      "Iteration 3790 || Loss: 7.7619 || 10iter: 2.6831 sec.\n",
      "-------------\n",
      "epoch 48 || Epoch_TRAIN_Loss:612.8062 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5810 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 49/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3800 || Loss: 7.0526 || 10iter: 8.9451 sec.\n",
      "Iteration 3810 || Loss: 7.5287 || 10iter: 5.4510 sec.\n",
      "Iteration 3820 || Loss: 7.8039 || 10iter: 4.8087 sec.\n",
      "Iteration 3830 || Loss: 7.6817 || 10iter: 4.5061 sec.\n",
      "Iteration 3840 || Loss: 7.3480 || 10iter: 4.9640 sec.\n",
      "Iteration 3850 || Loss: 7.3037 || 10iter: 4.7768 sec.\n",
      "Iteration 3860 || Loss: 8.3968 || 10iter: 3.6749 sec.\n",
      "Iteration 3870 || Loss: 7.5015 || 10iter: 2.6281 sec.\n",
      "-------------\n",
      "epoch 49 || Epoch_TRAIN_Loss:612.9924 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3102 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 50/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3880 || Loss: 7.7793 || 10iter: 9.8278 sec.\n",
      "Iteration 3890 || Loss: 6.9576 || 10iter: 5.0963 sec.\n",
      "Iteration 3900 || Loss: 7.5075 || 10iter: 4.7681 sec.\n",
      "Iteration 3910 || Loss: 7.8403 || 10iter: 4.8680 sec.\n",
      "Iteration 3920 || Loss: 7.6292 || 10iter: 4.4758 sec.\n",
      "Iteration 3930 || Loss: 7.6200 || 10iter: 4.7681 sec.\n",
      "Iteration 3940 || Loss: 7.6377 || 10iter: 3.7407 sec.\n",
      "Iteration 3950 || Loss: 7.5436 || 10iter: 2.5790 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 50 || Epoch_TRAIN_Loss:612.5504 ||Epoch_VAL_Loss:309.0907\n",
      "timer:  50.2682 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 51/200\n",
      "-------------\n",
      "train\n",
      "Iteration 3960 || Loss: 7.3868 || 10iter: 11.8384 sec.\n",
      "Iteration 3970 || Loss: 7.8455 || 10iter: 4.9044 sec.\n",
      "Iteration 3980 || Loss: 7.7221 || 10iter: 4.5198 sec.\n",
      "Iteration 3990 || Loss: 7.2680 || 10iter: 4.3716 sec.\n",
      "Iteration 4000 || Loss: 7.9492 || 10iter: 5.0535 sec.\n",
      "Iteration 4010 || Loss: 7.5622 || 10iter: 4.5078 sec.\n",
      "Iteration 4020 || Loss: 8.0189 || 10iter: 3.5109 sec.\n",
      "-------------\n",
      "epoch 51 || Epoch_TRAIN_Loss:611.4285 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.4008 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 52/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4030 || Loss: 8.8486 || 10iter: 4.7709 sec.\n",
      "Iteration 4040 || Loss: 7.9724 || 10iter: 6.1450 sec.\n",
      "Iteration 4050 || Loss: 7.5020 || 10iter: 4.8552 sec.\n",
      "Iteration 4060 || Loss: 8.1750 || 10iter: 4.7734 sec.\n",
      "Iteration 4070 || Loss: 7.6325 || 10iter: 4.5491 sec.\n",
      "Iteration 4080 || Loss: 7.8535 || 10iter: 4.5023 sec.\n",
      "Iteration 4090 || Loss: 8.3078 || 10iter: 4.7099 sec.\n",
      "Iteration 4100 || Loss: 7.3813 || 10iter: 3.4335 sec.\n",
      "-------------\n",
      "epoch 52 || Epoch_TRAIN_Loss:615.8939 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2001 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 53/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4110 || Loss: 7.8524 || 10iter: 5.4578 sec.\n",
      "Iteration 4120 || Loss: 7.3570 || 10iter: 6.2555 sec.\n",
      "Iteration 4130 || Loss: 8.4527 || 10iter: 4.7242 sec.\n",
      "Iteration 4140 || Loss: 7.7640 || 10iter: 4.6933 sec.\n",
      "Iteration 4150 || Loss: 8.0425 || 10iter: 4.5226 sec.\n",
      "Iteration 4160 || Loss: 7.8665 || 10iter: 4.7739 sec.\n",
      "Iteration 4170 || Loss: 8.0708 || 10iter: 4.8633 sec.\n",
      "Iteration 4180 || Loss: 6.8703 || 10iter: 2.9853 sec.\n",
      "-------------\n",
      "epoch 53 || Epoch_TRAIN_Loss:605.6774 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4407 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 54/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4190 || Loss: 7.8617 || 10iter: 5.9664 sec.\n",
      "Iteration 4200 || Loss: 7.7119 || 10iter: 6.1244 sec.\n",
      "Iteration 4210 || Loss: 7.2930 || 10iter: 4.7364 sec.\n",
      "Iteration 4220 || Loss: 7.6683 || 10iter: 4.7026 sec.\n",
      "Iteration 4230 || Loss: 7.2873 || 10iter: 4.7382 sec.\n",
      "Iteration 4240 || Loss: 7.8928 || 10iter: 4.4466 sec.\n",
      "Iteration 4250 || Loss: 7.9972 || 10iter: 4.9601 sec.\n",
      "Iteration 4260 || Loss: 7.7332 || 10iter: 2.8416 sec.\n",
      "-------------\n",
      "epoch 54 || Epoch_TRAIN_Loss:609.4069 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4012 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 55/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4270 || Loss: 8.4016 || 10iter: 6.8584 sec.\n",
      "Iteration 4280 || Loss: 7.5726 || 10iter: 5.7641 sec.\n",
      "Iteration 4290 || Loss: 7.9594 || 10iter: 4.9608 sec.\n",
      "Iteration 4300 || Loss: 7.2084 || 10iter: 4.4183 sec.\n",
      "Iteration 4310 || Loss: 8.2509 || 10iter: 5.6436 sec.\n",
      "Iteration 4320 || Loss: 7.3444 || 10iter: 4.9023 sec.\n",
      "Iteration 4330 || Loss: 7.8293 || 10iter: 4.6481 sec.\n",
      "Iteration 4340 || Loss: 7.8562 || 10iter: 2.7111 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 55 || Epoch_TRAIN_Loss:610.2241 ||Epoch_VAL_Loss:307.9434\n",
      "timer:  51.0086 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 56/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4350 || Loss: 8.1050 || 10iter: 7.1718 sec.\n",
      "Iteration 4360 || Loss: 7.5564 || 10iter: 5.9251 sec.\n",
      "Iteration 4370 || Loss: 7.3587 || 10iter: 4.6155 sec.\n",
      "Iteration 4380 || Loss: 7.9737 || 10iter: 4.8397 sec.\n",
      "Iteration 4390 || Loss: 7.3572 || 10iter: 4.3965 sec.\n",
      "Iteration 4400 || Loss: 7.9034 || 10iter: 4.8081 sec.\n",
      "Iteration 4410 || Loss: 7.1798 || 10iter: 4.5548 sec.\n",
      "Iteration 4420 || Loss: 8.2389 || 10iter: 2.6535 sec.\n",
      "-------------\n",
      "epoch 56 || Epoch_TRAIN_Loss:611.2812 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3536 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 57/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 4430 || Loss: 7.3362 || 10iter: 7.9547 sec.\n",
      "Iteration 4440 || Loss: 8.2832 || 10iter: 5.3994 sec.\n",
      "Iteration 4450 || Loss: 7.6814 || 10iter: 4.9281 sec.\n",
      "Iteration 4460 || Loss: 7.5738 || 10iter: 4.3818 sec.\n",
      "Iteration 4470 || Loss: 8.1004 || 10iter: 4.3034 sec.\n",
      "Iteration 4480 || Loss: 8.3416 || 10iter: 4.7159 sec.\n",
      "Iteration 4490 || Loss: 7.3484 || 10iter: 4.2141 sec.\n",
      "Iteration 4500 || Loss: 7.6546 || 10iter: 2.6682 sec.\n",
      "-------------\n",
      "epoch 57 || Epoch_TRAIN_Loss:602.7313 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.6745 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 58/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4510 || Loss: 7.9473 || 10iter: 8.6252 sec.\n",
      "Iteration 4520 || Loss: 7.4184 || 10iter: 5.7824 sec.\n",
      "Iteration 4530 || Loss: 7.3579 || 10iter: 4.4959 sec.\n",
      "Iteration 4540 || Loss: 7.3421 || 10iter: 4.5024 sec.\n",
      "Iteration 4550 || Loss: 8.0270 || 10iter: 4.8960 sec.\n",
      "Iteration 4560 || Loss: 8.6034 || 10iter: 4.6456 sec.\n",
      "Iteration 4570 || Loss: 8.4665 || 10iter: 3.8839 sec.\n",
      "Iteration 4580 || Loss: 7.6092 || 10iter: 2.6514 sec.\n",
      "-------------\n",
      "epoch 58 || Epoch_TRAIN_Loss:610.1791 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2890 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 59/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4590 || Loss: 7.8653 || 10iter: 8.8236 sec.\n",
      "Iteration 4600 || Loss: 7.7486 || 10iter: 5.8985 sec.\n",
      "Iteration 4610 || Loss: 7.6657 || 10iter: 4.1052 sec.\n",
      "Iteration 4620 || Loss: 7.4959 || 10iter: 4.7819 sec.\n",
      "Iteration 4630 || Loss: 7.4025 || 10iter: 4.7004 sec.\n",
      "Iteration 4640 || Loss: 7.8426 || 10iter: 4.4960 sec.\n",
      "Iteration 4650 || Loss: 7.4586 || 10iter: 4.3319 sec.\n",
      "Iteration 4660 || Loss: 7.5273 || 10iter: 2.6701 sec.\n",
      "-------------\n",
      "epoch 59 || Epoch_TRAIN_Loss:606.7539 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3195 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 60/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4670 || Loss: 8.1799 || 10iter: 10.2350 sec.\n",
      "Iteration 4680 || Loss: 7.2192 || 10iter: 4.7149 sec.\n",
      "Iteration 4690 || Loss: 6.9430 || 10iter: 4.5362 sec.\n",
      "Iteration 4700 || Loss: 7.3847 || 10iter: 4.6933 sec.\n",
      "Iteration 4710 || Loss: 7.9519 || 10iter: 4.9565 sec.\n",
      "Iteration 4720 || Loss: 7.7793 || 10iter: 4.7703 sec.\n",
      "Iteration 4730 || Loss: 7.6094 || 10iter: 3.3836 sec.\n",
      "Iteration 4740 || Loss: 7.4658 || 10iter: 2.5235 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 60 || Epoch_TRAIN_Loss:607.4519 ||Epoch_VAL_Loss:307.2602\n",
      "timer:  49.6576 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 61/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4750 || Loss: 8.4193 || 10iter: 10.9128 sec.\n",
      "Iteration 4760 || Loss: 7.6838 || 10iter: 4.8153 sec.\n",
      "Iteration 4770 || Loss: 7.6319 || 10iter: 4.6930 sec.\n",
      "Iteration 4780 || Loss: 7.8857 || 10iter: 4.5873 sec.\n",
      "Iteration 4790 || Loss: 7.9189 || 10iter: 4.9393 sec.\n",
      "Iteration 4800 || Loss: 7.4194 || 10iter: 4.8803 sec.\n",
      "Iteration 4810 || Loss: 7.9386 || 10iter: 3.4619 sec.\n",
      "-------------\n",
      "epoch 61 || Epoch_TRAIN_Loss:607.0594 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9131 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 62/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4820 || Loss: 8.1345 || 10iter: 4.4249 sec.\n",
      "Iteration 4830 || Loss: 7.5892 || 10iter: 6.7257 sec.\n",
      "Iteration 4840 || Loss: 8.2217 || 10iter: 4.9833 sec.\n",
      "Iteration 4850 || Loss: 7.8522 || 10iter: 4.4325 sec.\n",
      "Iteration 4860 || Loss: 8.0982 || 10iter: 4.8854 sec.\n",
      "Iteration 4870 || Loss: 7.2788 || 10iter: 4.5811 sec.\n",
      "Iteration 4880 || Loss: 7.1942 || 10iter: 4.7879 sec.\n",
      "Iteration 4890 || Loss: 7.8862 || 10iter: 3.2282 sec.\n",
      "-------------\n",
      "epoch 62 || Epoch_TRAIN_Loss:600.2807 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5414 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 63/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4900 || Loss: 7.4672 || 10iter: 5.3348 sec.\n",
      "Iteration 4910 || Loss: 7.4030 || 10iter: 6.4131 sec.\n",
      "Iteration 4920 || Loss: 7.5399 || 10iter: 4.9336 sec.\n",
      "Iteration 4930 || Loss: 7.6937 || 10iter: 4.5544 sec.\n",
      "Iteration 4940 || Loss: 7.3613 || 10iter: 4.9469 sec.\n",
      "Iteration 4950 || Loss: 7.8174 || 10iter: 4.6030 sec.\n",
      "Iteration 4960 || Loss: 8.3952 || 10iter: 4.7659 sec.\n",
      "Iteration 4970 || Loss: 7.7586 || 10iter: 3.0965 sec.\n",
      "-------------\n",
      "epoch 63 || Epoch_TRAIN_Loss:603.0328 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.8349 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 64/200\n",
      "-------------\n",
      "train\n",
      "Iteration 4980 || Loss: 8.1075 || 10iter: 6.0696 sec.\n",
      "Iteration 4990 || Loss: 7.3388 || 10iter: 6.4638 sec.\n",
      "Iteration 5000 || Loss: 7.3315 || 10iter: 5.0268 sec.\n",
      "Iteration 5010 || Loss: 7.7119 || 10iter: 4.5411 sec.\n",
      "Iteration 5020 || Loss: 7.6270 || 10iter: 4.8348 sec.\n",
      "Iteration 5030 || Loss: 7.6053 || 10iter: 4.3178 sec.\n",
      "Iteration 5040 || Loss: 7.7909 || 10iter: 4.6376 sec.\n",
      "Iteration 5050 || Loss: 7.7768 || 10iter: 2.8966 sec.\n",
      "-------------\n",
      "epoch 64 || Epoch_TRAIN_Loss:605.9991 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6484 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 65/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5060 || Loss: 8.2316 || 10iter: 6.5522 sec.\n",
      "Iteration 5070 || Loss: 7.7868 || 10iter: 5.9543 sec.\n",
      "Iteration 5080 || Loss: 7.4547 || 10iter: 5.1301 sec.\n",
      "Iteration 5090 || Loss: 7.3957 || 10iter: 4.6385 sec.\n",
      "Iteration 5100 || Loss: 7.7521 || 10iter: 4.4873 sec.\n",
      "Iteration 5110 || Loss: 6.9727 || 10iter: 4.5318 sec.\n",
      "Iteration 5120 || Loss: 8.2682 || 10iter: 4.6573 sec.\n",
      "Iteration 5130 || Loss: 7.8245 || 10iter: 2.7480 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 65 || Epoch_TRAIN_Loss:605.5872 ||Epoch_VAL_Loss:305.4843\n",
      "timer:  49.7076 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 66/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5140 || Loss: 7.9235 || 10iter: 7.0837 sec.\n",
      "Iteration 5150 || Loss: 7.5884 || 10iter: 6.1386 sec.\n",
      "Iteration 5160 || Loss: 7.5535 || 10iter: 4.5679 sec.\n",
      "Iteration 5170 || Loss: 7.1064 || 10iter: 4.4807 sec.\n",
      "Iteration 5180 || Loss: 7.2717 || 10iter: 4.6757 sec.\n",
      "Iteration 5190 || Loss: 7.2413 || 10iter: 4.2475 sec.\n",
      "Iteration 5200 || Loss: 7.5149 || 10iter: 4.8025 sec.\n",
      "Iteration 5210 || Loss: 7.6566 || 10iter: 2.6412 sec.\n",
      "-------------\n",
      "epoch 66 || Epoch_TRAIN_Loss:596.9970 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0152 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 67/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5220 || Loss: 7.3855 || 10iter: 7.3817 sec.\n",
      "Iteration 5230 || Loss: 7.5813 || 10iter: 6.0209 sec.\n",
      "Iteration 5240 || Loss: 8.2583 || 10iter: 4.7045 sec.\n",
      "Iteration 5250 || Loss: 7.7676 || 10iter: 4.8035 sec.\n",
      "Iteration 5260 || Loss: 8.0323 || 10iter: 4.7005 sec.\n",
      "Iteration 5270 || Loss: 7.7976 || 10iter: 4.5820 sec.\n",
      "Iteration 5280 || Loss: 7.5001 || 10iter: 4.2620 sec.\n",
      "Iteration 5290 || Loss: 7.2826 || 10iter: 2.6789 sec.\n",
      "-------------\n",
      "epoch 67 || Epoch_TRAIN_Loss:599.0679 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1980 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 68/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5300 || Loss: 7.9632 || 10iter: 8.1771 sec.\n",
      "Iteration 5310 || Loss: 7.7930 || 10iter: 6.7827 sec.\n",
      "Iteration 5320 || Loss: 7.7828 || 10iter: 4.5906 sec.\n",
      "Iteration 5330 || Loss: 8.2715 || 10iter: 4.7493 sec.\n",
      "Iteration 5340 || Loss: 7.8642 || 10iter: 4.4897 sec.\n",
      "Iteration 5350 || Loss: 7.8017 || 10iter: 4.5096 sec.\n",
      "Iteration 5360 || Loss: 7.2366 || 10iter: 4.1203 sec.\n",
      "Iteration 5370 || Loss: 7.8621 || 10iter: 2.6622 sec.\n",
      "-------------\n",
      "epoch 68 || Epoch_TRAIN_Loss:601.5639 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9103 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 69/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5380 || Loss: 7.6877 || 10iter: 9.3257 sec.\n",
      "Iteration 5390 || Loss: 7.0243 || 10iter: 5.6261 sec.\n",
      "Iteration 5400 || Loss: 7.8067 || 10iter: 4.3516 sec.\n",
      "Iteration 5410 || Loss: 7.1640 || 10iter: 4.6052 sec.\n",
      "Iteration 5420 || Loss: 8.1231 || 10iter: 4.4238 sec.\n",
      "Iteration 5430 || Loss: 7.5592 || 10iter: 5.0849 sec.\n",
      "Iteration 5440 || Loss: 8.0851 || 10iter: 3.9210 sec.\n",
      "Iteration 5450 || Loss: 7.4871 || 10iter: 2.6849 sec.\n",
      "-------------\n",
      "epoch 69 || Epoch_TRAIN_Loss:601.9795 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5834 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 70/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5460 || Loss: 7.6620 || 10iter: 10.4507 sec.\n",
      "Iteration 5470 || Loss: 8.1450 || 10iter: 4.4456 sec.\n",
      "Iteration 5480 || Loss: 8.3274 || 10iter: 4.7247 sec.\n",
      "Iteration 5490 || Loss: 7.8073 || 10iter: 4.4589 sec.\n",
      "Iteration 5500 || Loss: 7.6594 || 10iter: 4.6261 sec.\n",
      "Iteration 5510 || Loss: 7.6661 || 10iter: 4.6662 sec.\n",
      "Iteration 5520 || Loss: 7.1128 || 10iter: 3.6981 sec.\n",
      "Iteration 5530 || Loss: 11.5163 || 10iter: 2.5165 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 70 || Epoch_TRAIN_Loss:606.2083 ||Epoch_VAL_Loss:304.9086\n",
      "timer:  49.9402 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 71/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 5540 || Loss: 6.9874 || 10iter: 10.7431 sec.\n",
      "Iteration 5550 || Loss: 8.0100 || 10iter: 4.8436 sec.\n",
      "Iteration 5560 || Loss: 7.4643 || 10iter: 4.8327 sec.\n",
      "Iteration 5570 || Loss: 7.5365 || 10iter: 4.5915 sec.\n",
      "Iteration 5580 || Loss: 7.8349 || 10iter: 4.5827 sec.\n",
      "Iteration 5590 || Loss: 7.5802 || 10iter: 4.4773 sec.\n",
      "Iteration 5600 || Loss: 7.6261 || 10iter: 3.6598 sec.\n",
      "-------------\n",
      "epoch 71 || Epoch_TRAIN_Loss:598.6575 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3422 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 72/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5610 || Loss: 7.6670 || 10iter: 3.9917 sec.\n",
      "Iteration 5620 || Loss: 7.6091 || 10iter: 6.9906 sec.\n",
      "Iteration 5630 || Loss: 7.1984 || 10iter: 5.0881 sec.\n",
      "Iteration 5640 || Loss: 6.6879 || 10iter: 4.8719 sec.\n",
      "Iteration 5650 || Loss: 7.4332 || 10iter: 4.8356 sec.\n",
      "Iteration 5660 || Loss: 8.4924 || 10iter: 4.5809 sec.\n",
      "Iteration 5670 || Loss: 7.2329 || 10iter: 4.7111 sec.\n",
      "Iteration 5680 || Loss: 7.4658 || 10iter: 3.2002 sec.\n",
      "-------------\n",
      "epoch 72 || Epoch_TRAIN_Loss:598.8267 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.7906 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 73/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5690 || Loss: 7.1916 || 10iter: 5.1382 sec.\n",
      "Iteration 5700 || Loss: 7.3807 || 10iter: 6.2853 sec.\n",
      "Iteration 5710 || Loss: 7.9816 || 10iter: 4.8995 sec.\n",
      "Iteration 5720 || Loss: 8.7566 || 10iter: 4.6261 sec.\n",
      "Iteration 5730 || Loss: 7.5132 || 10iter: 4.8136 sec.\n",
      "Iteration 5740 || Loss: 7.1238 || 10iter: 4.3125 sec.\n",
      "Iteration 5750 || Loss: 7.8037 || 10iter: 4.5654 sec.\n",
      "Iteration 5760 || Loss: 7.4804 || 10iter: 3.0135 sec.\n",
      "-------------\n",
      "epoch 73 || Epoch_TRAIN_Loss:595.4869 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.8019 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 74/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5770 || Loss: 7.8788 || 10iter: 6.2117 sec.\n",
      "Iteration 5780 || Loss: 7.7680 || 10iter: 5.6093 sec.\n",
      "Iteration 5790 || Loss: 8.1710 || 10iter: 4.9951 sec.\n",
      "Iteration 5800 || Loss: 7.7145 || 10iter: 4.6113 sec.\n",
      "Iteration 5810 || Loss: 9.4656 || 10iter: 4.6585 sec.\n",
      "Iteration 5820 || Loss: 7.5766 || 10iter: 4.6306 sec.\n",
      "Iteration 5830 || Loss: 7.0565 || 10iter: 4.8758 sec.\n",
      "Iteration 5840 || Loss: 7.8601 || 10iter: 2.9475 sec.\n",
      "-------------\n",
      "epoch 74 || Epoch_TRAIN_Loss:599.7076 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4234 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 75/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5850 || Loss: 7.2253 || 10iter: 6.5576 sec.\n",
      "Iteration 5860 || Loss: 7.0670 || 10iter: 5.5979 sec.\n",
      "Iteration 5870 || Loss: 8.1096 || 10iter: 5.4291 sec.\n",
      "Iteration 5880 || Loss: 7.2717 || 10iter: 4.7995 sec.\n",
      "Iteration 5890 || Loss: 7.3563 || 10iter: 4.5038 sec.\n",
      "Iteration 5900 || Loss: 7.6114 || 10iter: 4.3344 sec.\n",
      "Iteration 5910 || Loss: 7.4263 || 10iter: 4.6734 sec.\n",
      "Iteration 5920 || Loss: 7.3953 || 10iter: 2.7983 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 75 || Epoch_TRAIN_Loss:593.4523 ||Epoch_VAL_Loss:304.6161\n",
      "timer:  50.1948 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 76/200\n",
      "-------------\n",
      "train\n",
      "Iteration 5930 || Loss: 7.3427 || 10iter: 7.0566 sec.\n",
      "Iteration 5940 || Loss: 6.7269 || 10iter: 5.8360 sec.\n",
      "Iteration 5950 || Loss: 7.7454 || 10iter: 5.1955 sec.\n",
      "Iteration 5960 || Loss: 7.3095 || 10iter: 4.5041 sec.\n",
      "Iteration 5970 || Loss: 7.2660 || 10iter: 4.8495 sec.\n",
      "Iteration 5980 || Loss: 7.5900 || 10iter: 5.3903 sec.\n",
      "Iteration 5990 || Loss: 7.4368 || 10iter: 4.5335 sec.\n",
      "Iteration 6000 || Loss: 7.8869 || 10iter: 2.6908 sec.\n",
      "-------------\n",
      "epoch 76 || Epoch_TRAIN_Loss:596.0704 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.4308 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 77/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6010 || Loss: 6.9872 || 10iter: 7.9215 sec.\n",
      "Iteration 6020 || Loss: 8.1726 || 10iter: 5.5944 sec.\n",
      "Iteration 6030 || Loss: 8.2826 || 10iter: 4.8219 sec.\n",
      "Iteration 6040 || Loss: 7.6663 || 10iter: 4.6477 sec.\n",
      "Iteration 6050 || Loss: 7.3749 || 10iter: 4.6005 sec.\n",
      "Iteration 6060 || Loss: 7.2480 || 10iter: 4.8600 sec.\n",
      "Iteration 6070 || Loss: 7.9802 || 10iter: 4.0920 sec.\n",
      "Iteration 6080 || Loss: 7.4302 || 10iter: 2.6548 sec.\n",
      "-------------\n",
      "epoch 77 || Epoch_TRAIN_Loss:597.2385 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3073 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 78/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6090 || Loss: 7.4801 || 10iter: 7.9732 sec.\n",
      "Iteration 6100 || Loss: 7.3637 || 10iter: 6.0433 sec.\n",
      "Iteration 6110 || Loss: 7.3901 || 10iter: 4.5500 sec.\n",
      "Iteration 6120 || Loss: 7.6993 || 10iter: 4.7100 sec.\n",
      "Iteration 6130 || Loss: 8.0506 || 10iter: 4.5928 sec.\n",
      "Iteration 6140 || Loss: 6.9379 || 10iter: 4.5046 sec.\n",
      "Iteration 6150 || Loss: 7.3513 || 10iter: 4.3023 sec.\n",
      "Iteration 6160 || Loss: 8.0851 || 10iter: 2.6630 sec.\n",
      "-------------\n",
      "epoch 78 || Epoch_TRAIN_Loss:592.3016 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1573 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 79/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6170 || Loss: 7.6806 || 10iter: 8.9813 sec.\n",
      "Iteration 6180 || Loss: 8.0036 || 10iter: 5.9172 sec.\n",
      "Iteration 6190 || Loss: 7.4226 || 10iter: 4.5016 sec.\n",
      "Iteration 6200 || Loss: 7.6652 || 10iter: 4.5543 sec.\n",
      "Iteration 6210 || Loss: 8.0060 || 10iter: 4.4886 sec.\n",
      "Iteration 6220 || Loss: 7.7286 || 10iter: 5.0451 sec.\n",
      "Iteration 6230 || Loss: 7.5842 || 10iter: 3.8323 sec.\n",
      "Iteration 6240 || Loss: 7.7411 || 10iter: 2.6698 sec.\n",
      "-------------\n",
      "epoch 79 || Epoch_TRAIN_Loss:598.5707 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5131 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 80/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6250 || Loss: 7.3241 || 10iter: 10.0187 sec.\n",
      "Iteration 6260 || Loss: 7.0917 || 10iter: 4.9211 sec.\n",
      "Iteration 6270 || Loss: 7.3912 || 10iter: 4.5894 sec.\n",
      "Iteration 6280 || Loss: 7.2823 || 10iter: 4.7216 sec.\n",
      "Iteration 6290 || Loss: 7.2544 || 10iter: 4.6695 sec.\n",
      "Iteration 6300 || Loss: 7.3054 || 10iter: 4.8698 sec.\n",
      "Iteration 6310 || Loss: 7.7237 || 10iter: 3.6381 sec.\n",
      "Iteration 6320 || Loss: 7.2427 || 10iter: 2.5289 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 80 || Epoch_TRAIN_Loss:592.2913 ||Epoch_VAL_Loss:301.2833\n",
      "timer:  50.4813 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 81/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6330 || Loss: 7.2858 || 10iter: 10.2000 sec.\n",
      "Iteration 6340 || Loss: 7.4950 || 10iter: 5.8020 sec.\n",
      "Iteration 6350 || Loss: 7.5016 || 10iter: 4.3927 sec.\n",
      "Iteration 6360 || Loss: 7.1275 || 10iter: 4.3526 sec.\n",
      "Iteration 6370 || Loss: 8.0257 || 10iter: 4.6875 sec.\n",
      "Iteration 6380 || Loss: 7.2103 || 10iter: 4.5888 sec.\n",
      "Iteration 6390 || Loss: 7.9016 || 10iter: 3.4633 sec.\n",
      "-------------\n",
      "epoch 81 || Epoch_TRAIN_Loss:594.5098 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1727 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 82/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6400 || Loss: 8.2245 || 10iter: 4.7732 sec.\n",
      "Iteration 6410 || Loss: 7.4392 || 10iter: 6.3561 sec.\n",
      "Iteration 6420 || Loss: 7.3483 || 10iter: 4.8321 sec.\n",
      "Iteration 6430 || Loss: 8.5729 || 10iter: 4.8680 sec.\n",
      "Iteration 6440 || Loss: 7.5390 || 10iter: 4.5982 sec.\n",
      "Iteration 6450 || Loss: 7.4940 || 10iter: 4.6462 sec.\n",
      "Iteration 6460 || Loss: 7.8047 || 10iter: 4.6240 sec.\n",
      "Iteration 6470 || Loss: 7.5847 || 10iter: 3.3906 sec.\n",
      "-------------\n",
      "epoch 82 || Epoch_TRAIN_Loss:595.8911 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5282 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 83/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6480 || Loss: 7.9593 || 10iter: 5.6672 sec.\n",
      "Iteration 6490 || Loss: 7.3720 || 10iter: 6.1283 sec.\n",
      "Iteration 6500 || Loss: 8.0157 || 10iter: 4.9989 sec.\n",
      "Iteration 6510 || Loss: 7.3210 || 10iter: 4.5869 sec.\n",
      "Iteration 6520 || Loss: 8.3185 || 10iter: 5.0072 sec.\n",
      "Iteration 6530 || Loss: 7.7094 || 10iter: 4.4873 sec.\n",
      "Iteration 6540 || Loss: 7.0763 || 10iter: 4.4648 sec.\n",
      "Iteration 6550 || Loss: 7.2460 || 10iter: 3.1863 sec.\n",
      "-------------\n",
      "epoch 83 || Epoch_TRAIN_Loss:591.0299 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6405 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 84/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6560 || Loss: 7.3580 || 10iter: 6.0140 sec.\n",
      "Iteration 6570 || Loss: 7.6299 || 10iter: 6.2965 sec.\n",
      "Iteration 6580 || Loss: 7.0247 || 10iter: 4.7919 sec.\n",
      "Iteration 6590 || Loss: 8.3569 || 10iter: 4.6086 sec.\n",
      "Iteration 6600 || Loss: 7.6951 || 10iter: 4.3512 sec.\n",
      "Iteration 6610 || Loss: 7.1548 || 10iter: 4.5696 sec.\n",
      "Iteration 6620 || Loss: 6.7393 || 10iter: 4.6451 sec.\n",
      "Iteration 6630 || Loss: 7.6764 || 10iter: 2.8796 sec.\n",
      "-------------\n",
      "epoch 84 || Epoch_TRAIN_Loss:589.2900 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0370 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 85/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6640 || Loss: 7.4343 || 10iter: 6.5432 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 6650 || Loss: 7.1282 || 10iter: 5.8680 sec.\n",
      "Iteration 6660 || Loss: 7.4400 || 10iter: 5.7051 sec.\n",
      "Iteration 6670 || Loss: 7.5945 || 10iter: 4.7411 sec.\n",
      "Iteration 6680 || Loss: 7.2295 || 10iter: 4.6780 sec.\n",
      "Iteration 6690 || Loss: 7.4645 || 10iter: 4.6156 sec.\n",
      "Iteration 6700 || Loss: 7.2158 || 10iter: 4.2716 sec.\n",
      "Iteration 6710 || Loss: 7.4447 || 10iter: 2.8125 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 85 || Epoch_TRAIN_Loss:588.3253 ||Epoch_VAL_Loss:302.2940\n",
      "timer:  50.9072 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 86/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6720 || Loss: 7.6274 || 10iter: 7.1147 sec.\n",
      "Iteration 6730 || Loss: 7.9492 || 10iter: 5.7072 sec.\n",
      "Iteration 6740 || Loss: 7.9173 || 10iter: 4.7434 sec.\n",
      "Iteration 6750 || Loss: 7.5764 || 10iter: 4.8336 sec.\n",
      "Iteration 6760 || Loss: 7.3553 || 10iter: 4.7206 sec.\n",
      "Iteration 6770 || Loss: 7.5243 || 10iter: 4.8761 sec.\n",
      "Iteration 6780 || Loss: 8.0348 || 10iter: 4.4266 sec.\n",
      "Iteration 6790 || Loss: 7.3649 || 10iter: 2.6742 sec.\n",
      "-------------\n",
      "epoch 86 || Epoch_TRAIN_Loss:587.4084 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3876 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 87/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6800 || Loss: 8.4055 || 10iter: 7.9971 sec.\n",
      "Iteration 6810 || Loss: 7.5703 || 10iter: 5.3684 sec.\n",
      "Iteration 6820 || Loss: 8.2246 || 10iter: 4.8147 sec.\n",
      "Iteration 6830 || Loss: 7.7213 || 10iter: 4.4563 sec.\n",
      "Iteration 6840 || Loss: 7.3665 || 10iter: 4.8027 sec.\n",
      "Iteration 6850 || Loss: 7.1639 || 10iter: 4.4715 sec.\n",
      "Iteration 6860 || Loss: 7.6096 || 10iter: 4.3855 sec.\n",
      "Iteration 6870 || Loss: 7.5653 || 10iter: 2.6749 sec.\n",
      "-------------\n",
      "epoch 87 || Epoch_TRAIN_Loss:591.7999 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0807 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 88/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6880 || Loss: 7.5126 || 10iter: 8.0536 sec.\n",
      "Iteration 6890 || Loss: 7.4563 || 10iter: 6.2976 sec.\n",
      "Iteration 6900 || Loss: 7.3432 || 10iter: 4.5195 sec.\n",
      "Iteration 6910 || Loss: 7.6542 || 10iter: 4.8174 sec.\n",
      "Iteration 6920 || Loss: 7.8594 || 10iter: 4.8109 sec.\n",
      "Iteration 6930 || Loss: 7.4927 || 10iter: 4.5522 sec.\n",
      "Iteration 6940 || Loss: 8.1031 || 10iter: 4.0310 sec.\n",
      "Iteration 6950 || Loss: 6.8886 || 10iter: 2.6603 sec.\n",
      "-------------\n",
      "epoch 88 || Epoch_TRAIN_Loss:593.2286 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4981 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 89/200\n",
      "-------------\n",
      "train\n",
      "Iteration 6960 || Loss: 7.4265 || 10iter: 9.0623 sec.\n",
      "Iteration 6970 || Loss: 7.2385 || 10iter: 5.7571 sec.\n",
      "Iteration 6980 || Loss: 7.6156 || 10iter: 4.4061 sec.\n",
      "Iteration 6990 || Loss: 6.9209 || 10iter: 5.5111 sec.\n",
      "Iteration 7000 || Loss: 7.7029 || 10iter: 4.7939 sec.\n",
      "Iteration 7010 || Loss: 7.5012 || 10iter: 4.6257 sec.\n",
      "Iteration 7020 || Loss: 7.1904 || 10iter: 3.7893 sec.\n",
      "Iteration 7030 || Loss: 7.8603 || 10iter: 2.6770 sec.\n",
      "-------------\n",
      "epoch 89 || Epoch_TRAIN_Loss:588.1983 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.2024 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 90/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7040 || Loss: 7.8889 || 10iter: 10.0181 sec.\n",
      "Iteration 7050 || Loss: 7.3936 || 10iter: 5.2477 sec.\n",
      "Iteration 7060 || Loss: 7.1667 || 10iter: 4.0348 sec.\n",
      "Iteration 7070 || Loss: 7.0037 || 10iter: 4.6929 sec.\n",
      "Iteration 7080 || Loss: 7.1000 || 10iter: 4.5382 sec.\n",
      "Iteration 7090 || Loss: 7.5476 || 10iter: 4.6328 sec.\n",
      "Iteration 7100 || Loss: 7.7933 || 10iter: 3.6597 sec.\n",
      "Iteration 7110 || Loss: 7.7456 || 10iter: 2.5525 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 90 || Epoch_TRAIN_Loss:588.2473 ||Epoch_VAL_Loss:300.5868\n",
      "timer:  49.3489 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 91/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7120 || Loss: 7.1525 || 10iter: 11.0205 sec.\n",
      "Iteration 7130 || Loss: 7.6671 || 10iter: 4.4371 sec.\n",
      "Iteration 7140 || Loss: 7.2101 || 10iter: 4.8342 sec.\n",
      "Iteration 7150 || Loss: 6.7828 || 10iter: 4.4088 sec.\n",
      "Iteration 7160 || Loss: 6.8164 || 10iter: 4.6547 sec.\n",
      "Iteration 7170 || Loss: 7.5702 || 10iter: 4.9417 sec.\n",
      "Iteration 7180 || Loss: 7.3769 || 10iter: 3.3931 sec.\n",
      "-------------\n",
      "epoch 91 || Epoch_TRAIN_Loss:594.3238 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3420 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 92/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7190 || Loss: 7.1531 || 10iter: 4.8461 sec.\n",
      "Iteration 7200 || Loss: 7.5216 || 10iter: 6.3221 sec.\n",
      "Iteration 7210 || Loss: 7.5019 || 10iter: 4.6058 sec.\n",
      "Iteration 7220 || Loss: 7.4939 || 10iter: 4.7907 sec.\n",
      "Iteration 7230 || Loss: 7.8313 || 10iter: 4.4148 sec.\n",
      "Iteration 7240 || Loss: 7.8278 || 10iter: 4.3965 sec.\n",
      "Iteration 7250 || Loss: 7.9252 || 10iter: 4.7971 sec.\n",
      "Iteration 7260 || Loss: 7.8511 || 10iter: 3.2320 sec.\n",
      "-------------\n",
      "epoch 92 || Epoch_TRAIN_Loss:590.1024 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.8833 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 93/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7270 || Loss: 8.1567 || 10iter: 5.8094 sec.\n",
      "Iteration 7280 || Loss: 7.4490 || 10iter: 5.7635 sec.\n",
      "Iteration 7290 || Loss: 6.7948 || 10iter: 5.0017 sec.\n",
      "Iteration 7300 || Loss: 7.7519 || 10iter: 5.0809 sec.\n",
      "Iteration 7310 || Loss: 7.3169 || 10iter: 4.6719 sec.\n",
      "Iteration 7320 || Loss: 7.3824 || 10iter: 5.3180 sec.\n",
      "Iteration 7330 || Loss: 7.8022 || 10iter: 5.1629 sec.\n",
      "Iteration 7340 || Loss: 7.2067 || 10iter: 3.3090 sec.\n",
      "-------------\n",
      "epoch 93 || Epoch_TRAIN_Loss:589.0617 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  42.2379 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 94/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7350 || Loss: 7.3135 || 10iter: 6.4171 sec.\n",
      "Iteration 7360 || Loss: 7.3832 || 10iter: 5.4432 sec.\n",
      "Iteration 7370 || Loss: 8.0509 || 10iter: 4.7487 sec.\n",
      "Iteration 7380 || Loss: 7.2982 || 10iter: 4.6942 sec.\n",
      "Iteration 7390 || Loss: 7.6694 || 10iter: 4.4722 sec.\n",
      "Iteration 7400 || Loss: 8.0655 || 10iter: 4.5824 sec.\n",
      "Iteration 7410 || Loss: 7.8464 || 10iter: 4.6825 sec.\n",
      "Iteration 7420 || Loss: 7.4310 || 10iter: 2.8394 sec.\n",
      "-------------\n",
      "epoch 94 || Epoch_TRAIN_Loss:591.8075 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.8126 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 95/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7430 || Loss: 7.8265 || 10iter: 6.4522 sec.\n",
      "Iteration 7440 || Loss: 6.8004 || 10iter: 6.1116 sec.\n",
      "Iteration 7450 || Loss: 7.4700 || 10iter: 4.4895 sec.\n",
      "Iteration 7460 || Loss: 7.6304 || 10iter: 5.0840 sec.\n",
      "Iteration 7470 || Loss: 7.4926 || 10iter: 4.6044 sec.\n",
      "Iteration 7480 || Loss: 8.0318 || 10iter: 4.7198 sec.\n",
      "Iteration 7490 || Loss: 7.4769 || 10iter: 4.6079 sec.\n",
      "Iteration 7500 || Loss: 7.0219 || 10iter: 2.7183 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 95 || Epoch_TRAIN_Loss:584.6330 ||Epoch_VAL_Loss:299.8308\n",
      "timer:  50.2514 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 96/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7510 || Loss: 7.7920 || 10iter: 7.2799 sec.\n",
      "Iteration 7520 || Loss: 7.1878 || 10iter: 5.8867 sec.\n",
      "Iteration 7530 || Loss: 7.1418 || 10iter: 4.3971 sec.\n",
      "Iteration 7540 || Loss: 6.9078 || 10iter: 4.7642 sec.\n",
      "Iteration 7550 || Loss: 7.1373 || 10iter: 4.7465 sec.\n",
      "Iteration 7560 || Loss: 7.8928 || 10iter: 4.3741 sec.\n",
      "Iteration 7570 || Loss: 7.2422 || 10iter: 4.3998 sec.\n",
      "Iteration 7580 || Loss: 7.4658 || 10iter: 2.6924 sec.\n",
      "-------------\n",
      "epoch 96 || Epoch_TRAIN_Loss:588.6770 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.9144 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 97/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7590 || Loss: 6.8922 || 10iter: 7.8447 sec.\n",
      "Iteration 7600 || Loss: 7.0259 || 10iter: 5.5095 sec.\n",
      "Iteration 7610 || Loss: 7.8724 || 10iter: 4.7792 sec.\n",
      "Iteration 7620 || Loss: 7.1891 || 10iter: 4.3904 sec.\n",
      "Iteration 7630 || Loss: 6.7990 || 10iter: 4.8152 sec.\n",
      "Iteration 7640 || Loss: 6.8947 || 10iter: 4.7112 sec.\n",
      "Iteration 7650 || Loss: 7.1174 || 10iter: 4.2562 sec.\n",
      "Iteration 7660 || Loss: 7.1915 || 10iter: 2.7568 sec.\n",
      "-------------\n",
      "epoch 97 || Epoch_TRAIN_Loss:582.2107 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1355 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 98/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7670 || Loss: 8.0279 || 10iter: 8.5787 sec.\n",
      "Iteration 7680 || Loss: 7.3392 || 10iter: 5.6803 sec.\n",
      "Iteration 7690 || Loss: 8.1201 || 10iter: 4.3986 sec.\n",
      "Iteration 7700 || Loss: 7.4282 || 10iter: 4.6881 sec.\n",
      "Iteration 7710 || Loss: 7.0516 || 10iter: 4.4236 sec.\n",
      "Iteration 7720 || Loss: 6.9764 || 10iter: 4.8024 sec.\n",
      "Iteration 7730 || Loss: 8.1782 || 10iter: 4.0660 sec.\n",
      "Iteration 7740 || Loss: 7.3472 || 10iter: 2.6712 sec.\n",
      "-------------\n",
      "epoch 98 || Epoch_TRAIN_Loss:590.0082 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1288 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 99/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 7750 || Loss: 7.1686 || 10iter: 8.9728 sec.\n",
      "Iteration 7760 || Loss: 7.1841 || 10iter: 5.9214 sec.\n",
      "Iteration 7770 || Loss: 7.6217 || 10iter: 4.2811 sec.\n",
      "Iteration 7780 || Loss: 7.0230 || 10iter: 4.7475 sec.\n",
      "Iteration 7790 || Loss: 7.3366 || 10iter: 4.6355 sec.\n",
      "Iteration 7800 || Loss: 7.2007 || 10iter: 4.5196 sec.\n",
      "Iteration 7810 || Loss: 7.0459 || 10iter: 3.8236 sec.\n",
      "Iteration 7820 || Loss: 6.9496 || 10iter: 2.6530 sec.\n",
      "-------------\n",
      "epoch 99 || Epoch_TRAIN_Loss:582.1453 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1773 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 100/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7830 || Loss: 7.2789 || 10iter: 9.8923 sec.\n",
      "Iteration 7840 || Loss: 7.0057 || 10iter: 5.2014 sec.\n",
      "Iteration 7850 || Loss: 7.7371 || 10iter: 4.4825 sec.\n",
      "Iteration 7860 || Loss: 7.2310 || 10iter: 4.4938 sec.\n",
      "Iteration 7870 || Loss: 7.8994 || 10iter: 4.8376 sec.\n",
      "Iteration 7880 || Loss: 6.9370 || 10iter: 4.5235 sec.\n",
      "Iteration 7890 || Loss: 8.2298 || 10iter: 3.7078 sec.\n",
      "Iteration 7900 || Loss: 6.5381 || 10iter: 2.5436 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 100 || Epoch_TRAIN_Loss:585.6948 ||Epoch_VAL_Loss:297.9459\n",
      "timer:  49.9995 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 101/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7910 || Loss: 7.0061 || 10iter: 10.9339 sec.\n",
      "Iteration 7920 || Loss: 7.4342 || 10iter: 4.4529 sec.\n",
      "Iteration 7930 || Loss: 7.5358 || 10iter: 4.7741 sec.\n",
      "Iteration 7940 || Loss: 6.7486 || 10iter: 4.6485 sec.\n",
      "Iteration 7950 || Loss: 7.3894 || 10iter: 4.7685 sec.\n",
      "Iteration 7960 || Loss: 7.2468 || 10iter: 4.7124 sec.\n",
      "Iteration 7970 || Loss: 8.1299 || 10iter: 3.5925 sec.\n",
      "-------------\n",
      "epoch 101 || Epoch_TRAIN_Loss:586.9174 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5207 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 102/200\n",
      "-------------\n",
      "train\n",
      "Iteration 7980 || Loss: 7.9309 || 10iter: 4.5197 sec.\n",
      "Iteration 7990 || Loss: 7.6436 || 10iter: 7.4241 sec.\n",
      "Iteration 8000 || Loss: 7.5531 || 10iter: 5.1391 sec.\n",
      "Iteration 8010 || Loss: 7.1852 || 10iter: 4.6171 sec.\n",
      "Iteration 8020 || Loss: 7.2623 || 10iter: 4.7045 sec.\n",
      "Iteration 8030 || Loss: 7.0771 || 10iter: 4.6627 sec.\n",
      "Iteration 8040 || Loss: 7.8654 || 10iter: 4.5670 sec.\n",
      "Iteration 8050 || Loss: 6.8265 || 10iter: 3.3531 sec.\n",
      "-------------\n",
      "epoch 102 || Epoch_TRAIN_Loss:589.9328 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.5106 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 103/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8060 || Loss: 7.6911 || 10iter: 4.9579 sec.\n",
      "Iteration 8070 || Loss: 7.0948 || 10iter: 6.6260 sec.\n",
      "Iteration 8080 || Loss: 7.3885 || 10iter: 4.9338 sec.\n",
      "Iteration 8090 || Loss: 6.9873 || 10iter: 4.3150 sec.\n",
      "Iteration 8100 || Loss: 7.3647 || 10iter: 4.4038 sec.\n",
      "Iteration 8110 || Loss: 7.7001 || 10iter: 5.1419 sec.\n",
      "Iteration 8120 || Loss: 7.0713 || 10iter: 4.5221 sec.\n",
      "Iteration 8130 || Loss: 8.5060 || 10iter: 3.0072 sec.\n",
      "-------------\n",
      "epoch 103 || Epoch_TRAIN_Loss:587.4097 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0935 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 104/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8140 || Loss: 8.0284 || 10iter: 5.4640 sec.\n",
      "Iteration 8150 || Loss: 8.2897 || 10iter: 6.6699 sec.\n",
      "Iteration 8160 || Loss: 7.7584 || 10iter: 4.7262 sec.\n",
      "Iteration 8170 || Loss: 7.3163 || 10iter: 5.0959 sec.\n",
      "Iteration 8180 || Loss: 7.1363 || 10iter: 4.3724 sec.\n",
      "Iteration 8190 || Loss: 7.8477 || 10iter: 4.7068 sec.\n",
      "Iteration 8200 || Loss: 7.0453 || 10iter: 4.6548 sec.\n",
      "Iteration 8210 || Loss: 7.4194 || 10iter: 2.9592 sec.\n",
      "-------------\n",
      "epoch 104 || Epoch_TRAIN_Loss:583.6524 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5728 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 105/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8220 || Loss: 8.2125 || 10iter: 6.3072 sec.\n",
      "Iteration 8230 || Loss: 7.6987 || 10iter: 6.0824 sec.\n",
      "Iteration 8240 || Loss: 7.8884 || 10iter: 4.8840 sec.\n",
      "Iteration 8250 || Loss: 8.0170 || 10iter: 4.6532 sec.\n",
      "Iteration 8260 || Loss: 7.1687 || 10iter: 4.7507 sec.\n",
      "Iteration 8270 || Loss: 6.9681 || 10iter: 4.7291 sec.\n",
      "Iteration 8280 || Loss: 7.4087 || 10iter: 4.8327 sec.\n",
      "Iteration 8290 || Loss: 8.0016 || 10iter: 2.6163 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 105 || Epoch_TRAIN_Loss:584.8776 ||Epoch_VAL_Loss:297.9449\n",
      "timer:  50.4294 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 106/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8300 || Loss: 7.4184 || 10iter: 7.4723 sec.\n",
      "Iteration 8310 || Loss: 7.0402 || 10iter: 5.9884 sec.\n",
      "Iteration 8320 || Loss: 7.7188 || 10iter: 5.5314 sec.\n",
      "Iteration 8330 || Loss: 7.6853 || 10iter: 4.8578 sec.\n",
      "Iteration 8340 || Loss: 7.3816 || 10iter: 4.7998 sec.\n",
      "Iteration 8350 || Loss: 7.4315 || 10iter: 4.5316 sec.\n",
      "Iteration 8360 || Loss: 7.3789 || 10iter: 4.4885 sec.\n",
      "Iteration 8370 || Loss: 6.9197 || 10iter: 2.6610 sec.\n",
      "-------------\n",
      "epoch 106 || Epoch_TRAIN_Loss:581.6724 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.7253 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 107/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8380 || Loss: 7.5654 || 10iter: 7.9297 sec.\n",
      "Iteration 8390 || Loss: 7.1721 || 10iter: 5.8055 sec.\n",
      "Iteration 8400 || Loss: 7.3177 || 10iter: 4.6130 sec.\n",
      "Iteration 8410 || Loss: 7.9862 || 10iter: 4.5659 sec.\n",
      "Iteration 8420 || Loss: 7.2521 || 10iter: 4.7104 sec.\n",
      "Iteration 8430 || Loss: 7.1869 || 10iter: 4.8806 sec.\n",
      "Iteration 8440 || Loss: 7.3009 || 10iter: 4.3360 sec.\n",
      "Iteration 8450 || Loss: 7.3075 || 10iter: 2.6683 sec.\n",
      "-------------\n",
      "epoch 107 || Epoch_TRAIN_Loss:582.4484 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5464 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 108/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8460 || Loss: 8.0750 || 10iter: 8.7815 sec.\n",
      "Iteration 8470 || Loss: 7.3857 || 10iter: 5.5180 sec.\n",
      "Iteration 8480 || Loss: 7.3587 || 10iter: 4.7484 sec.\n",
      "Iteration 8490 || Loss: 7.5071 || 10iter: 4.5591 sec.\n",
      "Iteration 8500 || Loss: 7.4119 || 10iter: 4.8834 sec.\n",
      "Iteration 8510 || Loss: 7.3710 || 10iter: 4.5139 sec.\n",
      "Iteration 8520 || Loss: 7.6835 || 10iter: 4.1828 sec.\n",
      "Iteration 8530 || Loss: 6.9512 || 10iter: 2.6520 sec.\n",
      "-------------\n",
      "epoch 108 || Epoch_TRAIN_Loss:579.6068 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6680 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 109/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8540 || Loss: 7.2365 || 10iter: 9.1060 sec.\n",
      "Iteration 8550 || Loss: 7.1102 || 10iter: 5.2763 sec.\n",
      "Iteration 8560 || Loss: 6.6974 || 10iter: 4.8266 sec.\n",
      "Iteration 8570 || Loss: 7.2366 || 10iter: 4.5178 sec.\n",
      "Iteration 8580 || Loss: 7.4367 || 10iter: 4.5565 sec.\n",
      "Iteration 8590 || Loss: 7.9589 || 10iter: 4.6594 sec.\n",
      "Iteration 8600 || Loss: 6.8796 || 10iter: 3.8225 sec.\n",
      "Iteration 8610 || Loss: 7.5427 || 10iter: 2.6948 sec.\n",
      "-------------\n",
      "epoch 109 || Epoch_TRAIN_Loss:582.7377 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.9797 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 110/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8620 || Loss: 7.1961 || 10iter: 10.0523 sec.\n",
      "Iteration 8630 || Loss: 7.2362 || 10iter: 5.2240 sec.\n",
      "Iteration 8640 || Loss: 7.5863 || 10iter: 4.4107 sec.\n",
      "Iteration 8650 || Loss: 7.7914 || 10iter: 4.7144 sec.\n",
      "Iteration 8660 || Loss: 6.5650 || 10iter: 4.7128 sec.\n",
      "Iteration 8670 || Loss: 7.5434 || 10iter: 5.5153 sec.\n",
      "Iteration 8680 || Loss: 7.7197 || 10iter: 3.9477 sec.\n",
      "Iteration 8690 || Loss: 8.7154 || 10iter: 2.5195 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 110 || Epoch_TRAIN_Loss:582.7751 ||Epoch_VAL_Loss:296.5615\n",
      "timer:  50.8577 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 111/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8700 || Loss: 7.5387 || 10iter: 10.9244 sec.\n",
      "Iteration 8710 || Loss: 7.5935 || 10iter: 4.6102 sec.\n",
      "Iteration 8720 || Loss: 7.1857 || 10iter: 4.6383 sec.\n",
      "Iteration 8730 || Loss: 7.0353 || 10iter: 4.6555 sec.\n",
      "Iteration 8740 || Loss: 7.9165 || 10iter: 4.5882 sec.\n",
      "Iteration 8750 || Loss: 6.5986 || 10iter: 4.5534 sec.\n",
      "Iteration 8760 || Loss: 7.5910 || 10iter: 3.2455 sec.\n",
      "-------------\n",
      "epoch 111 || Epoch_TRAIN_Loss:584.7460 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.8123 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 112/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8770 || Loss: 6.9833 || 10iter: 4.6547 sec.\n",
      "Iteration 8780 || Loss: 7.3931 || 10iter: 6.6737 sec.\n",
      "Iteration 8790 || Loss: 6.9895 || 10iter: 4.7221 sec.\n",
      "Iteration 8800 || Loss: 8.2199 || 10iter: 4.4521 sec.\n",
      "Iteration 8810 || Loss: 7.6151 || 10iter: 4.5617 sec.\n",
      "Iteration 8820 || Loss: 7.7746 || 10iter: 4.5815 sec.\n",
      "Iteration 8830 || Loss: 7.6845 || 10iter: 4.8833 sec.\n",
      "Iteration 8840 || Loss: 8.0711 || 10iter: 3.2133 sec.\n",
      "-------------\n",
      "epoch 112 || Epoch_TRAIN_Loss:578.3346 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1974 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 113/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 8850 || Loss: 7.0904 || 10iter: 4.6496 sec.\n",
      "Iteration 8860 || Loss: 7.1688 || 10iter: 6.5432 sec.\n",
      "Iteration 8870 || Loss: 7.2400 || 10iter: 5.0953 sec.\n",
      "Iteration 8880 || Loss: 7.0444 || 10iter: 4.3575 sec.\n",
      "Iteration 8890 || Loss: 7.4830 || 10iter: 5.0300 sec.\n",
      "Iteration 8900 || Loss: 7.9879 || 10iter: 4.3857 sec.\n",
      "Iteration 8910 || Loss: 6.8580 || 10iter: 5.1090 sec.\n",
      "Iteration 8920 || Loss: 7.3038 || 10iter: 3.0980 sec.\n",
      "-------------\n",
      "epoch 113 || Epoch_TRAIN_Loss:573.6355 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4196 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 114/200\n",
      "-------------\n",
      "train\n",
      "Iteration 8930 || Loss: 7.4717 || 10iter: 5.7147 sec.\n",
      "Iteration 8940 || Loss: 7.1344 || 10iter: 6.3528 sec.\n",
      "Iteration 8950 || Loss: 7.1244 || 10iter: 4.9913 sec.\n",
      "Iteration 8960 || Loss: 7.3127 || 10iter: 4.7169 sec.\n",
      "Iteration 8970 || Loss: 7.4502 || 10iter: 4.6688 sec.\n",
      "Iteration 8980 || Loss: 7.2919 || 10iter: 4.4717 sec.\n",
      "Iteration 8990 || Loss: 7.5356 || 10iter: 4.9085 sec.\n",
      "Iteration 9000 || Loss: 7.1884 || 10iter: 3.0516 sec.\n",
      "-------------\n",
      "epoch 114 || Epoch_TRAIN_Loss:578.7113 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.8499 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 115/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9010 || Loss: 6.5735 || 10iter: 6.7449 sec.\n",
      "Iteration 9020 || Loss: 7.0028 || 10iter: 5.6139 sec.\n",
      "Iteration 9030 || Loss: 7.4866 || 10iter: 4.6598 sec.\n",
      "Iteration 9040 || Loss: 6.7900 || 10iter: 4.5810 sec.\n",
      "Iteration 9050 || Loss: 7.6491 || 10iter: 4.5473 sec.\n",
      "Iteration 9060 || Loss: 7.5041 || 10iter: 4.7796 sec.\n",
      "Iteration 9070 || Loss: 6.8421 || 10iter: 4.6601 sec.\n",
      "Iteration 9080 || Loss: 7.0880 || 10iter: 2.7476 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 115 || Epoch_TRAIN_Loss:581.9218 ||Epoch_VAL_Loss:297.3774\n",
      "timer:  49.9349 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 116/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9090 || Loss: 8.0094 || 10iter: 7.4933 sec.\n",
      "Iteration 9100 || Loss: 7.0999 || 10iter: 5.4263 sec.\n",
      "Iteration 9110 || Loss: 7.1314 || 10iter: 4.7385 sec.\n",
      "Iteration 9120 || Loss: 7.2899 || 10iter: 4.6132 sec.\n",
      "Iteration 9130 || Loss: 7.6232 || 10iter: 4.7958 sec.\n",
      "Iteration 9140 || Loss: 7.5997 || 10iter: 4.5838 sec.\n",
      "Iteration 9150 || Loss: 7.0541 || 10iter: 4.4225 sec.\n",
      "Iteration 9160 || Loss: 7.3904 || 10iter: 2.6635 sec.\n",
      "-------------\n",
      "epoch 116 || Epoch_TRAIN_Loss:577.8390 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0619 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 117/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9170 || Loss: 7.4960 || 10iter: 7.7703 sec.\n",
      "Iteration 9180 || Loss: 6.9680 || 10iter: 5.6465 sec.\n",
      "Iteration 9190 || Loss: 7.5925 || 10iter: 4.9138 sec.\n",
      "Iteration 9200 || Loss: 6.7078 || 10iter: 4.6145 sec.\n",
      "Iteration 9210 || Loss: 7.8492 || 10iter: 4.9038 sec.\n",
      "Iteration 9220 || Loss: 7.7105 || 10iter: 4.7386 sec.\n",
      "Iteration 9230 || Loss: 6.9476 || 10iter: 4.2550 sec.\n",
      "Iteration 9240 || Loss: 6.7285 || 10iter: 2.6600 sec.\n",
      "-------------\n",
      "epoch 117 || Epoch_TRAIN_Loss:581.0266 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5597 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 118/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9250 || Loss: 7.2805 || 10iter: 8.2097 sec.\n",
      "Iteration 9260 || Loss: 6.5379 || 10iter: 6.2827 sec.\n",
      "Iteration 9270 || Loss: 7.4199 || 10iter: 4.6826 sec.\n",
      "Iteration 9280 || Loss: 7.0251 || 10iter: 4.3209 sec.\n",
      "Iteration 9290 || Loss: 7.1172 || 10iter: 4.5916 sec.\n",
      "Iteration 9300 || Loss: 7.5756 || 10iter: 4.6344 sec.\n",
      "Iteration 9310 || Loss: 7.3194 || 10iter: 4.0783 sec.\n",
      "Iteration 9320 || Loss: 7.4235 || 10iter: 2.6724 sec.\n",
      "-------------\n",
      "epoch 118 || Epoch_TRAIN_Loss:582.8869 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3415 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 119/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9330 || Loss: 7.5784 || 10iter: 9.4019 sec.\n",
      "Iteration 9340 || Loss: 7.1147 || 10iter: 6.2398 sec.\n",
      "Iteration 9350 || Loss: 6.6153 || 10iter: 4.5256 sec.\n",
      "Iteration 9360 || Loss: 7.2062 || 10iter: 4.6806 sec.\n",
      "Iteration 9370 || Loss: 6.9920 || 10iter: 4.8553 sec.\n",
      "Iteration 9380 || Loss: 7.0157 || 10iter: 4.2252 sec.\n",
      "Iteration 9390 || Loss: 7.3170 || 10iter: 4.0175 sec.\n",
      "Iteration 9400 || Loss: 7.4357 || 10iter: 2.6580 sec.\n",
      "-------------\n",
      "epoch 119 || Epoch_TRAIN_Loss:578.4390 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1539 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 120/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9410 || Loss: 7.6073 || 10iter: 10.1656 sec.\n",
      "Iteration 9420 || Loss: 7.1364 || 10iter: 5.3750 sec.\n",
      "Iteration 9430 || Loss: 7.4982 || 10iter: 4.5803 sec.\n",
      "Iteration 9440 || Loss: 8.0738 || 10iter: 4.9159 sec.\n",
      "Iteration 9450 || Loss: 6.8586 || 10iter: 4.3880 sec.\n",
      "Iteration 9460 || Loss: 7.7023 || 10iter: 4.6845 sec.\n",
      "Iteration 9470 || Loss: 7.4230 || 10iter: 3.8241 sec.\n",
      "Iteration 9480 || Loss: 8.0509 || 10iter: 2.5421 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 120 || Epoch_TRAIN_Loss:586.1423 ||Epoch_VAL_Loss:295.6194\n",
      "timer:  50.4794 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 121/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9490 || Loss: 7.5052 || 10iter: 10.8785 sec.\n",
      "Iteration 9500 || Loss: 7.5529 || 10iter: 5.0231 sec.\n",
      "Iteration 9510 || Loss: 8.1428 || 10iter: 4.2529 sec.\n",
      "Iteration 9520 || Loss: 7.6561 || 10iter: 4.5438 sec.\n",
      "Iteration 9530 || Loss: 7.2574 || 10iter: 4.9043 sec.\n",
      "Iteration 9540 || Loss: 6.9811 || 10iter: 4.6368 sec.\n",
      "Iteration 9550 || Loss: 7.1143 || 10iter: 3.4890 sec.\n",
      "-------------\n",
      "epoch 121 || Epoch_TRAIN_Loss:577.4806 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3520 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 122/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9560 || Loss: 7.3377 || 10iter: 3.8914 sec.\n",
      "Iteration 9570 || Loss: 7.2411 || 10iter: 7.1895 sec.\n",
      "Iteration 9580 || Loss: 7.4802 || 10iter: 4.8860 sec.\n",
      "Iteration 9590 || Loss: 6.8401 || 10iter: 4.3448 sec.\n",
      "Iteration 9600 || Loss: 7.1096 || 10iter: 4.6998 sec.\n",
      "Iteration 9610 || Loss: 7.7473 || 10iter: 4.7443 sec.\n",
      "Iteration 9620 || Loss: 7.2734 || 10iter: 4.5481 sec.\n",
      "Iteration 9630 || Loss: 7.3905 || 10iter: 3.1288 sec.\n",
      "-------------\n",
      "epoch 122 || Epoch_TRAIN_Loss:576.8615 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.8267 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 123/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9640 || Loss: 7.2474 || 10iter: 4.9962 sec.\n",
      "Iteration 9650 || Loss: 7.8926 || 10iter: 6.7003 sec.\n",
      "Iteration 9660 || Loss: 7.5177 || 10iter: 5.1105 sec.\n",
      "Iteration 9670 || Loss: 7.5248 || 10iter: 5.2920 sec.\n",
      "Iteration 9680 || Loss: 7.2700 || 10iter: 4.5630 sec.\n",
      "Iteration 9690 || Loss: 7.3461 || 10iter: 4.8856 sec.\n",
      "Iteration 9700 || Loss: 7.3916 || 10iter: 4.9438 sec.\n",
      "Iteration 9710 || Loss: 7.3521 || 10iter: 3.0220 sec.\n",
      "-------------\n",
      "epoch 123 || Epoch_TRAIN_Loss:576.8877 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.7024 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 124/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9720 || Loss: 6.9454 || 10iter: 5.7043 sec.\n",
      "Iteration 9730 || Loss: 7.1633 || 10iter: 6.3846 sec.\n",
      "Iteration 9740 || Loss: 6.8184 || 10iter: 4.9778 sec.\n",
      "Iteration 9750 || Loss: 7.4551 || 10iter: 4.8502 sec.\n",
      "Iteration 9760 || Loss: 6.9982 || 10iter: 4.4066 sec.\n",
      "Iteration 9770 || Loss: 7.1034 || 10iter: 4.8200 sec.\n",
      "Iteration 9780 || Loss: 6.8619 || 10iter: 4.7811 sec.\n",
      "Iteration 9790 || Loss: 6.9400 || 10iter: 2.8378 sec.\n",
      "-------------\n",
      "epoch 124 || Epoch_TRAIN_Loss:577.6837 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6003 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 125/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9800 || Loss: 7.1404 || 10iter: 6.3588 sec.\n",
      "Iteration 9810 || Loss: 7.2102 || 10iter: 6.4723 sec.\n",
      "Iteration 9820 || Loss: 7.2055 || 10iter: 4.5260 sec.\n",
      "Iteration 9830 || Loss: 7.4864 || 10iter: 4.5807 sec.\n",
      "Iteration 9840 || Loss: 7.0073 || 10iter: 4.6351 sec.\n",
      "Iteration 9850 || Loss: 6.8693 || 10iter: 4.5251 sec.\n",
      "Iteration 9860 || Loss: 6.7824 || 10iter: 4.7378 sec.\n",
      "Iteration 9870 || Loss: 6.6085 || 10iter: 2.7700 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 125 || Epoch_TRAIN_Loss:578.1831 ||Epoch_VAL_Loss:294.3004\n",
      "timer:  49.7358 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 126/200\n",
      "-------------\n",
      "train\n",
      "Iteration 9880 || Loss: 7.4379 || 10iter: 7.2917 sec.\n",
      "Iteration 9890 || Loss: 7.3694 || 10iter: 5.6487 sec.\n",
      "Iteration 9900 || Loss: 7.4402 || 10iter: 5.1293 sec.\n",
      "Iteration 9910 || Loss: 7.3626 || 10iter: 4.4805 sec.\n",
      "Iteration 9920 || Loss: 7.9053 || 10iter: 4.8712 sec.\n",
      "Iteration 9930 || Loss: 6.8989 || 10iter: 4.5426 sec.\n",
      "Iteration 9940 || Loss: 7.1150 || 10iter: 4.4307 sec.\n",
      "Iteration 9950 || Loss: 7.2746 || 10iter: 2.6435 sec.\n",
      "-------------\n",
      "epoch 126 || Epoch_TRAIN_Loss:574.1263 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4301 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 127/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 9960 || Loss: 7.4915 || 10iter: 8.0207 sec.\n",
      "Iteration 9970 || Loss: 7.2640 || 10iter: 5.1966 sec.\n",
      "Iteration 9980 || Loss: 7.7932 || 10iter: 4.6986 sec.\n",
      "Iteration 9990 || Loss: 7.2663 || 10iter: 4.9063 sec.\n",
      "Iteration 10000 || Loss: 8.0473 || 10iter: 5.3923 sec.\n",
      "Iteration 10010 || Loss: 7.2607 || 10iter: 5.1468 sec.\n",
      "Iteration 10020 || Loss: 6.8290 || 10iter: 4.1784 sec.\n",
      "Iteration 10030 || Loss: 6.8988 || 10iter: 2.6921 sec.\n",
      "-------------\n",
      "epoch 127 || Epoch_TRAIN_Loss:581.0813 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.3371 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 128/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10040 || Loss: 7.1013 || 10iter: 8.3769 sec.\n",
      "Iteration 10050 || Loss: 6.4887 || 10iter: 5.7794 sec.\n",
      "Iteration 10060 || Loss: 7.3477 || 10iter: 4.7893 sec.\n",
      "Iteration 10070 || Loss: 6.7089 || 10iter: 4.5065 sec.\n",
      "Iteration 10080 || Loss: 7.0928 || 10iter: 4.6466 sec.\n",
      "Iteration 10090 || Loss: 7.0040 || 10iter: 4.8737 sec.\n",
      "Iteration 10100 || Loss: 7.1648 || 10iter: 3.9944 sec.\n",
      "Iteration 10110 || Loss: 8.0661 || 10iter: 2.6508 sec.\n",
      "-------------\n",
      "epoch 128 || Epoch_TRAIN_Loss:570.1930 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4146 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 129/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10120 || Loss: 7.8368 || 10iter: 9.1177 sec.\n",
      "Iteration 10130 || Loss: 7.2585 || 10iter: 5.6688 sec.\n",
      "Iteration 10140 || Loss: 7.2272 || 10iter: 4.5414 sec.\n",
      "Iteration 10150 || Loss: 7.3399 || 10iter: 4.7686 sec.\n",
      "Iteration 10160 || Loss: 6.9820 || 10iter: 4.4366 sec.\n",
      "Iteration 10170 || Loss: 7.1948 || 10iter: 4.4944 sec.\n",
      "Iteration 10180 || Loss: 7.5993 || 10iter: 4.0607 sec.\n",
      "Iteration 10190 || Loss: 7.1963 || 10iter: 2.6481 sec.\n",
      "-------------\n",
      "epoch 129 || Epoch_TRAIN_Loss:574.4180 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2898 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 130/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10200 || Loss: 7.6128 || 10iter: 9.7032 sec.\n",
      "Iteration 10210 || Loss: 7.0194 || 10iter: 5.3639 sec.\n",
      "Iteration 10220 || Loss: 8.1170 || 10iter: 4.7185 sec.\n",
      "Iteration 10230 || Loss: 7.1299 || 10iter: 4.6638 sec.\n",
      "Iteration 10240 || Loss: 7.2783 || 10iter: 4.7093 sec.\n",
      "Iteration 10250 || Loss: 6.8511 || 10iter: 4.5574 sec.\n",
      "Iteration 10260 || Loss: 7.0119 || 10iter: 3.6808 sec.\n",
      "Iteration 10270 || Loss: 7.5455 || 10iter: 2.5191 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 130 || Epoch_TRAIN_Loss:574.1271 ||Epoch_VAL_Loss:294.1493\n",
      "timer:  49.7080 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 131/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10280 || Loss: 7.5568 || 10iter: 10.4558 sec.\n",
      "Iteration 10290 || Loss: 6.9898 || 10iter: 5.2647 sec.\n",
      "Iteration 10300 || Loss: 7.4288 || 10iter: 4.2340 sec.\n",
      "Iteration 10310 || Loss: 7.6609 || 10iter: 4.7874 sec.\n",
      "Iteration 10320 || Loss: 6.8791 || 10iter: 4.4914 sec.\n",
      "Iteration 10330 || Loss: 7.4166 || 10iter: 5.1718 sec.\n",
      "Iteration 10340 || Loss: 7.3713 || 10iter: 3.9489 sec.\n",
      "-------------\n",
      "epoch 131 || Epoch_TRAIN_Loss:574.6388 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.0346 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 132/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10350 || Loss: 6.7643 || 10iter: 4.3651 sec.\n",
      "Iteration 10360 || Loss: 6.8826 || 10iter: 6.9685 sec.\n",
      "Iteration 10370 || Loss: 7.4984 || 10iter: 4.6406 sec.\n",
      "Iteration 10380 || Loss: 7.1694 || 10iter: 4.3415 sec.\n",
      "Iteration 10390 || Loss: 7.4499 || 10iter: 4.5952 sec.\n",
      "Iteration 10400 || Loss: 7.5368 || 10iter: 4.5173 sec.\n",
      "Iteration 10410 || Loss: 7.3719 || 10iter: 4.6361 sec.\n",
      "Iteration 10420 || Loss: 7.2645 || 10iter: 3.2136 sec.\n",
      "-------------\n",
      "epoch 132 || Epoch_TRAIN_Loss:577.7045 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.7681 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 133/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10430 || Loss: 7.0711 || 10iter: 4.8243 sec.\n",
      "Iteration 10440 || Loss: 6.8363 || 10iter: 6.5630 sec.\n",
      "Iteration 10450 || Loss: 6.8707 || 10iter: 4.7938 sec.\n",
      "Iteration 10460 || Loss: 6.9782 || 10iter: 4.6211 sec.\n",
      "Iteration 10470 || Loss: 7.3873 || 10iter: 4.9647 sec.\n",
      "Iteration 10480 || Loss: 7.6755 || 10iter: 4.4075 sec.\n",
      "Iteration 10490 || Loss: 6.9611 || 10iter: 4.9051 sec.\n",
      "Iteration 10500 || Loss: 6.9539 || 10iter: 2.8840 sec.\n",
      "-------------\n",
      "epoch 133 || Epoch_TRAIN_Loss:574.6961 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1369 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 134/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10510 || Loss: 8.2428 || 10iter: 6.0974 sec.\n",
      "Iteration 10520 || Loss: 8.0292 || 10iter: 5.8745 sec.\n",
      "Iteration 10530 || Loss: 7.3751 || 10iter: 4.9967 sec.\n",
      "Iteration 10540 || Loss: 6.5333 || 10iter: 4.6988 sec.\n",
      "Iteration 10550 || Loss: 7.3667 || 10iter: 4.5995 sec.\n",
      "Iteration 10560 || Loss: 7.4979 || 10iter: 4.6200 sec.\n",
      "Iteration 10570 || Loss: 7.5134 || 10iter: 4.8203 sec.\n",
      "Iteration 10580 || Loss: 8.2726 || 10iter: 2.8416 sec.\n",
      "-------------\n",
      "epoch 134 || Epoch_TRAIN_Loss:576.6765 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3955 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 135/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10590 || Loss: 7.5500 || 10iter: 6.5075 sec.\n",
      "Iteration 10600 || Loss: 7.0274 || 10iter: 5.8295 sec.\n",
      "Iteration 10610 || Loss: 6.8754 || 10iter: 4.6513 sec.\n",
      "Iteration 10620 || Loss: 7.4388 || 10iter: 4.7799 sec.\n",
      "Iteration 10630 || Loss: 7.9481 || 10iter: 4.8668 sec.\n",
      "Iteration 10640 || Loss: 7.6512 || 10iter: 4.6008 sec.\n",
      "Iteration 10650 || Loss: 7.9528 || 10iter: 4.4808 sec.\n",
      "Iteration 10660 || Loss: 7.6799 || 10iter: 2.8125 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 135 || Epoch_TRAIN_Loss:577.3939 ||Epoch_VAL_Loss:293.8729\n",
      "timer:  49.7811 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 136/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10670 || Loss: 7.3225 || 10iter: 7.6848 sec.\n",
      "Iteration 10680 || Loss: 7.3994 || 10iter: 5.9731 sec.\n",
      "Iteration 10690 || Loss: 8.5678 || 10iter: 4.8383 sec.\n",
      "Iteration 10700 || Loss: 7.3316 || 10iter: 4.6404 sec.\n",
      "Iteration 10710 || Loss: 7.2396 || 10iter: 4.8554 sec.\n",
      "Iteration 10720 || Loss: 8.0349 || 10iter: 4.3027 sec.\n",
      "Iteration 10730 || Loss: 7.7985 || 10iter: 4.6788 sec.\n",
      "Iteration 10740 || Loss: 7.3710 || 10iter: 2.6695 sec.\n",
      "-------------\n",
      "epoch 136 || Epoch_TRAIN_Loss:576.8054 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.0035 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 137/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10750 || Loss: 7.2845 || 10iter: 8.0017 sec.\n",
      "Iteration 10760 || Loss: 7.4307 || 10iter: 5.3706 sec.\n",
      "Iteration 10770 || Loss: 7.6102 || 10iter: 4.9277 sec.\n",
      "Iteration 10780 || Loss: 7.0921 || 10iter: 4.5122 sec.\n",
      "Iteration 10790 || Loss: 7.4216 || 10iter: 4.6171 sec.\n",
      "Iteration 10800 || Loss: 7.4911 || 10iter: 4.4397 sec.\n",
      "Iteration 10810 || Loss: 7.4861 || 10iter: 4.1485 sec.\n",
      "Iteration 10820 || Loss: 6.8312 || 10iter: 2.7003 sec.\n",
      "-------------\n",
      "epoch 137 || Epoch_TRAIN_Loss:576.4015 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.8341 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 138/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10830 || Loss: 7.2817 || 10iter: 8.2794 sec.\n",
      "Iteration 10840 || Loss: 7.5178 || 10iter: 5.9025 sec.\n",
      "Iteration 10850 || Loss: 7.0730 || 10iter: 4.5513 sec.\n",
      "Iteration 10860 || Loss: 6.9496 || 10iter: 4.7009 sec.\n",
      "Iteration 10870 || Loss: 8.0611 || 10iter: 4.7081 sec.\n",
      "Iteration 10880 || Loss: 7.2695 || 10iter: 4.7038 sec.\n",
      "Iteration 10890 || Loss: 7.6475 || 10iter: 3.9774 sec.\n",
      "Iteration 10900 || Loss: 7.0555 || 10iter: 2.7080 sec.\n",
      "-------------\n",
      "epoch 138 || Epoch_TRAIN_Loss:571.0220 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3421 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 139/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10910 || Loss: 7.4452 || 10iter: 9.2618 sec.\n",
      "Iteration 10920 || Loss: 7.4211 || 10iter: 5.4712 sec.\n",
      "Iteration 10930 || Loss: 7.6358 || 10iter: 4.7151 sec.\n",
      "Iteration 10940 || Loss: 7.2285 || 10iter: 4.4167 sec.\n",
      "Iteration 10950 || Loss: 7.0817 || 10iter: 4.5845 sec.\n",
      "Iteration 10960 || Loss: 7.6436 || 10iter: 4.4574 sec.\n",
      "Iteration 10970 || Loss: 6.5682 || 10iter: 4.0542 sec.\n",
      "Iteration 10980 || Loss: 7.3023 || 10iter: 2.6286 sec.\n",
      "-------------\n",
      "epoch 139 || Epoch_TRAIN_Loss:571.5058 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1180 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 140/200\n",
      "-------------\n",
      "train\n",
      "Iteration 10990 || Loss: 7.5310 || 10iter: 10.2096 sec.\n",
      "Iteration 11000 || Loss: 6.4565 || 10iter: 4.7629 sec.\n",
      "Iteration 11010 || Loss: 7.2681 || 10iter: 4.7688 sec.\n",
      "Iteration 11020 || Loss: 7.7045 || 10iter: 5.0648 sec.\n",
      "Iteration 11030 || Loss: 7.3677 || 10iter: 4.8969 sec.\n",
      "Iteration 11040 || Loss: 7.8281 || 10iter: 4.8544 sec.\n",
      "Iteration 11050 || Loss: 7.3236 || 10iter: 3.4897 sec.\n",
      "Iteration 11060 || Loss: 7.3083 || 10iter: 2.5359 sec.\n",
      "-------------\n",
      "val\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------\n",
      "epoch 140 || Epoch_TRAIN_Loss:575.1957 ||Epoch_VAL_Loss:293.7180\n",
      "timer:  50.9668 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 141/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11070 || Loss: 6.9538 || 10iter: 10.8916 sec.\n",
      "Iteration 11080 || Loss: 7.7461 || 10iter: 5.0452 sec.\n",
      "Iteration 11090 || Loss: 6.8263 || 10iter: 4.3716 sec.\n",
      "Iteration 11100 || Loss: 7.1041 || 10iter: 4.7142 sec.\n",
      "Iteration 11110 || Loss: 7.4043 || 10iter: 4.5860 sec.\n",
      "Iteration 11120 || Loss: 7.3300 || 10iter: 4.8669 sec.\n",
      "Iteration 11130 || Loss: 7.9089 || 10iter: 3.4477 sec.\n",
      "-------------\n",
      "epoch 141 || Epoch_TRAIN_Loss:571.6904 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6133 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 142/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11140 || Loss: 7.2441 || 10iter: 4.6571 sec.\n",
      "Iteration 11150 || Loss: 7.4624 || 10iter: 6.4610 sec.\n",
      "Iteration 11160 || Loss: 7.6889 || 10iter: 4.9664 sec.\n",
      "Iteration 11170 || Loss: 7.5451 || 10iter: 4.5932 sec.\n",
      "Iteration 11180 || Loss: 6.8067 || 10iter: 4.6131 sec.\n",
      "Iteration 11190 || Loss: 7.6417 || 10iter: 4.5705 sec.\n",
      "Iteration 11200 || Loss: 7.7362 || 10iter: 4.9215 sec.\n",
      "Iteration 11210 || Loss: 7.7536 || 10iter: 3.2218 sec.\n",
      "-------------\n",
      "epoch 142 || Epoch_TRAIN_Loss:575.2959 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4410 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 143/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11220 || Loss: 7.3801 || 10iter: 4.9909 sec.\n",
      "Iteration 11230 || Loss: 8.2562 || 10iter: 6.5422 sec.\n",
      "Iteration 11240 || Loss: 7.1603 || 10iter: 5.1970 sec.\n",
      "Iteration 11250 || Loss: 7.4000 || 10iter: 4.3503 sec.\n",
      "Iteration 11260 || Loss: 7.1450 || 10iter: 4.6371 sec.\n",
      "Iteration 11270 || Loss: 7.3638 || 10iter: 4.6198 sec.\n",
      "Iteration 11280 || Loss: 6.8897 || 10iter: 4.9714 sec.\n",
      "Iteration 11290 || Loss: 7.7305 || 10iter: 2.9400 sec.\n",
      "-------------\n",
      "epoch 143 || Epoch_TRAIN_Loss:571.8088 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3479 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 144/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11300 || Loss: 7.2967 || 10iter: 6.3031 sec.\n",
      "Iteration 11310 || Loss: 7.6774 || 10iter: 5.8462 sec.\n",
      "Iteration 11320 || Loss: 7.1495 || 10iter: 4.7036 sec.\n",
      "Iteration 11330 || Loss: 7.3068 || 10iter: 4.9406 sec.\n",
      "Iteration 11340 || Loss: 7.4025 || 10iter: 4.5283 sec.\n",
      "Iteration 11350 || Loss: 7.0159 || 10iter: 5.4791 sec.\n",
      "Iteration 11360 || Loss: 7.7064 || 10iter: 4.9498 sec.\n",
      "Iteration 11370 || Loss: 7.0853 || 10iter: 2.8207 sec.\n",
      "-------------\n",
      "epoch 144 || Epoch_TRAIN_Loss:571.5646 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.5255 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 145/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11380 || Loss: 6.4576 || 10iter: 6.6446 sec.\n",
      "Iteration 11390 || Loss: 7.1834 || 10iter: 5.6504 sec.\n",
      "Iteration 11400 || Loss: 6.8581 || 10iter: 5.1062 sec.\n",
      "Iteration 11410 || Loss: 7.2510 || 10iter: 4.6492 sec.\n",
      "Iteration 11420 || Loss: 6.5396 || 10iter: 4.6643 sec.\n",
      "Iteration 11430 || Loss: 7.7049 || 10iter: 4.9042 sec.\n",
      "Iteration 11440 || Loss: 7.3298 || 10iter: 4.5027 sec.\n",
      "Iteration 11450 || Loss: 7.1344 || 10iter: 2.7500 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 145 || Epoch_TRAIN_Loss:573.8990 ||Epoch_VAL_Loss:293.9738\n",
      "timer:  50.2826 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 146/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11460 || Loss: 7.9838 || 10iter: 7.4219 sec.\n",
      "Iteration 11470 || Loss: 7.3126 || 10iter: 5.6876 sec.\n",
      "Iteration 11480 || Loss: 7.6843 || 10iter: 4.6114 sec.\n",
      "Iteration 11490 || Loss: 7.4630 || 10iter: 4.7901 sec.\n",
      "Iteration 11500 || Loss: 7.4489 || 10iter: 4.4778 sec.\n",
      "Iteration 11510 || Loss: 7.6281 || 10iter: 4.8287 sec.\n",
      "Iteration 11520 || Loss: 7.0300 || 10iter: 4.5938 sec.\n",
      "Iteration 11530 || Loss: 7.5662 || 10iter: 2.6716 sec.\n",
      "-------------\n",
      "epoch 146 || Epoch_TRAIN_Loss:573.9748 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4909 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 147/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11540 || Loss: 7.6485 || 10iter: 7.8809 sec.\n",
      "Iteration 11550 || Loss: 7.8599 || 10iter: 5.5774 sec.\n",
      "Iteration 11560 || Loss: 7.4093 || 10iter: 4.9319 sec.\n",
      "Iteration 11570 || Loss: 7.0768 || 10iter: 4.4903 sec.\n",
      "Iteration 11580 || Loss: 7.0405 || 10iter: 4.7251 sec.\n",
      "Iteration 11590 || Loss: 6.8575 || 10iter: 4.7948 sec.\n",
      "Iteration 11600 || Loss: 7.5986 || 10iter: 4.3146 sec.\n",
      "Iteration 11610 || Loss: 8.0684 || 10iter: 2.6854 sec.\n",
      "-------------\n",
      "epoch 147 || Epoch_TRAIN_Loss:576.9274 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4812 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 148/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11620 || Loss: 6.7740 || 10iter: 8.3167 sec.\n",
      "Iteration 11630 || Loss: 6.5482 || 10iter: 5.7667 sec.\n",
      "Iteration 11640 || Loss: 7.7408 || 10iter: 4.5262 sec.\n",
      "Iteration 11650 || Loss: 7.7748 || 10iter: 4.6730 sec.\n",
      "Iteration 11660 || Loss: 7.5204 || 10iter: 4.5150 sec.\n",
      "Iteration 11670 || Loss: 6.7127 || 10iter: 4.6722 sec.\n",
      "Iteration 11680 || Loss: 7.2542 || 10iter: 4.8663 sec.\n",
      "Iteration 11690 || Loss: 6.7240 || 10iter: 2.6216 sec.\n",
      "-------------\n",
      "epoch 148 || Epoch_TRAIN_Loss:570.1226 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.8032 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 149/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11700 || Loss: 7.1336 || 10iter: 9.4844 sec.\n",
      "Iteration 11710 || Loss: 7.5565 || 10iter: 5.3316 sec.\n",
      "Iteration 11720 || Loss: 7.0815 || 10iter: 4.3551 sec.\n",
      "Iteration 11730 || Loss: 6.7530 || 10iter: 4.6644 sec.\n",
      "Iteration 11740 || Loss: 6.7921 || 10iter: 4.7653 sec.\n",
      "Iteration 11750 || Loss: 7.7344 || 10iter: 4.5476 sec.\n",
      "Iteration 11760 || Loss: 7.1127 || 10iter: 3.8587 sec.\n",
      "Iteration 11770 || Loss: 7.1817 || 10iter: 2.6669 sec.\n",
      "-------------\n",
      "epoch 149 || Epoch_TRAIN_Loss:574.9215 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2201 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 150/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11780 || Loss: 7.1980 || 10iter: 9.7431 sec.\n",
      "Iteration 11790 || Loss: 6.8822 || 10iter: 5.6360 sec.\n",
      "Iteration 11800 || Loss: 7.2375 || 10iter: 4.6040 sec.\n",
      "Iteration 11810 || Loss: 7.5947 || 10iter: 4.7467 sec.\n",
      "Iteration 11820 || Loss: 7.1348 || 10iter: 4.3298 sec.\n",
      "Iteration 11830 || Loss: 7.3748 || 10iter: 4.7498 sec.\n",
      "Iteration 11840 || Loss: 7.1066 || 10iter: 3.6908 sec.\n",
      "Iteration 11850 || Loss: 6.3724 || 10iter: 2.5333 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 150 || Epoch_TRAIN_Loss:570.7947 ||Epoch_VAL_Loss:293.6096\n",
      "timer:  50.2487 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 151/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11860 || Loss: 7.3107 || 10iter: 10.5340 sec.\n",
      "Iteration 11870 || Loss: 6.9040 || 10iter: 5.0806 sec.\n",
      "Iteration 11880 || Loss: 7.6523 || 10iter: 4.6228 sec.\n",
      "Iteration 11890 || Loss: 7.1355 || 10iter: 4.7912 sec.\n",
      "Iteration 11900 || Loss: 7.4637 || 10iter: 4.7298 sec.\n",
      "Iteration 11910 || Loss: 7.0118 || 10iter: 4.8073 sec.\n",
      "Iteration 11920 || Loss: 6.9661 || 10iter: 3.4535 sec.\n",
      "-------------\n",
      "epoch 151 || Epoch_TRAIN_Loss:572.2335 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.6592 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 152/200\n",
      "-------------\n",
      "train\n",
      "Iteration 11930 || Loss: 7.6933 || 10iter: 5.1031 sec.\n",
      "Iteration 11940 || Loss: 7.0457 || 10iter: 6.1102 sec.\n",
      "Iteration 11950 || Loss: 7.2054 || 10iter: 4.8769 sec.\n",
      "Iteration 11960 || Loss: 6.7430 || 10iter: 4.3881 sec.\n",
      "Iteration 11970 || Loss: 7.0442 || 10iter: 4.7270 sec.\n",
      "Iteration 11980 || Loss: 6.8719 || 10iter: 4.5999 sec.\n",
      "Iteration 11990 || Loss: 7.2829 || 10iter: 4.7725 sec.\n",
      "Iteration 12000 || Loss: 7.6121 || 10iter: 3.2296 sec.\n",
      "-------------\n",
      "epoch 152 || Epoch_TRAIN_Loss:570.5452 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2269 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 153/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12010 || Loss: 7.1722 || 10iter: 5.5889 sec.\n",
      "Iteration 12020 || Loss: 7.2448 || 10iter: 6.8359 sec.\n",
      "Iteration 12030 || Loss: 7.9012 || 10iter: 4.8284 sec.\n",
      "Iteration 12040 || Loss: 6.8639 || 10iter: 4.5241 sec.\n",
      "Iteration 12050 || Loss: 7.2631 || 10iter: 4.6041 sec.\n",
      "Iteration 12060 || Loss: 7.9856 || 10iter: 4.4586 sec.\n",
      "Iteration 12070 || Loss: 6.7047 || 10iter: 4.8027 sec.\n",
      "Iteration 12080 || Loss: 7.4510 || 10iter: 3.3377 sec.\n",
      "-------------\n",
      "epoch 153 || Epoch_TRAIN_Loss:573.6658 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1721 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 154/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12090 || Loss: 7.0029 || 10iter: 6.0814 sec.\n",
      "Iteration 12100 || Loss: 7.3731 || 10iter: 6.1061 sec.\n",
      "Iteration 12110 || Loss: 7.4135 || 10iter: 4.6272 sec.\n",
      "Iteration 12120 || Loss: 7.7608 || 10iter: 4.7099 sec.\n",
      "Iteration 12130 || Loss: 7.5150 || 10iter: 4.8357 sec.\n",
      "Iteration 12140 || Loss: 7.8665 || 10iter: 4.5372 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 12150 || Loss: 7.3470 || 10iter: 4.6345 sec.\n",
      "Iteration 12160 || Loss: 7.1112 || 10iter: 2.8005 sec.\n",
      "-------------\n",
      "epoch 154 || Epoch_TRAIN_Loss:574.5252 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1482 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 155/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12170 || Loss: 7.3502 || 10iter: 7.0269 sec.\n",
      "Iteration 12180 || Loss: 6.7901 || 10iter: 5.8352 sec.\n",
      "Iteration 12190 || Loss: 7.3249 || 10iter: 4.6013 sec.\n",
      "Iteration 12200 || Loss: 7.6315 || 10iter: 4.7466 sec.\n",
      "Iteration 12210 || Loss: 6.5737 || 10iter: 4.8769 sec.\n",
      "Iteration 12220 || Loss: 7.8693 || 10iter: 4.7575 sec.\n",
      "Iteration 12230 || Loss: 7.1512 || 10iter: 4.6599 sec.\n",
      "Iteration 12240 || Loss: 7.3771 || 10iter: 2.7224 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 155 || Epoch_TRAIN_Loss:573.9549 ||Epoch_VAL_Loss:293.3489\n",
      "timer:  50.4585 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 156/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12250 || Loss: 7.5770 || 10iter: 7.4807 sec.\n",
      "Iteration 12260 || Loss: 6.8033 || 10iter: 5.6085 sec.\n",
      "Iteration 12270 || Loss: 6.9614 || 10iter: 4.9744 sec.\n",
      "Iteration 12280 || Loss: 7.7536 || 10iter: 4.7112 sec.\n",
      "Iteration 12290 || Loss: 6.8687 || 10iter: 4.4619 sec.\n",
      "Iteration 12300 || Loss: 7.4027 || 10iter: 4.7713 sec.\n",
      "Iteration 12310 || Loss: 7.4319 || 10iter: 4.6219 sec.\n",
      "Iteration 12320 || Loss: 7.4778 || 10iter: 2.7685 sec.\n",
      "-------------\n",
      "epoch 156 || Epoch_TRAIN_Loss:567.6059 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.7966 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 157/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12330 || Loss: 7.0238 || 10iter: 7.4892 sec.\n",
      "Iteration 12340 || Loss: 6.9099 || 10iter: 6.6905 sec.\n",
      "Iteration 12350 || Loss: 7.3351 || 10iter: 5.1382 sec.\n",
      "Iteration 12360 || Loss: 7.0254 || 10iter: 4.3937 sec.\n",
      "Iteration 12370 || Loss: 7.5438 || 10iter: 4.7434 sec.\n",
      "Iteration 12380 || Loss: 7.2341 || 10iter: 4.7463 sec.\n",
      "Iteration 12390 || Loss: 6.9628 || 10iter: 4.1426 sec.\n",
      "Iteration 12400 || Loss: 7.5895 || 10iter: 2.6583 sec.\n",
      "-------------\n",
      "epoch 157 || Epoch_TRAIN_Loss:569.3688 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1213 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 158/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12410 || Loss: 7.0473 || 10iter: 8.2171 sec.\n",
      "Iteration 12420 || Loss: 6.7383 || 10iter: 5.9174 sec.\n",
      "Iteration 12430 || Loss: 7.0010 || 10iter: 4.3728 sec.\n",
      "Iteration 12440 || Loss: 7.4937 || 10iter: 4.6825 sec.\n",
      "Iteration 12450 || Loss: 7.4006 || 10iter: 4.4077 sec.\n",
      "Iteration 12460 || Loss: 7.1754 || 10iter: 4.9198 sec.\n",
      "Iteration 12470 || Loss: 6.5733 || 10iter: 3.9430 sec.\n",
      "Iteration 12480 || Loss: 6.5590 || 10iter: 2.6658 sec.\n",
      "-------------\n",
      "epoch 158 || Epoch_TRAIN_Loss:573.8211 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.9519 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 159/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12490 || Loss: 6.8566 || 10iter: 9.3531 sec.\n",
      "Iteration 12500 || Loss: 7.4385 || 10iter: 5.5204 sec.\n",
      "Iteration 12510 || Loss: 7.2805 || 10iter: 4.4701 sec.\n",
      "Iteration 12520 || Loss: 7.2889 || 10iter: 4.8865 sec.\n",
      "Iteration 12530 || Loss: 7.0996 || 10iter: 4.7186 sec.\n",
      "Iteration 12540 || Loss: 7.4418 || 10iter: 4.7489 sec.\n",
      "Iteration 12550 || Loss: 6.8480 || 10iter: 4.0437 sec.\n",
      "Iteration 12560 || Loss: 7.9310 || 10iter: 2.6436 sec.\n",
      "-------------\n",
      "epoch 159 || Epoch_TRAIN_Loss:572.2935 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9591 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 160/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12570 || Loss: 7.0032 || 10iter: 10.2824 sec.\n",
      "Iteration 12580 || Loss: 7.4423 || 10iter: 5.2581 sec.\n",
      "Iteration 12590 || Loss: 6.8527 || 10iter: 4.4813 sec.\n",
      "Iteration 12600 || Loss: 6.9626 || 10iter: 4.7108 sec.\n",
      "Iteration 12610 || Loss: 7.5238 || 10iter: 4.6725 sec.\n",
      "Iteration 12620 || Loss: 6.8945 || 10iter: 4.5926 sec.\n",
      "Iteration 12630 || Loss: 7.4017 || 10iter: 3.8107 sec.\n",
      "Iteration 12640 || Loss: 8.3638 || 10iter: 2.4978 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 160 || Epoch_TRAIN_Loss:574.3362 ||Epoch_VAL_Loss:293.2446\n",
      "timer:  50.1496 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 161/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12650 || Loss: 7.4052 || 10iter: 10.9563 sec.\n",
      "Iteration 12660 || Loss: 6.6485 || 10iter: 4.7145 sec.\n",
      "Iteration 12670 || Loss: 7.5967 || 10iter: 5.2406 sec.\n",
      "Iteration 12680 || Loss: 7.3037 || 10iter: 5.0515 sec.\n",
      "Iteration 12690 || Loss: 7.5061 || 10iter: 4.8459 sec.\n",
      "Iteration 12700 || Loss: 6.8496 || 10iter: 4.8603 sec.\n",
      "Iteration 12710 || Loss: 7.1787 || 10iter: 3.4250 sec.\n",
      "-------------\n",
      "epoch 161 || Epoch_TRAIN_Loss:574.4441 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.7810 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 162/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12720 || Loss: 7.5228 || 10iter: 4.5667 sec.\n",
      "Iteration 12730 || Loss: 6.9586 || 10iter: 6.7942 sec.\n",
      "Iteration 12740 || Loss: 8.2422 || 10iter: 4.9903 sec.\n",
      "Iteration 12750 || Loss: 7.0525 || 10iter: 4.6187 sec.\n",
      "Iteration 12760 || Loss: 7.6231 || 10iter: 4.6536 sec.\n",
      "Iteration 12770 || Loss: 6.7453 || 10iter: 4.6673 sec.\n",
      "Iteration 12780 || Loss: 6.8578 || 10iter: 5.0190 sec.\n",
      "Iteration 12790 || Loss: 6.8606 || 10iter: 3.3168 sec.\n",
      "-------------\n",
      "epoch 162 || Epoch_TRAIN_Loss:572.6270 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1175 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 163/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12800 || Loss: 7.1732 || 10iter: 5.1481 sec.\n",
      "Iteration 12810 || Loss: 6.8914 || 10iter: 6.3752 sec.\n",
      "Iteration 12820 || Loss: 7.1483 || 10iter: 4.5886 sec.\n",
      "Iteration 12830 || Loss: 7.1781 || 10iter: 4.8525 sec.\n",
      "Iteration 12840 || Loss: 7.2689 || 10iter: 4.6878 sec.\n",
      "Iteration 12850 || Loss: 6.7128 || 10iter: 4.6449 sec.\n",
      "Iteration 12860 || Loss: 6.9443 || 10iter: 4.4681 sec.\n",
      "Iteration 12870 || Loss: 7.5362 || 10iter: 3.4282 sec.\n",
      "-------------\n",
      "epoch 163 || Epoch_TRAIN_Loss:574.7355 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3577 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 164/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12880 || Loss: 7.2142 || 10iter: 5.7730 sec.\n",
      "Iteration 12890 || Loss: 7.4065 || 10iter: 6.4030 sec.\n",
      "Iteration 12900 || Loss: 6.4970 || 10iter: 4.5465 sec.\n",
      "Iteration 12910 || Loss: 7.1834 || 10iter: 4.7167 sec.\n",
      "Iteration 12920 || Loss: 6.8629 || 10iter: 4.5630 sec.\n",
      "Iteration 12930 || Loss: 8.0348 || 10iter: 4.7142 sec.\n",
      "Iteration 12940 || Loss: 7.9080 || 10iter: 4.9027 sec.\n",
      "Iteration 12950 || Loss: 7.4381 || 10iter: 2.7278 sec.\n",
      "-------------\n",
      "epoch 164 || Epoch_TRAIN_Loss:573.1441 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3240 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 165/200\n",
      "-------------\n",
      "train\n",
      "Iteration 12960 || Loss: 7.4108 || 10iter: 6.5192 sec.\n",
      "Iteration 12970 || Loss: 6.9301 || 10iter: 5.9263 sec.\n",
      "Iteration 12980 || Loss: 7.5280 || 10iter: 4.7379 sec.\n",
      "Iteration 12990 || Loss: 7.3351 || 10iter: 4.5861 sec.\n",
      "Iteration 13000 || Loss: 7.3247 || 10iter: 4.5752 sec.\n",
      "Iteration 13010 || Loss: 6.5417 || 10iter: 4.5025 sec.\n",
      "Iteration 13020 || Loss: 7.3726 || 10iter: 5.6817 sec.\n",
      "Iteration 13030 || Loss: 6.9415 || 10iter: 2.7194 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 165 || Epoch_TRAIN_Loss:569.3042 ||Epoch_VAL_Loss:293.5182\n",
      "timer:  50.2022 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 166/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13040 || Loss: 8.2597 || 10iter: 7.2230 sec.\n",
      "Iteration 13050 || Loss: 6.9067 || 10iter: 5.7958 sec.\n",
      "Iteration 13060 || Loss: 6.9493 || 10iter: 4.6356 sec.\n",
      "Iteration 13070 || Loss: 7.3462 || 10iter: 4.7734 sec.\n",
      "Iteration 13080 || Loss: 6.6728 || 10iter: 4.7569 sec.\n",
      "Iteration 13090 || Loss: 7.6388 || 10iter: 4.6906 sec.\n",
      "Iteration 13100 || Loss: 6.8477 || 10iter: 4.4775 sec.\n",
      "Iteration 13110 || Loss: 7.0948 || 10iter: 2.6994 sec.\n",
      "-------------\n",
      "epoch 166 || Epoch_TRAIN_Loss:572.7244 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4128 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 167/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13120 || Loss: 7.1396 || 10iter: 8.2236 sec.\n",
      "Iteration 13130 || Loss: 6.8127 || 10iter: 5.4922 sec.\n",
      "Iteration 13140 || Loss: 7.6956 || 10iter: 4.8916 sec.\n",
      "Iteration 13150 || Loss: 7.3508 || 10iter: 4.6233 sec.\n",
      "Iteration 13160 || Loss: 7.6286 || 10iter: 4.5309 sec.\n",
      "Iteration 13170 || Loss: 7.1241 || 10iter: 4.3502 sec.\n",
      "Iteration 13180 || Loss: 6.7556 || 10iter: 4.3156 sec.\n",
      "Iteration 13190 || Loss: 7.2687 || 10iter: 2.6941 sec.\n",
      "-------------\n",
      "epoch 167 || Epoch_TRAIN_Loss:572.3019 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2238 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 168/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13200 || Loss: 7.4356 || 10iter: 8.3084 sec.\n",
      "Iteration 13210 || Loss: 7.3365 || 10iter: 5.9348 sec.\n",
      "Iteration 13220 || Loss: 7.2665 || 10iter: 4.6086 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 13230 || Loss: 7.8335 || 10iter: 4.6327 sec.\n",
      "Iteration 13240 || Loss: 7.8003 || 10iter: 5.1521 sec.\n",
      "Iteration 13250 || Loss: 7.9116 || 10iter: 4.6609 sec.\n",
      "Iteration 13260 || Loss: 7.4263 || 10iter: 3.9394 sec.\n",
      "Iteration 13270 || Loss: 7.2538 || 10iter: 2.6949 sec.\n",
      "-------------\n",
      "epoch 168 || Epoch_TRAIN_Loss:574.1076 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.7270 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 169/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13280 || Loss: 7.2439 || 10iter: 9.5531 sec.\n",
      "Iteration 13290 || Loss: 7.2440 || 10iter: 5.6237 sec.\n",
      "Iteration 13300 || Loss: 7.0751 || 10iter: 4.6113 sec.\n",
      "Iteration 13310 || Loss: 7.2790 || 10iter: 4.8045 sec.\n",
      "Iteration 13320 || Loss: 7.3349 || 10iter: 4.7545 sec.\n",
      "Iteration 13330 || Loss: 7.2753 || 10iter: 4.5395 sec.\n",
      "Iteration 13340 || Loss: 7.1332 || 10iter: 4.0341 sec.\n",
      "Iteration 13350 || Loss: 7.0883 || 10iter: 2.6836 sec.\n",
      "-------------\n",
      "epoch 169 || Epoch_TRAIN_Loss:573.5362 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.1576 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 170/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13360 || Loss: 8.2190 || 10iter: 10.6970 sec.\n",
      "Iteration 13370 || Loss: 6.6666 || 10iter: 5.0192 sec.\n",
      "Iteration 13380 || Loss: 6.9523 || 10iter: 4.7031 sec.\n",
      "Iteration 13390 || Loss: 6.9494 || 10iter: 4.6165 sec.\n",
      "Iteration 13400 || Loss: 7.2040 || 10iter: 4.6329 sec.\n",
      "Iteration 13410 || Loss: 7.3944 || 10iter: 4.9005 sec.\n",
      "Iteration 13420 || Loss: 6.5273 || 10iter: 3.4743 sec.\n",
      "Iteration 13430 || Loss: 7.5935 || 10iter: 2.5216 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 170 || Epoch_TRAIN_Loss:575.4545 ||Epoch_VAL_Loss:293.0885\n",
      "timer:  50.4535 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 171/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13440 || Loss: 6.8220 || 10iter: 11.0422 sec.\n",
      "Iteration 13450 || Loss: 7.3355 || 10iter: 4.7339 sec.\n",
      "Iteration 13460 || Loss: 6.8853 || 10iter: 4.5276 sec.\n",
      "Iteration 13470 || Loss: 7.6575 || 10iter: 4.5383 sec.\n",
      "Iteration 13480 || Loss: 7.0684 || 10iter: 4.7742 sec.\n",
      "Iteration 13490 || Loss: 7.0131 || 10iter: 4.6402 sec.\n",
      "Iteration 13500 || Loss: 7.8532 || 10iter: 3.5236 sec.\n",
      "-------------\n",
      "epoch 171 || Epoch_TRAIN_Loss:573.7741 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4581 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 172/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13510 || Loss: 7.2962 || 10iter: 4.5247 sec.\n",
      "Iteration 13520 || Loss: 7.1702 || 10iter: 6.4870 sec.\n",
      "Iteration 13530 || Loss: 6.5987 || 10iter: 4.6515 sec.\n",
      "Iteration 13540 || Loss: 6.8061 || 10iter: 4.6186 sec.\n",
      "Iteration 13550 || Loss: 6.8534 || 10iter: 4.7692 sec.\n",
      "Iteration 13560 || Loss: 7.7524 || 10iter: 4.3126 sec.\n",
      "Iteration 13570 || Loss: 7.2519 || 10iter: 4.9431 sec.\n",
      "Iteration 13580 || Loss: 7.5200 || 10iter: 3.3635 sec.\n",
      "-------------\n",
      "epoch 172 || Epoch_TRAIN_Loss:575.4766 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.0818 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 173/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13590 || Loss: 7.1035 || 10iter: 4.6472 sec.\n",
      "Iteration 13600 || Loss: 7.6123 || 10iter: 6.9405 sec.\n",
      "Iteration 13610 || Loss: 7.0211 || 10iter: 4.9175 sec.\n",
      "Iteration 13620 || Loss: 7.1169 || 10iter: 4.5714 sec.\n",
      "Iteration 13630 || Loss: 6.9131 || 10iter: 4.7059 sec.\n",
      "Iteration 13640 || Loss: 6.6215 || 10iter: 4.8294 sec.\n",
      "Iteration 13650 || Loss: 6.9274 || 10iter: 4.5674 sec.\n",
      "Iteration 13660 || Loss: 7.4952 || 10iter: 3.1771 sec.\n",
      "-------------\n",
      "epoch 173 || Epoch_TRAIN_Loss:572.3840 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5626 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 174/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13670 || Loss: 7.9468 || 10iter: 6.0232 sec.\n",
      "Iteration 13680 || Loss: 7.7249 || 10iter: 6.4260 sec.\n",
      "Iteration 13690 || Loss: 7.2122 || 10iter: 5.5992 sec.\n",
      "Iteration 13700 || Loss: 7.4502 || 10iter: 4.7509 sec.\n",
      "Iteration 13710 || Loss: 6.5580 || 10iter: 4.6361 sec.\n",
      "Iteration 13720 || Loss: 6.9367 || 10iter: 4.5769 sec.\n",
      "Iteration 13730 || Loss: 7.3547 || 10iter: 4.7624 sec.\n",
      "Iteration 13740 || Loss: 7.6706 || 10iter: 2.8669 sec.\n",
      "-------------\n",
      "epoch 174 || Epoch_TRAIN_Loss:575.6487 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.5018 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 175/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13750 || Loss: 7.5580 || 10iter: 6.8609 sec.\n",
      "Iteration 13760 || Loss: 7.1815 || 10iter: 6.1904 sec.\n",
      "Iteration 13770 || Loss: 6.9058 || 10iter: 4.4237 sec.\n",
      "Iteration 13780 || Loss: 6.8240 || 10iter: 4.8048 sec.\n",
      "Iteration 13790 || Loss: 6.9299 || 10iter: 4.7743 sec.\n",
      "Iteration 13800 || Loss: 7.3141 || 10iter: 4.4660 sec.\n",
      "Iteration 13810 || Loss: 7.1054 || 10iter: 4.8364 sec.\n",
      "Iteration 13820 || Loss: 7.2848 || 10iter: 2.7412 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 175 || Epoch_TRAIN_Loss:570.4557 ||Epoch_VAL_Loss:293.5524\n",
      "timer:  50.4637 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 176/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13830 || Loss: 7.6506 || 10iter: 7.5170 sec.\n",
      "Iteration 13840 || Loss: 7.1391 || 10iter: 5.6852 sec.\n",
      "Iteration 13850 || Loss: 6.9645 || 10iter: 4.5323 sec.\n",
      "Iteration 13860 || Loss: 7.2008 || 10iter: 4.3538 sec.\n",
      "Iteration 13870 || Loss: 6.6127 || 10iter: 4.8588 sec.\n",
      "Iteration 13880 || Loss: 6.3402 || 10iter: 4.9062 sec.\n",
      "Iteration 13890 || Loss: 7.5324 || 10iter: 4.3762 sec.\n",
      "Iteration 13900 || Loss: 7.0583 || 10iter: 2.7467 sec.\n",
      "-------------\n",
      "epoch 176 || Epoch_TRAIN_Loss:572.3106 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3618 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 177/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13910 || Loss: 6.7623 || 10iter: 8.1886 sec.\n",
      "Iteration 13920 || Loss: 7.4239 || 10iter: 5.5941 sec.\n",
      "Iteration 13930 || Loss: 7.6403 || 10iter: 4.5822 sec.\n",
      "Iteration 13940 || Loss: 7.7593 || 10iter: 4.4873 sec.\n",
      "Iteration 13950 || Loss: 7.1892 || 10iter: 4.8537 sec.\n",
      "Iteration 13960 || Loss: 7.1940 || 10iter: 4.7149 sec.\n",
      "Iteration 13970 || Loss: 7.9081 || 10iter: 4.1225 sec.\n",
      "Iteration 13980 || Loss: 6.7828 || 10iter: 2.6884 sec.\n",
      "-------------\n",
      "epoch 177 || Epoch_TRAIN_Loss:570.9037 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3004 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 178/200\n",
      "-------------\n",
      "train\n",
      "Iteration 13990 || Loss: 7.9541 || 10iter: 8.6084 sec.\n",
      "Iteration 14000 || Loss: 7.5784 || 10iter: 5.6806 sec.\n",
      "Iteration 14010 || Loss: 7.4648 || 10iter: 5.4821 sec.\n",
      "Iteration 14020 || Loss: 8.0001 || 10iter: 5.0008 sec.\n",
      "Iteration 14030 || Loss: 7.3344 || 10iter: 4.4759 sec.\n",
      "Iteration 14040 || Loss: 7.1634 || 10iter: 4.8440 sec.\n",
      "Iteration 14050 || Loss: 6.6594 || 10iter: 4.3378 sec.\n",
      "Iteration 14060 || Loss: 7.4844 || 10iter: 2.6612 sec.\n",
      "-------------\n",
      "epoch 178 || Epoch_TRAIN_Loss:568.5207 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.8957 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 179/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14070 || Loss: 7.3815 || 10iter: 8.8224 sec.\n",
      "Iteration 14080 || Loss: 7.3932 || 10iter: 5.4366 sec.\n",
      "Iteration 14090 || Loss: 8.0269 || 10iter: 4.4939 sec.\n",
      "Iteration 14100 || Loss: 7.0981 || 10iter: 4.6060 sec.\n",
      "Iteration 14110 || Loss: 7.2319 || 10iter: 4.7433 sec.\n",
      "Iteration 14120 || Loss: 7.0479 || 10iter: 4.6872 sec.\n",
      "Iteration 14130 || Loss: 7.2410 || 10iter: 4.0888 sec.\n",
      "Iteration 14140 || Loss: 7.1867 || 10iter: 2.6454 sec.\n",
      "-------------\n",
      "epoch 179 || Epoch_TRAIN_Loss:574.2057 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1208 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 180/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14150 || Loss: 8.2406 || 10iter: 10.2247 sec.\n",
      "Iteration 14160 || Loss: 7.3523 || 10iter: 5.2682 sec.\n",
      "Iteration 14170 || Loss: 6.3545 || 10iter: 4.3197 sec.\n",
      "Iteration 14180 || Loss: 7.0701 || 10iter: 4.5270 sec.\n",
      "Iteration 14190 || Loss: 6.9992 || 10iter: 4.8360 sec.\n",
      "Iteration 14200 || Loss: 7.1718 || 10iter: 4.9091 sec.\n",
      "Iteration 14210 || Loss: 6.9497 || 10iter: 3.4606 sec.\n",
      "Iteration 14220 || Loss: 6.7760 || 10iter: 2.5293 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 180 || Epoch_TRAIN_Loss:571.5631 ||Epoch_VAL_Loss:292.9650\n",
      "timer:  50.0040 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 181/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14230 || Loss: 7.4133 || 10iter: 10.7219 sec.\n",
      "Iteration 14240 || Loss: 6.6703 || 10iter: 4.6784 sec.\n",
      "Iteration 14250 || Loss: 6.8882 || 10iter: 4.5573 sec.\n",
      "Iteration 14260 || Loss: 7.0488 || 10iter: 4.3216 sec.\n",
      "Iteration 14270 || Loss: 6.8626 || 10iter: 4.9142 sec.\n",
      "Iteration 14280 || Loss: 7.0236 || 10iter: 4.7053 sec.\n",
      "Iteration 14290 || Loss: 7.8480 || 10iter: 3.3712 sec.\n",
      "-------------\n",
      "epoch 181 || Epoch_TRAIN_Loss:569.3026 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.9626 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 182/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14300 || Loss: 6.9595 || 10iter: 3.9415 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 14310 || Loss: 7.6364 || 10iter: 7.3449 sec.\n",
      "Iteration 14320 || Loss: 6.7037 || 10iter: 4.6634 sec.\n",
      "Iteration 14330 || Loss: 7.5829 || 10iter: 4.4787 sec.\n",
      "Iteration 14340 || Loss: 7.7472 || 10iter: 5.2835 sec.\n",
      "Iteration 14350 || Loss: 7.3572 || 10iter: 4.9510 sec.\n",
      "Iteration 14360 || Loss: 7.7453 || 10iter: 4.8470 sec.\n",
      "Iteration 14370 || Loss: 7.0304 || 10iter: 3.0764 sec.\n",
      "-------------\n",
      "epoch 182 || Epoch_TRAIN_Loss:575.7130 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.9863 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 183/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14380 || Loss: 7.2030 || 10iter: 5.0208 sec.\n",
      "Iteration 14390 || Loss: 7.0232 || 10iter: 6.4780 sec.\n",
      "Iteration 14400 || Loss: 7.0671 || 10iter: 4.5976 sec.\n",
      "Iteration 14410 || Loss: 6.8240 || 10iter: 4.6595 sec.\n",
      "Iteration 14420 || Loss: 6.4890 || 10iter: 4.5177 sec.\n",
      "Iteration 14430 || Loss: 7.0021 || 10iter: 4.4433 sec.\n",
      "Iteration 14440 || Loss: 7.1840 || 10iter: 4.7339 sec.\n",
      "Iteration 14450 || Loss: 7.8378 || 10iter: 3.1606 sec.\n",
      "-------------\n",
      "epoch 183 || Epoch_TRAIN_Loss:573.6929 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.7583 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 184/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14460 || Loss: 7.0136 || 10iter: 5.8015 sec.\n",
      "Iteration 14470 || Loss: 6.8866 || 10iter: 6.0467 sec.\n",
      "Iteration 14480 || Loss: 6.6355 || 10iter: 5.0119 sec.\n",
      "Iteration 14490 || Loss: 7.1089 || 10iter: 4.8843 sec.\n",
      "Iteration 14500 || Loss: 7.1422 || 10iter: 4.3245 sec.\n",
      "Iteration 14510 || Loss: 6.8201 || 10iter: 4.7248 sec.\n",
      "Iteration 14520 || Loss: 7.3549 || 10iter: 4.7388 sec.\n",
      "Iteration 14530 || Loss: 7.7164 || 10iter: 2.9931 sec.\n",
      "-------------\n",
      "epoch 184 || Epoch_TRAIN_Loss:568.9381 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.4769 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 185/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14540 || Loss: 7.9319 || 10iter: 7.1536 sec.\n",
      "Iteration 14550 || Loss: 6.6005 || 10iter: 5.6147 sec.\n",
      "Iteration 14560 || Loss: 6.9540 || 10iter: 4.8041 sec.\n",
      "Iteration 14570 || Loss: 7.7631 || 10iter: 4.6165 sec.\n",
      "Iteration 14580 || Loss: 7.2303 || 10iter: 4.8006 sec.\n",
      "Iteration 14590 || Loss: 6.7838 || 10iter: 4.5974 sec.\n",
      "Iteration 14600 || Loss: 7.1134 || 10iter: 4.3381 sec.\n",
      "Iteration 14610 || Loss: 6.7874 || 10iter: 2.7315 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 185 || Epoch_TRAIN_Loss:567.2715 ||Epoch_VAL_Loss:292.8814\n",
      "timer:  50.2699 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 186/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14620 || Loss: 7.2612 || 10iter: 6.9991 sec.\n",
      "Iteration 14630 || Loss: 7.1419 || 10iter: 6.1623 sec.\n",
      "Iteration 14640 || Loss: 7.2584 || 10iter: 4.7664 sec.\n",
      "Iteration 14650 || Loss: 7.2642 || 10iter: 4.5919 sec.\n",
      "Iteration 14660 || Loss: 6.7770 || 10iter: 4.6503 sec.\n",
      "Iteration 14670 || Loss: 7.3383 || 10iter: 4.9156 sec.\n",
      "Iteration 14680 || Loss: 7.0349 || 10iter: 5.1975 sec.\n",
      "Iteration 14690 || Loss: 7.5762 || 10iter: 2.6843 sec.\n",
      "-------------\n",
      "epoch 186 || Epoch_TRAIN_Loss:571.3230 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.3325 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 187/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14700 || Loss: 7.4594 || 10iter: 8.0318 sec.\n",
      "Iteration 14710 || Loss: 7.3843 || 10iter: 5.5474 sec.\n",
      "Iteration 14720 || Loss: 7.6373 || 10iter: 4.8650 sec.\n",
      "Iteration 14730 || Loss: 7.4640 || 10iter: 4.4978 sec.\n",
      "Iteration 14740 || Loss: 6.8900 || 10iter: 4.6450 sec.\n",
      "Iteration 14750 || Loss: 7.6530 || 10iter: 4.6896 sec.\n",
      "Iteration 14760 || Loss: 6.6599 || 10iter: 4.1360 sec.\n",
      "Iteration 14770 || Loss: 7.0455 || 10iter: 2.6584 sec.\n",
      "-------------\n",
      "epoch 187 || Epoch_TRAIN_Loss:575.0004 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.1879 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 188/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14780 || Loss: 6.7276 || 10iter: 7.9597 sec.\n",
      "Iteration 14790 || Loss: 7.2918 || 10iter: 5.8023 sec.\n",
      "Iteration 14800 || Loss: 7.3421 || 10iter: 4.8256 sec.\n",
      "Iteration 14810 || Loss: 7.3924 || 10iter: 4.7294 sec.\n",
      "Iteration 14820 || Loss: 7.1889 || 10iter: 4.6454 sec.\n",
      "Iteration 14830 || Loss: 7.0083 || 10iter: 4.6034 sec.\n",
      "Iteration 14840 || Loss: 7.9772 || 10iter: 4.1934 sec.\n",
      "Iteration 14850 || Loss: 6.9904 || 10iter: 2.6371 sec.\n",
      "-------------\n",
      "epoch 188 || Epoch_TRAIN_Loss:573.2946 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2053 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 189/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14860 || Loss: 7.3957 || 10iter: 9.4059 sec.\n",
      "Iteration 14870 || Loss: 7.2016 || 10iter: 5.4717 sec.\n",
      "Iteration 14880 || Loss: 7.6194 || 10iter: 4.6812 sec.\n",
      "Iteration 14890 || Loss: 6.8895 || 10iter: 4.4354 sec.\n",
      "Iteration 14900 || Loss: 6.7543 || 10iter: 4.3578 sec.\n",
      "Iteration 14910 || Loss: 7.3461 || 10iter: 4.9797 sec.\n",
      "Iteration 14920 || Loss: 6.8323 || 10iter: 3.7635 sec.\n",
      "Iteration 14930 || Loss: 7.0402 || 10iter: 2.6767 sec.\n",
      "-------------\n",
      "epoch 189 || Epoch_TRAIN_Loss:571.7940 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2889 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 190/200\n",
      "-------------\n",
      "train\n",
      "Iteration 14940 || Loss: 7.5815 || 10iter: 10.4368 sec.\n",
      "Iteration 14950 || Loss: 6.9362 || 10iter: 4.6738 sec.\n",
      "Iteration 14960 || Loss: 6.5106 || 10iter: 4.3587 sec.\n",
      "Iteration 14970 || Loss: 7.4325 || 10iter: 4.7540 sec.\n",
      "Iteration 14980 || Loss: 7.4984 || 10iter: 5.0342 sec.\n",
      "Iteration 14990 || Loss: 7.4721 || 10iter: 4.4281 sec.\n",
      "Iteration 15000 || Loss: 7.6131 || 10iter: 3.7512 sec.\n",
      "Iteration 15010 || Loss: 6.9195 || 10iter: 2.5338 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 190 || Epoch_TRAIN_Loss:572.0478 ||Epoch_VAL_Loss:293.3096\n",
      "timer:  50.2899 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 191/200\n",
      "-------------\n",
      "train\n",
      "Iteration 15020 || Loss: 7.5064 || 10iter: 11.5125 sec.\n",
      "Iteration 15030 || Loss: 7.0514 || 10iter: 4.6861 sec.\n",
      "Iteration 15040 || Loss: 8.2653 || 10iter: 4.5545 sec.\n",
      "Iteration 15050 || Loss: 6.6473 || 10iter: 4.7757 sec.\n",
      "Iteration 15060 || Loss: 7.4732 || 10iter: 4.3785 sec.\n",
      "Iteration 15070 || Loss: 6.6148 || 10iter: 4.4884 sec.\n",
      "Iteration 15080 || Loss: 7.3268 || 10iter: 3.7802 sec.\n",
      "-------------\n",
      "epoch 191 || Epoch_TRAIN_Loss:568.0997 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.8573 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 192/200\n",
      "-------------\n",
      "train\n",
      "Iteration 15090 || Loss: 7.8942 || 10iter: 4.0649 sec.\n",
      "Iteration 15100 || Loss: 7.5198 || 10iter: 7.4390 sec.\n",
      "Iteration 15110 || Loss: 8.0729 || 10iter: 4.8630 sec.\n",
      "Iteration 15120 || Loss: 8.4800 || 10iter: 4.3191 sec.\n",
      "Iteration 15130 || Loss: 6.8123 || 10iter: 4.6771 sec.\n",
      "Iteration 15140 || Loss: 7.5184 || 10iter: 4.9685 sec.\n",
      "Iteration 15150 || Loss: 7.5446 || 10iter: 4.6022 sec.\n",
      "Iteration 15160 || Loss: 6.8239 || 10iter: 3.1869 sec.\n",
      "-------------\n",
      "epoch 192 || Epoch_TRAIN_Loss:573.4597 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.5611 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 193/200\n",
      "-------------\n",
      "train\n",
      "Iteration 15170 || Loss: 6.8399 || 10iter: 4.7954 sec.\n",
      "Iteration 15180 || Loss: 6.8668 || 10iter: 6.7221 sec.\n",
      "Iteration 15190 || Loss: 7.0226 || 10iter: 4.5541 sec.\n",
      "Iteration 15200 || Loss: 7.3049 || 10iter: 4.6232 sec.\n",
      "Iteration 15210 || Loss: 7.1309 || 10iter: 4.7839 sec.\n",
      "Iteration 15220 || Loss: 7.6421 || 10iter: 4.4511 sec.\n",
      "Iteration 15230 || Loss: 7.5998 || 10iter: 4.5489 sec.\n",
      "Iteration 15240 || Loss: 7.0406 || 10iter: 2.9341 sec.\n",
      "-------------\n",
      "epoch 193 || Epoch_TRAIN_Loss:572.3722 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  39.5828 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 194/200\n",
      "-------------\n",
      "train\n",
      "Iteration 15250 || Loss: 7.0642 || 10iter: 6.3609 sec.\n",
      "Iteration 15260 || Loss: 7.4629 || 10iter: 5.7679 sec.\n",
      "Iteration 15270 || Loss: 6.6125 || 10iter: 4.9479 sec.\n",
      "Iteration 15280 || Loss: 6.8522 || 10iter: 4.5412 sec.\n",
      "Iteration 15290 || Loss: 7.6610 || 10iter: 4.8550 sec.\n",
      "Iteration 15300 || Loss: 7.3711 || 10iter: 4.5541 sec.\n",
      "Iteration 15310 || Loss: 7.2377 || 10iter: 4.6861 sec.\n",
      "Iteration 15320 || Loss: 7.5425 || 10iter: 2.7450 sec.\n",
      "-------------\n",
      "epoch 194 || Epoch_TRAIN_Loss:573.6693 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3540 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 195/200\n",
      "-------------\n",
      "train\n",
      "Iteration 15330 || Loss: 7.3782 || 10iter: 6.6871 sec.\n",
      "Iteration 15340 || Loss: 6.8738 || 10iter: 5.7452 sec.\n",
      "Iteration 15350 || Loss: 7.0425 || 10iter: 5.1742 sec.\n",
      "Iteration 15360 || Loss: 7.0918 || 10iter: 5.4359 sec.\n",
      "Iteration 15370 || Loss: 7.0899 || 10iter: 4.8299 sec.\n",
      "Iteration 15380 || Loss: 7.5025 || 10iter: 4.3902 sec.\n",
      "Iteration 15390 || Loss: 7.0752 || 10iter: 4.6849 sec.\n",
      "Iteration 15400 || Loss: 6.9695 || 10iter: 2.7440 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 195 || Epoch_TRAIN_Loss:569.9227 ||Epoch_VAL_Loss:293.1457\n",
      "timer:  51.0525 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 196/200\n",
      "-------------\n",
      "train\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 15410 || Loss: 7.6410 || 10iter: 7.5616 sec.\n",
      "Iteration 15420 || Loss: 7.3759 || 10iter: 5.5895 sec.\n",
      "Iteration 15430 || Loss: 7.6558 || 10iter: 4.8027 sec.\n",
      "Iteration 15440 || Loss: 7.0454 || 10iter: 4.4895 sec.\n",
      "Iteration 15450 || Loss: 7.3891 || 10iter: 4.6120 sec.\n",
      "Iteration 15460 || Loss: 7.1927 || 10iter: 4.6221 sec.\n",
      "Iteration 15470 || Loss: 7.1433 || 10iter: 4.6311 sec.\n",
      "Iteration 15480 || Loss: 7.2046 || 10iter: 2.6545 sec.\n",
      "-------------\n",
      "epoch 196 || Epoch_TRAIN_Loss:576.4735 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3655 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 197/200\n",
      "-------------\n",
      "train\n",
      "Iteration 15490 || Loss: 6.9888 || 10iter: 7.5767 sec.\n",
      "Iteration 15500 || Loss: 7.5062 || 10iter: 6.0118 sec.\n",
      "Iteration 15510 || Loss: 7.4012 || 10iter: 4.5765 sec.\n",
      "Iteration 15520 || Loss: 7.2515 || 10iter: 4.4953 sec.\n",
      "Iteration 15530 || Loss: 7.1827 || 10iter: 4.7971 sec.\n",
      "Iteration 15540 || Loss: 7.3055 || 10iter: 4.4811 sec.\n",
      "Iteration 15550 || Loss: 7.0113 || 10iter: 4.5126 sec.\n",
      "Iteration 15560 || Loss: 7.4985 || 10iter: 2.6438 sec.\n",
      "-------------\n",
      "epoch 197 || Epoch_TRAIN_Loss:571.6806 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2137 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 198/200\n",
      "-------------\n",
      "train\n",
      "Iteration 15570 || Loss: 7.5763 || 10iter: 8.4586 sec.\n",
      "Iteration 15580 || Loss: 7.3506 || 10iter: 5.7593 sec.\n",
      "Iteration 15590 || Loss: 8.2844 || 10iter: 4.5096 sec.\n",
      "Iteration 15600 || Loss: 7.1393 || 10iter: 4.6626 sec.\n",
      "Iteration 15610 || Loss: 7.8801 || 10iter: 4.5088 sec.\n",
      "Iteration 15620 || Loss: 7.2232 || 10iter: 4.7014 sec.\n",
      "Iteration 15630 || Loss: 6.9819 || 10iter: 4.1834 sec.\n",
      "Iteration 15640 || Loss: 7.4658 || 10iter: 2.6741 sec.\n",
      "-------------\n",
      "epoch 198 || Epoch_TRAIN_Loss:569.3093 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.3261 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 199/200\n",
      "-------------\n",
      "train\n",
      "Iteration 15650 || Loss: 7.6412 || 10iter: 9.0087 sec.\n",
      "Iteration 15660 || Loss: 7.4701 || 10iter: 5.9095 sec.\n",
      "Iteration 15670 || Loss: 7.3803 || 10iter: 4.3672 sec.\n",
      "Iteration 15680 || Loss: 7.8174 || 10iter: 4.6526 sec.\n",
      "Iteration 15690 || Loss: 6.9264 || 10iter: 4.8223 sec.\n",
      "Iteration 15700 || Loss: 7.1174 || 10iter: 5.4088 sec.\n",
      "Iteration 15710 || Loss: 7.5314 || 10iter: 3.6637 sec.\n",
      "Iteration 15720 || Loss: 7.7765 || 10iter: 2.6525 sec.\n",
      "-------------\n",
      "epoch 199 || Epoch_TRAIN_Loss:573.3405 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  41.0477 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 200/200\n",
      "-------------\n",
      "train\n",
      "Iteration 15730 || Loss: 7.1424 || 10iter: 9.5154 sec.\n",
      "Iteration 15740 || Loss: 7.5307 || 10iter: 5.5743 sec.\n",
      "Iteration 15750 || Loss: 7.1785 || 10iter: 4.3163 sec.\n",
      "Iteration 15760 || Loss: 7.0232 || 10iter: 4.6900 sec.\n",
      "Iteration 15770 || Loss: 7.9294 || 10iter: 4.6023 sec.\n",
      "Iteration 15780 || Loss: 7.0691 || 10iter: 4.7369 sec.\n",
      "Iteration 15790 || Loss: 7.9515 || 10iter: 3.6279 sec.\n",
      "Iteration 15800 || Loss: 7.8573 || 10iter: 2.5172 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 200 || Epoch_TRAIN_Loss:568.8524 ||Epoch_VAL_Loss:292.8755\n",
      "timer:  49.2043 sec.\n",
      "lr is: 1e-05\n",
      "-------------\n",
      "Epoch 201/200\n",
      "-------------\n",
      "train\n",
      "Iteration 15810 || Loss: 7.1843 || 10iter: 10.6043 sec.\n",
      "Iteration 15820 || Loss: 7.2308 || 10iter: 5.1298 sec.\n",
      "Iteration 15830 || Loss: 8.2273 || 10iter: 4.3853 sec.\n",
      "Iteration 15840 || Loss: 7.3061 || 10iter: 4.9136 sec.\n",
      "Iteration 15850 || Loss: 7.4018 || 10iter: 4.5649 sec.\n",
      "Iteration 15860 || Loss: 7.6735 || 10iter: 4.5424 sec.\n",
      "Iteration 15870 || Loss: 7.4854 || 10iter: 3.3850 sec.\n",
      "-------------\n",
      "epoch 201 || Epoch_TRAIN_Loss:570.0179 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  40.2099 sec.\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 200\n",
    "train_model(net, dataloaders_dict, criterion, optimizer, num_epochs=num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}