train_BlazeFace.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import stuff\n",
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "from itertools import product as product\n",
    "import time\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Function\n",
    "import pandas as pd\n",
    "\n",
    "# import dataset\n",
    "from utils.dataset import VOCDataset, DatasetTransform, make_datapath_list, Anno_xml2list, od_collate_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make dirs\n",
    "os.makedirs(\"weights\", exist_ok=True)\n",
    "os.makedirs(\"log\", exist_ok=True)\n",
    "input_size = 128\n",
    "batch_size = 32\n",
    "\n",
    "# Blazeface channels\n",
    "channels = 64\n",
    "\"\"\"\n",
    "The original blazeface use channels=24. \n",
    "However, it was not enough to detect persons in our dataset.\n",
    "Since detecting faces are quite a simple task compared to person detection, fewer channels were enabled.\n",
    "\"\"\"\n",
    "\n",
    "# Use focal loss or not\n",
    "focal = False\n",
    "\n",
    "# Use centernet-like Blazeface\n",
    "center = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# set up person only VOC dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "val not found\n",
      "trainlist:  6469\n",
      "vallist:  2097\n"
     ]
    }
   ],
   "source": [
    "# load files\n",
    "vocpath = os.path.join(\"..\", \"VOCdevkit\", \"VOC2007\")\n",
    "train_img_list, train_anno_list, val_img_list, val_anno_list = make_datapath_list(vocpath, cls=\"person\")\n",
    "\n",
    "# extend with VOC2012\n",
    "vocpath = \"../VOCdevkit/VOC2012\"\n",
    "train_img_list2, train_anno_list2, _, _ = make_datapath_list(vocpath, cls=\"person\", VOC2012=True)\n",
    "\n",
    "train_img_list.extend(train_img_list2)\n",
    "train_anno_list.extend(train_anno_list2)\n",
    "\n",
    "# make Dataset\n",
    "voc_classes = ['person']\n",
    "color_mean = (104, 117, 123)  # (BGR)の色の平均値\n",
    "\n",
    "print(\"trainlist: \", len(train_img_list))\n",
    "print(\"vallist: \", len(val_img_list))\n",
    "\n",
    "## DatasetTransformを適応\n",
    "transform = DatasetTransform(input_size, color_mean)\n",
    "transform_anno = Anno_xml2list(voc_classes)\n",
    "\n",
    "train_dataset = VOCDataset(train_img_list, train_anno_list, phase = \"train\", transform=transform, transform_anno = transform_anno)\n",
    "val_dataset = VOCDataset(val_img_list, val_anno_list, phase=\"val\", transform=DatasetTransform(\n",
    "    input_size, color_mean), transform_anno=Anno_xml2list(voc_classes))\n",
    "\n",
    "train_dataloader = data.DataLoader(\n",
    "    train_dataset, batch_size=batch_size, shuffle=True, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "val_dataloader = data.DataLoader(\n",
    "    val_dataset, batch_size=batch_size, shuffle=False, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "dataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 3, 128, 128])\n",
      "32\n",
      "torch.Size([3, 5])\n"
     ]
    }
   ],
   "source": [
    "# check operation\n",
    "batch_iterator = iter(dataloaders_dict[\"train\"])  # iter\n",
    "images, targets = next(batch_iterator)  # get first element\n",
    "print(images.size())  # torch.Size([4, 3, 300, 300])\n",
    "print(len(targets))\n",
    "print(targets[1].shape)  # check targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.4453, 0.4589, 0.8464, 1.0000, 0.0000],\n",
       "        [0.4453, 0.5749, 0.6536, 1.0000, 0.0000],\n",
       "        [0.8151, 0.7198, 0.8932, 0.9614, 0.0000]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test with ssd model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "if input_size==256:\n",
    "    from utils.blazeface import SSD256 as SSD\n",
    "    ssd_cfg = {\n",
    "        'num_classes': 2,  # 背景クラスを含めた合計クラス数\n",
    "        'input_size': input_size,  # 画像の入力サイズ\n",
    "        'bbox_aspect_num': [4, 6],  # 出力するDBoxのアスペクト比の種類\n",
    "        'feature_maps': [16, 8],  # 各sourceの画像サイズ\n",
    "        'steps': [8, 16],  # DBOXの大きさを決める\n",
    "        'min_sizes': [16, 32],  # DBOXの大きさを決める\n",
    "        'max_sizes': [32, 100],  # DBOXの大きさを決める\n",
    "        'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "    }\n",
    "elif input_size==128:\n",
    "    from utils.blazeface import SSD\n",
    "    ssd_cfg = {\n",
    "        'num_classes': 2,  # 背景クラスを含めた合計クラス数\n",
    "        'input_size': 128,  # 画像の入力サイズ\n",
    "        'bbox_aspect_num': [4, 6],  # 出力するDBoxのアスペクト比の種類\n",
    "        'feature_maps': [16, 8],  # 各sourceの画像サイズ\n",
    "        'steps': [4, 8],  # DBOXの大きさを決める\n",
    "        'min_sizes': [30, 60],  # DBOXの大きさを決める\n",
    "        'max_sizes': [60, 128],  # DBOXの大きさを決める\n",
    "        'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using: cuda:0\n",
      "set weights!\n"
     ]
    }
   ],
   "source": [
    "net = SSD(phase=\"train\", cfg=ssd_cfg, channels=channels)\n",
    "\n",
    "# SSDのweightsを設定\n",
    "\n",
    "def weights_init(m):\n",
    "    if isinstance(m, nn.Conv2d):\n",
    "        init.kaiming_normal_(m.weight.data)\n",
    "        if m.bias is not None:\n",
    "            nn.init.constant_(m.bias, 0.0)\n",
    "\n",
    "# set inits for loc and conf\n",
    "net.loc.apply(weights_init)\n",
    "net.conf.apply(weights_init)\n",
    "\n",
    "# GPUが使えるか確認\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"using:\", device)\n",
    "\n",
    "print(\"set weights!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SSD(\n",
      "  (blaze): BlazeFace(\n",
      "    (features): Sequential(\n",
      "      (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "      (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU(inplace=True)\n",
      "      (3): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=64)\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(64, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=64)\n",
      "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (4): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=64)\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(64, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=64)\n",
      "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (5): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=64)\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(128, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=128)\n",
      "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (6): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(128, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=128)\n",
      "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(128, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=128)\n",
      "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (7): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(128, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=128)\n",
      "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(128, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=128)\n",
      "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (8): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(128, 128, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=128)\n",
      "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(64, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=64)\n",
      "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (9): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(256, 256, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=256)\n",
      "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(64, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=64)\n",
      "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (10): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(256, 256, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=256)\n",
      "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(64, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=64)\n",
      "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (extra): BlazeFaceExtra(\n",
      "    (features): Sequential(\n",
      "      (0): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(256, 256, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=256)\n",
      "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(64, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=64)\n",
      "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (mp): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "      )\n",
      "      (1): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(256, 256, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=256)\n",
      "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(64, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=64)\n",
      "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "      (2): BlazeBlock(\n",
      "        (conv1): Sequential(\n",
      "          (0): Conv2d(256, 256, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=256)\n",
      "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "        (act): ReLU(inplace=True)\n",
      "        (conv2): Sequential(\n",
      "          (0): ReLU(inplace=True)\n",
      "          (1): Conv2d(64, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=64)\n",
      "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "          (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (loc): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(256, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Conv2d(256, 24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "  )\n",
      "  (conf): ModuleList(\n",
      "    (0): Sequential(\n",
      "      (0): Conv2d(256, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "    (1): Sequential(\n",
      "      (0): Conv2d(256, 12, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.ssd_model import MultiBoxLoss\n",
    "\n",
    "# define loss\n",
    "criterion = MultiBoxLoss(jaccard_thresh=0.5,neg_pos=3, device=device, focal=focal)\n",
    "\n",
    "# optim\n",
    "import torch.optim as optim\n",
    "optimizer = optim.Adam(net.parameters(), lr=1e-4, weight_decay=5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_current_lr(epoch):\n",
    "    lr = 1e-4\n",
    "    for i,lr_decay_epoch in enumerate([80,120]):\n",
    "        if epoch >= lr_decay_epoch:\n",
    "            lr *= 0.1\n",
    "    return lr\n",
    "\n",
    "def adjust_learning_rate(optimizer, epoch):\n",
    "    lr = get_current_lr(epoch)\n",
    "    print(\"lr is:\", lr)\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# モデルを学習させる関数を作成\n",
    "def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):\n",
    "\n",
    "    # GPUが使えるかを確認\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(\"used device：\", device)\n",
    "\n",
    "    # ネットワークをGPUへ\n",
    "    net.to(device)\n",
    "\n",
    "    # ネットワークがある程度固定であれば、高速化させる\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "    # イテレーションカウンタをセット\n",
    "    iteration = 1\n",
    "    epoch_train_loss = 0.0  # epochの損失和\n",
    "    epoch_val_loss = 0.0  # epochの損失和\n",
    "    logs = []\n",
    "\n",
    "    # epochのループ\n",
    "    for epoch in range(num_epochs+1):\n",
    "        \n",
    "        adjust_learning_rate(optimizer, epoch)\n",
    "        \n",
    "        # 開始時刻を保存\n",
    "        t_epoch_start = time.time()\n",
    "        t_iter_start = time.time()\n",
    "\n",
    "        print('-------------')\n",
    "        print('Epoch {}/{}'.format(epoch+1, num_epochs))\n",
    "        print('-------------')\n",
    "\n",
    "        # epochごとの訓練と検証のループ\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                net.train()  # モデルを訓練モードに\n",
    "                print('train')\n",
    "            else:\n",
    "                if((epoch+1) % 5 == 0):\n",
    "                    net.eval()   # モデルを検証モードに\n",
    "                    print('-------------')\n",
    "                    print('val')\n",
    "                else:\n",
    "                    # 検証は5回に1回だけ行う\n",
    "                    continue\n",
    "\n",
    "            # データローダーからminibatchずつ取り出すループ\n",
    "            for images, targets in dataloaders_dict[phase]:\n",
    "\n",
    "                # GPUが使えるならGPUにデータを送る\n",
    "                images = images.to(device)\n",
    "                targets = [ann.to(device)\n",
    "                           for ann in targets]  # リストの各要素のテンソルをGPUへ\n",
    "\n",
    "                # optimizerを初期化\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # 順伝搬（forward）計算\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    # 順伝搬（forward）計算\n",
    "                    outputs = net(images)\n",
    "\n",
    "                    # 損失の計算\n",
    "                    loss_l, loss_c = criterion(outputs, targets)\n",
    "                    loss = loss_l + loss_c\n",
    "\n",
    "                    # 訓練時はバックプロパゲーション\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()  # 勾配の計算\n",
    "\n",
    "                        # 勾配が大きくなりすぎると計算が不安定になるので、clipで最大でも勾配2.0に留める\n",
    "                        nn.utils.clip_grad_value_(\n",
    "                            net.parameters(), clip_value=2.0)\n",
    "\n",
    "                        optimizer.step()  # パラメータ更新\n",
    "\n",
    "                        if (iteration % 10 == 0):  # 10iterに1度、lossを表示\n",
    "                            t_iter_finish = time.time()\n",
    "                            duration = t_iter_finish - t_iter_start\n",
    "                            print('Iteration {} || Loss: {:.4f} || 10iter: {:.4f} sec.'.format(\n",
    "                                iteration, loss.item(), duration))\n",
    "                            t_iter_start = time.time()\n",
    "\n",
    "                        epoch_train_loss += loss.item()\n",
    "                        iteration += 1\n",
    "\n",
    "                    # 検証時\n",
    "                    else:\n",
    "                        epoch_val_loss += loss.item()\n",
    "\n",
    "        # epochのphaseごとのlossと正解率\n",
    "        t_epoch_finish = time.time()\n",
    "        print('-------------')\n",
    "        print('epoch {} || Epoch_TRAIN_Loss:{:.4f} ||Epoch_VAL_Loss:{:.4f}'.format(\n",
    "            epoch+1, epoch_train_loss, epoch_val_loss))\n",
    "        print('timer:  {:.4f} sec.'.format(t_epoch_finish - t_epoch_start))\n",
    "        t_epoch_start = time.time()\n",
    "\n",
    "        # ログを保存\n",
    "        log_epoch = {'epoch': epoch+1,\n",
    "                     'train_loss': epoch_train_loss, 'val_loss': epoch_val_loss}\n",
    "        logs.append(log_epoch)\n",
    "        df = pd.DataFrame(logs)\n",
    "        df.to_csv(\"log/log_output.csv\")\n",
    "\n",
    "        epoch_train_loss = 0.0  # epochの損失和\n",
    "        epoch_val_loss = 0.0  # epochの損失和\n",
    "\n",
    "        # ネットワークを保存する\n",
    "        if ((epoch+1) % 10 == 0):\n",
    "            torch.save(net.state_dict(), 'weights/blazeface{}_'.format(input_size) +\n",
    "                       str(epoch+1) + '.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# start training here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used device： cuda:0\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 1/150\n",
      "-------------\n",
      "train\n",
      "Iteration 10 || Loss: 22.5487 || 10iter: 7.5143 sec.\n",
      "Iteration 20 || Loss: 20.4253 || 10iter: 3.6372 sec.\n",
      "Iteration 30 || Loss: 18.7865 || 10iter: 3.7234 sec.\n",
      "Iteration 40 || Loss: 19.4787 || 10iter: 3.7046 sec.\n",
      "Iteration 50 || Loss: 16.7488 || 10iter: 3.6960 sec.\n",
      "Iteration 60 || Loss: 14.9106 || 10iter: 3.6627 sec.\n",
      "Iteration 70 || Loss: 16.1222 || 10iter: 3.6477 sec.\n",
      "Iteration 80 || Loss: 16.4054 || 10iter: 3.6974 sec.\n",
      "Iteration 90 || Loss: 15.2272 || 10iter: 3.6138 sec.\n",
      "Iteration 100 || Loss: 15.6662 || 10iter: 3.6454 sec.\n",
      "Iteration 110 || Loss: 15.4372 || 10iter: 3.7520 sec.\n",
      "Iteration 120 || Loss: 14.2358 || 10iter: 4.0816 sec.\n",
      "Iteration 130 || Loss: 14.4491 || 10iter: 4.0365 sec.\n",
      "Iteration 140 || Loss: 14.3126 || 10iter: 3.6645 sec.\n",
      "Iteration 150 || Loss: 19.1626 || 10iter: 3.7097 sec.\n",
      "Iteration 160 || Loss: 13.7796 || 10iter: 3.7511 sec.\n",
      "Iteration 170 || Loss: 17.5071 || 10iter: 3.7056 sec.\n",
      "Iteration 180 || Loss: 14.3437 || 10iter: 3.7253 sec.\n",
      "Iteration 190 || Loss: 13.6438 || 10iter: 3.5627 sec.\n",
      "Iteration 200 || Loss: 13.7172 || 10iter: 3.3501 sec.\n",
      "-------------\n",
      "epoch 1 || Epoch_TRAIN_Loss:3393.7682 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  79.1181 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 2/150\n",
      "-------------\n",
      "train\n",
      "Iteration 210 || Loss: 14.8495 || 10iter: 6.1780 sec.\n",
      "Iteration 220 || Loss: 12.7446 || 10iter: 3.7726 sec.\n",
      "Iteration 230 || Loss: 13.9678 || 10iter: 3.6722 sec.\n",
      "Iteration 240 || Loss: 11.8088 || 10iter: 3.6635 sec.\n",
      "Iteration 250 || Loss: 14.0492 || 10iter: 3.6833 sec.\n",
      "Iteration 260 || Loss: 14.0112 || 10iter: 3.6430 sec.\n",
      "Iteration 270 || Loss: 12.5184 || 10iter: 3.6438 sec.\n",
      "Iteration 280 || Loss: 16.5442 || 10iter: 3.7034 sec.\n",
      "Iteration 290 || Loss: 14.5278 || 10iter: 3.7049 sec.\n",
      "Iteration 300 || Loss: 12.3384 || 10iter: 3.7491 sec.\n",
      "Iteration 310 || Loss: 12.1479 || 10iter: 3.6433 sec.\n",
      "Iteration 320 || Loss: 12.1118 || 10iter: 3.7348 sec.\n",
      "Iteration 330 || Loss: 11.7679 || 10iter: 3.6941 sec.\n",
      "Iteration 340 || Loss: 12.6762 || 10iter: 3.7115 sec.\n",
      "Iteration 350 || Loss: 12.3955 || 10iter: 3.6919 sec.\n",
      "Iteration 360 || Loss: 10.7906 || 10iter: 3.6803 sec.\n",
      "Iteration 370 || Loss: 13.2989 || 10iter: 3.7497 sec.\n",
      "Iteration 380 || Loss: 12.7676 || 10iter: 3.6955 sec.\n",
      "Iteration 390 || Loss: 11.9035 || 10iter: 3.7472 sec.\n",
      "Iteration 400 || Loss: 11.0480 || 10iter: 3.3447 sec.\n",
      "-------------\n",
      "epoch 2 || Epoch_TRAIN_Loss:2660.4273 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  77.9275 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 3/150\n",
      "-------------\n",
      "train\n",
      "Iteration 410 || Loss: 13.6322 || 10iter: 4.7412 sec.\n",
      "Iteration 420 || Loss: 11.9111 || 10iter: 4.1115 sec.\n",
      "Iteration 430 || Loss: 12.1449 || 10iter: 3.7747 sec.\n",
      "Iteration 440 || Loss: 9.7894 || 10iter: 3.7494 sec.\n",
      "Iteration 450 || Loss: 12.3495 || 10iter: 3.7967 sec.\n",
      "Iteration 460 || Loss: 11.4153 || 10iter: 3.8357 sec.\n",
      "Iteration 470 || Loss: 10.4655 || 10iter: 3.6958 sec.\n",
      "Iteration 480 || Loss: 10.6566 || 10iter: 3.6921 sec.\n",
      "Iteration 490 || Loss: 10.9808 || 10iter: 3.6961 sec.\n",
      "Iteration 500 || Loss: 11.5464 || 10iter: 3.7421 sec.\n",
      "Iteration 510 || Loss: 11.5250 || 10iter: 3.6962 sec.\n",
      "Iteration 520 || Loss: 10.6723 || 10iter: 3.7027 sec.\n",
      "Iteration 530 || Loss: 11.7232 || 10iter: 3.7636 sec.\n",
      "Iteration 540 || Loss: 11.0821 || 10iter: 3.7532 sec.\n",
      "Iteration 550 || Loss: 10.3688 || 10iter: 3.7241 sec.\n",
      "Iteration 560 || Loss: 10.0894 || 10iter: 3.6858 sec.\n",
      "Iteration 570 || Loss: 12.0440 || 10iter: 3.7670 sec.\n",
      "Iteration 580 || Loss: 12.0366 || 10iter: 3.7261 sec.\n",
      "Iteration 590 || Loss: 10.9555 || 10iter: 4.1657 sec.\n",
      "Iteration 600 || Loss: 10.7895 || 10iter: 3.4877 sec.\n",
      "-------------\n",
      "epoch 3 || Epoch_TRAIN_Loss:2383.1194 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  79.1697 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 4/150\n",
      "-------------\n",
      "train\n",
      "Iteration 610 || Loss: 12.6594 || 10iter: 2.9424 sec.\n",
      "Iteration 620 || Loss: 11.7079 || 10iter: 4.7964 sec.\n",
      "Iteration 630 || Loss: 11.5099 || 10iter: 3.6256 sec.\n",
      "Iteration 640 || Loss: 9.7712 || 10iter: 3.7328 sec.\n",
      "Iteration 650 || Loss: 10.4589 || 10iter: 3.7103 sec.\n",
      "Iteration 660 || Loss: 11.5097 || 10iter: 3.6986 sec.\n",
      "Iteration 670 || Loss: 11.3667 || 10iter: 3.6695 sec.\n",
      "Iteration 680 || Loss: 11.2946 || 10iter: 3.6858 sec.\n",
      "Iteration 690 || Loss: 9.0729 || 10iter: 3.6735 sec.\n",
      "Iteration 700 || Loss: 9.3220 || 10iter: 3.6979 sec.\n",
      "Iteration 710 || Loss: 10.5712 || 10iter: 3.7310 sec.\n",
      "Iteration 720 || Loss: 10.2825 || 10iter: 3.7869 sec.\n",
      "Iteration 730 || Loss: 10.5675 || 10iter: 3.6866 sec.\n",
      "Iteration 740 || Loss: 9.2522 || 10iter: 3.7191 sec.\n",
      "Iteration 750 || Loss: 11.2401 || 10iter: 3.6745 sec.\n",
      "Iteration 760 || Loss: 10.1342 || 10iter: 3.6661 sec.\n",
      "Iteration 770 || Loss: 10.5420 || 10iter: 3.7038 sec.\n",
      "Iteration 780 || Loss: 8.0688 || 10iter: 3.6653 sec.\n",
      "Iteration 790 || Loss: 9.2815 || 10iter: 3.6615 sec.\n",
      "Iteration 800 || Loss: 9.8428 || 10iter: 3.5167 sec.\n",
      "Iteration 810 || Loss: 10.8593 || 10iter: 3.3968 sec.\n",
      "-------------\n",
      "epoch 4 || Epoch_TRAIN_Loss:2168.6042 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  77.9275 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 5/150\n",
      "-------------\n",
      "train\n",
      "Iteration 820 || Loss: 9.5277 || 10iter: 6.5565 sec.\n",
      "Iteration 830 || Loss: 10.5073 || 10iter: 3.8273 sec.\n",
      "Iteration 840 || Loss: 10.1626 || 10iter: 3.7588 sec.\n",
      "Iteration 850 || Loss: 10.3846 || 10iter: 3.6742 sec.\n",
      "Iteration 860 || Loss: 12.9533 || 10iter: 3.7828 sec.\n",
      "Iteration 870 || Loss: 10.2510 || 10iter: 3.6828 sec.\n",
      "Iteration 880 || Loss: 10.5014 || 10iter: 3.6816 sec.\n",
      "Iteration 890 || Loss: 9.8378 || 10iter: 3.6740 sec.\n",
      "Iteration 900 || Loss: 8.6193 || 10iter: 3.7278 sec.\n",
      "Iteration 910 || Loss: 10.4999 || 10iter: 3.7299 sec.\n",
      "Iteration 920 || Loss: 10.5771 || 10iter: 3.6672 sec.\n",
      "Iteration 930 || Loss: 8.9392 || 10iter: 3.7901 sec.\n",
      "Iteration 940 || Loss: 11.1137 || 10iter: 3.7390 sec.\n",
      "Iteration 950 || Loss: 10.5550 || 10iter: 3.6694 sec.\n",
      "Iteration 960 || Loss: 11.0979 || 10iter: 3.6734 sec.\n",
      "Iteration 970 || Loss: 10.6657 || 10iter: 3.6461 sec.\n",
      "Iteration 980 || Loss: 9.5648 || 10iter: 3.6849 sec.\n",
      "Iteration 990 || Loss: 9.9373 || 10iter: 3.6906 sec.\n",
      "Iteration 1000 || Loss: 10.3537 || 10iter: 3.7027 sec.\n",
      "Iteration 1010 || Loss: 10.3155 || 10iter: 3.4159 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 5 || Epoch_TRAIN_Loss:2034.5860 ||Epoch_VAL_Loss:675.7839\n",
      "timer:  89.4556 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 6/150\n",
      "-------------\n",
      "train\n",
      "Iteration 1020 || Loss: 8.5028 || 10iter: 5.5503 sec.\n",
      "Iteration 1030 || Loss: 9.7737 || 10iter: 4.3113 sec.\n",
      "Iteration 1040 || Loss: 10.8222 || 10iter: 3.7630 sec.\n",
      "Iteration 1050 || Loss: 9.7726 || 10iter: 3.7402 sec.\n",
      "Iteration 1060 || Loss: 9.3968 || 10iter: 3.7570 sec.\n",
      "Iteration 1070 || Loss: 8.4893 || 10iter: 3.6504 sec.\n",
      "Iteration 1080 || Loss: 8.9208 || 10iter: 3.6629 sec.\n",
      "Iteration 1090 || Loss: 10.5808 || 10iter: 3.6717 sec.\n",
      "Iteration 1100 || Loss: 9.4065 || 10iter: 3.6953 sec.\n",
      "Iteration 1110 || Loss: 8.2725 || 10iter: 3.7389 sec.\n",
      "Iteration 1120 || Loss: 9.0933 || 10iter: 3.7058 sec.\n",
      "Iteration 1130 || Loss: 8.4038 || 10iter: 3.7057 sec.\n",
      "Iteration 1140 || Loss: 9.7452 || 10iter: 3.6937 sec.\n",
      "Iteration 1150 || Loss: 8.1577 || 10iter: 3.6938 sec.\n",
      "Iteration 1160 || Loss: 9.1904 || 10iter: 3.6391 sec.\n",
      "Iteration 1170 || Loss: 9.8317 || 10iter: 3.6723 sec.\n",
      "Iteration 1180 || Loss: 9.3573 || 10iter: 3.7283 sec.\n",
      "Iteration 1190 || Loss: 8.0732 || 10iter: 3.7118 sec.\n",
      "Iteration 1200 || Loss: 8.1336 || 10iter: 3.7498 sec.\n",
      "Iteration 1210 || Loss: 8.5818 || 10iter: 3.4738 sec.\n",
      "-------------\n",
      "epoch 6 || Epoch_TRAIN_Loss:1908.5910 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  78.8586 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 7/150\n",
      "-------------\n",
      "train\n",
      "Iteration 1220 || Loss: 9.5252 || 10iter: 3.6545 sec.\n",
      "Iteration 1230 || Loss: 8.3581 || 10iter: 4.5701 sec.\n",
      "Iteration 1240 || Loss: 9.6602 || 10iter: 3.5936 sec.\n",
      "Iteration 1250 || Loss: 10.4722 || 10iter: 3.6871 sec.\n",
      "Iteration 1260 || Loss: 8.6971 || 10iter: 3.6595 sec.\n",
      "Iteration 1270 || Loss: 10.4249 || 10iter: 3.6253 sec.\n",
      "Iteration 1280 || Loss: 9.0668 || 10iter: 3.6783 sec.\n",
      "Iteration 1290 || Loss: 8.8525 || 10iter: 3.7130 sec.\n",
      "Iteration 1300 || Loss: 9.7897 || 10iter: 3.7114 sec.\n",
      "Iteration 1310 || Loss: 8.8281 || 10iter: 3.6472 sec.\n",
      "Iteration 1320 || Loss: 9.4618 || 10iter: 3.7294 sec.\n",
      "Iteration 1330 || Loss: 9.1377 || 10iter: 3.6430 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1340 || Loss: 8.1670 || 10iter: 3.6865 sec.\n",
      "Iteration 1350 || Loss: 10.7478 || 10iter: 3.7634 sec.\n",
      "Iteration 1360 || Loss: 9.5503 || 10iter: 3.7305 sec.\n",
      "Iteration 1370 || Loss: 8.5046 || 10iter: 3.6910 sec.\n",
      "Iteration 1380 || Loss: 9.4814 || 10iter: 3.7139 sec.\n",
      "Iteration 1390 || Loss: 8.8221 || 10iter: 3.6337 sec.\n",
      "Iteration 1400 || Loss: 8.8724 || 10iter: 3.7073 sec.\n",
      "Iteration 1410 || Loss: 8.7612 || 10iter: 3.5459 sec.\n",
      "Iteration 1420 || Loss: 8.2808 || 10iter: 3.4160 sec.\n",
      "-------------\n",
      "epoch 7 || Epoch_TRAIN_Loss:1820.9165 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  77.9438 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 8/150\n",
      "-------------\n",
      "train\n",
      "Iteration 1430 || Loss: 9.0718 || 10iter: 6.7785 sec.\n",
      "Iteration 1440 || Loss: 10.1622 || 10iter: 3.7324 sec.\n",
      "Iteration 1450 || Loss: 9.8537 || 10iter: 3.7132 sec.\n",
      "Iteration 1460 || Loss: 8.3607 || 10iter: 3.6899 sec.\n",
      "Iteration 1470 || Loss: 8.3677 || 10iter: 3.7450 sec.\n",
      "Iteration 1480 || Loss: 8.2059 || 10iter: 3.6585 sec.\n",
      "Iteration 1490 || Loss: 11.7468 || 10iter: 4.0845 sec.\n",
      "Iteration 1500 || Loss: 8.5191 || 10iter: 3.6565 sec.\n",
      "Iteration 1510 || Loss: 7.5577 || 10iter: 3.6637 sec.\n",
      "Iteration 1520 || Loss: 8.8684 || 10iter: 3.7349 sec.\n",
      "Iteration 1530 || Loss: 8.2085 || 10iter: 3.7173 sec.\n",
      "Iteration 1540 || Loss: 8.3099 || 10iter: 3.7127 sec.\n",
      "Iteration 1550 || Loss: 8.6931 || 10iter: 3.6698 sec.\n",
      "Iteration 1560 || Loss: 9.0733 || 10iter: 3.6899 sec.\n",
      "Iteration 1570 || Loss: 8.2620 || 10iter: 3.6772 sec.\n",
      "Iteration 1580 || Loss: 7.8406 || 10iter: 3.6047 sec.\n",
      "Iteration 1590 || Loss: 7.9292 || 10iter: 3.6722 sec.\n",
      "Iteration 1600 || Loss: 8.1073 || 10iter: 3.6982 sec.\n",
      "Iteration 1610 || Loss: 7.3982 || 10iter: 3.6141 sec.\n",
      "Iteration 1620 || Loss: 8.2398 || 10iter: 3.4061 sec.\n",
      "-------------\n",
      "epoch 8 || Epoch_TRAIN_Loss:1737.4124 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  78.0914 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 9/150\n",
      "-------------\n",
      "train\n",
      "Iteration 1630 || Loss: 8.5099 || 10iter: 5.8363 sec.\n",
      "Iteration 1640 || Loss: 9.1940 || 10iter: 3.7257 sec.\n",
      "Iteration 1650 || Loss: 8.5753 || 10iter: 3.7745 sec.\n",
      "Iteration 1660 || Loss: 8.9083 || 10iter: 3.7956 sec.\n",
      "Iteration 1670 || Loss: 8.7264 || 10iter: 3.7007 sec.\n",
      "Iteration 1680 || Loss: 8.7676 || 10iter: 3.6787 sec.\n",
      "Iteration 1690 || Loss: 8.1691 || 10iter: 3.6930 sec.\n",
      "Iteration 1700 || Loss: 8.2775 || 10iter: 3.7067 sec.\n",
      "Iteration 1710 || Loss: 7.9014 || 10iter: 3.6625 sec.\n",
      "Iteration 1720 || Loss: 7.7835 || 10iter: 3.7828 sec.\n",
      "Iteration 1730 || Loss: 9.5028 || 10iter: 3.7257 sec.\n",
      "Iteration 1740 || Loss: 8.2347 || 10iter: 3.7808 sec.\n",
      "Iteration 1750 || Loss: 7.8091 || 10iter: 3.6795 sec.\n",
      "Iteration 1760 || Loss: 7.1864 || 10iter: 3.6891 sec.\n",
      "Iteration 1770 || Loss: 8.4940 || 10iter: 3.7631 sec.\n",
      "Iteration 1780 || Loss: 7.3133 || 10iter: 3.7215 sec.\n",
      "Iteration 1790 || Loss: 8.2673 || 10iter: 3.7223 sec.\n",
      "Iteration 1800 || Loss: 7.8582 || 10iter: 3.7609 sec.\n",
      "Iteration 1810 || Loss: 9.1774 || 10iter: 3.6832 sec.\n",
      "Iteration 1820 || Loss: 8.9593 || 10iter: 3.4278 sec.\n",
      "-------------\n",
      "epoch 9 || Epoch_TRAIN_Loss:1665.4210 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  78.5051 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 10/150\n",
      "-------------\n",
      "train\n",
      "Iteration 1830 || Loss: 8.0980 || 10iter: 4.1657 sec.\n",
      "Iteration 1840 || Loss: 7.7851 || 10iter: 4.3428 sec.\n",
      "Iteration 1850 || Loss: 8.2199 || 10iter: 3.6944 sec.\n",
      "Iteration 1860 || Loss: 8.0817 || 10iter: 3.7193 sec.\n",
      "Iteration 1870 || Loss: 8.9930 || 10iter: 3.6679 sec.\n",
      "Iteration 1880 || Loss: 7.2181 || 10iter: 3.7269 sec.\n",
      "Iteration 1890 || Loss: 8.9111 || 10iter: 3.6812 sec.\n",
      "Iteration 1900 || Loss: 6.6686 || 10iter: 3.7001 sec.\n",
      "Iteration 1910 || Loss: 7.5189 || 10iter: 3.7102 sec.\n",
      "Iteration 1920 || Loss: 8.4194 || 10iter: 3.7463 sec.\n",
      "Iteration 1930 || Loss: 7.3692 || 10iter: 3.7276 sec.\n",
      "Iteration 1940 || Loss: 7.8322 || 10iter: 3.7035 sec.\n",
      "Iteration 1950 || Loss: 7.1636 || 10iter: 3.7540 sec.\n",
      "Iteration 1960 || Loss: 7.6693 || 10iter: 4.0963 sec.\n",
      "Iteration 1970 || Loss: 7.7333 || 10iter: 3.7092 sec.\n",
      "Iteration 1980 || Loss: 6.7248 || 10iter: 3.6745 sec.\n",
      "Iteration 1990 || Loss: 7.6229 || 10iter: 3.6751 sec.\n",
      "Iteration 2000 || Loss: 7.5608 || 10iter: 3.7621 sec.\n",
      "Iteration 2010 || Loss: 7.6534 || 10iter: 3.6767 sec.\n",
      "Iteration 2020 || Loss: 8.4941 || 10iter: 3.5377 sec.\n",
      "Iteration 2030 || Loss: 14.6393 || 10iter: 3.1410 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 10 || Epoch_TRAIN_Loss:1619.8951 ||Epoch_VAL_Loss:536.4415\n",
      "timer:  89.3801 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 11/150\n",
      "-------------\n",
      "train\n",
      "Iteration 2040 || Loss: 7.7393 || 10iter: 7.4414 sec.\n",
      "Iteration 2050 || Loss: 10.2427 || 10iter: 3.6015 sec.\n",
      "Iteration 2060 || Loss: 9.1427 || 10iter: 3.7351 sec.\n",
      "Iteration 2070 || Loss: 6.9870 || 10iter: 3.7013 sec.\n",
      "Iteration 2080 || Loss: 7.1127 || 10iter: 3.7384 sec.\n",
      "Iteration 2090 || Loss: 7.8186 || 10iter: 3.6646 sec.\n",
      "Iteration 2100 || Loss: 6.9949 || 10iter: 3.6591 sec.\n",
      "Iteration 2110 || Loss: 8.4575 || 10iter: 3.6825 sec.\n",
      "Iteration 2120 || Loss: 8.9016 || 10iter: 3.6639 sec.\n",
      "Iteration 2130 || Loss: 7.2826 || 10iter: 3.6790 sec.\n",
      "Iteration 2140 || Loss: 9.9309 || 10iter: 3.7218 sec.\n",
      "Iteration 2150 || Loss: 7.7177 || 10iter: 3.6993 sec.\n",
      "Iteration 2160 || Loss: 7.3491 || 10iter: 3.7076 sec.\n",
      "Iteration 2170 || Loss: 7.3418 || 10iter: 3.6376 sec.\n",
      "Iteration 2180 || Loss: 7.5233 || 10iter: 3.7128 sec.\n",
      "Iteration 2190 || Loss: 7.2580 || 10iter: 3.6821 sec.\n",
      "Iteration 2200 || Loss: 8.1267 || 10iter: 3.6461 sec.\n",
      "Iteration 2210 || Loss: 8.1897 || 10iter: 3.7327 sec.\n",
      "Iteration 2220 || Loss: 7.0425 || 10iter: 3.5725 sec.\n",
      "Iteration 2230 || Loss: 8.9730 || 10iter: 3.4458 sec.\n",
      "-------------\n",
      "epoch 11 || Epoch_TRAIN_Loss:1597.4715 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  77.9588 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 12/150\n",
      "-------------\n",
      "train\n",
      "Iteration 2240 || Loss: 8.3763 || 10iter: 6.1771 sec.\n",
      "Iteration 2250 || Loss: 7.5468 || 10iter: 3.8574 sec.\n",
      "Iteration 2260 || Loss: 6.6665 || 10iter: 3.7710 sec.\n",
      "Iteration 2270 || Loss: 7.0636 || 10iter: 3.7707 sec.\n",
      "Iteration 2280 || Loss: 7.0528 || 10iter: 3.7230 sec.\n",
      "Iteration 2290 || Loss: 6.7473 || 10iter: 3.6306 sec.\n",
      "Iteration 2300 || Loss: 7.8578 || 10iter: 3.7856 sec.\n",
      "Iteration 2310 || Loss: 6.6563 || 10iter: 3.6558 sec.\n",
      "Iteration 2320 || Loss: 7.2961 || 10iter: 3.6743 sec.\n",
      "Iteration 2330 || Loss: 6.8784 || 10iter: 3.6891 sec.\n",
      "Iteration 2340 || Loss: 6.6777 || 10iter: 3.6862 sec.\n",
      "Iteration 2350 || Loss: 9.0656 || 10iter: 3.7479 sec.\n",
      "Iteration 2360 || Loss: 6.9341 || 10iter: 3.6613 sec.\n",
      "Iteration 2370 || Loss: 8.1664 || 10iter: 3.7464 sec.\n",
      "Iteration 2380 || Loss: 8.1820 || 10iter: 3.7074 sec.\n",
      "Iteration 2390 || Loss: 6.9863 || 10iter: 3.7545 sec.\n",
      "Iteration 2400 || Loss: 8.3008 || 10iter: 4.1332 sec.\n",
      "Iteration 2410 || Loss: 7.7859 || 10iter: 3.7166 sec.\n",
      "Iteration 2420 || Loss: 9.2023 || 10iter: 3.7041 sec.\n",
      "Iteration 2430 || Loss: 7.3851 || 10iter: 3.3805 sec.\n",
      "-------------\n",
      "epoch 12 || Epoch_TRAIN_Loss:1544.0009 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  78.8025 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 13/150\n",
      "-------------\n",
      "train\n",
      "Iteration 2440 || Loss: 7.4362 || 10iter: 4.7007 sec.\n",
      "Iteration 2450 || Loss: 7.7052 || 10iter: 4.1170 sec.\n",
      "Iteration 2460 || Loss: 7.4400 || 10iter: 3.7026 sec.\n",
      "Iteration 2470 || Loss: 6.6144 || 10iter: 3.7789 sec.\n",
      "Iteration 2480 || Loss: 7.7242 || 10iter: 3.6379 sec.\n",
      "Iteration 2490 || Loss: 7.3377 || 10iter: 3.7000 sec.\n",
      "Iteration 2500 || Loss: 7.3322 || 10iter: 3.7253 sec.\n",
      "Iteration 2510 || Loss: 7.5276 || 10iter: 3.7115 sec.\n",
      "Iteration 2520 || Loss: 6.6453 || 10iter: 3.7732 sec.\n",
      "Iteration 2530 || Loss: 7.0819 || 10iter: 3.7317 sec.\n",
      "Iteration 2540 || Loss: 6.5508 || 10iter: 3.6864 sec.\n",
      "Iteration 2550 || Loss: 6.6096 || 10iter: 3.7690 sec.\n",
      "Iteration 2560 || Loss: 7.8793 || 10iter: 3.6729 sec.\n",
      "Iteration 2570 || Loss: 6.7947 || 10iter: 3.7423 sec.\n",
      "Iteration 2580 || Loss: 7.3851 || 10iter: 3.7126 sec.\n",
      "Iteration 2590 || Loss: 7.6771 || 10iter: 3.7147 sec.\n",
      "Iteration 2600 || Loss: 8.5196 || 10iter: 3.6806 sec.\n",
      "Iteration 2610 || Loss: 6.5212 || 10iter: 3.6734 sec.\n",
      "Iteration 2620 || Loss: 7.4159 || 10iter: 3.7153 sec.\n",
      "Iteration 2630 || Loss: 6.8512 || 10iter: 3.4851 sec.\n",
      "-------------\n",
      "epoch 13 || Epoch_TRAIN_Loss:1495.0367 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  78.3036 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 14/150\n",
      "-------------\n",
      "train\n",
      "Iteration 2640 || Loss: 6.6330 || 10iter: 2.9284 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 2650 || Loss: 7.3811 || 10iter: 4.8936 sec.\n",
      "Iteration 2660 || Loss: 6.9956 || 10iter: 3.6586 sec.\n",
      "Iteration 2670 || Loss: 7.2370 || 10iter: 3.7618 sec.\n",
      "Iteration 2680 || Loss: 9.3859 || 10iter: 3.6937 sec.\n",
      "Iteration 2690 || Loss: 8.5226 || 10iter: 3.6454 sec.\n",
      "Iteration 2700 || Loss: 6.6618 || 10iter: 3.6676 sec.\n",
      "Iteration 2710 || Loss: 7.3833 || 10iter: 3.6418 sec.\n",
      "Iteration 2720 || Loss: 7.6083 || 10iter: 3.7231 sec.\n",
      "Iteration 2730 || Loss: 6.5915 || 10iter: 3.6188 sec.\n",
      "Iteration 2740 || Loss: 7.0925 || 10iter: 3.6939 sec.\n",
      "Iteration 2750 || Loss: 6.8639 || 10iter: 3.6755 sec.\n",
      "Iteration 2760 || Loss: 6.4041 || 10iter: 3.7471 sec.\n",
      "Iteration 2770 || Loss: 7.9480 || 10iter: 3.7545 sec.\n",
      "Iteration 2780 || Loss: 6.3801 || 10iter: 3.6705 sec.\n",
      "Iteration 2790 || Loss: 7.4054 || 10iter: 3.7307 sec.\n",
      "Iteration 2800 || Loss: 7.2402 || 10iter: 3.7251 sec.\n",
      "Iteration 2810 || Loss: 7.8042 || 10iter: 3.7182 sec.\n",
      "Iteration 2820 || Loss: 8.3246 || 10iter: 3.6670 sec.\n",
      "Iteration 2830 || Loss: 7.7231 || 10iter: 3.5178 sec.\n",
      "Iteration 2840 || Loss: 8.4561 || 10iter: 3.3731 sec.\n",
      "-------------\n",
      "epoch 14 || Epoch_TRAIN_Loss:1460.1673 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  77.9891 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 15/150\n",
      "-------------\n",
      "train\n",
      "Iteration 2850 || Loss: 6.0244 || 10iter: 6.6598 sec.\n",
      "Iteration 2860 || Loss: 6.9309 || 10iter: 4.0742 sec.\n",
      "Iteration 2870 || Loss: 6.0894 || 10iter: 3.7107 sec.\n",
      "Iteration 2880 || Loss: 7.2623 || 10iter: 3.6865 sec.\n",
      "Iteration 2890 || Loss: 6.6023 || 10iter: 3.7197 sec.\n",
      "Iteration 2900 || Loss: 7.1347 || 10iter: 3.6782 sec.\n",
      "Iteration 2910 || Loss: 9.4876 || 10iter: 3.6656 sec.\n",
      "Iteration 2920 || Loss: 6.9368 || 10iter: 3.7169 sec.\n",
      "Iteration 2930 || Loss: 7.3579 || 10iter: 3.7077 sec.\n",
      "Iteration 2940 || Loss: 6.8470 || 10iter: 3.7200 sec.\n",
      "Iteration 2950 || Loss: 6.9705 || 10iter: 3.7649 sec.\n",
      "Iteration 2960 || Loss: 7.9624 || 10iter: 3.7770 sec.\n",
      "Iteration 2970 || Loss: 6.9033 || 10iter: 3.7193 sec.\n",
      "Iteration 2980 || Loss: 7.2067 || 10iter: 3.7207 sec.\n",
      "Iteration 2990 || Loss: 7.4374 || 10iter: 3.7151 sec.\n",
      "Iteration 3000 || Loss: 6.4362 || 10iter: 3.6593 sec.\n",
      "Iteration 3010 || Loss: 7.3118 || 10iter: 3.7060 sec.\n",
      "Iteration 3020 || Loss: 7.8195 || 10iter: 3.7431 sec.\n",
      "Iteration 3030 || Loss: 9.3239 || 10iter: 3.7114 sec.\n",
      "Iteration 3040 || Loss: 6.7063 || 10iter: 3.3844 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 15 || Epoch_TRAIN_Loss:1448.6412 ||Epoch_VAL_Loss:473.9629\n",
      "timer:  89.3634 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 16/150\n",
      "-------------\n",
      "train\n",
      "Iteration 3050 || Loss: 6.3400 || 10iter: 5.3814 sec.\n",
      "Iteration 3060 || Loss: 7.8766 || 10iter: 3.8581 sec.\n",
      "Iteration 3070 || Loss: 6.0648 || 10iter: 3.7430 sec.\n",
      "Iteration 3080 || Loss: 6.8589 || 10iter: 3.6705 sec.\n",
      "Iteration 3090 || Loss: 6.1681 || 10iter: 3.6475 sec.\n",
      "Iteration 3100 || Loss: 6.7379 || 10iter: 3.7582 sec.\n",
      "Iteration 3110 || Loss: 6.5340 || 10iter: 3.6928 sec.\n",
      "Iteration 3120 || Loss: 6.3563 || 10iter: 3.6902 sec.\n",
      "Iteration 3130 || Loss: 7.3566 || 10iter: 3.7404 sec.\n",
      "Iteration 3140 || Loss: 6.6442 || 10iter: 3.7836 sec.\n",
      "Iteration 3150 || Loss: 8.0594 || 10iter: 3.7314 sec.\n",
      "Iteration 3160 || Loss: 6.6080 || 10iter: 3.7878 sec.\n",
      "Iteration 3170 || Loss: 6.3923 || 10iter: 3.7240 sec.\n",
      "Iteration 3180 || Loss: 5.9358 || 10iter: 3.7063 sec.\n",
      "Iteration 3190 || Loss: 8.1331 || 10iter: 3.7261 sec.\n",
      "Iteration 3200 || Loss: 7.2034 || 10iter: 3.7310 sec.\n",
      "Iteration 3210 || Loss: 6.6403 || 10iter: 3.6688 sec.\n",
      "Iteration 3220 || Loss: 8.6917 || 10iter: 3.7409 sec.\n",
      "Iteration 3230 || Loss: 7.3304 || 10iter: 3.6761 sec.\n",
      "Iteration 3240 || Loss: 7.3357 || 10iter: 3.4186 sec.\n",
      "-------------\n",
      "epoch 16 || Epoch_TRAIN_Loss:1407.0539 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  78.3731 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 17/150\n",
      "-------------\n",
      "train\n",
      "Iteration 3250 || Loss: 7.8072 || 10iter: 3.4765 sec.\n",
      "Iteration 3260 || Loss: 7.2572 || 10iter: 4.7138 sec.\n",
      "Iteration 3270 || Loss: 7.0407 || 10iter: 3.7374 sec.\n",
      "Iteration 3280 || Loss: 6.6440 || 10iter: 3.7450 sec.\n",
      "Iteration 3290 || Loss: 6.1413 || 10iter: 3.7628 sec.\n",
      "Iteration 3300 || Loss: 6.0140 || 10iter: 4.0169 sec.\n",
      "Iteration 3310 || Loss: 6.6174 || 10iter: 3.6970 sec.\n",
      "Iteration 3320 || Loss: 6.9563 || 10iter: 3.7291 sec.\n",
      "Iteration 3330 || Loss: 6.2270 || 10iter: 3.7543 sec.\n",
      "Iteration 3340 || Loss: 6.7501 || 10iter: 3.7326 sec.\n",
      "Iteration 3350 || Loss: 7.1831 || 10iter: 3.7497 sec.\n",
      "Iteration 3360 || Loss: 6.9876 || 10iter: 3.7179 sec.\n",
      "Iteration 3370 || Loss: 7.3574 || 10iter: 3.7240 sec.\n",
      "Iteration 3380 || Loss: 7.4201 || 10iter: 3.7484 sec.\n",
      "Iteration 3390 || Loss: 6.4245 || 10iter: 3.9315 sec.\n",
      "Iteration 3400 || Loss: 6.4968 || 10iter: 4.1296 sec.\n",
      "Iteration 3410 || Loss: 8.5030 || 10iter: 3.7941 sec.\n",
      "Iteration 3420 || Loss: 7.0972 || 10iter: 3.6593 sec.\n",
      "Iteration 3430 || Loss: 7.2004 || 10iter: 3.6908 sec.\n",
      "Iteration 3440 || Loss: 6.0971 || 10iter: 3.5804 sec.\n",
      "Iteration 3450 || Loss: 7.4060 || 10iter: 3.4393 sec.\n",
      "-------------\n",
      "epoch 17 || Epoch_TRAIN_Loss:1386.0934 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  79.6802 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 18/150\n",
      "-------------\n",
      "train\n",
      "Iteration 3460 || Loss: 6.3807 || 10iter: 6.8505 sec.\n",
      "Iteration 3470 || Loss: 6.9118 || 10iter: 3.6616 sec.\n",
      "Iteration 3480 || Loss: 7.2696 || 10iter: 3.7291 sec.\n",
      "Iteration 3490 || Loss: 6.7431 || 10iter: 3.7633 sec.\n",
      "Iteration 3500 || Loss: 6.3927 || 10iter: 3.7618 sec.\n",
      "Iteration 3510 || Loss: 6.8484 || 10iter: 3.6756 sec.\n",
      "Iteration 3520 || Loss: 7.3701 || 10iter: 3.8278 sec.\n",
      "Iteration 3530 || Loss: 7.0996 || 10iter: 3.6449 sec.\n",
      "Iteration 3540 || Loss: 6.1194 || 10iter: 3.6699 sec.\n",
      "Iteration 3550 || Loss: 6.3272 || 10iter: 3.7010 sec.\n",
      "Iteration 3560 || Loss: 6.4220 || 10iter: 3.7192 sec.\n",
      "Iteration 3570 || Loss: 5.7725 || 10iter: 3.8979 sec.\n",
      "Iteration 3580 || Loss: 7.5219 || 10iter: 4.2883 sec.\n",
      "Iteration 3590 || Loss: 7.2845 || 10iter: 4.4800 sec.\n",
      "Iteration 3600 || Loss: 7.5489 || 10iter: 4.3412 sec.\n",
      "Iteration 3610 || Loss: 7.3834 || 10iter: 4.3358 sec.\n",
      "Iteration 3620 || Loss: 5.6678 || 10iter: 4.3582 sec.\n",
      "Iteration 3630 || Loss: 7.8453 || 10iter: 4.3930 sec.\n",
      "Iteration 3640 || Loss: 5.6233 || 10iter: 4.2360 sec.\n",
      "Iteration 3650 || Loss: 6.3956 || 10iter: 3.6278 sec.\n",
      "-------------\n",
      "epoch 18 || Epoch_TRAIN_Loss:1371.1585 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  83.1286 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 19/150\n",
      "-------------\n",
      "train\n",
      "Iteration 3660 || Loss: 5.9004 || 10iter: 5.8487 sec.\n",
      "Iteration 3670 || Loss: 7.4085 || 10iter: 3.7408 sec.\n",
      "Iteration 3680 || Loss: 6.2014 || 10iter: 3.8222 sec.\n",
      "Iteration 3690 || Loss: 6.5688 || 10iter: 3.7516 sec.\n",
      "Iteration 3700 || Loss: 6.2532 || 10iter: 3.7624 sec.\n",
      "Iteration 3710 || Loss: 6.6979 || 10iter: 3.7609 sec.\n",
      "Iteration 3720 || Loss: 6.2950 || 10iter: 3.7287 sec.\n",
      "Iteration 3730 || Loss: 5.9928 || 10iter: 3.7668 sec.\n",
      "Iteration 3740 || Loss: 5.9990 || 10iter: 3.7313 sec.\n",
      "Iteration 3750 || Loss: 6.0600 || 10iter: 4.1525 sec.\n",
      "Iteration 3760 || Loss: 6.8959 || 10iter: 3.8053 sec.\n",
      "Iteration 3770 || Loss: 6.5461 || 10iter: 3.8168 sec.\n",
      "Iteration 3780 || Loss: 8.8223 || 10iter: 3.6936 sec.\n",
      "Iteration 3790 || Loss: 7.2102 || 10iter: 3.6818 sec.\n",
      "Iteration 3800 || Loss: 6.6057 || 10iter: 3.6767 sec.\n",
      "Iteration 3810 || Loss: 7.4133 || 10iter: 3.7008 sec.\n",
      "Iteration 3820 || Loss: 6.0434 || 10iter: 3.6281 sec.\n",
      "Iteration 3830 || Loss: 6.5833 || 10iter: 3.6739 sec.\n",
      "Iteration 3840 || Loss: 7.2243 || 10iter: 4.6986 sec.\n",
      "Iteration 3850 || Loss: 6.6238 || 10iter: 5.4873 sec.\n",
      "-------------\n",
      "epoch 19 || Epoch_TRAIN_Loss:1343.9155 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  82.1068 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 20/150\n",
      "-------------\n",
      "train\n",
      "Iteration 3860 || Loss: 6.0703 || 10iter: 4.2817 sec.\n",
      "Iteration 3870 || Loss: 5.8293 || 10iter: 4.2590 sec.\n",
      "Iteration 3880 || Loss: 5.8300 || 10iter: 3.6270 sec.\n",
      "Iteration 3890 || Loss: 7.1104 || 10iter: 3.7094 sec.\n",
      "Iteration 3900 || Loss: 7.7592 || 10iter: 3.6461 sec.\n",
      "Iteration 3910 || Loss: 7.6017 || 10iter: 3.6729 sec.\n",
      "Iteration 3920 || Loss: 6.2090 || 10iter: 3.6628 sec.\n",
      "Iteration 3930 || Loss: 6.5203 || 10iter: 3.6879 sec.\n",
      "Iteration 3940 || Loss: 6.7071 || 10iter: 3.6928 sec.\n",
      "Iteration 3950 || Loss: 5.8901 || 10iter: 3.6336 sec.\n",
      "Iteration 3960 || Loss: 6.7507 || 10iter: 3.7363 sec.\n",
      "Iteration 3970 || Loss: 6.8133 || 10iter: 3.7579 sec.\n",
      "Iteration 3980 || Loss: 6.2767 || 10iter: 3.7649 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 3990 || Loss: 5.9829 || 10iter: 3.7686 sec.\n",
      "Iteration 4000 || Loss: 5.7105 || 10iter: 3.6965 sec.\n",
      "Iteration 4010 || Loss: 5.4220 || 10iter: 3.7337 sec.\n",
      "Iteration 4020 || Loss: 7.3890 || 10iter: 3.6648 sec.\n",
      "Iteration 4030 || Loss: 6.2232 || 10iter: 3.7293 sec.\n",
      "Iteration 4040 || Loss: 6.4296 || 10iter: 3.7058 sec.\n",
      "Iteration 4050 || Loss: 6.5877 || 10iter: 3.5883 sec.\n",
      "Iteration 4060 || Loss: 6.4441 || 10iter: 3.1801 sec.\n",
      "-------------\n",
      "val\n",
      "-------------\n",
      "epoch 20 || Epoch_TRAIN_Loss:1328.0876 ||Epoch_VAL_Loss:443.7957\n",
      "timer:  88.7218 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 21/150\n",
      "-------------\n",
      "train\n",
      "Iteration 4070 || Loss: 7.0428 || 10iter: 7.4494 sec.\n",
      "Iteration 4080 || Loss: 6.1590 || 10iter: 3.5673 sec.\n",
      "Iteration 4090 || Loss: 7.3442 || 10iter: 3.7159 sec.\n",
      "Iteration 4100 || Loss: 6.8254 || 10iter: 3.6860 sec.\n",
      "Iteration 4110 || Loss: 6.1635 || 10iter: 3.6744 sec.\n",
      "Iteration 4120 || Loss: 6.0537 || 10iter: 3.6815 sec.\n",
      "Iteration 4130 || Loss: 6.0340 || 10iter: 3.7479 sec.\n",
      "Iteration 4140 || Loss: 6.8953 || 10iter: 3.7122 sec.\n",
      "Iteration 4150 || Loss: 5.9969 || 10iter: 3.7596 sec.\n",
      "Iteration 4160 || Loss: 5.8750 || 10iter: 3.7089 sec.\n",
      "Iteration 4170 || Loss: 9.2795 || 10iter: 3.7350 sec.\n",
      "Iteration 4180 || Loss: 7.7047 || 10iter: 4.0099 sec.\n",
      "Iteration 4190 || Loss: 6.5469 || 10iter: 3.9050 sec.\n",
      "Iteration 4200 || Loss: 6.5492 || 10iter: 3.7440 sec.\n",
      "Iteration 4210 || Loss: 7.0222 || 10iter: 3.6845 sec.\n",
      "Iteration 4220 || Loss: 5.8379 || 10iter: 3.6718 sec.\n",
      "Iteration 4230 || Loss: 6.2137 || 10iter: 3.6922 sec.\n",
      "Iteration 4240 || Loss: 7.8749 || 10iter: 3.6716 sec.\n",
      "Iteration 4250 || Loss: 6.1672 || 10iter: 3.5410 sec.\n",
      "Iteration 4260 || Loss: 8.8909 || 10iter: 3.3635 sec.\n",
      "-------------\n",
      "epoch 21 || Epoch_TRAIN_Loss:1312.8116 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  78.5495 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 22/150\n",
      "-------------\n",
      "train\n",
      "Iteration 4270 || Loss: 5.5495 || 10iter: 6.2124 sec.\n",
      "Iteration 4280 || Loss: 6.9745 || 10iter: 3.7230 sec.\n",
      "Iteration 4290 || Loss: 6.0317 || 10iter: 3.7564 sec.\n",
      "Iteration 4300 || Loss: 6.7650 || 10iter: 3.6608 sec.\n",
      "Iteration 4310 || Loss: 6.6165 || 10iter: 3.7022 sec.\n",
      "Iteration 4320 || Loss: 5.9231 || 10iter: 3.6095 sec.\n",
      "Iteration 4330 || Loss: 6.5469 || 10iter: 3.6695 sec.\n",
      "Iteration 4340 || Loss: 6.4045 || 10iter: 3.7089 sec.\n",
      "Iteration 4350 || Loss: 6.8817 || 10iter: 3.6401 sec.\n",
      "Iteration 4360 || Loss: 6.2660 || 10iter: 3.7155 sec.\n",
      "Iteration 4370 || Loss: 6.6867 || 10iter: 3.7447 sec.\n",
      "Iteration 4380 || Loss: 6.4531 || 10iter: 3.6923 sec.\n",
      "Iteration 4390 || Loss: 6.0172 || 10iter: 3.7023 sec.\n",
      "Iteration 4400 || Loss: 5.8508 || 10iter: 3.6975 sec.\n",
      "Iteration 4410 || Loss: 6.4747 || 10iter: 3.6442 sec.\n",
      "Iteration 4420 || Loss: 5.8894 || 10iter: 3.6938 sec.\n",
      "Iteration 4430 || Loss: 5.8768 || 10iter: 3.7264 sec.\n",
      "Iteration 4440 || Loss: 6.2507 || 10iter: 3.6923 sec.\n",
      "Iteration 4450 || Loss: 6.8746 || 10iter: 3.6603 sec.\n",
      "Iteration 4460 || Loss: 6.5680 || 10iter: 3.4087 sec.\n",
      "-------------\n",
      "epoch 22 || Epoch_TRAIN_Loss:1294.7365 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  77.8937 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 23/150\n",
      "-------------\n",
      "train\n",
      "Iteration 4470 || Loss: 5.8892 || 10iter: 4.5751 sec.\n",
      "Iteration 4480 || Loss: 6.3441 || 10iter: 4.2324 sec.\n",
      "Iteration 4490 || Loss: 7.0330 || 10iter: 3.6894 sec.\n",
      "Iteration 4500 || Loss: 6.4366 || 10iter: 3.7169 sec.\n",
      "Iteration 4510 || Loss: 7.1660 || 10iter: 3.7450 sec.\n",
      "Iteration 4520 || Loss: 6.3465 || 10iter: 3.6481 sec.\n",
      "Iteration 4530 || Loss: 6.4875 || 10iter: 3.6920 sec.\n",
      "Iteration 4540 || Loss: 6.0351 || 10iter: 3.7075 sec.\n",
      "Iteration 4550 || Loss: 6.6024 || 10iter: 3.6786 sec.\n",
      "Iteration 4560 || Loss: 6.2854 || 10iter: 3.6962 sec.\n",
      "Iteration 4570 || Loss: 6.8636 || 10iter: 3.6831 sec.\n",
      "Iteration 4580 || Loss: 5.9380 || 10iter: 3.6920 sec.\n",
      "Iteration 4590 || Loss: 6.3749 || 10iter: 3.7784 sec.\n",
      "Iteration 4600 || Loss: 5.0004 || 10iter: 3.7272 sec.\n",
      "Iteration 4610 || Loss: 5.8148 || 10iter: 3.7922 sec.\n",
      "Iteration 4620 || Loss: 7.0155 || 10iter: 5.3641 sec.\n",
      "Iteration 4630 || Loss: 6.5821 || 10iter: 6.5998 sec.\n",
      "Iteration 4640 || Loss: 6.8226 || 10iter: 5.4766 sec.\n",
      "Iteration 4650 || Loss: 7.1818 || 10iter: 9.8342 sec.\n",
      "Iteration 4660 || Loss: 6.5743 || 10iter: 9.0452 sec.\n",
      "-------------\n",
      "epoch 23 || Epoch_TRAIN_Loss:1284.3434 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  100.7576 sec.\n",
      "lr is: 0.0001\n",
      "-------------\n",
      "Epoch 24/150\n",
      "-------------\n",
      "train\n",
      "Iteration 4670 || Loss: 5.0948 || 10iter: 4.3473 sec.\n",
      "Iteration 4680 || Loss: 6.6090 || 10iter: 9.4685 sec.\n",
      "Iteration 4690 || Loss: 5.7356 || 10iter: 9.0905 sec.\n",
      "Iteration 4700 || Loss: 6.3354 || 10iter: 8.8396 sec.\n",
      "Iteration 4710 || Loss: 5.8043 || 10iter: 8.9113 sec.\n",
      "Iteration 4720 || Loss: 6.0878 || 10iter: 9.0417 sec.\n",
      "Iteration 4730 || Loss: 5.9673 || 10iter: 9.0285 sec.\n",
      "Iteration 4740 || Loss: 6.2185 || 10iter: 8.8859 sec.\n",
      "Iteration 4750 || Loss: 5.5712 || 10iter: 9.0473 sec.\n",
      "Iteration 4760 || Loss: 5.3946 || 10iter: 9.0808 sec.\n",
      "Iteration 4770 || Loss: 6.7708 || 10iter: 9.0546 sec.\n",
      "Iteration 4780 || Loss: 5.7064 || 10iter: 8.9430 sec.\n",
      "Iteration 4790 || Loss: 6.5230 || 10iter: 9.4664 sec.\n",
      "Iteration 4800 || Loss: 7.2651 || 10iter: 9.8223 sec.\n",
      "Iteration 4810 || Loss: 5.6373 || 10iter: 9.1189 sec.\n",
      "Iteration 4820 || Loss: 5.8847 || 10iter: 9.0414 sec.\n",
      "Iteration 4830 || Loss: 5.9783 || 10iter: 9.0342 sec.\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 150\n",
    "train_model(net, dataloaders_dict, criterion, optimizer, num_epochs=num_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That's all :)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}