lab-10-3-mnist_nn_dropout.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Eager mode on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checkpoint function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load & pre-processing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=labels))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)\n",
    "\n",
    "def dropout(rate) :\n",
    "    return tf.keras.layers.Dropout(rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (class version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(4):\n",
    "            self.model.add(dense(512, weight_init))\n",
    "            self.model.add(relu())\n",
    "            self.model.add(dropout(rate=0.5))\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (function version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(4) :\n",
    "        model.add(dense(512, weight_init))\n",
    "        model.add(relu())\n",
    "        model.add(dropout(rate=0.5))\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define data & hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size).\\\n",
    "    repeat()\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x)).\\\n",
    "    repeat()\n",
    "\n",
    "train_iterator = train_dataset.make_one_shot_iterator()\n",
    "test_iterator = test_dataset.make_one_shot_iterator()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model & optimizer & writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_dropout'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restore checkpoint & start train or test phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 0.4892, train_loss: 2.35166407, train_accuracy: 0.2266, test_Accuracy: 0.1755\n",
      "Epoch: [ 0] [    1/  468] time: 0.7493, train_loss: 2.31816411, train_accuracy: 0.3281, test_Accuracy: 0.2692\n",
      "Epoch: [ 0] [    2/  468] time: 1.0163, train_loss: 2.27257347, train_accuracy: 0.3047, test_Accuracy: 0.3348\n",
      "Epoch: [ 0] [    3/  468] time: 1.2659, train_loss: 2.30772448, train_accuracy: 0.3906, test_Accuracy: 0.3769\n",
      "Epoch: [ 0] [    4/  468] time: 1.5214, train_loss: 2.30245805, train_accuracy: 0.3906, test_Accuracy: 0.4140\n",
      "Epoch: [ 0] [    5/  468] time: 1.7696, train_loss: 2.21979237, train_accuracy: 0.4141, test_Accuracy: 0.4624\n",
      "Epoch: [ 0] [    6/  468] time: 2.0366, train_loss: 2.22849751, train_accuracy: 0.5078, test_Accuracy: 0.5241\n",
      "Epoch: [ 0] [    7/  468] time: 2.2963, train_loss: 2.17310953, train_accuracy: 0.6016, test_Accuracy: 0.5381\n",
      "Epoch: [ 0] [    8/  468] time: 2.5491, train_loss: 2.22101831, train_accuracy: 0.5312, test_Accuracy: 0.5453\n",
      "Epoch: [ 0] [    9/  468] time: 2.8047, train_loss: 2.16039658, train_accuracy: 0.5469, test_Accuracy: 0.5383\n",
      "Epoch: [ 0] [   10/  468] time: 3.0549, train_loss: 2.14023185, train_accuracy: 0.5938, test_Accuracy: 0.5645\n",
      "Epoch: [ 0] [   11/  468] time: 3.3119, train_loss: 2.16180134, train_accuracy: 0.5547, test_Accuracy: 0.5972\n",
      "Epoch: [ 0] [   12/  468] time: 3.5805, train_loss: 2.04477596, train_accuracy: 0.5938, test_Accuracy: 0.6091\n",
      "Epoch: [ 0] [   13/  468] time: 3.8409, train_loss: 2.12729621, train_accuracy: 0.6016, test_Accuracy: 0.6054\n",
      "Epoch: [ 0] [   14/  468] time: 4.0958, train_loss: 1.97647572, train_accuracy: 0.5547, test_Accuracy: 0.5946\n",
      "Epoch: [ 0] [   15/  468] time: 4.3486, train_loss: 1.97797561, train_accuracy: 0.5938, test_Accuracy: 0.5858\n",
      "Epoch: [ 0] [   16/  468] time: 4.6013, train_loss: 1.86957383, train_accuracy: 0.5391, test_Accuracy: 0.5818\n",
      "Epoch: [ 0] [   17/  468] time: 4.8688, train_loss: 1.84999371, train_accuracy: 0.5859, test_Accuracy: 0.5796\n",
      "Epoch: [ 0] [   18/  468] time: 5.1382, train_loss: 1.83039165, train_accuracy: 0.6641, test_Accuracy: 0.5840\n",
      "Epoch: [ 0] [   19/  468] time: 5.3895, train_loss: 1.76467848, train_accuracy: 0.6172, test_Accuracy: 0.5939\n",
      "Epoch: [ 0] [   20/  468] time: 5.6446, train_loss: 1.64892292, train_accuracy: 0.6641, test_Accuracy: 0.6076\n",
      "Epoch: [ 0] [   21/  468] time: 5.8964, train_loss: 1.62264895, train_accuracy: 0.5859, test_Accuracy: 0.6372\n",
      "Epoch: [ 0] [   22/  468] time: 6.1388, train_loss: 1.52448225, train_accuracy: 0.6484, test_Accuracy: 0.6432\n",
      "Epoch: [ 0] [   23/  468] time: 6.3814, train_loss: 1.57604134, train_accuracy: 0.6484, test_Accuracy: 0.6524\n",
      "Epoch: [ 0] [   24/  468] time: 6.6338, train_loss: 1.39687681, train_accuracy: 0.6953, test_Accuracy: 0.6728\n",
      "Epoch: [ 0] [   25/  468] time: 6.8764, train_loss: 1.62612474, train_accuracy: 0.6562, test_Accuracy: 0.6942\n",
      "Epoch: [ 0] [   26/  468] time: 7.1321, train_loss: 1.33804989, train_accuracy: 0.7500, test_Accuracy: 0.7068\n",
      "Epoch: [ 0] [   27/  468] time: 7.3796, train_loss: 1.32129550, train_accuracy: 0.6953, test_Accuracy: 0.7012\n",
      "Epoch: [ 0] [   28/  468] time: 7.6186, train_loss: 1.37315309, train_accuracy: 0.6250, test_Accuracy: 0.6855\n",
      "Epoch: [ 0] [   29/  468] time: 7.8547, train_loss: 1.29034197, train_accuracy: 0.6406, test_Accuracy: 0.6852\n",
      "Epoch: [ 0] [   30/  468] time: 8.0969, train_loss: 1.11016214, train_accuracy: 0.7656, test_Accuracy: 0.6922\n",
      "Epoch: [ 0] [   31/  468] time: 8.3440, train_loss: 0.99452913, train_accuracy: 0.7578, test_Accuracy: 0.7031\n",
      "Epoch: [ 0] [   32/  468] time: 8.5813, train_loss: 1.18841004, train_accuracy: 0.7969, test_Accuracy: 0.7216\n",
      "Epoch: [ 0] [   33/  468] time: 8.8162, train_loss: 1.15699029, train_accuracy: 0.7266, test_Accuracy: 0.7629\n",
      "Epoch: [ 0] [   34/  468] time: 9.0646, train_loss: 1.13805616, train_accuracy: 0.7969, test_Accuracy: 0.7845\n",
      "Epoch: [ 0] [   35/  468] time: 9.3286, train_loss: 1.05537868, train_accuracy: 0.7812, test_Accuracy: 0.8046\n",
      "Epoch: [ 0] [   36/  468] time: 9.5765, train_loss: 1.09025693, train_accuracy: 0.7734, test_Accuracy: 0.7948\n",
      "Epoch: [ 0] [   37/  468] time: 9.8128, train_loss: 1.20280290, train_accuracy: 0.7656, test_Accuracy: 0.7715\n",
      "Epoch: [ 0] [   38/  468] time: 10.0672, train_loss: 1.01069069, train_accuracy: 0.7422, test_Accuracy: 0.7616\n",
      "Epoch: [ 0] [   39/  468] time: 10.3014, train_loss: 1.09623027, train_accuracy: 0.7109, test_Accuracy: 0.7604\n",
      "Epoch: [ 0] [   40/  468] time: 10.5628, train_loss: 1.06042922, train_accuracy: 0.7188, test_Accuracy: 0.7843\n",
      "Epoch: [ 0] [   41/  468] time: 10.8016, train_loss: 0.91645384, train_accuracy: 0.7734, test_Accuracy: 0.8028\n",
      "Epoch: [ 0] [   42/  468] time: 11.0445, train_loss: 0.81922615, train_accuracy: 0.8125, test_Accuracy: 0.8100\n",
      "Epoch: [ 0] [   43/  468] time: 11.2951, train_loss: 0.86774564, train_accuracy: 0.8047, test_Accuracy: 0.8061\n",
      "Epoch: [ 0] [   44/  468] time: 11.5353, train_loss: 0.76969570, train_accuracy: 0.8516, test_Accuracy: 0.7970\n",
      "Epoch: [ 0] [   45/  468] time: 11.7768, train_loss: 0.95573699, train_accuracy: 0.7656, test_Accuracy: 0.7924\n",
      "Epoch: [ 0] [   46/  468] time: 12.0288, train_loss: 0.77348953, train_accuracy: 0.7656, test_Accuracy: 0.7951\n",
      "Epoch: [ 0] [   47/  468] time: 12.2654, train_loss: 0.94310784, train_accuracy: 0.8203, test_Accuracy: 0.7991\n",
      "Epoch: [ 0] [   48/  468] time: 12.5047, train_loss: 0.82521325, train_accuracy: 0.8047, test_Accuracy: 0.8090\n",
      "Epoch: [ 0] [   49/  468] time: 12.7458, train_loss: 0.85912508, train_accuracy: 0.8516, test_Accuracy: 0.8165\n",
      "Epoch: [ 0] [   50/  468] time: 12.9869, train_loss: 0.73905206, train_accuracy: 0.8594, test_Accuracy: 0.8265\n",
      "Epoch: [ 0] [   51/  468] time: 13.2310, train_loss: 0.74961883, train_accuracy: 0.8828, test_Accuracy: 0.8382\n",
      "Epoch: [ 0] [   52/  468] time: 13.4845, train_loss: 0.85728168, train_accuracy: 0.8281, test_Accuracy: 0.8471\n",
      "Epoch: [ 0] [   53/  468] time: 13.7241, train_loss: 0.78012478, train_accuracy: 0.8359, test_Accuracy: 0.8534\n",
      "Epoch: [ 0] [   54/  468] time: 13.9668, train_loss: 0.77256429, train_accuracy: 0.8125, test_Accuracy: 0.8546\n",
      "Epoch: [ 0] [   55/  468] time: 14.2176, train_loss: 0.81701088, train_accuracy: 0.8438, test_Accuracy: 0.8572\n",
      "Epoch: [ 0] [   56/  468] time: 14.4672, train_loss: 0.73891366, train_accuracy: 0.8516, test_Accuracy: 0.8576\n",
      "Epoch: [ 0] [   57/  468] time: 14.7146, train_loss: 0.64071631, train_accuracy: 0.8984, test_Accuracy: 0.8596\n",
      "Epoch: [ 0] [   58/  468] time: 14.9646, train_loss: 0.64435887, train_accuracy: 0.8750, test_Accuracy: 0.8615\n",
      "Epoch: [ 0] [   59/  468] time: 15.2154, train_loss: 0.72509122, train_accuracy: 0.8438, test_Accuracy: 0.8597\n",
      "Epoch: [ 0] [   60/  468] time: 15.4619, train_loss: 0.74073493, train_accuracy: 0.8438, test_Accuracy: 0.8591\n",
      "Epoch: [ 0] [   61/  468] time: 15.6968, train_loss: 0.62443161, train_accuracy: 0.8750, test_Accuracy: 0.8587\n",
      "Epoch: [ 0] [   62/  468] time: 15.9485, train_loss: 0.60390902, train_accuracy: 0.8516, test_Accuracy: 0.8570\n",
      "Epoch: [ 0] [   63/  468] time: 16.1930, train_loss: 0.65012634, train_accuracy: 0.8594, test_Accuracy: 0.8583\n",
      "Epoch: [ 0] [   64/  468] time: 16.4348, train_loss: 0.45524049, train_accuracy: 0.8906, test_Accuracy: 0.8655\n",
      "Epoch: [ 0] [   65/  468] time: 16.6805, train_loss: 0.57473397, train_accuracy: 0.8984, test_Accuracy: 0.8701\n",
      "Epoch: [ 0] [   66/  468] time: 16.9299, train_loss: 0.59452939, train_accuracy: 0.8828, test_Accuracy: 0.8707\n",
      "Epoch: [ 0] [   67/  468] time: 17.1780, train_loss: 0.81400239, train_accuracy: 0.8594, test_Accuracy: 0.8686\n",
      "Epoch: [ 0] [   68/  468] time: 17.4239, train_loss: 0.53491282, train_accuracy: 0.8516, test_Accuracy: 0.8635\n",
      "Epoch: [ 0] [   69/  468] time: 17.6654, train_loss: 0.58852851, train_accuracy: 0.8047, test_Accuracy: 0.8596\n",
      "Epoch: [ 0] [   70/  468] time: 17.9177, train_loss: 0.63813418, train_accuracy: 0.8125, test_Accuracy: 0.8603\n",
      "Epoch: [ 0] [   71/  468] time: 18.1653, train_loss: 0.64944589, train_accuracy: 0.8828, test_Accuracy: 0.8649\n",
      "Epoch: [ 0] [   72/  468] time: 18.4052, train_loss: 0.52546024, train_accuracy: 0.8672, test_Accuracy: 0.8643\n",
      "Epoch: [ 0] [   73/  468] time: 18.6540, train_loss: 0.69215631, train_accuracy: 0.8516, test_Accuracy: 0.8688\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   74/  468] time: 18.9003, train_loss: 0.71976399, train_accuracy: 0.8516, test_Accuracy: 0.8722\n",
      "Epoch: [ 0] [   75/  468] time: 19.1427, train_loss: 0.62141907, train_accuracy: 0.8984, test_Accuracy: 0.8755\n",
      "Epoch: [ 0] [   76/  468] time: 19.3877, train_loss: 0.46489429, train_accuracy: 0.9062, test_Accuracy: 0.8752\n",
      "Epoch: [ 0] [   77/  468] time: 19.6312, train_loss: 0.56345868, train_accuracy: 0.8672, test_Accuracy: 0.8741\n",
      "Epoch: [ 0] [   78/  468] time: 19.8710, train_loss: 0.57509935, train_accuracy: 0.8906, test_Accuracy: 0.8757\n",
      "Epoch: [ 0] [   79/  468] time: 20.1191, train_loss: 0.69914323, train_accuracy: 0.8906, test_Accuracy: 0.8769\n",
      "Epoch: [ 0] [   80/  468] time: 20.3568, train_loss: 0.68760693, train_accuracy: 0.8984, test_Accuracy: 0.8827\n",
      "Epoch: [ 0] [   81/  468] time: 20.6043, train_loss: 0.47655237, train_accuracy: 0.9062, test_Accuracy: 0.8904\n",
      "Epoch: [ 0] [   82/  468] time: 20.8731, train_loss: 0.42906165, train_accuracy: 0.8984, test_Accuracy: 0.8948\n",
      "Epoch: [ 0] [   83/  468] time: 21.1164, train_loss: 0.41126582, train_accuracy: 0.9219, test_Accuracy: 0.8941\n",
      "Epoch: [ 0] [   84/  468] time: 21.3760, train_loss: 0.45814008, train_accuracy: 0.8984, test_Accuracy: 0.8916\n",
      "Epoch: [ 0] [   85/  468] time: 21.6163, train_loss: 0.43905562, train_accuracy: 0.9453, test_Accuracy: 0.8885\n",
      "Epoch: [ 0] [   86/  468] time: 21.8625, train_loss: 0.38462260, train_accuracy: 0.9219, test_Accuracy: 0.8878\n",
      "Epoch: [ 0] [   87/  468] time: 22.1115, train_loss: 0.62416697, train_accuracy: 0.9062, test_Accuracy: 0.8885\n",
      "Epoch: [ 0] [   88/  468] time: 22.3526, train_loss: 0.59016126, train_accuracy: 0.8594, test_Accuracy: 0.8922\n",
      "Epoch: [ 0] [   89/  468] time: 22.5931, train_loss: 0.62140715, train_accuracy: 0.9062, test_Accuracy: 0.8962\n",
      "Epoch: [ 0] [   90/  468] time: 22.8399, train_loss: 0.45845512, train_accuracy: 0.9375, test_Accuracy: 0.9005\n",
      "Epoch: [ 0] [   91/  468] time: 23.0893, train_loss: 0.45127520, train_accuracy: 0.9297, test_Accuracy: 0.9029\n",
      "Epoch: [ 0] [   92/  468] time: 23.3369, train_loss: 0.26811162, train_accuracy: 0.9219, test_Accuracy: 0.9034\n",
      "Epoch: [ 0] [   93/  468] time: 23.5780, train_loss: 0.65063655, train_accuracy: 0.8828, test_Accuracy: 0.9018\n",
      "Epoch: [ 0] [   94/  468] time: 23.8167, train_loss: 0.55218077, train_accuracy: 0.8672, test_Accuracy: 0.9019\n",
      "Epoch: [ 0] [   95/  468] time: 24.0579, train_loss: 0.31599236, train_accuracy: 0.9297, test_Accuracy: 0.9024\n",
      "Epoch: [ 0] [   96/  468] time: 24.3032, train_loss: 0.52770197, train_accuracy: 0.8828, test_Accuracy: 0.9028\n",
      "Epoch: [ 0] [   97/  468] time: 24.5555, train_loss: 0.63011038, train_accuracy: 0.9141, test_Accuracy: 0.9047\n",
      "Epoch: [ 0] [   98/  468] time: 24.7941, train_loss: 0.42567647, train_accuracy: 0.9297, test_Accuracy: 0.9072\n",
      "Epoch: [ 0] [   99/  468] time: 25.0387, train_loss: 0.41413444, train_accuracy: 0.9219, test_Accuracy: 0.9093\n",
      "Epoch: [ 0] [  100/  468] time: 25.2817, train_loss: 0.44839120, train_accuracy: 0.8984, test_Accuracy: 0.9105\n",
      "Epoch: [ 0] [  101/  468] time: 25.5200, train_loss: 0.36459714, train_accuracy: 0.9297, test_Accuracy: 0.9114\n",
      "Epoch: [ 0] [  102/  468] time: 25.7690, train_loss: 0.46875757, train_accuracy: 0.8984, test_Accuracy: 0.9109\n",
      "Epoch: [ 0] [  103/  468] time: 26.0125, train_loss: 0.62729216, train_accuracy: 0.8906, test_Accuracy: 0.9120\n",
      "Epoch: [ 0] [  104/  468] time: 26.2548, train_loss: 0.44138491, train_accuracy: 0.9453, test_Accuracy: 0.9122\n",
      "Epoch: [ 0] [  105/  468] time: 26.5017, train_loss: 0.45282000, train_accuracy: 0.9062, test_Accuracy: 0.9125\n",
      "Epoch: [ 0] [  106/  468] time: 26.7444, train_loss: 0.40508834, train_accuracy: 0.9141, test_Accuracy: 0.9120\n",
      "Epoch: [ 0] [  107/  468] time: 26.9882, train_loss: 0.51962435, train_accuracy: 0.8906, test_Accuracy: 0.9108\n",
      "Epoch: [ 0] [  108/  468] time: 27.2291, train_loss: 0.44988045, train_accuracy: 0.9141, test_Accuracy: 0.9104\n",
      "Epoch: [ 0] [  109/  468] time: 27.4700, train_loss: 0.66583335, train_accuracy: 0.8750, test_Accuracy: 0.9122\n",
      "Epoch: [ 0] [  110/  468] time: 27.7125, train_loss: 0.42763686, train_accuracy: 0.9297, test_Accuracy: 0.9085\n",
      "Epoch: [ 0] [  111/  468] time: 27.9587, train_loss: 0.37104881, train_accuracy: 0.9141, test_Accuracy: 0.9095\n",
      "Epoch: [ 0] [  112/  468] time: 28.2388, train_loss: 0.39960644, train_accuracy: 0.9375, test_Accuracy: 0.9104\n",
      "Epoch: [ 0] [  113/  468] time: 28.4851, train_loss: 0.51437569, train_accuracy: 0.9062, test_Accuracy: 0.9118\n",
      "Epoch: [ 0] [  114/  468] time: 28.7233, train_loss: 0.45849979, train_accuracy: 0.8906, test_Accuracy: 0.9141\n",
      "Epoch: [ 0] [  115/  468] time: 28.9672, train_loss: 0.46436703, train_accuracy: 0.9062, test_Accuracy: 0.9138\n",
      "Epoch: [ 0] [  116/  468] time: 29.2167, train_loss: 0.53517401, train_accuracy: 0.9219, test_Accuracy: 0.9136\n",
      "Epoch: [ 0] [  117/  468] time: 29.4641, train_loss: 0.54252136, train_accuracy: 0.9141, test_Accuracy: 0.9125\n",
      "Epoch: [ 0] [  118/  468] time: 29.7130, train_loss: 0.67444360, train_accuracy: 0.8672, test_Accuracy: 0.9139\n",
      "Epoch: [ 0] [  119/  468] time: 29.9639, train_loss: 0.37548476, train_accuracy: 0.9375, test_Accuracy: 0.9139\n",
      "Epoch: [ 0] [  120/  468] time: 30.2163, train_loss: 0.41118199, train_accuracy: 0.9219, test_Accuracy: 0.9138\n",
      "Epoch: [ 0] [  121/  468] time: 30.4597, train_loss: 0.52426779, train_accuracy: 0.9062, test_Accuracy: 0.9150\n",
      "Epoch: [ 0] [  122/  468] time: 30.7078, train_loss: 0.44432792, train_accuracy: 0.9219, test_Accuracy: 0.9139\n",
      "Epoch: [ 0] [  123/  468] time: 30.9522, train_loss: 0.38637266, train_accuracy: 0.9375, test_Accuracy: 0.9117\n",
      "Epoch: [ 0] [  124/  468] time: 31.1986, train_loss: 0.48553473, train_accuracy: 0.8906, test_Accuracy: 0.9117\n",
      "Epoch: [ 0] [  125/  468] time: 31.4515, train_loss: 0.44734165, train_accuracy: 0.9219, test_Accuracy: 0.9141\n",
      "Epoch: [ 0] [  126/  468] time: 31.6942, train_loss: 0.37106374, train_accuracy: 0.9453, test_Accuracy: 0.9167\n",
      "Epoch: [ 0] [  127/  468] time: 31.9386, train_loss: 0.47147429, train_accuracy: 0.9062, test_Accuracy: 0.9210\n",
      "Epoch: [ 0] [  128/  468] time: 32.1867, train_loss: 0.35326776, train_accuracy: 0.9609, test_Accuracy: 0.9224\n",
      "Epoch: [ 0] [  129/  468] time: 32.4349, train_loss: 0.50945085, train_accuracy: 0.9062, test_Accuracy: 0.9214\n",
      "Epoch: [ 0] [  130/  468] time: 32.6810, train_loss: 0.52586144, train_accuracy: 0.8984, test_Accuracy: 0.9218\n",
      "Epoch: [ 0] [  131/  468] time: 32.9235, train_loss: 0.37755305, train_accuracy: 0.9297, test_Accuracy: 0.9220\n",
      "Epoch: [ 0] [  132/  468] time: 33.1623, train_loss: 0.37765691, train_accuracy: 0.9375, test_Accuracy: 0.9228\n",
      "Epoch: [ 0] [  133/  468] time: 33.4076, train_loss: 0.32885650, train_accuracy: 0.9453, test_Accuracy: 0.9229\n",
      "Epoch: [ 0] [  134/  468] time: 33.6441, train_loss: 0.39978439, train_accuracy: 0.9141, test_Accuracy: 0.9223\n",
      "Epoch: [ 0] [  135/  468] time: 33.8836, train_loss: 0.69827294, train_accuracy: 0.8672, test_Accuracy: 0.9227\n",
      "Epoch: [ 0] [  136/  468] time: 34.1224, train_loss: 0.40441817, train_accuracy: 0.9453, test_Accuracy: 0.9229\n",
      "Epoch: [ 0] [  137/  468] time: 34.3627, train_loss: 0.34028032, train_accuracy: 0.9141, test_Accuracy: 0.9228\n",
      "Epoch: [ 0] [  138/  468] time: 34.6081, train_loss: 0.36490846, train_accuracy: 0.9297, test_Accuracy: 0.9227\n",
      "Epoch: [ 0] [  139/  468] time: 34.8447, train_loss: 0.31729570, train_accuracy: 0.9531, test_Accuracy: 0.9217\n",
      "Epoch: [ 0] [  140/  468] time: 35.0940, train_loss: 0.40953225, train_accuracy: 0.8906, test_Accuracy: 0.9205\n",
      "Epoch: [ 0] [  141/  468] time: 35.3390, train_loss: 0.48812598, train_accuracy: 0.9062, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [  142/  468] time: 35.5790, train_loss: 0.30659917, train_accuracy: 0.9453, test_Accuracy: 0.9202\n",
      "Epoch: [ 0] [  143/  468] time: 35.8345, train_loss: 0.38416445, train_accuracy: 0.9219, test_Accuracy: 0.9218\n",
      "Epoch: [ 0] [  144/  468] time: 36.0971, train_loss: 0.38724408, train_accuracy: 0.9141, test_Accuracy: 0.9248\n",
      "Epoch: [ 0] [  145/  468] time: 36.3395, train_loss: 0.27139699, train_accuracy: 0.9531, test_Accuracy: 0.9264\n",
      "Epoch: [ 0] [  146/  468] time: 36.5828, train_loss: 0.39812726, train_accuracy: 0.9375, test_Accuracy: 0.9261\n",
      "Epoch: [ 0] [  147/  468] time: 36.8255, train_loss: 0.32830662, train_accuracy: 0.9297, test_Accuracy: 0.9237\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  148/  468] time: 37.0641, train_loss: 0.40230501, train_accuracy: 0.8672, test_Accuracy: 0.9222\n",
      "Epoch: [ 0] [  149/  468] time: 37.3070, train_loss: 0.34811944, train_accuracy: 0.9375, test_Accuracy: 0.9216\n",
      "Epoch: [ 0] [  150/  468] time: 37.5529, train_loss: 0.35084498, train_accuracy: 0.8984, test_Accuracy: 0.9216\n",
      "Epoch: [ 0] [  151/  468] time: 37.8077, train_loss: 0.24494763, train_accuracy: 0.9531, test_Accuracy: 0.9219\n",
      "Epoch: [ 0] [  152/  468] time: 38.0480, train_loss: 0.37574947, train_accuracy: 0.9453, test_Accuracy: 0.9232\n",
      "Epoch: [ 0] [  153/  468] time: 38.2901, train_loss: 0.28041494, train_accuracy: 0.9531, test_Accuracy: 0.9251\n",
      "Epoch: [ 0] [  154/  468] time: 38.5334, train_loss: 0.37605911, train_accuracy: 0.9688, test_Accuracy: 0.9266\n",
      "Epoch: [ 0] [  155/  468] time: 38.7816, train_loss: 0.46547490, train_accuracy: 0.8906, test_Accuracy: 0.9272\n",
      "Epoch: [ 0] [  156/  468] time: 39.0264, train_loss: 0.35544017, train_accuracy: 0.9688, test_Accuracy: 0.9281\n",
      "Epoch: [ 0] [  157/  468] time: 39.2740, train_loss: 0.24738625, train_accuracy: 0.9609, test_Accuracy: 0.9286\n",
      "Epoch: [ 0] [  158/  468] time: 39.5155, train_loss: 0.36197770, train_accuracy: 0.9375, test_Accuracy: 0.9285\n",
      "Epoch: [ 0] [  159/  468] time: 39.7587, train_loss: 0.42672342, train_accuracy: 0.9141, test_Accuracy: 0.9277\n",
      "Epoch: [ 0] [  160/  468] time: 40.0031, train_loss: 0.31201231, train_accuracy: 0.9531, test_Accuracy: 0.9263\n",
      "Epoch: [ 0] [  161/  468] time: 40.2499, train_loss: 0.34107250, train_accuracy: 0.9062, test_Accuracy: 0.9276\n",
      "Epoch: [ 0] [  162/  468] time: 40.4893, train_loss: 0.32112527, train_accuracy: 0.9531, test_Accuracy: 0.9284\n",
      "Epoch: [ 0] [  163/  468] time: 40.7310, train_loss: 0.23648287, train_accuracy: 0.9219, test_Accuracy: 0.9295\n",
      "Epoch: [ 0] [  164/  468] time: 40.9817, train_loss: 0.40242612, train_accuracy: 0.9062, test_Accuracy: 0.9307\n",
      "Epoch: [ 0] [  165/  468] time: 41.2370, train_loss: 0.43395466, train_accuracy: 0.9297, test_Accuracy: 0.9319\n",
      "Epoch: [ 0] [  166/  468] time: 41.5047, train_loss: 0.46942136, train_accuracy: 0.9141, test_Accuracy: 0.9308\n",
      "Epoch: [ 0] [  167/  468] time: 41.7591, train_loss: 0.48704457, train_accuracy: 0.9062, test_Accuracy: 0.9308\n",
      "Epoch: [ 0] [  168/  468] time: 42.0017, train_loss: 0.29814154, train_accuracy: 0.9531, test_Accuracy: 0.9322\n",
      "Epoch: [ 0] [  169/  468] time: 42.2387, train_loss: 0.36687425, train_accuracy: 0.9062, test_Accuracy: 0.9324\n",
      "Epoch: [ 0] [  170/  468] time: 42.4881, train_loss: 0.49865556, train_accuracy: 0.9062, test_Accuracy: 0.9319\n",
      "Epoch: [ 0] [  171/  468] time: 42.7382, train_loss: 0.39057451, train_accuracy: 0.9219, test_Accuracy: 0.9316\n",
      "Epoch: [ 0] [  172/  468] time: 42.9794, train_loss: 0.32108295, train_accuracy: 0.9375, test_Accuracy: 0.9312\n",
      "Epoch: [ 0] [  173/  468] time: 43.2281, train_loss: 0.50560504, train_accuracy: 0.8984, test_Accuracy: 0.9314\n",
      "Epoch: [ 0] [  174/  468] time: 43.4819, train_loss: 0.24301511, train_accuracy: 0.9453, test_Accuracy: 0.9316\n",
      "Epoch: [ 0] [  175/  468] time: 43.7288, train_loss: 0.47971442, train_accuracy: 0.9141, test_Accuracy: 0.9318\n",
      "Epoch: [ 0] [  176/  468] time: 43.9717, train_loss: 0.43046018, train_accuracy: 0.8984, test_Accuracy: 0.9313\n",
      "Epoch: [ 0] [  177/  468] time: 44.2189, train_loss: 0.35289407, train_accuracy: 0.9297, test_Accuracy: 0.9307\n",
      "Epoch: [ 0] [  178/  468] time: 44.4649, train_loss: 0.23339561, train_accuracy: 0.9766, test_Accuracy: 0.9309\n",
      "Epoch: [ 0] [  179/  468] time: 44.7083, train_loss: 0.43220407, train_accuracy: 0.9062, test_Accuracy: 0.9329\n",
      "Epoch: [ 0] [  180/  468] time: 44.9545, train_loss: 0.49509597, train_accuracy: 0.9062, test_Accuracy: 0.9345\n",
      "Epoch: [ 0] [  181/  468] time: 45.1981, train_loss: 0.30226427, train_accuracy: 0.9453, test_Accuracy: 0.9353\n",
      "Epoch: [ 0] [  182/  468] time: 45.4439, train_loss: 0.43603235, train_accuracy: 0.8672, test_Accuracy: 0.9351\n",
      "Epoch: [ 0] [  183/  468] time: 45.6914, train_loss: 0.37539589, train_accuracy: 0.9219, test_Accuracy: 0.9337\n",
      "Epoch: [ 0] [  184/  468] time: 45.9648, train_loss: 0.45908725, train_accuracy: 0.9219, test_Accuracy: 0.9321\n",
      "Epoch: [ 0] [  185/  468] time: 46.2170, train_loss: 0.45767713, train_accuracy: 0.8984, test_Accuracy: 0.9317\n",
      "Epoch: [ 0] [  186/  468] time: 46.4669, train_loss: 0.37614557, train_accuracy: 0.9141, test_Accuracy: 0.9322\n",
      "Epoch: [ 0] [  187/  468] time: 46.7349, train_loss: 0.33167428, train_accuracy: 0.9375, test_Accuracy: 0.9336\n",
      "Epoch: [ 0] [  188/  468] time: 46.9803, train_loss: 0.33601549, train_accuracy: 0.9062, test_Accuracy: 0.9343\n",
      "Epoch: [ 0] [  189/  468] time: 47.2197, train_loss: 0.31684649, train_accuracy: 0.9453, test_Accuracy: 0.9345\n",
      "Epoch: [ 0] [  190/  468] time: 47.4713, train_loss: 0.34496582, train_accuracy: 0.9453, test_Accuracy: 0.9370\n",
      "Epoch: [ 0] [  191/  468] time: 47.7161, train_loss: 0.25248677, train_accuracy: 0.9844, test_Accuracy: 0.9369\n",
      "Epoch: [ 0] [  192/  468] time: 47.9643, train_loss: 0.52852660, train_accuracy: 0.9062, test_Accuracy: 0.9364\n",
      "Epoch: [ 0] [  193/  468] time: 48.2143, train_loss: 0.48983860, train_accuracy: 0.8906, test_Accuracy: 0.9369\n",
      "Epoch: [ 0] [  194/  468] time: 48.4634, train_loss: 0.34920147, train_accuracy: 0.9531, test_Accuracy: 0.9361\n",
      "Epoch: [ 0] [  195/  468] time: 48.7125, train_loss: 0.19963247, train_accuracy: 0.9766, test_Accuracy: 0.9367\n",
      "Epoch: [ 0] [  196/  468] time: 48.9559, train_loss: 0.31531152, train_accuracy: 0.9375, test_Accuracy: 0.9369\n",
      "Epoch: [ 0] [  197/  468] time: 49.1944, train_loss: 0.38752988, train_accuracy: 0.9141, test_Accuracy: 0.9381\n",
      "Epoch: [ 0] [  198/  468] time: 49.4387, train_loss: 0.30530584, train_accuracy: 0.9297, test_Accuracy: 0.9389\n",
      "Epoch: [ 0] [  199/  468] time: 49.6820, train_loss: 0.37153327, train_accuracy: 0.9531, test_Accuracy: 0.9394\n",
      "Epoch: [ 0] [  200/  468] time: 49.9190, train_loss: 0.29692101, train_accuracy: 0.9453, test_Accuracy: 0.9393\n",
      "Epoch: [ 0] [  201/  468] time: 50.1634, train_loss: 0.21370217, train_accuracy: 0.9609, test_Accuracy: 0.9385\n",
      "Epoch: [ 0] [  202/  468] time: 50.4120, train_loss: 0.31315970, train_accuracy: 0.9219, test_Accuracy: 0.9373\n",
      "Epoch: [ 0] [  203/  468] time: 50.6617, train_loss: 0.27120757, train_accuracy: 0.9531, test_Accuracy: 0.9380\n",
      "Epoch: [ 0] [  204/  468] time: 50.9062, train_loss: 0.32472131, train_accuracy: 0.9922, test_Accuracy: 0.9388\n",
      "Epoch: [ 0] [  205/  468] time: 51.1534, train_loss: 0.24341266, train_accuracy: 0.9688, test_Accuracy: 0.9384\n",
      "Epoch: [ 0] [  206/  468] time: 51.3982, train_loss: 0.18848693, train_accuracy: 0.9453, test_Accuracy: 0.9389\n",
      "Epoch: [ 0] [  207/  468] time: 51.6421, train_loss: 0.28718537, train_accuracy: 0.9375, test_Accuracy: 0.9396\n",
      "Epoch: [ 0] [  208/  468] time: 51.8845, train_loss: 0.25905359, train_accuracy: 0.9453, test_Accuracy: 0.9393\n",
      "Epoch: [ 0] [  209/  468] time: 52.1268, train_loss: 0.48603520, train_accuracy: 0.8984, test_Accuracy: 0.9395\n",
      "Epoch: [ 0] [  210/  468] time: 52.3688, train_loss: 0.22943412, train_accuracy: 0.9609, test_Accuracy: 0.9388\n",
      "Epoch: [ 0] [  211/  468] time: 52.6110, train_loss: 0.30440444, train_accuracy: 0.9766, test_Accuracy: 0.9390\n",
      "Epoch: [ 0] [  212/  468] time: 52.8511, train_loss: 0.22641499, train_accuracy: 0.9453, test_Accuracy: 0.9394\n",
      "Epoch: [ 0] [  213/  468] time: 53.0957, train_loss: 0.34742796, train_accuracy: 0.9453, test_Accuracy: 0.9384\n",
      "Epoch: [ 0] [  214/  468] time: 53.3373, train_loss: 0.27095720, train_accuracy: 0.9609, test_Accuracy: 0.9389\n",
      "Epoch: [ 0] [  215/  468] time: 53.5868, train_loss: 0.48233679, train_accuracy: 0.9062, test_Accuracy: 0.9388\n",
      "Epoch: [ 0] [  216/  468] time: 53.8343, train_loss: 0.39086485, train_accuracy: 0.9531, test_Accuracy: 0.9394\n",
      "Epoch: [ 0] [  217/  468] time: 54.0775, train_loss: 0.26513693, train_accuracy: 0.9844, test_Accuracy: 0.9390\n",
      "Epoch: [ 0] [  218/  468] time: 54.3193, train_loss: 0.48383212, train_accuracy: 0.8594, test_Accuracy: 0.9397\n",
      "Epoch: [ 0] [  219/  468] time: 54.5639, train_loss: 0.42804855, train_accuracy: 0.9297, test_Accuracy: 0.9401\n",
      "Epoch: [ 0] [  220/  468] time: 54.8109, train_loss: 0.33335638, train_accuracy: 0.9375, test_Accuracy: 0.9380\n",
      "Epoch: [ 0] [  221/  468] time: 55.0509, train_loss: 0.34305808, train_accuracy: 0.9453, test_Accuracy: 0.9394\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  222/  468] time: 55.2946, train_loss: 0.23482168, train_accuracy: 0.9766, test_Accuracy: 0.9379\n",
      "Epoch: [ 0] [  223/  468] time: 55.5344, train_loss: 0.30466810, train_accuracy: 0.9297, test_Accuracy: 0.9367\n",
      "Epoch: [ 0] [  224/  468] time: 55.7768, train_loss: 0.35650253, train_accuracy: 0.9297, test_Accuracy: 0.9360\n",
      "Epoch: [ 0] [  225/  468] time: 56.0207, train_loss: 0.34423637, train_accuracy: 0.9453, test_Accuracy: 0.9365\n",
      "Epoch: [ 0] [  226/  468] time: 56.2686, train_loss: 0.30451143, train_accuracy: 0.9375, test_Accuracy: 0.9372\n",
      "Epoch: [ 0] [  227/  468] time: 56.5169, train_loss: 0.43092716, train_accuracy: 0.9219, test_Accuracy: 0.9383\n",
      "Epoch: [ 0] [  228/  468] time: 56.7660, train_loss: 0.31617412, train_accuracy: 0.9531, test_Accuracy: 0.9394\n",
      "Epoch: [ 0] [  229/  468] time: 57.0209, train_loss: 0.35764739, train_accuracy: 0.9297, test_Accuracy: 0.9397\n",
      "Epoch: [ 0] [  230/  468] time: 57.2646, train_loss: 0.39516395, train_accuracy: 0.9453, test_Accuracy: 0.9412\n",
      "Epoch: [ 0] [  231/  468] time: 57.5087, train_loss: 0.32598963, train_accuracy: 0.9219, test_Accuracy: 0.9436\n",
      "Epoch: [ 0] [  232/  468] time: 57.7501, train_loss: 0.41747510, train_accuracy: 0.9141, test_Accuracy: 0.9445\n",
      "Epoch: [ 0] [  233/  468] time: 57.9928, train_loss: 0.35651755, train_accuracy: 0.9531, test_Accuracy: 0.9436\n",
      "Epoch: [ 0] [  234/  468] time: 58.2491, train_loss: 0.24005108, train_accuracy: 0.9688, test_Accuracy: 0.9429\n",
      "Epoch: [ 0] [  235/  468] time: 58.4958, train_loss: 0.18812180, train_accuracy: 0.9766, test_Accuracy: 0.9427\n",
      "Epoch: [ 0] [  236/  468] time: 58.7405, train_loss: 0.22973648, train_accuracy: 0.9766, test_Accuracy: 0.9413\n",
      "Epoch: [ 0] [  237/  468] time: 58.9889, train_loss: 0.27710456, train_accuracy: 0.9375, test_Accuracy: 0.9400\n",
      "Epoch: [ 0] [  238/  468] time: 59.2389, train_loss: 0.40454787, train_accuracy: 0.9375, test_Accuracy: 0.9401\n",
      "Epoch: [ 0] [  239/  468] time: 59.4907, train_loss: 0.41430148, train_accuracy: 0.9297, test_Accuracy: 0.9394\n",
      "Epoch: [ 0] [  240/  468] time: 59.7395, train_loss: 0.43377778, train_accuracy: 0.9297, test_Accuracy: 0.9409\n",
      "Epoch: [ 0] [  241/  468] time: 59.9792, train_loss: 0.42348385, train_accuracy: 0.9453, test_Accuracy: 0.9417\n",
      "Epoch: [ 0] [  242/  468] time: 60.2262, train_loss: 0.33332688, train_accuracy: 0.9297, test_Accuracy: 0.9420\n",
      "Epoch: [ 0] [  243/  468] time: 60.4701, train_loss: 0.37282443, train_accuracy: 0.9219, test_Accuracy: 0.9422\n",
      "Epoch: [ 0] [  244/  468] time: 60.7091, train_loss: 0.40317678, train_accuracy: 0.9609, test_Accuracy: 0.9415\n",
      "Epoch: [ 0] [  245/  468] time: 60.9563, train_loss: 0.17393421, train_accuracy: 0.9609, test_Accuracy: 0.9423\n",
      "Epoch: [ 0] [  246/  468] time: 61.2002, train_loss: 0.30110148, train_accuracy: 0.9609, test_Accuracy: 0.9432\n",
      "Epoch: [ 0] [  247/  468] time: 61.4407, train_loss: 0.36647254, train_accuracy: 0.9609, test_Accuracy: 0.9432\n",
      "Epoch: [ 0] [  248/  468] time: 61.6813, train_loss: 0.34739494, train_accuracy: 0.9375, test_Accuracy: 0.9430\n",
      "Epoch: [ 0] [  249/  468] time: 61.9226, train_loss: 0.19899911, train_accuracy: 0.9766, test_Accuracy: 0.9436\n",
      "Epoch: [ 0] [  250/  468] time: 62.1646, train_loss: 0.27616948, train_accuracy: 0.9453, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [  251/  468] time: 62.4081, train_loss: 0.39166743, train_accuracy: 0.9375, test_Accuracy: 0.9432\n",
      "Epoch: [ 0] [  252/  468] time: 62.6511, train_loss: 0.34141362, train_accuracy: 0.9531, test_Accuracy: 0.9436\n",
      "Epoch: [ 0] [  253/  468] time: 62.8967, train_loss: 0.18597198, train_accuracy: 0.9844, test_Accuracy: 0.9439\n",
      "Epoch: [ 0] [  254/  468] time: 63.1341, train_loss: 0.35532966, train_accuracy: 0.9062, test_Accuracy: 0.9420\n",
      "Epoch: [ 0] [  255/  468] time: 63.3763, train_loss: 0.20422596, train_accuracy: 0.9688, test_Accuracy: 0.9405\n",
      "Epoch: [ 0] [  256/  468] time: 63.6192, train_loss: 0.27555245, train_accuracy: 0.9531, test_Accuracy: 0.9400\n",
      "Epoch: [ 0] [  257/  468] time: 63.8574, train_loss: 0.26639104, train_accuracy: 0.9453, test_Accuracy: 0.9393\n",
      "Epoch: [ 0] [  258/  468] time: 64.0979, train_loss: 0.40226442, train_accuracy: 0.9219, test_Accuracy: 0.9401\n",
      "Epoch: [ 0] [  259/  468] time: 64.3403, train_loss: 0.36155915, train_accuracy: 0.9375, test_Accuracy: 0.9409\n",
      "Epoch: [ 0] [  260/  468] time: 64.5823, train_loss: 0.38677830, train_accuracy: 0.9609, test_Accuracy: 0.9417\n",
      "Epoch: [ 0] [  261/  468] time: 64.8230, train_loss: 0.32947111, train_accuracy: 0.8906, test_Accuracy: 0.9414\n",
      "Epoch: [ 0] [  262/  468] time: 65.0687, train_loss: 0.23024678, train_accuracy: 0.9453, test_Accuracy: 0.9414\n",
      "Epoch: [ 0] [  263/  468] time: 65.3079, train_loss: 0.29120669, train_accuracy: 0.9766, test_Accuracy: 0.9402\n",
      "Epoch: [ 0] [  264/  468] time: 65.5504, train_loss: 0.36228794, train_accuracy: 0.9766, test_Accuracy: 0.9417\n",
      "Epoch: [ 0] [  265/  468] time: 65.7915, train_loss: 0.30505601, train_accuracy: 0.9297, test_Accuracy: 0.9447\n",
      "Epoch: [ 0] [  266/  468] time: 66.0314, train_loss: 0.26982206, train_accuracy: 0.9297, test_Accuracy: 0.9472\n",
      "Epoch: [ 0] [  267/  468] time: 66.2757, train_loss: 0.30878848, train_accuracy: 0.9531, test_Accuracy: 0.9466\n",
      "Epoch: [ 0] [  268/  468] time: 66.5192, train_loss: 0.37597710, train_accuracy: 0.9141, test_Accuracy: 0.9452\n",
      "Epoch: [ 0] [  269/  468] time: 66.7557, train_loss: 0.25967395, train_accuracy: 0.9531, test_Accuracy: 0.9424\n",
      "Epoch: [ 0] [  270/  468] time: 66.9973, train_loss: 0.36861607, train_accuracy: 0.9219, test_Accuracy: 0.9386\n",
      "Epoch: [ 0] [  271/  468] time: 67.2387, train_loss: 0.29389340, train_accuracy: 0.9453, test_Accuracy: 0.9354\n",
      "Epoch: [ 0] [  272/  468] time: 67.4869, train_loss: 0.36627358, train_accuracy: 0.9297, test_Accuracy: 0.9344\n",
      "Epoch: [ 0] [  273/  468] time: 67.7278, train_loss: 0.20050517, train_accuracy: 0.9688, test_Accuracy: 0.9356\n",
      "Epoch: [ 0] [  274/  468] time: 67.9685, train_loss: 0.35715425, train_accuracy: 0.9375, test_Accuracy: 0.9368\n",
      "Epoch: [ 0] [  275/  468] time: 68.2127, train_loss: 0.23454255, train_accuracy: 0.9609, test_Accuracy: 0.9398\n",
      "Epoch: [ 0] [  276/  468] time: 68.4577, train_loss: 0.32051244, train_accuracy: 0.9531, test_Accuracy: 0.9428\n",
      "Epoch: [ 0] [  277/  468] time: 68.6976, train_loss: 0.33573940, train_accuracy: 0.9141, test_Accuracy: 0.9448\n",
      "Epoch: [ 0] [  278/  468] time: 68.9398, train_loss: 0.28232270, train_accuracy: 0.9766, test_Accuracy: 0.9474\n",
      "Epoch: [ 0] [  279/  468] time: 69.1814, train_loss: 0.24062684, train_accuracy: 0.9453, test_Accuracy: 0.9484\n",
      "Epoch: [ 0] [  280/  468] time: 69.4283, train_loss: 0.32813513, train_accuracy: 0.9375, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [  281/  468] time: 69.6725, train_loss: 0.28549221, train_accuracy: 0.9531, test_Accuracy: 0.9461\n",
      "Epoch: [ 0] [  282/  468] time: 69.9141, train_loss: 0.38570166, train_accuracy: 0.9375, test_Accuracy: 0.9446\n",
      "Epoch: [ 0] [  283/  468] time: 70.1552, train_loss: 0.33959651, train_accuracy: 0.9375, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [  284/  468] time: 70.3949, train_loss: 0.25516897, train_accuracy: 0.9375, test_Accuracy: 0.9436\n",
      "Epoch: [ 0] [  285/  468] time: 70.6354, train_loss: 0.34770417, train_accuracy: 0.9219, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  286/  468] time: 70.8747, train_loss: 0.27364677, train_accuracy: 0.9453, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  287/  468] time: 71.1165, train_loss: 0.27615049, train_accuracy: 0.9531, test_Accuracy: 0.9485\n",
      "Epoch: [ 0] [  288/  468] time: 71.3586, train_loss: 0.31046718, train_accuracy: 0.9453, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  289/  468] time: 71.6028, train_loss: 0.50075245, train_accuracy: 0.9375, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  290/  468] time: 71.8438, train_loss: 0.29847819, train_accuracy: 0.9453, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  291/  468] time: 72.0894, train_loss: 0.29369736, train_accuracy: 0.9609, test_Accuracy: 0.9470\n",
      "Epoch: [ 0] [  292/  468] time: 72.3339, train_loss: 0.49302042, train_accuracy: 0.8906, test_Accuracy: 0.9460\n",
      "Epoch: [ 0] [  293/  468] time: 72.5832, train_loss: 0.30806440, train_accuracy: 0.9609, test_Accuracy: 0.9461\n",
      "Epoch: [ 0] [  294/  468] time: 72.8179, train_loss: 0.12656185, train_accuracy: 0.9844, test_Accuracy: 0.9464\n",
      "Epoch: [ 0] [  295/  468] time: 73.0599, train_loss: 0.26742405, train_accuracy: 0.9609, test_Accuracy: 0.9466\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  296/  468] time: 73.3030, train_loss: 0.31580269, train_accuracy: 0.9531, test_Accuracy: 0.9459\n",
      "Epoch: [ 0] [  297/  468] time: 73.5446, train_loss: 0.37518737, train_accuracy: 0.9375, test_Accuracy: 0.9460\n",
      "Epoch: [ 0] [  298/  468] time: 73.7842, train_loss: 0.27873743, train_accuracy: 0.9297, test_Accuracy: 0.9447\n",
      "Epoch: [ 0] [  299/  468] time: 74.0228, train_loss: 0.39344108, train_accuracy: 0.9453, test_Accuracy: 0.9447\n",
      "Epoch: [ 0] [  300/  468] time: 74.2611, train_loss: 0.32158673, train_accuracy: 0.9531, test_Accuracy: 0.9443\n",
      "Epoch: [ 0] [  301/  468] time: 74.5017, train_loss: 0.27024978, train_accuracy: 0.9609, test_Accuracy: 0.9445\n",
      "Epoch: [ 0] [  302/  468] time: 74.7459, train_loss: 0.18463610, train_accuracy: 0.9766, test_Accuracy: 0.9454\n",
      "Epoch: [ 0] [  303/  468] time: 74.9879, train_loss: 0.30676609, train_accuracy: 0.9219, test_Accuracy: 0.9456\n",
      "Epoch: [ 0] [  304/  468] time: 75.2311, train_loss: 0.23281193, train_accuracy: 0.9609, test_Accuracy: 0.9441\n",
      "Epoch: [ 0] [  305/  468] time: 75.4748, train_loss: 0.27065477, train_accuracy: 0.9375, test_Accuracy: 0.9426\n",
      "Epoch: [ 0] [  306/  468] time: 75.7214, train_loss: 0.37124351, train_accuracy: 0.9297, test_Accuracy: 0.9415\n",
      "Epoch: [ 0] [  307/  468] time: 75.9602, train_loss: 0.20420420, train_accuracy: 0.9219, test_Accuracy: 0.9424\n",
      "Epoch: [ 0] [  308/  468] time: 76.2093, train_loss: 0.37699085, train_accuracy: 0.9453, test_Accuracy: 0.9442\n",
      "Epoch: [ 0] [  309/  468] time: 76.4554, train_loss: 0.26928896, train_accuracy: 0.9688, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  310/  468] time: 76.7012, train_loss: 0.18439575, train_accuracy: 0.9375, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  311/  468] time: 76.9448, train_loss: 0.38791269, train_accuracy: 0.8984, test_Accuracy: 0.9482\n",
      "Epoch: [ 0] [  312/  468] time: 77.1992, train_loss: 0.19406164, train_accuracy: 0.9766, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  313/  468] time: 77.4407, train_loss: 0.25829914, train_accuracy: 0.9453, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  314/  468] time: 77.6882, train_loss: 0.29440543, train_accuracy: 0.9609, test_Accuracy: 0.9457\n",
      "Epoch: [ 0] [  315/  468] time: 77.9359, train_loss: 0.31955332, train_accuracy: 0.9531, test_Accuracy: 0.9450\n",
      "Epoch: [ 0] [  316/  468] time: 78.1854, train_loss: 0.58786017, train_accuracy: 0.9297, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  317/  468] time: 78.4314, train_loss: 0.23656324, train_accuracy: 0.9375, test_Accuracy: 0.9459\n",
      "Epoch: [ 0] [  318/  468] time: 78.6841, train_loss: 0.19304794, train_accuracy: 0.9453, test_Accuracy: 0.9475\n",
      "Epoch: [ 0] [  319/  468] time: 78.9327, train_loss: 0.31205374, train_accuracy: 0.9062, test_Accuracy: 0.9484\n",
      "Epoch: [ 0] [  320/  468] time: 79.1722, train_loss: 0.45244482, train_accuracy: 0.9219, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  321/  468] time: 79.4203, train_loss: 0.24979690, train_accuracy: 0.9219, test_Accuracy: 0.9487\n",
      "Epoch: [ 0] [  322/  468] time: 79.6674, train_loss: 0.31345314, train_accuracy: 0.9062, test_Accuracy: 0.9491\n",
      "Epoch: [ 0] [  323/  468] time: 79.9142, train_loss: 0.33785361, train_accuracy: 0.9453, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  324/  468] time: 80.1583, train_loss: 0.23529994, train_accuracy: 0.9688, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  325/  468] time: 80.4014, train_loss: 0.25603160, train_accuracy: 0.9531, test_Accuracy: 0.9469\n",
      "Epoch: [ 0] [  326/  468] time: 80.6457, train_loss: 0.20123595, train_accuracy: 0.9688, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  327/  468] time: 80.8892, train_loss: 0.23586492, train_accuracy: 0.9688, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  328/  468] time: 81.1306, train_loss: 0.12995762, train_accuracy: 0.9609, test_Accuracy: 0.9448\n",
      "Epoch: [ 0] [  329/  468] time: 81.3709, train_loss: 0.22890915, train_accuracy: 0.9609, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  330/  468] time: 81.6147, train_loss: 0.31884709, train_accuracy: 0.9453, test_Accuracy: 0.9475\n",
      "Epoch: [ 0] [  331/  468] time: 81.8541, train_loss: 0.33769208, train_accuracy: 0.9609, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  332/  468] time: 82.0931, train_loss: 0.34753615, train_accuracy: 0.9375, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  333/  468] time: 82.3362, train_loss: 0.22117397, train_accuracy: 0.9766, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  334/  468] time: 82.5794, train_loss: 0.34561533, train_accuracy: 0.9219, test_Accuracy: 0.9506\n",
      "Epoch: [ 0] [  335/  468] time: 82.8216, train_loss: 0.16295764, train_accuracy: 0.9688, test_Accuracy: 0.9500\n",
      "Epoch: [ 0] [  336/  468] time: 83.0709, train_loss: 0.25041437, train_accuracy: 0.9453, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  337/  468] time: 83.3255, train_loss: 0.21541718, train_accuracy: 0.9609, test_Accuracy: 0.9476\n",
      "Epoch: [ 0] [  338/  468] time: 83.5631, train_loss: 0.23733380, train_accuracy: 0.9453, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [  339/  468] time: 83.8189, train_loss: 0.17518699, train_accuracy: 0.9609, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  340/  468] time: 84.0685, train_loss: 0.21215056, train_accuracy: 0.9609, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  341/  468] time: 84.3117, train_loss: 0.37593585, train_accuracy: 0.9219, test_Accuracy: 0.9486\n",
      "Epoch: [ 0] [  342/  468] time: 84.5565, train_loss: 0.36244202, train_accuracy: 0.9297, test_Accuracy: 0.9494\n",
      "Epoch: [ 0] [  343/  468] time: 84.8020, train_loss: 0.27791828, train_accuracy: 0.9531, test_Accuracy: 0.9500\n",
      "Epoch: [ 0] [  344/  468] time: 85.0459, train_loss: 0.17411706, train_accuracy: 0.9766, test_Accuracy: 0.9489\n",
      "Epoch: [ 0] [  345/  468] time: 85.2876, train_loss: 0.24948438, train_accuracy: 0.9766, test_Accuracy: 0.9476\n",
      "Epoch: [ 0] [  346/  468] time: 85.5294, train_loss: 0.28734410, train_accuracy: 0.9219, test_Accuracy: 0.9481\n",
      "Epoch: [ 0] [  347/  468] time: 85.7706, train_loss: 0.32917112, train_accuracy: 0.9375, test_Accuracy: 0.9485\n",
      "Epoch: [ 0] [  348/  468] time: 86.0088, train_loss: 0.27880892, train_accuracy: 0.9453, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  349/  468] time: 86.2507, train_loss: 0.32444856, train_accuracy: 0.9453, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  350/  468] time: 86.5024, train_loss: 0.32092547, train_accuracy: 0.9297, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  351/  468] time: 86.7396, train_loss: 0.20113677, train_accuracy: 0.9531, test_Accuracy: 0.9518\n",
      "Epoch: [ 0] [  352/  468] time: 86.9778, train_loss: 0.27735519, train_accuracy: 0.9609, test_Accuracy: 0.9501\n",
      "Epoch: [ 0] [  353/  468] time: 87.2217, train_loss: 0.25270632, train_accuracy: 0.9531, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  354/  468] time: 87.4638, train_loss: 0.34534112, train_accuracy: 0.9297, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  355/  468] time: 87.7037, train_loss: 0.22657469, train_accuracy: 0.9375, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  356/  468] time: 87.9408, train_loss: 0.26886550, train_accuracy: 0.9609, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  357/  468] time: 88.1832, train_loss: 0.27886629, train_accuracy: 0.9531, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  358/  468] time: 88.4192, train_loss: 0.32846794, train_accuracy: 0.9062, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  359/  468] time: 88.6595, train_loss: 0.26433432, train_accuracy: 0.9688, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  360/  468] time: 88.9096, train_loss: 0.20442829, train_accuracy: 0.9688, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  361/  468] time: 89.1565, train_loss: 0.25442061, train_accuracy: 0.9531, test_Accuracy: 0.9527\n",
      "Epoch: [ 0] [  362/  468] time: 89.4001, train_loss: 0.28800195, train_accuracy: 0.9609, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  363/  468] time: 89.6346, train_loss: 0.26052848, train_accuracy: 0.9688, test_Accuracy: 0.9501\n",
      "Epoch: [ 0] [  364/  468] time: 89.8782, train_loss: 0.22139062, train_accuracy: 0.9609, test_Accuracy: 0.9502\n",
      "Epoch: [ 0] [  365/  468] time: 90.1173, train_loss: 0.22480750, train_accuracy: 0.9609, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  366/  468] time: 90.3592, train_loss: 0.22495028, train_accuracy: 0.9453, test_Accuracy: 0.9521\n",
      "Epoch: [ 0] [  367/  468] time: 90.5972, train_loss: 0.13481826, train_accuracy: 0.9688, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  368/  468] time: 90.8432, train_loss: 0.36260220, train_accuracy: 0.9531, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  369/  468] time: 91.0871, train_loss: 0.16601065, train_accuracy: 0.9688, test_Accuracy: 0.9535\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  370/  468] time: 91.3308, train_loss: 0.19224104, train_accuracy: 0.9609, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  371/  468] time: 91.5643, train_loss: 0.16831174, train_accuracy: 0.9531, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  372/  468] time: 91.8032, train_loss: 0.18280905, train_accuracy: 0.9766, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  373/  468] time: 92.0412, train_loss: 0.15925537, train_accuracy: 0.9766, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  374/  468] time: 92.2828, train_loss: 0.25645590, train_accuracy: 0.9609, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  375/  468] time: 92.5230, train_loss: 0.25996539, train_accuracy: 0.9453, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  376/  468] time: 92.7568, train_loss: 0.31770265, train_accuracy: 0.9375, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  377/  468] time: 92.9925, train_loss: 0.35784179, train_accuracy: 0.9531, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  378/  468] time: 93.2293, train_loss: 0.22785175, train_accuracy: 0.9453, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  379/  468] time: 93.4701, train_loss: 0.28505605, train_accuracy: 0.9531, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  380/  468] time: 93.7097, train_loss: 0.23359008, train_accuracy: 0.9688, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  381/  468] time: 93.9505, train_loss: 0.23887010, train_accuracy: 0.9531, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  382/  468] time: 94.1843, train_loss: 0.28569537, train_accuracy: 0.9453, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  383/  468] time: 94.4219, train_loss: 0.24678287, train_accuracy: 0.9766, test_Accuracy: 0.9521\n",
      "Epoch: [ 0] [  384/  468] time: 94.6624, train_loss: 0.26010412, train_accuracy: 0.9375, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  385/  468] time: 94.9046, train_loss: 0.21470815, train_accuracy: 0.9766, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  386/  468] time: 95.1424, train_loss: 0.23702583, train_accuracy: 0.9531, test_Accuracy: 0.9525\n",
      "Epoch: [ 0] [  387/  468] time: 95.3852, train_loss: 0.18702318, train_accuracy: 0.9688, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  388/  468] time: 95.6274, train_loss: 0.27438593, train_accuracy: 0.9375, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  389/  468] time: 95.8684, train_loss: 0.24261771, train_accuracy: 0.9609, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  390/  468] time: 96.1119, train_loss: 0.33984837, train_accuracy: 0.9375, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  391/  468] time: 96.3501, train_loss: 0.18464769, train_accuracy: 0.9688, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  392/  468] time: 96.5886, train_loss: 0.31244636, train_accuracy: 0.9297, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  393/  468] time: 96.8243, train_loss: 0.44147760, train_accuracy: 0.9141, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  394/  468] time: 97.0625, train_loss: 0.35989398, train_accuracy: 0.9609, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  395/  468] time: 97.3034, train_loss: 0.28471884, train_accuracy: 0.9531, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  396/  468] time: 97.5387, train_loss: 0.30826807, train_accuracy: 0.9688, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  397/  468] time: 97.7752, train_loss: 0.23621427, train_accuracy: 0.9375, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  398/  468] time: 98.0128, train_loss: 0.31743819, train_accuracy: 0.9766, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  399/  468] time: 98.2597, train_loss: 0.27798158, train_accuracy: 0.9688, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  400/  468] time: 98.4982, train_loss: 0.27075318, train_accuracy: 0.9766, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  401/  468] time: 98.7321, train_loss: 0.31258532, train_accuracy: 0.9219, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  402/  468] time: 98.9684, train_loss: 0.22993264, train_accuracy: 0.9609, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  403/  468] time: 99.2059, train_loss: 0.30584115, train_accuracy: 0.9531, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  404/  468] time: 99.4465, train_loss: 0.34379169, train_accuracy: 0.9375, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  405/  468] time: 99.6834, train_loss: 0.32687661, train_accuracy: 0.9766, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  406/  468] time: 99.9298, train_loss: 0.19834203, train_accuracy: 0.9766, test_Accuracy: 0.9510\n",
      "Epoch: [ 0] [  407/  468] time: 100.1672, train_loss: 0.22714078, train_accuracy: 0.9531, test_Accuracy: 0.9505\n",
      "Epoch: [ 0] [  408/  468] time: 100.4087, train_loss: 0.47928464, train_accuracy: 0.9219, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  409/  468] time: 100.6523, train_loss: 0.31923872, train_accuracy: 0.9453, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  410/  468] time: 100.8935, train_loss: 0.17166114, train_accuracy: 0.9766, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  411/  468] time: 101.1329, train_loss: 0.28567246, train_accuracy: 0.9453, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  412/  468] time: 101.3691, train_loss: 0.25518310, train_accuracy: 0.9531, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  413/  468] time: 101.6138, train_loss: 0.28934011, train_accuracy: 0.9531, test_Accuracy: 0.9550\n",
      "Epoch: [ 0] [  414/  468] time: 101.8504, train_loss: 0.25761274, train_accuracy: 0.9609, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  415/  468] time: 102.0883, train_loss: 0.27683610, train_accuracy: 0.9375, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  416/  468] time: 102.3279, train_loss: 0.47177327, train_accuracy: 0.9375, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  417/  468] time: 102.5692, train_loss: 0.27572876, train_accuracy: 0.9688, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  418/  468] time: 102.8166, train_loss: 0.36007500, train_accuracy: 0.9531, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  419/  468] time: 103.0595, train_loss: 0.31294802, train_accuracy: 0.9688, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  420/  468] time: 103.3018, train_loss: 0.28806391, train_accuracy: 0.9375, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  421/  468] time: 103.5420, train_loss: 0.32152259, train_accuracy: 0.9609, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  422/  468] time: 103.7794, train_loss: 0.19283020, train_accuracy: 0.9688, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  423/  468] time: 104.0183, train_loss: 0.26361048, train_accuracy: 0.9219, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  424/  468] time: 104.2645, train_loss: 0.10676025, train_accuracy: 0.9844, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  425/  468] time: 104.5049, train_loss: 0.21146175, train_accuracy: 0.9453, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  426/  468] time: 104.7467, train_loss: 0.29838699, train_accuracy: 0.9375, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  427/  468] time: 104.9861, train_loss: 0.28940791, train_accuracy: 0.9219, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  428/  468] time: 105.2280, train_loss: 0.27221692, train_accuracy: 0.9219, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  429/  468] time: 105.4696, train_loss: 0.18611613, train_accuracy: 0.9766, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  430/  468] time: 105.7134, train_loss: 0.37815776, train_accuracy: 0.9297, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  431/  468] time: 105.9596, train_loss: 0.20920351, train_accuracy: 0.9453, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  432/  468] time: 106.2024, train_loss: 0.17453642, train_accuracy: 0.9688, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  433/  468] time: 106.4457, train_loss: 0.17842403, train_accuracy: 0.9766, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  434/  468] time: 106.6862, train_loss: 0.17147142, train_accuracy: 0.9766, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  435/  468] time: 106.9335, train_loss: 0.36243549, train_accuracy: 0.9531, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  436/  468] time: 107.1753, train_loss: 0.15879130, train_accuracy: 0.9453, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  437/  468] time: 107.4177, train_loss: 0.31797400, train_accuracy: 0.9453, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  438/  468] time: 107.6591, train_loss: 0.33799276, train_accuracy: 0.9297, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  439/  468] time: 107.9012, train_loss: 0.33533841, train_accuracy: 0.9062, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  440/  468] time: 108.1409, train_loss: 0.27102989, train_accuracy: 0.9766, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  441/  468] time: 108.3809, train_loss: 0.24828097, train_accuracy: 0.9609, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  442/  468] time: 108.6274, train_loss: 0.27515462, train_accuracy: 0.9531, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  443/  468] time: 108.8638, train_loss: 0.17252566, train_accuracy: 0.9844, test_Accuracy: 0.9579\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  444/  468] time: 109.1102, train_loss: 0.20996577, train_accuracy: 0.9688, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  445/  468] time: 109.3558, train_loss: 0.25112832, train_accuracy: 0.9453, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  446/  468] time: 109.5950, train_loss: 0.21572490, train_accuracy: 0.9297, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  447/  468] time: 109.8398, train_loss: 0.19312790, train_accuracy: 0.9766, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  448/  468] time: 110.0855, train_loss: 0.21895725, train_accuracy: 0.9766, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  449/  468] time: 110.3324, train_loss: 0.30323431, train_accuracy: 0.9609, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  450/  468] time: 110.5747, train_loss: 0.45926964, train_accuracy: 0.9375, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  451/  468] time: 110.8187, train_loss: 0.25460136, train_accuracy: 0.9766, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  452/  468] time: 111.0610, train_loss: 0.25121906, train_accuracy: 0.9609, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  453/  468] time: 111.3100, train_loss: 0.19322535, train_accuracy: 0.9453, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  454/  468] time: 111.5611, train_loss: 0.39260274, train_accuracy: 0.9766, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  455/  468] time: 111.8120, train_loss: 0.25677297, train_accuracy: 0.9688, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  456/  468] time: 112.0573, train_loss: 0.29166359, train_accuracy: 0.9453, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  457/  468] time: 112.3119, train_loss: 0.21213335, train_accuracy: 0.9844, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  458/  468] time: 112.5607, train_loss: 0.24359199, train_accuracy: 0.9531, test_Accuracy: 0.9566\n",
      "Epoch: [ 0] [  459/  468] time: 112.8128, train_loss: 0.28908992, train_accuracy: 0.9609, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  460/  468] time: 113.0556, train_loss: 0.25873101, train_accuracy: 0.9688, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  461/  468] time: 113.3027, train_loss: 0.22146364, train_accuracy: 0.9453, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  462/  468] time: 113.5471, train_loss: 0.27700448, train_accuracy: 0.9766, test_Accuracy: 0.9550\n",
      "Epoch: [ 0] [  463/  468] time: 113.8024, train_loss: 0.26848540, train_accuracy: 0.9375, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  464/  468] time: 114.0391, train_loss: 0.16459459, train_accuracy: 0.9531, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  465/  468] time: 114.2864, train_loss: 0.17364690, train_accuracy: 0.9609, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  466/  468] time: 114.5242, train_loss: 0.19587857, train_accuracy: 0.9609, test_Accuracy: 0.9506\n",
      "Epoch: [ 0] [  467/  468] time: 114.7632, train_loss: 0.34247974, train_accuracy: 0.8906, test_Accuracy: 0.9492\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.contrib.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)\n",
    "    global_step = tf.train.create_global_step()\n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)\n",
    "        start_iteration = checkpoint_counter - start_epoch * training_iterations\n",
    "        counter = checkpoint_counter\n",
    "        global_step.assign(checkpoint_counter)\n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default(), tf.contrib.summary.always_record_summaries():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx in range(start_iteration, training_iterations):\n",
    "                train_input, train_label = train_iterator.get_next()\n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables), global_step=global_step)\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "\n",
    "                test_input, test_label = test_iterator.get_next()\n",
    "                test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.contrib.summary.scalar(name='train_loss', tensor=train_loss)\n",
    "                tf.contrib.summary.scalar(name='train_accuracy', tensor=train_accuracy)\n",
    "                tf.contrib.summary.scalar(name='test_accuracy', tensor=test_accuracy)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1\n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    test_input, test_label = test_iterator.get_next()\n",
    "    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test accuracy : 95.91 %"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}