RESNET_CIFAR100_fromScratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pretrained model, and generate new transformed dataset by passing images through pretrained model \n",
    "import os\n",
    "import numpy as np\n",
    "import pickle\n",
    "import keras\n",
    "from keras.datasets import cifar100\n",
    "from keras.layers import Dense, Conv2D, BatchNormalization, Activation\n",
    "from keras.layers import AveragePooling2D, Input, Flatten\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K\n",
    "from keras.models import Model, load_model\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training parameters\n",
    "batch_size = 128\n",
    "epochs = 200\n",
    "num_classes = 100\n",
    "\n",
    "data_augmentation = True\n",
    "subtract_pixel_mean = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model parameter (CIFAR-10)\n",
    "# ----------------------------------------------------------------------------\n",
    "#           |      | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch\n",
    "# Model     |  n   | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti\n",
    "#           |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2)\n",
    "# ----------------------------------------------------------------------------\n",
    "# ResNet20  | 3 (2)| 92.16     | 91.25     | -----     | -----     | 35 (---)\n",
    "# ResNet32  | 5(NA)| 92.46     | 92.49     | NA        | NA        | 50 ( NA)\n",
    "# ResNet44  | 7(NA)| 92.50     | 92.83     | NA        | NA        | 70 ( NA)\n",
    "# ResNet56  | 9 (6)| 92.71     | 93.03     | 93.01     | NA        | 90 (100)\n",
    "# ResNet110 |18(12)| 92.65     | 93.39+-.16| 93.15     | 93.63     | 165(180)\n",
    "# ResNet164 |27(18)| -----     | 94.07     | -----     | 94.54     | ---(---)\n",
    "# ResNet1001| (111)| -----     | 92.39     | -----     | 95.08+-.14| ---(---)\n",
    "# ---------------------------------------------------------------------------\n",
    "\n",
    "version = 2\n",
    "n = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "depth = n * 9 + 2\n",
    "model_type = 'ResNet%dv%d' % (depth, version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = cifar100.load_data()\n",
    "x_train = x_train.astype(float)\n",
    "x_test = x_test.astype(float)\n",
    "\n",
    "if subtract_pixel_mean:\n",
    "    x_train_mean = np.mean(x_train, axis=0)\n",
    "    x_train -= x_train_mean\n",
    "    x_test -= x_train_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = to_categorical(y_train, num_classes)\n",
    "y_test = to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "x_test shape: (10000, 32, 32, 3)\n",
      "y_train shape: (50000, 100)\n",
      "y_test shape: (10000, 100)\n"
     ]
    }
   ],
   "source": [
    "input_shape = x_train[0].shape\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print('x_test shape:', x_test.shape)\n",
    "print('y_train shape:', y_train.shape)\n",
    "print('y_test shape:', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_layer(inputs,\n",
    "                 num_filters=16,\n",
    "                 kernel_size=3,\n",
    "                 strides=1,\n",
    "                 activation='relu',\n",
    "                 batch_normalization=True,\n",
    "                 conv_first=True):\n",
    "    \"\"\"2D Convolution-Batch Normalization-Activation stack builder\n",
    "    # Arguments\n",
    "        inputs (tensor): input tensor from input image or previous layer\n",
    "        num_filters (int): Conv2D number of filters\n",
    "        kernel_size (int): Conv2D square kernel dimensions\n",
    "        strides (int): Conv2D square stride dimensions\n",
    "        activation (string): activation name\n",
    "        batch_normalization (bool): whether to include batch normalization\n",
    "        conv_first (bool): conv-bn-activation (True) or\n",
    "            bn-activation-conv (False)\n",
    "    # Returns\n",
    "        x (tensor): tensor as input to the next layer\n",
    "    \"\"\"\n",
    "    conv = Conv2D(num_filters,\n",
    "                  kernel_size=kernel_size,\n",
    "                  strides=strides,\n",
    "                  padding='same',\n",
    "                  kernel_initializer='he_normal',\n",
    "                  kernel_regularizer=l2(1e-4))\n",
    "\n",
    "    x = inputs\n",
    "    if conv_first:\n",
    "        x = conv(x)\n",
    "        if batch_normalization:\n",
    "            x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "    else:\n",
    "        if batch_normalization:\n",
    "            x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "        x = conv(x)\n",
    "    \n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_v2(input_shape, depth, num_classes=100):\n",
    "    \"\"\"ResNet Version 2 Model builder [b]\n",
    "    Stacks of (1 x 1)-(3 x 3)-(1 x 1) BN-ReLU-Conv2D or also known as\n",
    "    bottleneck layer\n",
    "    First shortcut connection per layer is 1 x 1 Conv2D.\n",
    "    Second and onwards shortcut connection is identity.\n",
    "    At the beginning of each stage, the feature map size is halved (downsampled)\n",
    "    by a convolutional layer with strides=2, while the number of filter maps is\n",
    "    doubled. Within each stage, the layers have the same number filters and the\n",
    "    same filter map sizes.\n",
    "    Features maps sizes:\n",
    "    conv1  : 32x32,  16\n",
    "    stage 0: 32x32,  64\n",
    "    stage 1: 16x16, 128\n",
    "    stage 2:  8x8,  256\n",
    "    # Arguments\n",
    "        input_shape (tensor): shape of input image tensor\n",
    "        depth (int): number of core convolutional layers\n",
    "        num_classes (int): number of classes (CIFAR10 has 10)\n",
    "    # Returns\n",
    "        model (Model): Keras model instance\n",
    "    \"\"\"\n",
    "    if (depth - 2) % 9 != 0:\n",
    "        raise ValueError('depth should be 9n+2 (eg 56 or 110 in [b])')\n",
    "    # Start model definition.\n",
    "    num_filters_in = 16\n",
    "    num_res_blocks = int((depth - 2) / 9)\n",
    "\n",
    "    inputs = Input(shape=input_shape)\n",
    "    # v2 performs Conv2D with BN-ReLU on input before splitting into 2 paths\n",
    "    x = resnet_layer(inputs=inputs,\n",
    "                     num_filters=num_filters_in,\n",
    "                     conv_first=True)\n",
    "\n",
    "    # Instantiate the stack of residual units\n",
    "    for stage in range(3):\n",
    "        for res_block in range(num_res_blocks):\n",
    "            activation = 'relu'\n",
    "            batch_normalization = True\n",
    "            strides = 1\n",
    "            if stage == 0:\n",
    "                num_filters_out = num_filters_in * 4\n",
    "                if res_block == 0:  # first layer and first stage\n",
    "                    activation = None\n",
    "                    batch_normalization = False\n",
    "            else:\n",
    "                num_filters_out = num_filters_in * 2\n",
    "                if res_block == 0:  # first layer but not first stage\n",
    "                    strides = 2    # downsample\n",
    "\n",
    "            # bottleneck residual unit\n",
    "            y = resnet_layer(inputs=x,\n",
    "                             num_filters=num_filters_in,\n",
    "                             kernel_size=1,\n",
    "                             strides=strides,\n",
    "                             activation=activation,\n",
    "                             batch_normalization=batch_normalization,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_in,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_out,\n",
    "                             kernel_size=1,\n",
    "                             conv_first=False)\n",
    "            if res_block == 0:\n",
    "                # linear projection residual shortcut connection to match\n",
    "                # changed dims\n",
    "                x = resnet_layer(inputs=x,\n",
    "                                 num_filters=num_filters_out,\n",
    "                                 kernel_size=1,\n",
    "                                 strides=strides,\n",
    "                                 activation=None,\n",
    "                                 batch_normalization=False)\n",
    "            x = keras.layers.add([x, y])\n",
    "\n",
    "        num_filters_in = num_filters_out\n",
    "\n",
    "    # Add classifier on top.\n",
    "    # v2 has BN-ReLU before Pooling\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = AveragePooling2D(pool_size=8)(x)\n",
    "    \n",
    "    y = Flatten()(x)\n",
    "    outputs = Dense(num_classes,\n",
    "                    activation='softmax',\n",
    "                    kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Instantiate model.\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_schedule(epoch):\n",
    "    \"\"\"Learning Rate Schedule\n",
    "    Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\n",
    "    Called automatically every epoch as part of callbacks during training.\n",
    "    # Arguments\n",
    "        epoch (int): The number of epochs\n",
    "    # Returns\n",
    "        lr (float32): learning rate\n",
    "    \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 180:\n",
    "        lr *= 0.5e-3\n",
    "    elif epoch > 160:\n",
    "        lr *= 1e-3\n",
    "    elif epoch > 120:\n",
    "        lr *= 1e-2\n",
    "    elif epoch > 80:\n",
    "        lr *= 1e-1\n",
    "    elif epoch > 40:\n",
    "        lr *= 5e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    \n",
    "    return lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_dir = os.path.join(os.getcwd(), 'saved_models/%s/baseline/' % model_type)\n",
    "model_name = 'cifar100_%s_model.{epoch:03d}.h5' % model_type\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "filepath = os.path.join(save_dir, model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = ModelCheckpoint(filepath=filepath,\n",
    "                             monitor='val_acc',\n",
    "                             verbose=1,\n",
    "                             save_best_only=True)\n",
    "\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=5,\n",
    "                               min_lr=0.5e-6)\n",
    "\n",
    "callbacks = [checkpoint, lr_reducer, lr_scheduler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 32, 32, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 32, 32, 16)   448         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 32, 32, 16)   64          conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 32, 32, 16)   0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 32, 32, 16)   272         activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 32, 32, 16)   64          conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 32, 32, 16)   0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 32, 32, 16)   2320        activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 32, 32, 16)   64          conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 32, 32, 16)   0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 32, 32, 64)   1088        activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 32, 32, 64)   1088        activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 32, 32, 64)   0           conv2d_5[0][0]                   \n",
      "                                                                 conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 32, 32, 64)   256         add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 32, 32, 64)   0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 32, 32, 16)   1040        activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 32, 32, 16)   64          conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 32, 32, 16)   0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 32, 32, 16)   2320        activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 32, 32, 16)   64          conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 32, 32, 16)   0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 32, 32, 64)   1088        activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 32, 32, 64)   0           add_1[0][0]                      \n",
      "                                                                 conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 32, 32, 64)   256         add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 32, 32, 64)   0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 16, 16, 64)   4160        activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 16, 16, 64)   256         conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 16, 16, 64)   0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 16, 16, 64)   36928       activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 16, 16, 64)   256         conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 16, 16, 64)   0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 16, 16, 128)  8320        add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 16, 16, 128)  8320        activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 16, 16, 128)  0           conv2d_12[0][0]                  \n",
      "                                                                 conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 16, 16, 128)  512         add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 16, 16, 128)  0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 16, 16, 64)   8256        activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 16, 16, 64)   256         conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 16, 16, 64)   0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 16, 16, 64)   36928       activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 16, 16, 64)   256         conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 16, 16, 64)   0           batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 16, 16, 128)  8320        activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 16, 16, 128)  0           add_3[0][0]                      \n",
      "                                                                 conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 16, 16, 128)  512         add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 16, 16, 128)  0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 8, 8, 128)    16512       activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 8, 8, 128)    512         conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 8, 8, 128)    0           batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 8, 8, 128)    147584      activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 8, 8, 128)    512         conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 8, 8, 128)    0           batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 8, 8, 256)    33024       add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 8, 8, 256)    33024       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 8, 8, 256)    0           conv2d_19[0][0]                  \n",
      "                                                                 conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 8, 8, 256)    1024        add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 8, 8, 256)    0           batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 8, 8, 128)    32896       activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 8, 8, 128)    512         conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 8, 8, 128)    0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, 8, 8, 128)    147584      activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 8, 8, 128)    512         conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 8, 8, 128)    0           batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)              (None, 8, 8, 256)    33024       activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 8, 8, 256)    0           add_5[0][0]                      \n",
      "                                                                 conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 8, 8, 256)    1024        add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 8, 8, 256)    0           batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_1 (AveragePoo (None, 1, 1, 256)    0           activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 256)          0           average_pooling2d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 100)          25700       flatten_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 597,220\n",
      "Trainable params: 593,732\n",
      "Non-trainable params: 3,488\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = resnet_v2(input_shape=input_shape, depth=depth)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=Adam(lr=lr_schedule(0)),\n",
    "              metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "if data_augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        # set input mean to 0 over the dataset\n",
    "        featurewise_center=False,\n",
    "        # set each sample mean to 0\n",
    "        samplewise_center=False,\n",
    "        # divide inputs by std of dataset\n",
    "        featurewise_std_normalization=False,\n",
    "        # divide each input by its std\n",
    "        samplewise_std_normalization=False,\n",
    "        # apply ZCA whitening\n",
    "        zca_whitening=False,\n",
    "        # epsilon for ZCA whitening\n",
    "        zca_epsilon=1e-06,\n",
    "        # randomly rotate images in the range (deg 0 to 180)\n",
    "        rotation_range=0,\n",
    "        # randomly shift images horizontally\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically\n",
    "        height_shift_range=0.1,\n",
    "        # set range for random shear\n",
    "        shear_range=0.,\n",
    "        # set range for random zoom\n",
    "        zoom_range=0.,\n",
    "        # set range for random channel shifts\n",
    "        channel_shift_range=0.,\n",
    "        # set mode for filling points outside the input boundaries\n",
    "        fill_mode='nearest',\n",
    "        # value used for fill_mode = \"constant\"\n",
    "        cval=0.,\n",
    "        # randomly flip images\n",
    "        horizontal_flip=True,\n",
    "        # randomly flip images\n",
    "        vertical_flip=False,\n",
    "        # set rescaling factor (applied before any other transformation)\n",
    "        rescale=None,\n",
    "        # set function that will be applied on each input\n",
    "        preprocessing_function=None,\n",
    "        # image data format, either \"channels_first\" or \"channels_last\"\n",
    "        data_format=None,\n",
    "        # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "        validation_split=0.0\n",
    "    )\n",
    "else:\n",
    "    datagen = ImageDataGenerator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 21s 55ms/step - loss: 4.1884 - acc: 0.1249 - val_loss: 3.9548 - val_acc: 0.1468\n",
      "\n",
      "Epoch 00001: val_acc improved from -inf to 0.14680, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.001.h5\n",
      "Epoch 2/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 42ms/step - loss: 3.5041 - acc: 0.2230 - val_loss: 3.9482 - val_acc: 0.1507\n",
      "\n",
      "Epoch 00002: val_acc improved from 0.14680 to 0.15070, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.002.h5\n",
      "Epoch 3/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 3.1236 - acc: 0.2884 - val_loss: 3.3909 - val_acc: 0.2308\n",
      "\n",
      "Epoch 00003: val_acc improved from 0.15070 to 0.23080, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.003.h5\n",
      "Epoch 4/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 42ms/step - loss: 2.8633 - acc: 0.3367 - val_loss: 2.9783 - val_acc: 0.3079\n",
      "\n",
      "Epoch 00004: val_acc improved from 0.23080 to 0.30790, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.004.h5\n",
      "Epoch 5/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 2.6675 - acc: 0.3759 - val_loss: 2.8759 - val_acc: 0.3366\n",
      "\n",
      "Epoch 00005: val_acc improved from 0.30790 to 0.33660, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.005.h5\n",
      "Epoch 6/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 2.5166 - acc: 0.4094 - val_loss: 2.8202 - val_acc: 0.3501\n",
      "\n",
      "Epoch 00006: val_acc improved from 0.33660 to 0.35010, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.006.h5\n",
      "Epoch 7/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 2.3840 - acc: 0.4399 - val_loss: 2.6806 - val_acc: 0.3836\n",
      "\n",
      "Epoch 00007: val_acc improved from 0.35010 to 0.38360, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.007.h5\n",
      "Epoch 8/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 2.2872 - acc: 0.4618 - val_loss: 2.5345 - val_acc: 0.4125\n",
      "\n",
      "Epoch 00008: val_acc improved from 0.38360 to 0.41250, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.008.h5\n",
      "Epoch 9/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 2.1935 - acc: 0.4804 - val_loss: 2.5809 - val_acc: 0.4064\n",
      "\n",
      "Epoch 00009: val_acc did not improve from 0.41250\n",
      "Epoch 10/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 2.1100 - acc: 0.5029 - val_loss: 2.5251 - val_acc: 0.4244\n",
      "\n",
      "Epoch 00010: val_acc improved from 0.41250 to 0.42440, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.010.h5\n",
      "Epoch 11/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 2.0449 - acc: 0.5186 - val_loss: 2.4526 - val_acc: 0.4369\n",
      "\n",
      "Epoch 00011: val_acc improved from 0.42440 to 0.43690, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.011.h5\n",
      "Epoch 12/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 42ms/step - loss: 1.9722 - acc: 0.5359 - val_loss: 2.3381 - val_acc: 0.4609\n",
      "\n",
      "Epoch 00012: val_acc improved from 0.43690 to 0.46090, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.012.h5\n",
      "Epoch 13/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 1.9216 - acc: 0.5491 - val_loss: 2.4232 - val_acc: 0.4478\n",
      "\n",
      "Epoch 00013: val_acc did not improve from 0.46090\n",
      "Epoch 14/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 1.8754 - acc: 0.5573 - val_loss: 2.2061 - val_acc: 0.4876\n",
      "\n",
      "Epoch 00014: val_acc improved from 0.46090 to 0.48760, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.014.h5\n",
      "Epoch 15/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 1.8257 - acc: 0.5734 - val_loss: 2.4150 - val_acc: 0.4588\n",
      "\n",
      "Epoch 00015: val_acc did not improve from 0.48760\n",
      "Epoch 16/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 1.7753 - acc: 0.5853 - val_loss: 2.6324 - val_acc: 0.4432\n",
      "\n",
      "Epoch 00016: val_acc did not improve from 0.48760\n",
      "Epoch 17/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 42ms/step - loss: 1.7476 - acc: 0.5939 - val_loss: 2.1637 - val_acc: 0.5033\n",
      "\n",
      "Epoch 00017: val_acc improved from 0.48760 to 0.50330, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.017.h5\n",
      "Epoch 18/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 1.7019 - acc: 0.6036 - val_loss: 2.3127 - val_acc: 0.4858\n",
      "\n",
      "Epoch 00018: val_acc did not improve from 0.50330\n",
      "Epoch 19/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 16s 42ms/step - loss: 1.6733 - acc: 0.6128 - val_loss: 2.3063 - val_acc: 0.4886\n",
      "\n",
      "Epoch 00019: val_acc did not improve from 0.50330\n",
      "Epoch 20/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 1.6315 - acc: 0.6217 - val_loss: 2.0227 - val_acc: 0.5466\n",
      "\n",
      "Epoch 00020: val_acc improved from 0.50330 to 0.54660, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.020.h5\n",
      "Epoch 21/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 42ms/step - loss: 1.6128 - acc: 0.6270 - val_loss: 2.2619 - val_acc: 0.5077\n",
      "\n",
      "Epoch 00021: val_acc did not improve from 0.54660\n",
      "Epoch 22/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 1.5752 - acc: 0.6376 - val_loss: 2.1342 - val_acc: 0.5251\n",
      "\n",
      "Epoch 00022: val_acc did not improve from 0.54660\n",
      "Epoch 23/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 16s 42ms/step - loss: 1.5512 - acc: 0.6437 - val_loss: 2.0590 - val_acc: 0.5456\n",
      "\n",
      "Epoch 00023: val_acc did not improve from 0.54660\n",
      "Epoch 24/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 42ms/step - loss: 1.5348 - acc: 0.6461 - val_loss: 2.0473 - val_acc: 0.5415\n",
      "\n",
      "Epoch 00024: val_acc did not improve from 0.54660\n",
      "Epoch 25/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 16s 42ms/step - loss: 1.5097 - acc: 0.6551 - val_loss: 2.0882 - val_acc: 0.5445\n",
      "\n",
      "Epoch 00025: val_acc did not improve from 0.54660\n",
      "Epoch 26/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 16s 42ms/step - loss: 1.4883 - acc: 0.6622 - val_loss: 1.9875 - val_acc: 0.5623\n",
      "\n",
      "Epoch 00026: val_acc improved from 0.54660 to 0.56230, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.026.h5\n",
      "Epoch 27/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 16s 42ms/step - loss: 1.4667 - acc: 0.6655 - val_loss: 2.1195 - val_acc: 0.5447\n",
      "\n",
      "Epoch 00027: val_acc did not improve from 0.56230\n",
      "Epoch 28/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 42ms/step - loss: 1.4486 - acc: 0.6709 - val_loss: 2.0150 - val_acc: 0.5503\n",
      "\n",
      "Epoch 00028: val_acc did not improve from 0.56230\n",
      "Epoch 29/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 16s 42ms/step - loss: 1.4208 - acc: 0.6797 - val_loss: 2.0899 - val_acc: 0.5432\n",
      "\n",
      "Epoch 00029: val_acc did not improve from 0.56230\n",
      "Epoch 30/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 16s 42ms/step - loss: 1.4197 - acc: 0.6785 - val_loss: 2.1466 - val_acc: 0.5491\n",
      "\n",
      "Epoch 00030: val_acc did not improve from 0.56230\n",
      "Epoch 31/200\n",
      "Learning rate:  0.001\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "390/390 [==============================] - 16s 42ms/step - loss: 1.3898 - acc: 0.6869 - val_loss: 1.9658 - val_acc: 0.5707\n",
      "\n",
      "Epoch 00031: val_acc improved from 0.56230 to 0.57070, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.031.h5\n",
      "Epoch 32/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 16s 42ms/step - loss: 1.3704 - acc: 0.6935 - val_loss: 2.2160 - val_acc: 0.5322\n",
      "\n",
      "Epoch 00032: val_acc did not improve from 0.57070\n",
      "Epoch 33/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 16s 42ms/step - loss: 1.3591 - acc: 0.6991 - val_loss: 2.0338 - val_acc: 0.5628\n",
      "\n",
      "Epoch 00033: val_acc did not improve from 0.57070\n",
      "Epoch 34/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 16s 42ms/step - loss: 1.3416 - acc: 0.7010 - val_loss: 1.9009 - val_acc: 0.5803\n",
      "\n",
      "Epoch 00034: val_acc improved from 0.57070 to 0.58030, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.034.h5\n",
      "Epoch 35/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 1.3379 - acc: 0.7004 - val_loss: 1.9303 - val_acc: 0.5867\n",
      "\n",
      "Epoch 00035: val_acc improved from 0.58030 to 0.58670, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.035.h5\n",
      "Epoch 36/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 17s 43ms/step - loss: 1.3177 - acc: 0.7079 - val_loss: 2.0140 - val_acc: 0.5704\n",
      "\n",
      "Epoch 00036: val_acc did not improve from 0.58670\n",
      "Epoch 37/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 1.3011 - acc: 0.7128 - val_loss: 1.9591 - val_acc: 0.5833\n",
      "\n",
      "Epoch 00037: val_acc did not improve from 0.58670\n",
      "Epoch 38/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 1.2939 - acc: 0.7141 - val_loss: 2.1791 - val_acc: 0.5444\n",
      "\n",
      "Epoch 00038: val_acc did not improve from 0.58670\n",
      "Epoch 39/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 1.2782 - acc: 0.7199 - val_loss: 2.0402 - val_acc: 0.5761\n",
      "\n",
      "Epoch 00039: val_acc did not improve from 0.58670\n",
      "Epoch 40/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 1.2682 - acc: 0.7234 - val_loss: 1.9236 - val_acc: 0.5828\n",
      "\n",
      "Epoch 00040: val_acc did not improve from 0.58670\n",
      "Epoch 41/200\n",
      "Learning rate:  0.001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 1.2607 - acc: 0.7269 - val_loss: 2.2255 - val_acc: 0.5434\n",
      "\n",
      "Epoch 00041: val_acc did not improve from 0.58670\n",
      "Epoch 42/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 1.0906 - acc: 0.7746 - val_loss: 1.7296 - val_acc: 0.6355\n",
      "\n",
      "Epoch 00042: val_acc improved from 0.58670 to 0.63550, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.042.h5\n",
      "Epoch 43/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 1.0397 - acc: 0.7885 - val_loss: 1.7559 - val_acc: 0.6294\n",
      "\n",
      "Epoch 00043: val_acc did not improve from 0.63550\n",
      "Epoch 44/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 1.0183 - acc: 0.7913 - val_loss: 1.8330 - val_acc: 0.6142\n",
      "\n",
      "Epoch 00044: val_acc did not improve from 0.63550\n",
      "Epoch 45/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 1.0063 - acc: 0.7919 - val_loss: 1.7012 - val_acc: 0.6318\n",
      "\n",
      "Epoch 00045: val_acc did not improve from 0.63550\n",
      "Epoch 46/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.9772 - acc: 0.8005 - val_loss: 1.8268 - val_acc: 0.6179\n",
      "\n",
      "Epoch 00046: val_acc did not improve from 0.63550\n",
      "Epoch 47/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.9720 - acc: 0.8019 - val_loss: 1.9081 - val_acc: 0.6091\n",
      "\n",
      "Epoch 00047: val_acc did not improve from 0.63550\n",
      "Epoch 48/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.9547 - acc: 0.8064 - val_loss: 1.8572 - val_acc: 0.6151\n",
      "\n",
      "Epoch 00048: val_acc did not improve from 0.63550\n",
      "Epoch 49/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.9410 - acc: 0.8082 - val_loss: 1.8789 - val_acc: 0.6124\n",
      "\n",
      "Epoch 00049: val_acc did not improve from 0.63550\n",
      "Epoch 50/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 17s 45ms/step - loss: 0.9286 - acc: 0.8112 - val_loss: 1.7994 - val_acc: 0.6272\n",
      "\n",
      "Epoch 00050: val_acc did not improve from 0.63550\n",
      "Epoch 51/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.9167 - acc: 0.8143 - val_loss: 1.8385 - val_acc: 0.6225\n",
      "\n",
      "Epoch 00051: val_acc did not improve from 0.63550\n",
      "Epoch 52/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.9123 - acc: 0.8132 - val_loss: 1.8617 - val_acc: 0.6229\n",
      "\n",
      "Epoch 00052: val_acc did not improve from 0.63550\n",
      "Epoch 53/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.8946 - acc: 0.8175 - val_loss: 1.8202 - val_acc: 0.6288\n",
      "\n",
      "Epoch 00053: val_acc did not improve from 0.63550\n",
      "Epoch 54/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.8821 - acc: 0.8215 - val_loss: 1.9876 - val_acc: 0.6037\n",
      "\n",
      "Epoch 00054: val_acc did not improve from 0.63550\n",
      "Epoch 55/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.8761 - acc: 0.8230 - val_loss: 1.8182 - val_acc: 0.6262\n",
      "\n",
      "Epoch 00055: val_acc did not improve from 0.63550\n",
      "Epoch 56/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.8650 - acc: 0.8259 - val_loss: 1.7808 - val_acc: 0.6394\n",
      "\n",
      "Epoch 00056: val_acc improved from 0.63550 to 0.63940, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.056.h5\n",
      "Epoch 57/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.8600 - acc: 0.8266 - val_loss: 1.9159 - val_acc: 0.6172\n",
      "\n",
      "Epoch 00057: val_acc did not improve from 0.63940\n",
      "Epoch 58/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.8511 - acc: 0.8287 - val_loss: 1.8397 - val_acc: 0.6228\n",
      "\n",
      "Epoch 00058: val_acc did not improve from 0.63940\n",
      "Epoch 59/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.8373 - acc: 0.8321 - val_loss: 1.9312 - val_acc: 0.6166\n",
      "\n",
      "Epoch 00059: val_acc did not improve from 0.63940\n",
      "Epoch 60/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.8279 - acc: 0.8343 - val_loss: 1.8275 - val_acc: 0.6317\n",
      "\n",
      "Epoch 00060: val_acc did not improve from 0.63940\n",
      "Epoch 61/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.8228 - acc: 0.8364 - val_loss: 1.9194 - val_acc: 0.6134\n",
      "\n",
      "Epoch 00061: val_acc did not improve from 0.63940\n",
      "Epoch 62/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.8135 - acc: 0.8360 - val_loss: 1.8899 - val_acc: 0.6188\n",
      "\n",
      "Epoch 00062: val_acc did not improve from 0.63940\n",
      "Epoch 63/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.8107 - acc: 0.8368 - val_loss: 1.9122 - val_acc: 0.6192\n",
      "\n",
      "Epoch 00063: val_acc did not improve from 0.63940\n",
      "Epoch 64/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.8030 - acc: 0.8399 - val_loss: 1.8744 - val_acc: 0.6235\n",
      "\n",
      "Epoch 00064: val_acc did not improve from 0.63940\n",
      "Epoch 65/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.7917 - acc: 0.8428 - val_loss: 1.8490 - val_acc: 0.6289\n",
      "\n",
      "Epoch 00065: val_acc did not improve from 0.63940\n",
      "Epoch 66/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.7803 - acc: 0.8455 - val_loss: 1.8835 - val_acc: 0.6252\n",
      "\n",
      "Epoch 00066: val_acc did not improve from 0.63940\n",
      "Epoch 67/200\n",
      "Learning rate:  0.0005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "390/390 [==============================] - 18s 46ms/step - loss: 0.7811 - acc: 0.8446 - val_loss: 1.8999 - val_acc: 0.6221\n",
      "\n",
      "Epoch 00067: val_acc did not improve from 0.63940\n",
      "Epoch 68/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.7736 - acc: 0.8479 - val_loss: 1.8826 - val_acc: 0.6233\n",
      "\n",
      "Epoch 00068: val_acc did not improve from 0.63940\n",
      "Epoch 69/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.7705 - acc: 0.8474 - val_loss: 1.8622 - val_acc: 0.6210\n",
      "\n",
      "Epoch 00069: val_acc did not improve from 0.63940\n",
      "Epoch 70/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.7594 - acc: 0.8520 - val_loss: 1.9432 - val_acc: 0.6236\n",
      "\n",
      "Epoch 00070: val_acc did not improve from 0.63940\n",
      "Epoch 71/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.7592 - acc: 0.8489 - val_loss: 2.0703 - val_acc: 0.6074\n",
      "\n",
      "Epoch 00071: val_acc did not improve from 0.63940\n",
      "Epoch 72/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.7512 - acc: 0.8527 - val_loss: 1.9704 - val_acc: 0.6106\n",
      "\n",
      "Epoch 00072: val_acc did not improve from 0.63940\n",
      "Epoch 73/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.7541 - acc: 0.8528 - val_loss: 1.9350 - val_acc: 0.6196\n",
      "\n",
      "Epoch 00073: val_acc did not improve from 0.63940\n",
      "Epoch 74/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.7386 - acc: 0.8555 - val_loss: 1.9498 - val_acc: 0.6192\n",
      "\n",
      "Epoch 00074: val_acc did not improve from 0.63940\n",
      "Epoch 75/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.7327 - acc: 0.8574 - val_loss: 1.9190 - val_acc: 0.6237\n",
      "\n",
      "Epoch 00075: val_acc did not improve from 0.63940\n",
      "Epoch 76/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.7366 - acc: 0.8575 - val_loss: 1.9793 - val_acc: 0.6207\n",
      "\n",
      "Epoch 00076: val_acc did not improve from 0.63940\n",
      "Epoch 77/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.7256 - acc: 0.8619 - val_loss: 1.9518 - val_acc: 0.6169\n",
      "\n",
      "Epoch 00077: val_acc did not improve from 0.63940\n",
      "Epoch 78/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.7193 - acc: 0.8630 - val_loss: 2.0089 - val_acc: 0.6185\n",
      "\n",
      "Epoch 00078: val_acc did not improve from 0.63940\n",
      "Epoch 79/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.7153 - acc: 0.8622 - val_loss: 2.0006 - val_acc: 0.6208\n",
      "\n",
      "Epoch 00079: val_acc did not improve from 0.63940\n",
      "Epoch 80/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.7121 - acc: 0.8647 - val_loss: 2.0643 - val_acc: 0.6134\n",
      "\n",
      "Epoch 00080: val_acc did not improve from 0.63940\n",
      "Epoch 81/200\n",
      "Learning rate:  0.0005\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.7119 - acc: 0.8628 - val_loss: 1.9923 - val_acc: 0.6184\n",
      "\n",
      "Epoch 00081: val_acc did not improve from 0.63940\n",
      "Epoch 82/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.5921 - acc: 0.9078 - val_loss: 1.7155 - val_acc: 0.6606\n",
      "\n",
      "Epoch 00082: val_acc improved from 0.63940 to 0.66060, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.082.h5\n",
      "Epoch 83/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.5419 - acc: 0.9268 - val_loss: 1.7364 - val_acc: 0.6591\n",
      "\n",
      "Epoch 00083: val_acc did not improve from 0.66060\n",
      "Epoch 84/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.5256 - acc: 0.9313 - val_loss: 1.7414 - val_acc: 0.6551\n",
      "\n",
      "Epoch 00084: val_acc did not improve from 0.66060\n",
      "Epoch 85/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.5133 - acc: 0.9361 - val_loss: 1.7474 - val_acc: 0.6570\n",
      "\n",
      "Epoch 00085: val_acc did not improve from 0.66060\n",
      "Epoch 86/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.5107 - acc: 0.9355 - val_loss: 1.7494 - val_acc: 0.6566\n",
      "\n",
      "Epoch 00086: val_acc did not improve from 0.66060\n",
      "Epoch 87/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.5041 - acc: 0.9372 - val_loss: 1.7616 - val_acc: 0.6601\n",
      "\n",
      "Epoch 00087: val_acc did not improve from 0.66060\n",
      "Epoch 88/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4997 - acc: 0.9378 - val_loss: 1.7652 - val_acc: 0.6589\n",
      "\n",
      "Epoch 00088: val_acc did not improve from 0.66060\n",
      "Epoch 89/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4913 - acc: 0.9402 - val_loss: 1.7674 - val_acc: 0.6592\n",
      "\n",
      "Epoch 00089: val_acc did not improve from 0.66060\n",
      "Epoch 90/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4874 - acc: 0.9413 - val_loss: 1.7732 - val_acc: 0.6622\n",
      "\n",
      "Epoch 00090: val_acc improved from 0.66060 to 0.66220, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.090.h5\n",
      "Epoch 91/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4811 - acc: 0.9425 - val_loss: 1.7697 - val_acc: 0.6637\n",
      "\n",
      "Epoch 00091: val_acc improved from 0.66220 to 0.66370, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/ResNet20v2/baseline/cifar100_ResNet20v2_model.091.h5\n",
      "Epoch 92/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4792 - acc: 0.9427 - val_loss: 1.8295 - val_acc: 0.6550\n",
      "\n",
      "Epoch 00092: val_acc did not improve from 0.66370\n",
      "Epoch 93/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4720 - acc: 0.9441 - val_loss: 1.7859 - val_acc: 0.6567\n",
      "\n",
      "Epoch 00093: val_acc did not improve from 0.66370\n",
      "Epoch 94/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4697 - acc: 0.9444 - val_loss: 1.8134 - val_acc: 0.6574\n",
      "\n",
      "Epoch 00094: val_acc did not improve from 0.66370\n",
      "Epoch 95/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4608 - acc: 0.9480 - val_loss: 1.8008 - val_acc: 0.6567\n",
      "\n",
      "Epoch 00095: val_acc did not improve from 0.66370\n",
      "Epoch 96/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4634 - acc: 0.9464 - val_loss: 1.8372 - val_acc: 0.6561\n",
      "\n",
      "Epoch 00096: val_acc did not improve from 0.66370\n",
      "Epoch 97/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4583 - acc: 0.9477 - val_loss: 1.8271 - val_acc: 0.6583\n",
      "\n",
      "Epoch 00097: val_acc did not improve from 0.66370\n",
      "Epoch 98/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4544 - acc: 0.9489 - val_loss: 1.8251 - val_acc: 0.6582\n",
      "\n",
      "Epoch 00098: val_acc did not improve from 0.66370\n",
      "Epoch 99/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4510 - acc: 0.9492 - val_loss: 1.8244 - val_acc: 0.6524\n",
      "\n",
      "Epoch 00099: val_acc did not improve from 0.66370\n",
      "Epoch 100/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4482 - acc: 0.9488 - val_loss: 1.8572 - val_acc: 0.6542\n",
      "\n",
      "Epoch 00100: val_acc did not improve from 0.66370\n",
      "Epoch 101/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.4458 - acc: 0.9499 - val_loss: 1.8574 - val_acc: 0.6557\n",
      "\n",
      "Epoch 00101: val_acc did not improve from 0.66370\n",
      "Epoch 102/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4406 - acc: 0.9510 - val_loss: 1.8545 - val_acc: 0.6558\n",
      "\n",
      "Epoch 00102: val_acc did not improve from 0.66370\n",
      "Epoch 103/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.4397 - acc: 0.9512 - val_loss: 1.8402 - val_acc: 0.6553\n",
      "\n",
      "Epoch 00103: val_acc did not improve from 0.66370\n",
      "Epoch 104/200\n",
      "Learning rate:  0.0001\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4360 - acc: 0.9518 - val_loss: 1.8614 - val_acc: 0.6535\n",
      "\n",
      "Epoch 00104: val_acc did not improve from 0.66370\n",
      "Epoch 105/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4326 - acc: 0.9519 - val_loss: 1.8840 - val_acc: 0.6465\n",
      "\n",
      "Epoch 00105: val_acc did not improve from 0.66370\n",
      "Epoch 106/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4295 - acc: 0.9535 - val_loss: 1.8702 - val_acc: 0.6523\n",
      "\n",
      "Epoch 00106: val_acc did not improve from 0.66370\n",
      "Epoch 107/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4292 - acc: 0.9529 - val_loss: 1.8874 - val_acc: 0.6546\n",
      "\n",
      "Epoch 00107: val_acc did not improve from 0.66370\n",
      "Epoch 108/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4242 - acc: 0.9541 - val_loss: 1.8570 - val_acc: 0.6584\n",
      "\n",
      "Epoch 00108: val_acc did not improve from 0.66370\n",
      "Epoch 109/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4209 - acc: 0.9541 - val_loss: 1.8409 - val_acc: 0.6550\n",
      "\n",
      "Epoch 00109: val_acc did not improve from 0.66370\n",
      "Epoch 110/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4177 - acc: 0.9559 - val_loss: 1.8802 - val_acc: 0.6560\n",
      "\n",
      "Epoch 00110: val_acc did not improve from 0.66370\n",
      "Epoch 111/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4156 - acc: 0.9559 - val_loss: 1.8638 - val_acc: 0.6560\n",
      "\n",
      "Epoch 00111: val_acc did not improve from 0.66370\n",
      "Epoch 112/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4127 - acc: 0.9560 - val_loss: 1.8636 - val_acc: 0.6569\n",
      "\n",
      "Epoch 00112: val_acc did not improve from 0.66370\n",
      "Epoch 113/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4122 - acc: 0.9567 - val_loss: 1.8703 - val_acc: 0.6557\n",
      "\n",
      "Epoch 00113: val_acc did not improve from 0.66370\n",
      "Epoch 114/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4075 - acc: 0.9578 - val_loss: 1.8776 - val_acc: 0.6573\n",
      "\n",
      "Epoch 00114: val_acc did not improve from 0.66370\n",
      "Epoch 115/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.4073 - acc: 0.9579 - val_loss: 1.8797 - val_acc: 0.6580\n",
      "\n",
      "Epoch 00115: val_acc did not improve from 0.66370\n",
      "Epoch 116/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4019 - acc: 0.9588 - val_loss: 1.8836 - val_acc: 0.6537\n",
      "\n",
      "Epoch 00116: val_acc did not improve from 0.66370\n",
      "Epoch 117/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.4009 - acc: 0.9595 - val_loss: 1.8840 - val_acc: 0.6553\n",
      "\n",
      "Epoch 00117: val_acc did not improve from 0.66370\n",
      "Epoch 118/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.4011 - acc: 0.9579 - val_loss: 1.8991 - val_acc: 0.6570\n",
      "\n",
      "Epoch 00118: val_acc did not improve from 0.66370\n",
      "Epoch 119/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 19s 48ms/step - loss: 0.3962 - acc: 0.9597 - val_loss: 1.9027 - val_acc: 0.6574\n",
      "\n",
      "Epoch 00119: val_acc did not improve from 0.66370\n",
      "Epoch 120/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 20s 51ms/step - loss: 0.3964 - acc: 0.9597 - val_loss: 1.9320 - val_acc: 0.6528\n",
      "\n",
      "Epoch 00120: val_acc did not improve from 0.66370\n",
      "Epoch 121/200\n",
      "Learning rate:  0.0001\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3944 - acc: 0.9597 - val_loss: 1.9337 - val_acc: 0.6563\n",
      "\n",
      "Epoch 00121: val_acc did not improve from 0.66370\n",
      "Epoch 122/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3793 - acc: 0.9648 - val_loss: 1.8735 - val_acc: 0.6596\n",
      "\n",
      "Epoch 00122: val_acc did not improve from 0.66370\n",
      "Epoch 123/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3696 - acc: 0.9684 - val_loss: 1.8719 - val_acc: 0.6593\n",
      "\n",
      "Epoch 00123: val_acc did not improve from 0.66370\n",
      "Epoch 124/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3653 - acc: 0.9703 - val_loss: 1.8655 - val_acc: 0.6587\n",
      "\n",
      "Epoch 00124: val_acc did not improve from 0.66370\n",
      "Epoch 125/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3662 - acc: 0.9701 - val_loss: 1.8679 - val_acc: 0.6607\n",
      "\n",
      "Epoch 00125: val_acc did not improve from 0.66370\n",
      "Epoch 126/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3630 - acc: 0.9711 - val_loss: 1.8704 - val_acc: 0.6596\n",
      "\n",
      "Epoch 00126: val_acc did not improve from 0.66370\n",
      "Epoch 127/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3614 - acc: 0.9718 - val_loss: 1.8710 - val_acc: 0.6593\n",
      "\n",
      "Epoch 00127: val_acc did not improve from 0.66370\n",
      "Epoch 128/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3624 - acc: 0.9710 - val_loss: 1.8716 - val_acc: 0.6596\n",
      "\n",
      "Epoch 00128: val_acc did not improve from 0.66370\n",
      "Epoch 129/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3612 - acc: 0.9722 - val_loss: 1.8670 - val_acc: 0.6601\n",
      "\n",
      "Epoch 00129: val_acc did not improve from 0.66370\n",
      "Epoch 130/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3602 - acc: 0.9733 - val_loss: 1.8665 - val_acc: 0.6613\n",
      "\n",
      "Epoch 00130: val_acc did not improve from 0.66370\n",
      "Epoch 131/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3606 - acc: 0.9716 - val_loss: 1.8694 - val_acc: 0.6596\n",
      "\n",
      "Epoch 00131: val_acc did not improve from 0.66370\n",
      "Epoch 132/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3594 - acc: 0.9719 - val_loss: 1.8769 - val_acc: 0.6581\n",
      "\n",
      "Epoch 00132: val_acc did not improve from 0.66370\n",
      "Epoch 133/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3570 - acc: 0.9730 - val_loss: 1.8732 - val_acc: 0.6606\n",
      "\n",
      "Epoch 00133: val_acc did not improve from 0.66370\n",
      "Epoch 134/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3554 - acc: 0.9737 - val_loss: 1.8681 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00134: val_acc did not improve from 0.66370\n",
      "Epoch 135/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3573 - acc: 0.9726 - val_loss: 1.8773 - val_acc: 0.6597\n",
      "\n",
      "Epoch 00135: val_acc did not improve from 0.66370\n",
      "Epoch 136/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3564 - acc: 0.9736 - val_loss: 1.8700 - val_acc: 0.6594\n",
      "\n",
      "Epoch 00136: val_acc did not improve from 0.66370\n",
      "Epoch 137/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3562 - acc: 0.9734 - val_loss: 1.8761 - val_acc: 0.6587\n",
      "\n",
      "Epoch 00137: val_acc did not improve from 0.66370\n",
      "Epoch 138/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3562 - acc: 0.9732 - val_loss: 1.8739 - val_acc: 0.6597\n",
      "\n",
      "Epoch 00138: val_acc did not improve from 0.66370\n",
      "Epoch 139/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3547 - acc: 0.9728 - val_loss: 1.8748 - val_acc: 0.6607\n",
      "\n",
      "Epoch 00139: val_acc did not improve from 0.66370\n",
      "Epoch 140/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3521 - acc: 0.9759 - val_loss: 1.8793 - val_acc: 0.6600\n",
      "\n",
      "Epoch 00140: val_acc did not improve from 0.66370\n",
      "Epoch 141/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3536 - acc: 0.9740 - val_loss: 1.8784 - val_acc: 0.6592\n",
      "\n",
      "Epoch 00141: val_acc did not improve from 0.66370\n",
      "Epoch 142/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3505 - acc: 0.9748 - val_loss: 1.8741 - val_acc: 0.6607\n",
      "\n",
      "Epoch 00142: val_acc did not improve from 0.66370\n",
      "Epoch 143/200\n",
      "Learning rate:  1e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3517 - acc: 0.9750 - val_loss: 1.8747 - val_acc: 0.6610\n",
      "\n",
      "Epoch 00143: val_acc did not improve from 0.66370\n",
      "Epoch 144/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3516 - acc: 0.9753 - val_loss: 1.8771 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00144: val_acc did not improve from 0.66370\n",
      "Epoch 145/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3513 - acc: 0.9746 - val_loss: 1.8773 - val_acc: 0.6595\n",
      "\n",
      "Epoch 00145: val_acc did not improve from 0.66370\n",
      "Epoch 146/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3530 - acc: 0.9739 - val_loss: 1.8797 - val_acc: 0.6612\n",
      "\n",
      "Epoch 00146: val_acc did not improve from 0.66370\n",
      "Epoch 147/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3518 - acc: 0.9741 - val_loss: 1.8846 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00147: val_acc did not improve from 0.66370\n",
      "Epoch 148/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3522 - acc: 0.9739 - val_loss: 1.8831 - val_acc: 0.6606\n",
      "\n",
      "Epoch 00148: val_acc did not improve from 0.66370\n",
      "Epoch 149/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3516 - acc: 0.9739 - val_loss: 1.8820 - val_acc: 0.6590\n",
      "\n",
      "Epoch 00149: val_acc did not improve from 0.66370\n",
      "Epoch 150/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3492 - acc: 0.9747 - val_loss: 1.8833 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00150: val_acc did not improve from 0.66370\n",
      "Epoch 151/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3486 - acc: 0.9758 - val_loss: 1.8809 - val_acc: 0.6612\n",
      "\n",
      "Epoch 00151: val_acc did not improve from 0.66370\n",
      "Epoch 152/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3503 - acc: 0.9745 - val_loss: 1.8806 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00152: val_acc did not improve from 0.66370\n",
      "Epoch 153/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3494 - acc: 0.9742 - val_loss: 1.8851 - val_acc: 0.6610\n",
      "\n",
      "Epoch 00153: val_acc did not improve from 0.66370\n",
      "Epoch 154/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3479 - acc: 0.9754 - val_loss: 1.8742 - val_acc: 0.6608\n",
      "\n",
      "Epoch 00154: val_acc did not improve from 0.66370\n",
      "Epoch 155/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3469 - acc: 0.9759 - val_loss: 1.8803 - val_acc: 0.6602\n",
      "\n",
      "Epoch 00155: val_acc did not improve from 0.66370\n",
      "Epoch 156/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3464 - acc: 0.9760 - val_loss: 1.8824 - val_acc: 0.6592\n",
      "\n",
      "Epoch 00156: val_acc did not improve from 0.66370\n",
      "Epoch 157/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3487 - acc: 0.9744 - val_loss: 1.8751 - val_acc: 0.6612\n",
      "\n",
      "Epoch 00157: val_acc did not improve from 0.66370\n",
      "Epoch 158/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3474 - acc: 0.9753 - val_loss: 1.8822 - val_acc: 0.6594\n",
      "\n",
      "Epoch 00158: val_acc did not improve from 0.66370\n",
      "Epoch 159/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3462 - acc: 0.9759 - val_loss: 1.8845 - val_acc: 0.6632\n",
      "\n",
      "Epoch 00159: val_acc did not improve from 0.66370\n",
      "Epoch 160/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3455 - acc: 0.9758 - val_loss: 1.8813 - val_acc: 0.6618\n",
      "\n",
      "Epoch 00160: val_acc did not improve from 0.66370\n",
      "Epoch 161/200\n",
      "Learning rate:  1e-05\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3460 - acc: 0.9752 - val_loss: 1.8850 - val_acc: 0.6610\n",
      "\n",
      "Epoch 00161: val_acc did not improve from 0.66370\n",
      "Epoch 162/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3456 - acc: 0.9758 - val_loss: 1.8835 - val_acc: 0.6613\n",
      "\n",
      "Epoch 00162: val_acc did not improve from 0.66370\n",
      "Epoch 163/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3453 - acc: 0.9757 - val_loss: 1.8818 - val_acc: 0.6608\n",
      "\n",
      "Epoch 00163: val_acc did not improve from 0.66370\n",
      "Epoch 164/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3435 - acc: 0.9764 - val_loss: 1.8826 - val_acc: 0.6614\n",
      "\n",
      "Epoch 00164: val_acc did not improve from 0.66370\n",
      "Epoch 165/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3441 - acc: 0.9754 - val_loss: 1.8835 - val_acc: 0.6606\n",
      "\n",
      "Epoch 00165: val_acc did not improve from 0.66370\n",
      "Epoch 166/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3442 - acc: 0.9758 - val_loss: 1.8824 - val_acc: 0.6605\n",
      "\n",
      "Epoch 00166: val_acc did not improve from 0.66370\n",
      "Epoch 167/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3447 - acc: 0.9758 - val_loss: 1.8827 - val_acc: 0.6605\n",
      "\n",
      "Epoch 00167: val_acc did not improve from 0.66370\n",
      "Epoch 168/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3425 - acc: 0.9767 - val_loss: 1.8818 - val_acc: 0.6603\n",
      "\n",
      "Epoch 00168: val_acc did not improve from 0.66370\n",
      "Epoch 169/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3425 - acc: 0.9767 - val_loss: 1.8830 - val_acc: 0.6605\n",
      "\n",
      "Epoch 00169: val_acc did not improve from 0.66370\n",
      "Epoch 170/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 19s 48ms/step - loss: 0.3428 - acc: 0.9758 - val_loss: 1.8836 - val_acc: 0.6602\n",
      "\n",
      "Epoch 00170: val_acc did not improve from 0.66370\n",
      "Epoch 171/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3455 - acc: 0.9759 - val_loss: 1.8850 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00171: val_acc did not improve from 0.66370\n",
      "Epoch 172/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3414 - acc: 0.9771 - val_loss: 1.8840 - val_acc: 0.6602\n",
      "\n",
      "Epoch 00172: val_acc did not improve from 0.66370\n",
      "Epoch 173/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3420 - acc: 0.9770 - val_loss: 1.8839 - val_acc: 0.6599\n",
      "\n",
      "Epoch 00173: val_acc did not improve from 0.66370\n",
      "Epoch 174/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3425 - acc: 0.9772 - val_loss: 1.8822 - val_acc: 0.6609\n",
      "\n",
      "Epoch 00174: val_acc did not improve from 0.66370\n",
      "Epoch 175/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 19s 48ms/step - loss: 0.3423 - acc: 0.9766 - val_loss: 1.8831 - val_acc: 0.6606\n",
      "\n",
      "Epoch 00175: val_acc did not improve from 0.66370\n",
      "Epoch 176/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3426 - acc: 0.9767 - val_loss: 1.8819 - val_acc: 0.6600\n",
      "\n",
      "Epoch 00176: val_acc did not improve from 0.66370\n",
      "Epoch 177/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3414 - acc: 0.9769 - val_loss: 1.8815 - val_acc: 0.6606\n",
      "\n",
      "Epoch 00177: val_acc did not improve from 0.66370\n",
      "Epoch 178/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3432 - acc: 0.9768 - val_loss: 1.8836 - val_acc: 0.6600\n",
      "\n",
      "Epoch 00178: val_acc did not improve from 0.66370\n",
      "Epoch 179/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3433 - acc: 0.9767 - val_loss: 1.8837 - val_acc: 0.6603\n",
      "\n",
      "Epoch 00179: val_acc did not improve from 0.66370\n",
      "Epoch 180/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3422 - acc: 0.9769 - val_loss: 1.8829 - val_acc: 0.6601\n",
      "\n",
      "Epoch 00180: val_acc did not improve from 0.66370\n",
      "Epoch 181/200\n",
      "Learning rate:  1e-06\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3419 - acc: 0.9772 - val_loss: 1.8801 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00181: val_acc did not improve from 0.66370\n",
      "Epoch 182/200\n",
      "Learning rate:  5e-07\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3447 - acc: 0.9755 - val_loss: 1.8808 - val_acc: 0.6609\n",
      "\n",
      "Epoch 00182: val_acc did not improve from 0.66370\n",
      "Epoch 183/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3420 - acc: 0.9771 - val_loss: 1.8804 - val_acc: 0.6603\n",
      "\n",
      "Epoch 00183: val_acc did not improve from 0.66370\n",
      "Epoch 184/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3426 - acc: 0.9768 - val_loss: 1.8836 - val_acc: 0.6596\n",
      "\n",
      "Epoch 00184: val_acc did not improve from 0.66370\n",
      "Epoch 185/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3433 - acc: 0.9754 - val_loss: 1.8797 - val_acc: 0.6598\n",
      "\n",
      "Epoch 00185: val_acc did not improve from 0.66370\n",
      "Epoch 186/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3420 - acc: 0.9769 - val_loss: 1.8807 - val_acc: 0.6597\n",
      "\n",
      "Epoch 00186: val_acc did not improve from 0.66370\n",
      "Epoch 187/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3417 - acc: 0.9765 - val_loss: 1.8842 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00187: val_acc did not improve from 0.66370\n",
      "Epoch 188/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3435 - acc: 0.9763 - val_loss: 1.8822 - val_acc: 0.6603\n",
      "\n",
      "Epoch 00188: val_acc did not improve from 0.66370\n",
      "Epoch 189/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3405 - acc: 0.9775 - val_loss: 1.8820 - val_acc: 0.6599\n",
      "\n",
      "Epoch 00189: val_acc did not improve from 0.66370\n",
      "Epoch 190/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3429 - acc: 0.9763 - val_loss: 1.8814 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00190: val_acc did not improve from 0.66370\n",
      "Epoch 191/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3406 - acc: 0.9779 - val_loss: 1.8835 - val_acc: 0.6597\n",
      "\n",
      "Epoch 00191: val_acc did not improve from 0.66370\n",
      "Epoch 192/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3442 - acc: 0.9753 - val_loss: 1.8843 - val_acc: 0.6606\n",
      "\n",
      "Epoch 00192: val_acc did not improve from 0.66370\n",
      "Epoch 193/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3408 - acc: 0.9764 - val_loss: 1.8840 - val_acc: 0.6599\n",
      "\n",
      "Epoch 00193: val_acc did not improve from 0.66370\n",
      "Epoch 194/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3434 - acc: 0.9761 - val_loss: 1.8805 - val_acc: 0.6599\n",
      "\n",
      "Epoch 00194: val_acc did not improve from 0.66370\n",
      "Epoch 195/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3413 - acc: 0.9769 - val_loss: 1.8830 - val_acc: 0.6600\n",
      "\n",
      "Epoch 00195: val_acc did not improve from 0.66370\n",
      "Epoch 196/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3441 - acc: 0.9762 - val_loss: 1.8815 - val_acc: 0.6607\n",
      "\n",
      "Epoch 00196: val_acc did not improve from 0.66370\n",
      "Epoch 197/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3409 - acc: 0.9774 - val_loss: 1.8854 - val_acc: 0.6601\n",
      "\n",
      "Epoch 00197: val_acc did not improve from 0.66370\n",
      "Epoch 198/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 46ms/step - loss: 0.3424 - acc: 0.9769 - val_loss: 1.8795 - val_acc: 0.6612\n",
      "\n",
      "Epoch 00198: val_acc did not improve from 0.66370\n",
      "Epoch 199/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 47ms/step - loss: 0.3406 - acc: 0.9782 - val_loss: 1.8831 - val_acc: 0.6604\n",
      "\n",
      "Epoch 00199: val_acc did not improve from 0.66370\n",
      "Epoch 200/200\n",
      "Learning rate:  5e-07\n",
      "390/390 [==============================] - 18s 45ms/step - loss: 0.3419 - acc: 0.9767 - val_loss: 1.8828 - val_acc: 0.6609\n",
      "\n",
      "Epoch 00200: val_acc did not improve from 0.66370\n"
     ]
    }
   ],
   "source": [
    "history = model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\n",
    "                              epochs=epochs, steps_per_epoch=x_train.shape[0] // batch_size,\n",
    "                              validation_data=(x_test, y_test),\n",
    "                              verbose=1, workers=4, callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "history_filepath = os.path.join(save_dir, 'cifar100_%s_history_temp.pk' % (model_type))\n",
    "pickle.dump(history, open(history_filepath, \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 3s 256us/step\n",
      "Test loss: 1.7697390132904052\n",
      "Test accuracy: 0.6637\n"
     ]
    }
   ],
   "source": [
    "best_model_filepath = os.path.join(save_dir, 'cifar100_%s_model.%03d.h5' % (model_type, 91))\n",
    "best_model = load_model(best_model_filepath)\n",
    "\n",
    "scores = best_model.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.663700\n",
      "Top 5 accuracy: 0.893600\n"
     ]
    }
   ],
   "source": [
    "def acc(top_k, y_true):\n",
    "    correct = 0\n",
    "    for i in range(y_true.shape[0]):\n",
    "        if y_true[i] in top_k[i]:\n",
    "            correct += 1\n",
    "    return correct/float(y_true.shape[0])\n",
    "\n",
    "y_pred = best_model.predict(x_test)\n",
    "print(\"Accuracy: %f\" % (np.sum(np.argmax(y_pred, axis=1) == np.argmax(y_test, axis=1)) / float(len(y_test))))\n",
    "\n",
    "k = 5\n",
    "y_pred_top_k = y_pred.argsort()[:,-k:]\n",
    "print(\"Top %d accuracy: %f\" % (k, acc(y_pred_top_k, np.argmax(y_test, axis=1))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting layer: add_6\n"
     ]
    }
   ],
   "source": [
    "print(\"Extracting layer: %s\" % (best_model.get_layer(index=-6).name))\n",
    "feat_extractor_model = Model(inputs= best_model.input, outputs = best_model.get_layer(index=-6).output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train features: (50000, 8, 8, 256)\n",
      "test features: (10000, 8, 8, 256)\n"
     ]
    }
   ],
   "source": [
    "x_train_feat = feat_extractor_model.predict(x_train)\n",
    "print(\"train features:\", x_train_feat.shape)\n",
    "x_test_feat = feat_extractor_model.predict(x_test)\n",
    "print(\"test features:\", x_test_feat.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_dir_feat = os.path.join(os.getcwd(), 'saved_models/%s/extracted_feat/' % (model_type))\n",
    "if not os.path.isdir(save_dir_feat):\n",
    "    os.makedirs(save_dir_feat)\n",
    "    \n",
    "np.save(os.path.join(save_dir_feat, 'x_train_feat_cifar100_%s.npy' % (model_type)), x_train_feat)\n",
    "np.save(os.path.join(save_dir_feat, 'x_test_feat_cifar100_%s.npy' % (model_type)), x_test_feat)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}