ZSL_node2vec_transfer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from keras.models import Model, load_model\n",
    "from keras.layers import Input, Dense, BatchNormalization, Activation, AveragePooling2D, Flatten\n",
    "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler\n",
    "from keras.datasets import cifar100\n",
    "from keras import backend as K\n",
    "import pickle\n",
    "import sklearn\n",
    "from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_type = 'ResNet56v2'\n",
    "embedding_type = 'node2vec'\n",
    "seed = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_embeddings(fname):   \n",
    "    file_path = os.path.join(os.getcwd(), 'wiki/%s.emb' % (fname))\n",
    "    \n",
    "    embeddings = {}\n",
    "    with open(file_path) as file:\n",
    "        for line in file:\n",
    "            line = line.strip().split()\n",
    "            if int(line[0]) >= 100:\n",
    "                continue\n",
    "            embeddings[int(line[0])] = np.array(line[1:]).astype(float)\n",
    "\n",
    "    return embeddings\n",
    "\n",
    "label_embeddings = load_embeddings('link_edges')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cat_dict = pickle.load(open(os.path.join(os.getcwd(), 'wiki/link_edges_dict_cat.pkl'), \"rb\"))\n",
    "graph_to_cifar_dict = pickle.load(open(os.path.join(os.getcwd(), 'wiki/graph_to_cifar_labels.pkl'), \"rb\"))\n",
    "\n",
    "data_dir = os.path.join(os.getcwd(), 'Data/cifar-100-python/')\n",
    "meta = pickle.load(open(os.path.join(data_dir, 'meta'), \"rb\"), encoding='latin1')\n",
    "fine_label_names = meta['fine_label_names']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading data and embeddings\n",
    "embedding_len = len(label_embeddings[0])\n",
    "\n",
    "label_embeddings_arr = np.zeros((100, embedding_len))\n",
    "for i in range(100):\n",
    "    label_embeddings_arr[i] = label_embeddings[i]\n",
    "\n",
    "save_dir_feat = os.path.join(os.getcwd(), 'saved_models/zsl/%s/type1/seed%s/extracted_feat/' % (model_type, seed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_feat = np.load(os.path.join(save_dir_feat, 'X_train_feat_cifar100_%s.npy' % (model_type)))\n",
    "X_test_seen_feat = np.load(os.path.join(save_dir_feat, 'X_test_seen_feat_cifar100_%s.npy' % (model_type)))\n",
    "X_test_unseen_feat = np.load(os.path.join(save_dir_feat, 'X_test_unseen_feat_cifar100_%s.npy' % (model_type)))\n",
    "X_test_all_feat = np.load(os.path.join(save_dir_feat, 'X_test_all_feat_cifar100_%s.npy' % (model_type)))\n",
    "\n",
    "input_shape = X_train_feat[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np.load(os.path.join(save_dir_feat, 'y_train_cifar100_%s.npy' % (model_type)))\n",
    "y_test_seen = np.load(os.path.join(save_dir_feat, 'y_test_seen_cifar100_%s.npy' % (model_type)))\n",
    "y_test_unseen = np.load(os.path.join(save_dir_feat, 'y_test_unseen_cifar100_%s.npy' % (model_type)))\n",
    "y_test_all = np.load(os.path.join(save_dir_feat, 'y_test_all_cifar100_%s.npy' % (model_type)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_embeddings = np.zeros((len(y_train), embedding_len))\n",
    "for i in range(len(y_train)):\n",
    "    y_train_embeddings[i] = label_embeddings[int(y_train[i])]\n",
    "    \n",
    "y_test_seen_embeddings = np.zeros((len(y_test_seen), embedding_len))\n",
    "for i in range(len(y_test_seen)):\n",
    "    y_test_seen_embeddings[i] = label_embeddings[int(y_test_seen[i])]\n",
    "    \n",
    "y_test_unseen_embeddings = np.zeros((len(y_test_unseen), embedding_len))\n",
    "for i in range(len(y_test_unseen)):\n",
    "    y_test_unseen_embeddings[i] = label_embeddings[int(y_test_unseen[i])]\n",
    "    \n",
    "y_test_all_embeddings = np.zeros((len(y_test_all), embedding_len))\n",
    "for i in range(len(y_test_all)):\n",
    "    y_test_all_embeddings[i] = label_embeddings[int(y_test_all[i])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(input_shape=input_shape, embedding_len=embedding_len):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    x = BatchNormalization()(inputs)\n",
    "    x = Activation('relu')(x)\n",
    "    x = AveragePooling2D(pool_size=2)(x)\n",
    "    \n",
    "    x = Flatten()(x)\n",
    "    \n",
    "    x = Dense(1024) (x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    x = Dense(512) (x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    x = Dense(embedding_len) (x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    outputs = Dense(embedding_len,\n",
    "                    kernel_initializer='he_normal')(x)\n",
    "    \n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "epochs = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 8, 8, 256)         1024      \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_1 (Average (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1024)              4195328   \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 1024)              4096      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               524800    \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               16512     \n",
      "=================================================================\n",
      "Total params: 4,809,984\n",
      "Trainable params: 4,806,144\n",
      "Non-trainable params: 3,840\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = build_model()\n",
    "model.compile(loss='cosine_proximity', optimizer='adam', metrics=['mse'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_dir = os.path.join(os.getcwd(), 'saved_models/zsl/%s/type1/seed%s/%s_transfer/' % (model_type, seed, embedding_type))\n",
    "model_name = 'cifar100_%s_model.{epoch:03d}.h5' % (model_type)\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "filepath = os.path.join(save_dir, model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_schedule(epoch):\n",
    "    \"\"\"Learning Rate Schedule\n",
    "    Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\n",
    "    Called automatically every epoch as part of callbacks during training.\n",
    "    # Arguments\n",
    "        epoch (int): The number of epochs\n",
    "    # Returns\n",
    "        lr (float32): learning rate\n",
    "    \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 120:\n",
    "        lr *= 0.5e-3\n",
    "    elif epoch > 80:\n",
    "        lr *= 1e-3\n",
    "    elif epoch > 40:\n",
    "        lr *= 1e-2\n",
    "    elif epoch > 20:\n",
    "        lr *= 1e-1\n",
    "    print('Learning rate: ', lr)\n",
    "\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = ModelCheckpoint(filepath=filepath,\n",
    "                             monitor='val_loss',\n",
    "                             verbose=1,\n",
    "                             save_best_only=True)\n",
    "\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=5,\n",
    "                               min_lr=0.5e-6)\n",
    "\n",
    "callbacks = [checkpoint, lr_reducer, lr_scheduler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 8000 samples\n",
      "Epoch 1/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 7s 178us/step - loss: -0.7829 - mean_squared_error: 2.2023 - val_loss: -0.8667 - val_mean_squared_error: 5.5278\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to -0.86668, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.001.h5\n",
      "Epoch 2/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 130us/step - loss: -0.8917 - mean_squared_error: 0.6840 - val_loss: -0.8992 - val_mean_squared_error: 0.4079\n",
      "\n",
      "Epoch 00002: val_loss improved from -0.86668 to -0.89923, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.002.h5\n",
      "Epoch 3/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 131us/step - loss: -0.9061 - mean_squared_error: 0.4801 - val_loss: -0.9062 - val_mean_squared_error: 0.4499\n",
      "\n",
      "Epoch 00003: val_loss improved from -0.89923 to -0.90619, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.003.h5\n",
      "Epoch 4/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 136us/step - loss: -0.9152 - mean_squared_error: 0.4463 - val_loss: -0.9122 - val_mean_squared_error: 0.4857\n",
      "\n",
      "Epoch 00004: val_loss improved from -0.90619 to -0.91224, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.004.h5\n",
      "Epoch 5/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 134us/step - loss: -0.9246 - mean_squared_error: 0.5158 - val_loss: -0.9160 - val_mean_squared_error: 0.6271\n",
      "\n",
      "Epoch 00005: val_loss improved from -0.91224 to -0.91601, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.005.h5\n",
      "Epoch 6/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 131us/step - loss: -0.9330 - mean_squared_error: 0.6346 - val_loss: -0.9155 - val_mean_squared_error: 0.9533\n",
      "\n",
      "Epoch 00006: val_loss did not improve from -0.91601\n",
      "Epoch 7/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 6s 139us/step - loss: -0.9412 - mean_squared_error: 0.8005 - val_loss: -0.9206 - val_mean_squared_error: 0.7315\n",
      "\n",
      "Epoch 00007: val_loss improved from -0.91601 to -0.92063, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.007.h5\n",
      "Epoch 8/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 134us/step - loss: -0.9485 - mean_squared_error: 0.9422 - val_loss: -0.9187 - val_mean_squared_error: 1.2372\n",
      "\n",
      "Epoch 00008: val_loss did not improve from -0.92063\n",
      "Epoch 9/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 137us/step - loss: -0.9537 - mean_squared_error: 1.1163 - val_loss: -0.9149 - val_mean_squared_error: 11.8357\n",
      "\n",
      "Epoch 00009: val_loss did not improve from -0.92063\n",
      "Epoch 10/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 131us/step - loss: -0.9588 - mean_squared_error: 1.2386 - val_loss: -0.9207 - val_mean_squared_error: 1.4869\n",
      "\n",
      "Epoch 00010: val_loss improved from -0.92063 to -0.92075, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.010.h5\n",
      "Epoch 11/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 6s 139us/step - loss: -0.9626 - mean_squared_error: 1.4162 - val_loss: -0.9183 - val_mean_squared_error: 1.4355\n",
      "\n",
      "Epoch 00011: val_loss did not improve from -0.92075\n",
      "Epoch 12/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 130us/step - loss: -0.9656 - mean_squared_error: 1.5220 - val_loss: -0.9208 - val_mean_squared_error: 1.9294\n",
      "\n",
      "Epoch 00012: val_loss improved from -0.92075 to -0.92077, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.012.h5\n",
      "Epoch 13/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 133us/step - loss: -0.9680 - mean_squared_error: 1.7005 - val_loss: -0.9191 - val_mean_squared_error: 2.0434\n",
      "\n",
      "Epoch 00013: val_loss did not improve from -0.92077\n",
      "Epoch 14/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 133us/step - loss: -0.9707 - mean_squared_error: 1.7408 - val_loss: -0.9208 - val_mean_squared_error: 1.8462\n",
      "\n",
      "Epoch 00014: val_loss improved from -0.92077 to -0.92083, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.014.h5\n",
      "Epoch 15/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 130us/step - loss: -0.9740 - mean_squared_error: 1.8856 - val_loss: -0.9197 - val_mean_squared_error: 1.9660\n",
      "\n",
      "Epoch 00015: val_loss did not improve from -0.92083\n",
      "Epoch 16/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 132us/step - loss: -0.9755 - mean_squared_error: 2.0241 - val_loss: -0.9201 - val_mean_squared_error: 2.1193\n",
      "\n",
      "Epoch 00016: val_loss did not improve from -0.92083\n",
      "Epoch 17/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 133us/step - loss: -0.9780 - mean_squared_error: 2.0914 - val_loss: -0.9197 - val_mean_squared_error: 2.1983\n",
      "\n",
      "Epoch 00017: val_loss did not improve from -0.92083\n",
      "Epoch 18/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 6s 139us/step - loss: -0.9802 - mean_squared_error: 2.3334 - val_loss: -0.9165 - val_mean_squared_error: 2.3902\n",
      "\n",
      "Epoch 00018: val_loss did not improve from -0.92083\n",
      "Epoch 19/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 129us/step - loss: -0.9824 - mean_squared_error: 2.3300 - val_loss: -0.9214 - val_mean_squared_error: 2.2352\n",
      "\n",
      "Epoch 00019: val_loss improved from -0.92083 to -0.92144, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.019.h5\n",
      "Epoch 20/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 131us/step - loss: -0.9840 - mean_squared_error: 2.4701 - val_loss: -0.9206 - val_mean_squared_error: 2.4029\n",
      "\n",
      "Epoch 00020: val_loss did not improve from -0.92144\n",
      "Epoch 21/40\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 128us/step - loss: -0.9854 - mean_squared_error: 2.5554 - val_loss: -0.9170 - val_mean_squared_error: 2.6787\n",
      "\n",
      "Epoch 00021: val_loss did not improve from -0.92144\n",
      "Epoch 22/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 131us/step - loss: -0.9886 - mean_squared_error: 2.6741 - val_loss: -0.9225 - val_mean_squared_error: 2.4232\n",
      "\n",
      "Epoch 00022: val_loss improved from -0.92144 to -0.92246, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.022.h5\n",
      "Epoch 23/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 133us/step - loss: -0.9904 - mean_squared_error: 2.5991 - val_loss: -0.9229 - val_mean_squared_error: 2.4652\n",
      "\n",
      "Epoch 00023: val_loss improved from -0.92246 to -0.92293, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.023.h5\n",
      "Epoch 24/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 136us/step - loss: -0.9912 - mean_squared_error: 2.5639 - val_loss: -0.9228 - val_mean_squared_error: 2.3996\n",
      "\n",
      "Epoch 00024: val_loss did not improve from -0.92293\n",
      "Epoch 25/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 130us/step - loss: -0.9919 - mean_squared_error: 2.5292 - val_loss: -0.9230 - val_mean_squared_error: 2.2988\n",
      "\n",
      "Epoch 00025: val_loss improved from -0.92293 to -0.92297, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.025.h5\n",
      "Epoch 26/40\n",
      "Learning rate:  0.0001\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 5s 130us/step - loss: -0.9921 - mean_squared_error: 2.5372 - val_loss: -0.9231 - val_mean_squared_error: 2.3252\n",
      "\n",
      "Epoch 00026: val_loss improved from -0.92297 to -0.92306, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.026.h5\n",
      "Epoch 27/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 126us/step - loss: -0.9926 - mean_squared_error: 2.5356 - val_loss: -0.9227 - val_mean_squared_error: 2.3151\n",
      "\n",
      "Epoch 00027: val_loss did not improve from -0.92306\n",
      "Epoch 28/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 128us/step - loss: -0.9929 - mean_squared_error: 2.5256 - val_loss: -0.9216 - val_mean_squared_error: 2.2551\n",
      "\n",
      "Epoch 00028: val_loss did not improve from -0.92306\n",
      "Epoch 29/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 125us/step - loss: -0.9931 - mean_squared_error: 2.4949 - val_loss: -0.9225 - val_mean_squared_error: 2.3850\n",
      "\n",
      "Epoch 00029: val_loss did not improve from -0.92306\n",
      "Epoch 30/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 126us/step - loss: -0.9934 - mean_squared_error: 2.5090 - val_loss: -0.9228 - val_mean_squared_error: 2.1440\n",
      "\n",
      "Epoch 00030: val_loss did not improve from -0.92306\n",
      "Epoch 31/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 130us/step - loss: -0.9937 - mean_squared_error: 2.5017 - val_loss: -0.9232 - val_mean_squared_error: 2.2418\n",
      "\n",
      "Epoch 00031: val_loss improved from -0.92306 to -0.92324, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed6/node2vec_transfer/cifar100_ResNet56v2_model.031.h5\n",
      "Epoch 32/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 124us/step - loss: -0.9939 - mean_squared_error: 2.4838 - val_loss: -0.9230 - val_mean_squared_error: 2.2358\n",
      "\n",
      "Epoch 00032: val_loss did not improve from -0.92324\n",
      "Epoch 33/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 126us/step - loss: -0.9941 - mean_squared_error: 2.4649 - val_loss: -0.9222 - val_mean_squared_error: 2.2208\n",
      "\n",
      "Epoch 00033: val_loss did not improve from -0.92324\n",
      "Epoch 34/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 5s 130us/step - loss: -0.9945 - mean_squared_error: 2.4840 - val_loss: -0.9217 - val_mean_squared_error: 2.1395\n",
      "\n",
      "Epoch 00034: val_loss did not improve from -0.92324\n",
      "Epoch 35/40\n",
      "Learning rate:  0.0001\n",
      "40000/40000 [==============================] - 6s 138us/step - loss: -0.9945 - mean_squared_error: 2.4736 - val_loss: -0.9224 - val_mean_squared_error: 2.1932\n",
      "\n",
      "Epoch 00035: val_loss did not improve from -0.92324\n",
      "Epoch 36/40\n",
      "Learning rate:  0.0001\n",
      "27264/40000 [===================>..........] - ETA: 1s - loss: -0.9947 - mean_squared_error: 2.4704"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-977d62f2d6c5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m           \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m           \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m           \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m          )\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1397\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1398\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1399\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1400\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1401\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(X_train_feat, y_train_embeddings,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          validation_data=(X_test_seen_feat, y_test_seen_embeddings),\n",
    "          shuffle=True,\n",
    "          callbacks=callbacks,\n",
    "          verbose=1\n",
    "         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.DataFrame([cat_dict.keys(), cat_dict.values()]).transpose()\n",
    "df1.columns = ['graph_label', 'graph_label_name']\n",
    "\n",
    "df2 = pd.DataFrame([graph_to_cifar_dict.keys(), graph_to_cifar_dict.values()]).transpose()\n",
    "df2.columns = ['graph_label_name', 'cifar_label_name']\n",
    "\n",
    "df3 = pd.DataFrame([fine_label_names]).transpose().reset_index()\n",
    "df3.columns = ['cifar_label', 'cifar_label_name']\n",
    "\n",
    "df_labels = df1.merge(df2, left_on='graph_label_name', right_on='graph_label_name')\n",
    "df_labels = df_labels.merge(df3, left_on='cifar_label_name', right_on='cifar_label_name')\n",
    "\n",
    "label_dict = df_labels[['graph_label', 'cifar_label']].set_index('graph_label').to_dict()['cifar_label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Top K Predictions\n",
    "def pred_top_k(y_test_pred, label_embeddings_arr=label_embeddings_arr, k=5):\n",
    "    sim_table = cosine_similarity(y_test_pred, label_embeddings_arr)\n",
    "    top_k_guesses = np.argpartition(sim_table,range(99-k+1,100),axis=1)[:,99-k+1:]\n",
    "    return sim_table, top_k_guesses\n",
    "\n",
    "#Top k Accuracy\n",
    "def calc_top_k_acc(top_k, y_test):\n",
    "    correct = 0\n",
    "    for i in range(y_test.shape[0]):\n",
    "        if np.squeeze(y_test)[i] in top_k[i]:\n",
    "            correct += 1\n",
    "    return correct/float(y_test.shape[0])\n",
    "\n",
    "def evaluate(model, x_test, y_test, label_embeddings_arr, k=5):\n",
    "    y_test_pred = model.predict(x_test)\n",
    "    sim_table, top_k_guesses = pred_top_k(y_test_pred, label_embeddings_arr, k=k)\n",
    "    top_k_guesses = [[label_dict[i] for i in top_k_guesses[j]] for j in range(len(top_k_guesses))]\n",
    "    \n",
    "    #Top prediction\n",
    "    label_predictions = sim_table.argmax(axis=1)\n",
    "    label_predictions = [label_dict[i] for i in label_predictions]\n",
    "    \n",
    "    #Accuracy\n",
    "    acc = np.sum((np.squeeze(y_test) == label_predictions)) / float(y_test.shape[0])\n",
    "    top_k_acc = calc_top_k_acc(top_k_guesses, y_test)\n",
    "    print(\"Accuracy: \" + str(acc))\n",
    "    print(\"Top \" + str(k) + \" Accuracy: \" + str(top_k_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model_filepath = os.path.join(save_dir, 'cifar100_%s_model.%03d.h5' % (model_type, 31))\n",
    "best_model = load_model(best_model_filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.075375\n",
      "Top 5 Accuracy: 0.135375\n"
     ]
    }
   ],
   "source": [
    "evaluate(best_model, X_test_seen_feat, y_test_seen, label_embeddings_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.057\n",
      "Top 5 Accuracy: 0.2555\n"
     ]
    }
   ],
   "source": [
    "# Regular ZSL setting where we only need to consider choose between the 20 unseen classes, rather than all 100\n",
    "\n",
    "# Just replace the word vectors for \"seen\" labels with something really far so it won't be close to any predicted vector\n",
    "unseen_labels = np.unique(y_test_unseen)\n",
    "label_embeddings_arr_unseen = np.copy(label_embeddings_arr)\n",
    "for i in range(100):\n",
    "    if label_dict[i] in unseen_labels:\n",
    "        continue\n",
    "    label_embeddings_arr_unseen[i] = np.ones(label_embeddings_arr[0].shape) * 1000\n",
    "\n",
    "evaluate(best_model, X_test_unseen_feat, y_test_unseen, label_embeddings_arr_unseen) # Significant improvement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.013666666666666667\n",
      "Top 5 Accuracy: 0.06\n"
     ]
    }
   ],
   "source": [
    "# Although accuracy is still 0%, we seen an improvement in top 5 % accuracy (0% -> 15%). Sign that w2v is useful for ZSL\n",
    "\n",
    "evaluate(best_model, X_test_unseen_feat, y_test_unseen, label_embeddings_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nx = np.linspace(0,99,100) \\ny = np.zeros(100)\\ny_test_pred = model.predict(x_test)\\n\\nfor i in range(100):\\n    k = i+1\\n    sim_table, top_k_guesses = pred_top_k(y_test_pred, k=k)\\n    y[i] = calc_top_k_acc(top_k_guesses, y_test)\\n    \\nplt.plot(x,y)\\nplt.title(\"Top K Accuracy\")\\n'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "x = np.linspace(0,99,100) \n",
    "y = np.zeros(100)\n",
    "y_test_pred = model.predict(x_test)\n",
    "\n",
    "for i in range(100):\n",
    "    k = i+1\n",
    "    sim_table, top_k_guesses = pred_top_k(y_test_pred, k=k)\n",
    "    y[i] = calc_top_k_acc(top_k_guesses, y_test)\n",
    "    \n",
    "plt.plot(x,y)\n",
    "plt.title(\"Top K Accuracy\")\n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}