ZSL_w2v_transfer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from keras.models import Model, load_model\n",
    "from keras.layers import Input, Dense, BatchNormalization, Activation, AveragePooling2D, Flatten\n",
    "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler\n",
    "from keras.datasets import cifar100\n",
    "from keras import backend as K\n",
    "import pickle\n",
    "import sklearn\n",
    "from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_type = 'ResNet56v2'\n",
    "seed = 4\n",
    "\n",
    "embedding_type = 'glove'\n",
    "dim = 100\n",
    "save_dir = 'Data/Embeddings/Full/%s/' % (embedding_type)\n",
    "data_dir = os.path.join(os.getcwd(), save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading data and embeddings\n",
    "label_embeddings = pickle.load(open(os.path.join(data_dir, '%s_%s_labels_to_embeddings.pk' % (embedding_type, dim)), \"rb\"))\n",
    "embedding_len = len(label_embeddings[0])\n",
    "\n",
    "label_embeddings_arr = np.zeros((100, embedding_len))\n",
    "for i in range(100):\n",
    "    label_embeddings_arr[i] = label_embeddings[i]\n",
    "    \n",
    "save_dir_feat = os.path.join(os.getcwd(), 'saved_models/zsl/%s/type1/seed%s/extracted_feat/' % (model_type, seed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_feat = np.load(os.path.join(save_dir_feat, 'X_train_feat_cifar100_%s.npy' % (model_type)))\n",
    "X_test_seen_feat = np.load(os.path.join(save_dir_feat, 'X_test_seen_feat_cifar100_%s.npy' % (model_type)))\n",
    "X_test_unseen_feat = np.load(os.path.join(save_dir_feat, 'X_test_unseen_feat_cifar100_%s.npy' % (model_type)))\n",
    "X_test_all_feat = np.load(os.path.join(save_dir_feat, 'X_test_all_feat_cifar100_%s.npy' % (model_type)))\n",
    "\n",
    "input_shape = X_train_feat[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np.load(os.path.join(save_dir_feat, 'y_train_cifar100_%s.npy' % (model_type)))\n",
    "y_test_seen = np.load(os.path.join(save_dir_feat, 'y_test_seen_cifar100_%s.npy' % (model_type)))\n",
    "y_test_unseen = np.load(os.path.join(save_dir_feat, 'y_test_unseen_cifar100_%s.npy' % (model_type)))\n",
    "y_test_all = np.load(os.path.join(save_dir_feat, 'y_test_all_cifar100_%s.npy' % (model_type)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_embeddings = np.zeros((len(y_train), embedding_len))\n",
    "for i in range(len(y_train)):\n",
    "    y_train_embeddings[i] = label_embeddings[int(y_train[i])]\n",
    "    \n",
    "y_test_seen_embeddings = np.zeros((len(y_test_seen), embedding_len))\n",
    "for i in range(len(y_test_seen)):\n",
    "    y_test_seen_embeddings[i] = label_embeddings[int(y_test_seen[i])]\n",
    "    \n",
    "y_test_unseen_embeddings = np.zeros((len(y_test_unseen), embedding_len))\n",
    "for i in range(len(y_test_unseen)):\n",
    "    y_test_unseen_embeddings[i] = label_embeddings[int(y_test_unseen[i])]\n",
    "    \n",
    "y_test_all_embeddings = np.zeros((len(y_test_all), embedding_len))\n",
    "for i in range(len(y_test_all)):\n",
    "    y_test_all_embeddings[i] = label_embeddings[int(y_test_all[i])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(input_shape=input_shape, embedding_len=embedding_len):\n",
    "    inputs = Input(shape=input_shape)\n",
    "\n",
    "    x = BatchNormalization()(inputs)\n",
    "    x = Activation('relu')(x)\n",
    "    x = AveragePooling2D(pool_size=2)(x)\n",
    "    \n",
    "    x = Flatten()(x)\n",
    "    \n",
    "    x = Dense(1024) (x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    x = Dense(512) (x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    x = Dense(embedding_len) (x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    outputs = Dense(embedding_len,\n",
    "                    kernel_initializer='he_normal')(x)\n",
    "    \n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "epochs = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 8, 8, 256)         1024      \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_1 (Average (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1024)              4195328   \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 1024)              4096      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               524800    \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 100)               51300     \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 100)               10100     \n",
      "=================================================================\n",
      "Total params: 4,789,096\n",
      "Trainable params: 4,785,312\n",
      "Non-trainable params: 3,784\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = build_model()\n",
    "model.compile(loss='cosine_proximity', optimizer='adam', metrics=['mse'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_dir = os.path.join(os.getcwd(), 'saved_models/zsl/%s/type1/seed%s/%s_transfer/' % (model_type, seed, embedding_type))\n",
    "model_name = 'cifar100_%s_model.{epoch:03d}.h5' % (model_type)\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "filepath = os.path.join(save_dir, model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_schedule(epoch):\n",
    "    \"\"\"Learning Rate Schedule\n",
    "    Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\n",
    "    Called automatically every epoch as part of callbacks during training.\n",
    "    # Arguments\n",
    "        epoch (int): The number of epochs\n",
    "    # Returns\n",
    "        lr (float32): learning rate\n",
    "    \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 120:\n",
    "        lr *= 0.5e-3\n",
    "    elif epoch > 80:\n",
    "        lr *= 1e-3\n",
    "    elif epoch > 40:\n",
    "        lr *= 1e-2\n",
    "    elif epoch > 20:\n",
    "        lr *= 1e-1\n",
    "    print('Learning rate: ', lr)\n",
    "\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = ModelCheckpoint(filepath=filepath,\n",
    "                             monitor='val_loss',\n",
    "                             verbose=1,\n",
    "                             save_best_only=True)\n",
    "\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=5,\n",
    "                               min_lr=0.5e-6)\n",
    "\n",
    "callbacks = [checkpoint, lr_reducer, lr_scheduler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 8000 samples\n",
      "Epoch 1/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 6s 151us/step - loss: -0.6321 - mean_squared_error: 1.9111 - val_loss: -0.6750 - val_mean_squared_error: 1.3013\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to -0.67504, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed4/glove_transfer/cifar100_ResNet56v2_model.001.h5\n",
      "Epoch 2/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 113us/step - loss: -0.8075 - mean_squared_error: 1.0527 - val_loss: -0.6878 - val_mean_squared_error: 1.3003\n",
      "\n",
      "Epoch 00002: val_loss improved from -0.67504 to -0.68780, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed4/glove_transfer/cifar100_ResNet56v2_model.002.h5\n",
      "Epoch 3/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 4s 112us/step - loss: -0.8605 - mean_squared_error: 1.2503 - val_loss: -0.7066 - val_mean_squared_error: 1.6176\n",
      "\n",
      "Epoch 00003: val_loss improved from -0.68780 to -0.70657, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed4/glove_transfer/cifar100_ResNet56v2_model.003.h5\n",
      "Epoch 4/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 116us/step - loss: -0.8935 - mean_squared_error: 1.4319 - val_loss: -0.7155 - val_mean_squared_error: 1.4241\n",
      "\n",
      "Epoch 00004: val_loss improved from -0.70657 to -0.71546, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed4/glove_transfer/cifar100_ResNet56v2_model.004.h5\n",
      "Epoch 5/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 4s 109us/step - loss: -0.9142 - mean_squared_error: 1.5657 - val_loss: -0.7152 - val_mean_squared_error: 1.6935\n",
      "\n",
      "Epoch 00005: val_loss did not improve from -0.71546\n",
      "Epoch 6/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 4s 110us/step - loss: -0.9286 - mean_squared_error: 1.6319 - val_loss: -0.7145 - val_mean_squared_error: 1.8390\n",
      "\n",
      "Epoch 00006: val_loss did not improve from -0.71546\n",
      "Epoch 7/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 4s 110us/step - loss: -0.9387 - mean_squared_error: 1.6804 - val_loss: -0.7097 - val_mean_squared_error: 2.0277\n",
      "\n",
      "Epoch 00007: val_loss did not improve from -0.71546\n",
      "Epoch 8/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 113us/step - loss: -0.9470 - mean_squared_error: 1.7169 - val_loss: -0.7105 - val_mean_squared_error: 1.9179\n",
      "\n",
      "Epoch 00008: val_loss did not improve from -0.71546\n",
      "Epoch 9/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 4s 110us/step - loss: -0.9525 - mean_squared_error: 1.7562 - val_loss: -0.7155 - val_mean_squared_error: 1.9623\n",
      "\n",
      "Epoch 00009: val_loss improved from -0.71546 to -0.71552, saving model to /home/tliu/Dev/CMU/10-715/10715_Project/saved_models/zsl/ResNet56v2/type1/seed4/glove_transfer/cifar100_ResNet56v2_model.009.h5\n",
      "Epoch 10/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 113us/step - loss: -0.9570 - mean_squared_error: 1.7895 - val_loss: -0.7149 - val_mean_squared_error: 2.1801\n",
      "\n",
      "Epoch 00010: val_loss did not improve from -0.71552\n",
      "Epoch 11/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 5s 113us/step - loss: -0.9624 - mean_squared_error: 1.8155 - val_loss: -0.7043 - val_mean_squared_error: 1.8071\n",
      "\n",
      "Epoch 00011: val_loss did not improve from -0.71552\n",
      "Epoch 12/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 4s 110us/step - loss: -0.9656 - mean_squared_error: 1.8477 - val_loss: -0.7104 - val_mean_squared_error: 2.0882\n",
      "\n",
      "Epoch 00012: val_loss did not improve from -0.71552\n",
      "Epoch 13/20\n",
      "Learning rate:  0.001\n",
      "40000/40000 [==============================] - 4s 111us/step - loss: -0.9640 - mean_squared_error: 1.8976 - val_loss: -0.7064 - val_mean_squared_error: 2.7793\n",
      "\n",
      "Epoch 00013: val_loss did not improve from -0.71552\n",
      "Epoch 14/20\n",
      "Learning rate:  0.001\n",
      "19456/40000 [=============>................] - ETA: 2s - loss: -0.9705 - mean_squared_error: 1.9186"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-977d62f2d6c5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m           \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m           \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m           \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m          )\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1397\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1398\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1399\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1400\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1401\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(X_train_feat, y_train_embeddings,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          validation_data=(X_test_seen_feat, y_test_seen_embeddings),\n",
    "          shuffle=True,\n",
    "          callbacks=callbacks,\n",
    "          verbose=1\n",
    "         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model_filepath = os.path.join(save_dir, 'cifar100_%s_model.%03d.h5' % (model_type, 9))\n",
    "best_model = load_model(best_model_filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Top K Predictions\n",
    "def pred_top_k(y_test_pred, label_embeddings_arr=label_embeddings_arr, k=5):\n",
    "    sim_table = cosine_similarity(y_test_pred, label_embeddings_arr)\n",
    "    top_k_guesses = np.argpartition(sim_table,range(99-k+1,100),axis=1)[:,99-k+1:]\n",
    "    return sim_table, top_k_guesses\n",
    "\n",
    "#Top k Accuracy\n",
    "def calc_top_k_acc(top_k, y_test):\n",
    "    correct = 0\n",
    "    for i in range(y_test.shape[0]):\n",
    "        if np.squeeze(y_test)[i] in top_k[i]:\n",
    "            correct += 1\n",
    "    return correct/float(y_test.shape[0])\n",
    "\n",
    "def evaluate(model, x_test, y_test, label_embeddings_arr, k=5):\n",
    "    \n",
    "    y_test_pred = model.predict(x_test)\n",
    "    sim_table, top_k_guesses = pred_top_k(y_test_pred, label_embeddings_arr, k=k)\n",
    "\n",
    "    #Top prediction\n",
    "    label_predictions = sim_table.argmax(axis=1)\n",
    "    \n",
    "    #Accuracy\n",
    "    acc = np.sum((np.squeeze(y_test) == label_predictions)) / float(y_test.shape[0])\n",
    "    top_k_acc = calc_top_k_acc(top_k_guesses, y_test)\n",
    "    print(\"Accuracy: \" + str(acc))\n",
    "    print(\"Top \" + str(k) + \" Accuracy: \" + str(top_k_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.498\n",
      "Top 5 Accuracy: 0.661125\n"
     ]
    }
   ],
   "source": [
    "evaluate(best_model, X_test_seen_feat, y_test_seen, label_embeddings_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.24266666666666667\n",
      "Top 5 Accuracy: 0.5626666666666666\n"
     ]
    }
   ],
   "source": [
    "# Regular ZSL setting where we only need to consider choose between the 20 unseen classes, rather than all 100\n",
    "\n",
    "# Just replace the word vectors for \"seen\" labels with something really far so it won't be close to any predicted vector\n",
    "unseen_labels = np.unique(y_test_unseen)\n",
    "label_embeddings_arr_unseen = np.copy(label_embeddings_arr)\n",
    "for i in range(100):\n",
    "    if i in unseen_labels:\n",
    "        continue\n",
    "    label_embeddings_arr_unseen[i] = np.ones(label_embeddings_arr[0].shape) * 1000\n",
    "\n",
    "evaluate(best_model, X_test_unseen_feat, y_test_unseen, label_embeddings_arr_unseen) # Significant improvement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.0004166666666666667\n",
      "Top 5 Accuracy: 0.186\n"
     ]
    }
   ],
   "source": [
    "# Although accuracy is still 0%, we seen an improvement in top 5 % accuracy (0% -> 15%). Sign that w2v is useful for ZSL\n",
    "\n",
    "evaluate(best_model, X_test_unseen_feat, y_test_unseen, label_embeddings_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nx = np.linspace(0,99,100) \\ny = np.zeros(100)\\ny_test_pred = model.predict(x_test)\\n\\nfor i in range(100):\\n    k = i+1\\n    sim_table, top_k_guesses = pred_top_k(y_test_pred, k=k)\\n    y[i] = calc_top_k_acc(top_k_guesses, y_test)\\n    \\nplt.plot(x,y)\\nplt.title(\"Top K Accuracy\")\\n'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "x = np.linspace(0,99,100) \n",
    "y = np.zeros(100)\n",
    "y_test_pred = model.predict(x_test)\n",
    "\n",
    "for i in range(100):\n",
    "    k = i+1\n",
    "    sim_table, top_k_guesses = pred_top_k(y_test_pred, k=k)\n",
    "    y[i] = calc_top_k_acc(top_k_guesses, y_test)\n",
    "    \n",
    "plt.plot(x,y)\n",
    "plt.title(\"Top K Accuracy\")\n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}