Using Inception  and PyTorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import torch\n",
    "import cv2\n",
    "from glob import glob\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torchvision.models as models\n",
    "from torch.autograd import Variable\n",
    "import torchvision.transforms as transforms\n",
    "from PIL import Image, ImageFile\n",
    "from torch.utils.data import TensorDataset, DataLoader,Dataset\n",
    "from sklearn.model_selection import train_test_split\n",
    "import copy \n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "ImageFile.LOAD_TRUNCATED_IMAGES = True\n",
    "#print(os.listdir(\"../input\"))\n",
    "#../input/train/train\n",
    "#../input/train/test\n",
    "\n",
    "# Any results you write to the current directory are saved as output.\n",
    "use_cuda = torch.cuda.is_available()\n",
    "if not use_cuda:\n",
    "    print('No GPU found. Please use a GPU to train your neural network.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('train.csv')\n",
    "df_test = pd.read_csv('sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18528, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_id  category\n",
       "0         0        77\n",
       "1         1        81\n",
       "2         2        52\n",
       "3         3        72\n",
       "4         4        58"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = df_train[:18528]\n",
    "print(df_train.shape)\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOTE: class is inherited from Dataset\n",
    "class ImageLabelDataset(Dataset):\n",
    "    def __init__(self, df_data, prediction, folder=\"train\", transformtype=\"train\"):\n",
    "        super().__init__()\n",
    "        self.df = df_data.values\n",
    "        self.prediction = prediction.values\n",
    "        self.folder = folder\n",
    "        self.transformtype = transformtype\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        tensorimage = self.preprocess_image(self.df[index], self.transformtype)\n",
    "        label = self.prediction[index]\n",
    "        label_tensor = self.get_dummies(label)\n",
    "        #label_tensor = label\n",
    "        #x = np.squeeze(label_tensor.detach().cpu().numpy())\n",
    "        #print(x.argsort()[-6:][::-1])\n",
    "        #print(len(label_tensor))\n",
    "        #print(label_tensor.shape)\n",
    "        return tensorimage, label_tensor\n",
    "    \n",
    "    def preprocess_image(self, img_path, transformtype):\n",
    "        data_transform = transforms.Compose([transforms.ToPILImage(),\n",
    "                                             transforms.Scale(299),\n",
    "                                             transforms.RandomResizedCrop(299), \n",
    "                                             transforms.RandomHorizontalFlip(),\n",
    "                                             transforms.RandomVerticalFlip(),\n",
    "                                             transforms.RandomRotation(degrees=30), \n",
    "                                             transforms.ToTensor(),\n",
    "                                             transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "                                            ])\n",
    "        \n",
    "        test_transform = transforms.Compose([transforms.ToPILImage(),\n",
    "                                             transforms.Scale(299),\n",
    "                                             transforms.CenterCrop(299),\n",
    "                                             transforms.ToTensor(),\n",
    "                                             transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "                                            ])\n",
    "        image = cv2.imread(\"{}/{}.jpg\".format(self.folder, img_path))\n",
    "        if(transformtype == \"train\"):\n",
    "            image = data_transform(image)\n",
    "        else:\n",
    "            image = test_transform(image)\n",
    "        return image\n",
    "    \n",
    "    def get_dummies(self, attribute_id):\n",
    "        label_tensor = torch.zeros((1, 103))\n",
    "        #for label in attribute_id.split():\n",
    "        label_tensor[0, int(attribute_id)-1] = 1\n",
    "        return label_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_train.head()\n",
    "batch_size = 32\n",
    "train_image = df_train[\"image_id\"]\n",
    "#target = df_train.drop(['id', 'attribute_ids'],axis=1)\n",
    "target = df_train[\"category\"]\n",
    "X_train, X_val, y_train, y_val = train_test_split(train_image, target, random_state=42, test_size=0.1)\n",
    "\n",
    "test_image = df_test[\"image_id\"]\n",
    "test_target = df_test[\"category\"]\n",
    "#test_target = df_test.drop(['id', 'attribute_ids'],axis=1)\n",
    "\n",
    "train_set = ImageLabelDataset(df_data=X_train, prediction=y_train, folder=\"train\", transformtype=\"train\")\n",
    "val_set = ImageLabelDataset(df_data=X_val, prediction=y_val, folder=\"train\", transformtype=\"test\")\n",
    "predict_set = ImageLabelDataset(df_data=test_image, prediction=test_target, folder=\"test\", transformtype=\"test\")\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=1)\n",
    "val_loader = torch.utils.data.DataLoader(val_set, batch_size=batch_size, shuffle=True, num_workers=1)\n",
    "test_val_loader = torch.utils.data.DataLoader(val_set, batch_size=1, shuffle=False, num_workers=1)\n",
    "test_loader = torch.utils.data.DataLoader(predict_set, batch_size=1, num_workers=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "n_output=1\n",
    "# Number of Epochs\n",
    "num_epochs = 10\n",
    "# Learning Rate\n",
    "learning_rate = 0.0001\n",
    "# Model parameters\n",
    "\n",
    "# Show stats for every n number of batches\n",
    "show_every_n_batches = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_rnn(model, batch_size, optimizer, criterion, n_epochs, show_every_n_batches=100):\n",
    "    train_loss = 0\n",
    "    valid_loss = 0\n",
    "    batch_losses = []\n",
    "    val_batch_losses = []\n",
    "    valid_loss_min = np.Inf\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    previousLoss = np.Inf\n",
    "    minLoss = np.Inf\n",
    "\n",
    "    print(\"Training for %d epoch(s)...\" % n_epochs)\n",
    "    for epoch_i in range(1, n_epochs + 1):\n",
    "        model.train()\n",
    "        for batch_idx, (data, target) in enumerate(train_loader):\n",
    "            #print(batch_idx + len(train_loader.dataset))\n",
    "            if use_cuda:\n",
    "                data, target = data.cuda(), target.cuda()\n",
    "            # clear the gradients of all optimized variables\n",
    "            target = np.squeeze(target)\n",
    "            #target = target.view(1,-1)\n",
    "            target= target.long()\n",
    "            #print(np.squeeze(target))\n",
    "            #targs = target.view(-1).long()\n",
    "            optimizer.zero_grad()\n",
    "            # forward pass: Train compute predicted outputs by passing inputs to the model\n",
    "            output = model.forward(data)\n",
    "            output, _ = output\n",
    "            #print(output.squeeze(1))\n",
    "            #print(type(output))\n",
    "            #_, preds = torch.max(output.data, 1)\n",
    "            loss = criterion(output, torch.max(target, 1)[1])\n",
    "            # backward pass: compute gradient of the loss with respect to model parameters\n",
    "            loss.backward()\n",
    "            # perform a single optimization step (parameter update)\n",
    "            optimizer.step()\n",
    "            #print(loss)\n",
    "            # update training loss\n",
    "            ## find the loss and update the model parameters accordingly       \n",
    "            train_loss += loss.item()\n",
    "            #print(loss.item())\n",
    "            \n",
    "        model.eval()\n",
    "        for batch_idx, (data, target) in enumerate(val_loader):\n",
    "            # move to GPU\n",
    "            if use_cuda:\n",
    "                data, target = data.cuda(), target.cuda()\n",
    "            ## update the average validation loss\n",
    "            # forward pass: compute predicted outputs by passing inputs to the model\n",
    "            target = np.squeeze(target)\n",
    "            target= target.long()\n",
    "            #target = target.view(1,-1)\n",
    "            \n",
    "            output = model.forward(data)\n",
    "            #print(output)\n",
    "            #print(target)\n",
    "            #output, _ = output\n",
    "            # calculate the Val batch loss\n",
    "            loss = criterion(output, torch.max(target, 1)[1])\n",
    "            # update average validation loss \n",
    "            #val_batch_losses.append(loss.item())\n",
    "            #valid_loss += ((1 / (batch_idx + 1)) * (loss.item() - valid_loss))\n",
    "            valid_loss += loss.item()\n",
    "            #print(loss.item())\n",
    "        \n",
    "        valid_loss = valid_loss/len(val_loader.dataset)\n",
    "        train_loss = train_loss/len(train_loader.dataset)\n",
    "        \n",
    "        # print training/validation statistics \n",
    "        if epoch_i%show_every_n_batches == 0:\n",
    "            print('Epoch: {} \\tTraining Loss: {:.6f} \\tValidation Loss: {:.6f}'.format(epoch_i, train_loss, valid_loss))\n",
    "        \n",
    "            ## TODO: save the model if validation loss has decreased\n",
    "            # save model if validation loss has decreased\n",
    "            if valid_loss < valid_loss_min:\n",
    "                print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
    "                valid_loss_min,\n",
    "                valid_loss))\n",
    "                with open('trained_rnn_new', 'wb') as pickle_file:\n",
    "                    torch.save(model.state_dict(), 'save/trained_rnn_new')\n",
    "                valid_loss_min = valid_loss\n",
    "                train_loss = 0\n",
    "                valid_loss = 0\n",
    "                #batch_losses = []\n",
    "                #val_batch_losses = []\n",
    "  \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inception3(\n",
      "  (Conv2d_1a_3x3): BasicConv2d(\n",
      "    (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "    (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (Conv2d_2a_3x3): BasicConv2d(\n",
      "    (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (Conv2d_2b_3x3): BasicConv2d(\n",
      "    (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (Conv2d_3b_1x1): BasicConv2d(\n",
      "    (conv): Conv2d(64, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(80, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (Conv2d_4a_3x3): BasicConv2d(\n",
      "    (conv): Conv2d(80, 192, kernel_size=(3, 3), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (Mixed_5b): InceptionA(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_1): BasicConv2d(\n",
      "      (conv): Conv2d(192, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_2): BasicConv2d(\n",
      "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_5c): InceptionA(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_1): BasicConv2d(\n",
      "      (conv): Conv2d(256, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_2): BasicConv2d(\n",
      "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_5d): InceptionA(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_1): BasicConv2d(\n",
      "      (conv): Conv2d(288, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_2): BasicConv2d(\n",
      "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6a): InceptionB(\n",
      "    (branch3x3): BasicConv2d(\n",
      "      (conv): Conv2d(288, 384, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6b): InceptionC(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d(\n",
      "      (conv): Conv2d(128, 128, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d(\n",
      "      (conv): Conv2d(128, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(128, 128, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(128, 128, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d(\n",
      "      (conv): Conv2d(128, 128, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d(\n",
      "      (conv): Conv2d(128, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6c): InceptionC(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d(\n",
      "      (conv): Conv2d(160, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d(\n",
      "      (conv): Conv2d(160, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6d): InceptionC(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d(\n",
      "      (conv): Conv2d(160, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d(\n",
      "      (conv): Conv2d(160, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6e): InceptionC(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (AuxLogits): InceptionAux(\n",
      "    (conv0): BasicConv2d(\n",
      "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (conv1): BasicConv2d(\n",
      "      (conv): Conv2d(128, 768, kernel_size=(5, 5), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(768, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (fc): Linear(in_features=768, out_features=1000, bias=True)\n",
      "  )\n",
      "  (Mixed_7a): InceptionD(\n",
      "    (branch3x3_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_2): BasicConv2d(\n",
      "      (conv): Conv2d(192, 320, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7x3_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7x3_2): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7x3_3): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7x3_4): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_7b): InceptionE(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(1280, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_1): BasicConv2d(\n",
      "      (conv): Conv2d(1280, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_2a): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_2b): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(1280, 448, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(448, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(448, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3a): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3b): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(1280, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_7c): InceptionE(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(2048, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_1): BasicConv2d(\n",
      "      (conv): Conv2d(2048, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_2a): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_2b): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(2048, 448, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(448, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(448, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3a): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3b): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(2048, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=2048, out_features=1000, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model_transfer = models.inception_v3(pretrained=True)\n",
    "print(model_transfer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for 10 epoch(s)...\n",
      "Epoch: 1 \tTraining Loss: 0.048963 \tValidation Loss: 0.035549\n",
      "Validation loss decreased (inf --> 0.035549).  Saving model ...\n",
      "Epoch: 2 \tTraining Loss: 0.048180 \tValidation Loss: 0.035746\n",
      "Epoch: 3 \tTraining Loss: 0.048594 \tValidation Loss: 0.035054\n",
      "Validation loss decreased (0.035549 --> 0.035054).  Saving model ...\n",
      "Epoch: 4 \tTraining Loss: 0.048142 \tValidation Loss: 0.035022\n",
      "Validation loss decreased (0.035054 --> 0.035022).  Saving model ...\n",
      "Epoch: 5 \tTraining Loss: 0.048809 \tValidation Loss: 0.032463\n",
      "Validation loss decreased (0.035022 --> 0.032463).  Saving model ...\n",
      "Epoch: 6 \tTraining Loss: 0.047482 \tValidation Loss: 0.034268\n",
      "Epoch: 7 \tTraining Loss: 0.048381 \tValidation Loss: 0.034375\n",
      "Epoch: 8 \tTraining Loss: 0.048544 \tValidation Loss: 0.033418\n",
      "Epoch: 9 \tTraining Loss: 0.047943 \tValidation Loss: 0.035477\n",
      "Epoch: 10 \tTraining Loss: 0.047770 \tValidation Loss: 0.035886\n"
     ]
    }
   ],
   "source": [
    "# Freeze training for all \"features\" layers\n",
    "#for param in model_transfer.features.parameters():\n",
    "    #param.requires_grad = False\n",
    "\n",
    "## Freezing all layers\n",
    "for params in model_transfer.parameters():\n",
    "    params.requires_grad = False\n",
    "\n",
    "## Freezing the first few layers. Here I am freezing the first 7 layers \n",
    "        \n",
    "custom_model = nn.Sequential(nn.Linear(2048, 1024), \n",
    "                  nn.ReLU(),\n",
    "                  nn.Dropout(p=0.5), \n",
    "                  nn.Linear(1024, 102)\n",
    "                 )\n",
    "\n",
    "model_transfer.fc = custom_model\n",
    "#model_transfer.classifier = custom_model\n",
    "\n",
    "model_transfer.load_state_dict(torch.load('save/trained_rnn_new'))\n",
    "\n",
    "if use_cuda:\n",
    "    model_transfer = model_transfer.cuda()\n",
    "\n",
    "# print(model_transfer)\n",
    "\n",
    "# specify loss function\n",
    "criterion_scratch = nn.CrossEntropyLoss()\n",
    "#criterion_scratch = nn.BCELoss()\n",
    "#criterion_scratch = nn.BCELoss(reduction=\"mean\").to('cuda:0')\n",
    "\n",
    "# specify optimizer\n",
    "optimizer_scratch = optim.SGD(model_transfer.parameters(), lr=learning_rate, momentum=0.9)\n",
    "#optimizer_scratch = optim.Adam(model_transfer.classifier.parameters(), lr=learning_rate)\n",
    "#optimizer_scratch = optim.SGD(list(filter(lambda p: p.requires_grad, model_transfer.parameters())), lr=learning_rate, momentum=0.9)\n",
    "\n",
    "\n",
    "trained_rnn = train_rnn(model_transfer, batch_size, optimizer_scratch, criterion_scratch, num_epochs, show_every_n_batches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Inception3(\n",
       "  (Conv2d_1a_3x3): BasicConv2d(\n",
       "    (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
       "    (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (Conv2d_2a_3x3): BasicConv2d(\n",
       "    (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), bias=False)\n",
       "    (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (Conv2d_2b_3x3): BasicConv2d(\n",
       "    (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (Conv2d_3b_1x1): BasicConv2d(\n",
       "    (conv): Conv2d(64, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "    (bn): BatchNorm2d(80, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (Conv2d_4a_3x3): BasicConv2d(\n",
       "    (conv): Conv2d(80, 192, kernel_size=(3, 3), stride=(1, 1), bias=False)\n",
       "    (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (Mixed_5b): InceptionA(\n",
       "    (branch1x1): BasicConv2d(\n",
       "      (conv): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch5x5_1): BasicConv2d(\n",
       "      (conv): Conv2d(192, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch5x5_2): BasicConv2d(\n",
       "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_1): BasicConv2d(\n",
       "      (conv): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_2): BasicConv2d(\n",
       "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_3): BasicConv2d(\n",
       "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch_pool): BasicConv2d(\n",
       "      (conv): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (Mixed_5c): InceptionA(\n",
       "    (branch1x1): BasicConv2d(\n",
       "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch5x5_1): BasicConv2d(\n",
       "      (conv): Conv2d(256, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch5x5_2): BasicConv2d(\n",
       "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_1): BasicConv2d(\n",
       "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_2): BasicConv2d(\n",
       "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_3): BasicConv2d(\n",
       "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch_pool): BasicConv2d(\n",
       "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (Mixed_5d): InceptionA(\n",
       "    (branch1x1): BasicConv2d(\n",
       "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch5x5_1): BasicConv2d(\n",
       "      (conv): Conv2d(288, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch5x5_2): BasicConv2d(\n",
       "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_1): BasicConv2d(\n",
       "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_2): BasicConv2d(\n",
       "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_3): BasicConv2d(\n",
       "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch_pool): BasicConv2d(\n",
       "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (Mixed_6a): InceptionB(\n",
       "    (branch3x3): BasicConv2d(\n",
       "      (conv): Conv2d(288, 384, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_1): BasicConv2d(\n",
       "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_2): BasicConv2d(\n",
       "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_3): BasicConv2d(\n",
       "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
       "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (Mixed_6b): InceptionC(\n",
       "    (branch1x1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_2): BasicConv2d(\n",
       "      (conv): Conv2d(128, 128, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_3): BasicConv2d(\n",
       "      (conv): Conv2d(128, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_2): BasicConv2d(\n",
       "      (conv): Conv2d(128, 128, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_3): BasicConv2d(\n",
       "      (conv): Conv2d(128, 128, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_4): BasicConv2d(\n",
       "      (conv): Conv2d(128, 128, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_5): BasicConv2d(\n",
       "      (conv): Conv2d(128, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch_pool): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (Mixed_6c): InceptionC(\n",
       "    (branch1x1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_2): BasicConv2d(\n",
       "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_3): BasicConv2d(\n",
       "      (conv): Conv2d(160, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_2): BasicConv2d(\n",
       "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_3): BasicConv2d(\n",
       "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_4): BasicConv2d(\n",
       "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_5): BasicConv2d(\n",
       "      (conv): Conv2d(160, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch_pool): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (Mixed_6d): InceptionC(\n",
       "    (branch1x1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_2): BasicConv2d(\n",
       "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_3): BasicConv2d(\n",
       "      (conv): Conv2d(160, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_2): BasicConv2d(\n",
       "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_3): BasicConv2d(\n",
       "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_4): BasicConv2d(\n",
       "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_5): BasicConv2d(\n",
       "      (conv): Conv2d(160, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch_pool): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (Mixed_6e): InceptionC(\n",
       "    (branch1x1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_2): BasicConv2d(\n",
       "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7_3): BasicConv2d(\n",
       "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_2): BasicConv2d(\n",
       "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_3): BasicConv2d(\n",
       "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_4): BasicConv2d(\n",
       "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7dbl_5): BasicConv2d(\n",
       "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch_pool): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (AuxLogits): InceptionAux(\n",
       "    (conv0): BasicConv2d(\n",
       "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (conv1): BasicConv2d(\n",
       "      (conv): Conv2d(128, 768, kernel_size=(5, 5), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(768, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (fc): Linear(in_features=768, out_features=1000, bias=True)\n",
       "  )\n",
       "  (Mixed_7a): InceptionD(\n",
       "    (branch3x3_1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3_2): BasicConv2d(\n",
       "      (conv): Conv2d(192, 320, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
       "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7x3_1): BasicConv2d(\n",
       "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7x3_2): BasicConv2d(\n",
       "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7x3_3): BasicConv2d(\n",
       "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch7x7x3_4): BasicConv2d(\n",
       "      (conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (Mixed_7b): InceptionE(\n",
       "    (branch1x1): BasicConv2d(\n",
       "      (conv): Conv2d(1280, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3_1): BasicConv2d(\n",
       "      (conv): Conv2d(1280, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3_2a): BasicConv2d(\n",
       "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3_2b): BasicConv2d(\n",
       "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_1): BasicConv2d(\n",
       "      (conv): Conv2d(1280, 448, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(448, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_2): BasicConv2d(\n",
       "      (conv): Conv2d(448, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_3a): BasicConv2d(\n",
       "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_3b): BasicConv2d(\n",
       "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch_pool): BasicConv2d(\n",
       "      (conv): Conv2d(1280, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (Mixed_7c): InceptionE(\n",
       "    (branch1x1): BasicConv2d(\n",
       "      (conv): Conv2d(2048, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3_1): BasicConv2d(\n",
       "      (conv): Conv2d(2048, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3_2a): BasicConv2d(\n",
       "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3_2b): BasicConv2d(\n",
       "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_1): BasicConv2d(\n",
       "      (conv): Conv2d(2048, 448, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(448, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_2): BasicConv2d(\n",
       "      (conv): Conv2d(448, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_3a): BasicConv2d(\n",
       "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch3x3dbl_3b): BasicConv2d(\n",
       "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
       "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (branch_pool): BasicConv2d(\n",
       "      (conv): Conv2d(2048, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (fc): Sequential(\n",
       "    (0): Linear(in_features=2048, out_features=1024, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Dropout(p=0.5)\n",
       "    (3): Linear(in_features=1024, out_features=102, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_transfer.load_state_dict(torch.load('save/trained_rnn_new'))\n",
    "model_transfer.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[57, 80, 47, 12, 30, 73, 26, 51, 72, 94, 76, 57, 81, 43, 46, 58, 46, 94, 72, 77, 8, 98, 81, 98, 24, 42, 76, 15, 11, 95, 37, 37, 83, 73, 73, 19, 5, 75, 2, 91, 76, 8, 77, 51, 56, 89, 16, 28, 77, 55, 41, 83, 31, 73, 73, 44, 49, 60, 70, 52, 83, 89, 56, 63, 26, 46, 13, 75, 77, 78, 19, 43, 8, 52, 88, 25, 44, 94, 71, 84, 56, 78, 5, 48, 98, 8, 11, 44, 36, 50, 76, 79, 8, 65, 43, 58, 12, 72, 14, 5]\n",
      "[56, 79, 73, 11, 29, 72, 25, 50, 71, 93, 54, 56, 94, 42, 10, 52, 14, 93, 74, 76, 7, 97, 80, 69, 49, 41, 75, 41, 10, 71, 36, 36, 82, 80, 72, 18, 4, 74, 1, 90, 75, 7, 76, 75, 95, 88, 15, 27, 76, 54, 40, 82, 30, 77, 6, 43, 48, 59, 69, 51, 82, 88, 55, 62, 25, 45, 12, 74, 76, 86, 18, 23, 7, 51, 87, 24, 43, 93, 70, 83, 55, 77, 4, 15, 97, 7, 50, 43, 35, 49, 75, 78, 16, 41, 52, 57, 11, 71, 13, 26]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.4648664587566038"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "preds = []\n",
    "targetpred = []\n",
    "for batch_idx, (data, target) in enumerate(test_val_loader):\n",
    "    # move to GPU\n",
    "    #print(np.squeeze(target.detach().cpu().numpy()))\n",
    "    if use_cuda:\n",
    "        data, target = data.cuda(), target.cuda()\n",
    "    ## update the average validation loss\n",
    "    # forward pass: compute predicted outputs by passing inputs to the model\n",
    "    target = np.squeeze(target)\n",
    "    #target= target.long()\n",
    "    output = model_transfer.forward(data)    \n",
    "    #print(target)\n",
    "    #print(torch.max(target, 0))\n",
    "    #print(output)\n",
    "    #print(torch.max(output, 1)[1].detach().cpu().numpy())\n",
    "    #values, indices = output.max(1)    \n",
    "    pr = torch.max(output, 1)[1].detach().cpu().numpy()\n",
    "    tpr = torch.max(target, 0)[1].detach().cpu().numpy()\n",
    "    #print(tpr)\n",
    "    preds.append(int(pr[0]) + 1)\n",
    "    targetpred.append(int(tpr))\n",
    "    #print(pr.detach().cpu())\n",
    "print(preds[:100])\n",
    "print(targetpred[:100])\n",
    "r2_score(targetpred, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   image_id  category\n",
      "0     18540         9\n",
      "1     18541        40\n",
      "2     18542        78\n",
      "3     18543        72\n",
      "4     18544        44\n"
     ]
    }
   ],
   "source": [
    "preds = []\n",
    "np.set_printoptions(threshold=100)\n",
    "submission = pd.read_csv('sample_submission.csv')\n",
    "for batch_idx, (data, target) in enumerate(test_loader):\n",
    "    # move to GPU\n",
    "    #print(np.squeeze(target.detach().cpu().numpy()))\n",
    "    if use_cuda:\n",
    "        data, target = data.cuda(), target.cuda()\n",
    "    ## update the average validation loss\n",
    "    # forward pass: compute predicted outputs by passing inputs to the model\n",
    "    output = model_transfer.forward(data)\n",
    "    #print(target)\n",
    "    #print(output)\n",
    "    #print(torch.max(output, 1)[1].detach().cpu().numpy())\n",
    "    #values, indices = output.max(1)    \n",
    "    pr = torch.max(output, 1)[1].detach().cpu().numpy()\n",
    "    preds.append(int(pr[0]) + 1)\n",
    "    #for i in pr:\n",
    "        #preds.append(i)\n",
    "    #print(batch_idx)\n",
    "    #print(pr)\n",
    "    #print(x.argsort()[-6:][::-1])\n",
    "    #print(pr.argsort()[-6:][::-1])\n",
    "    #print(values)\n",
    "    # calculate the batch loss\n",
    "submission[\"category\"] = preds\n",
    "print(submission.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}