Using Inception  and PyTorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import torch\n",
    "import cv2\n",
    "from glob import glob\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torchvision.models as models\n",
    "from torch.autograd import Variable\n",
    "import torchvision.transforms as transforms\n",
    "from PIL import Image, ImageFile\n",
    "from torch.utils.data import TensorDataset, DataLoader,Dataset\n",
    "from sklearn.model_selection import train_test_split\n",
    "import copy \n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "ImageFile.LOAD_TRUNCATED_IMAGES = True\n",
    "#print(os.listdir(\"../input\"))\n",
    "#../input/train/train\n",
    "#../input/train/test\n",
    "\n",
    "# Any results you write to the current directory are saved as output.\n",
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "if not use_cuda:\n",
    "    print('No GPU found. Please use a GPU to train your neural network.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('train.csv')\n",
    "df_test = pd.read_csv('sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18528, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_id  category\n",
       "0         0        77\n",
       "1         1        81\n",
       "2         2        52\n",
       "3         3        72\n",
       "4         4        58"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = df_train[:18528]\n",
    "print(df_train.shape)\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOTE: class is inherited from Dataset\n",
    "class ImageLabelDataset(Dataset):\n",
    "    def __init__(self, df_data, prediction, folder=\"train\", transformtype=\"train\"):\n",
    "        super().__init__()\n",
    "        self.df = df_data.values\n",
    "        self.prediction = prediction.values\n",
    "        self.folder = folder\n",
    "        self.transformtype = transformtype\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        tensorimage = self.preprocess_image(self.df[index], self.transformtype)\n",
    "        label = self.prediction[index]\n",
    "        label_tensor = self.get_dummies(label)\n",
    "        #label_tensor = [int(label)]\n",
    "        #x = np.squeeze(label_tensor.detach().cpu().numpy())\n",
    "        #print(x.argsort()[-6:][::-1])\n",
    "        #print(len(label_tensor))\n",
    "        #print(label_tensor.shape)\n",
    "        return tensorimage, label_tensor\n",
    "    \n",
    "    def preprocess_image(self, img_path, transformtype):\n",
    "        data_transform = transforms.Compose([transforms.ToPILImage(),\n",
    "                                             transforms.RandomResizedCrop(299), \n",
    "                                             transforms.RandomHorizontalFlip(),\n",
    "                                             transforms.RandomVerticalFlip(),\n",
    "                                             transforms.RandomRotation(degrees=30), \n",
    "                                             transforms.ToTensor(),\n",
    "                                             transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "                                            ])\n",
    "        \n",
    "        test_transform = transforms.Compose([transforms.ToPILImage(),\n",
    "                                             transforms.Resize(299),\n",
    "                                             transforms.CenterCrop(299),\n",
    "                                             transforms.ToTensor(),\n",
    "                                             transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n",
    "                                            ])\n",
    "        image = cv2.imread(\"{}/{}.jpg\".format(self.folder, img_path))\n",
    "        if(transformtype == \"train\"):\n",
    "            image = data_transform(image)\n",
    "        else:\n",
    "            image = test_transform(image)\n",
    "        return image\n",
    "    \n",
    "    def get_dummies(self, attribute_id):\n",
    "        label_tensor = torch.zeros((1, 102))\n",
    "        #label_tensor = torch.zeros((1, 1))\n",
    "        #for label in attribute_id.split():\n",
    "        label_tensor[0, int(attribute_id)-1] = 1\n",
    "        #label_tensor[0, 0] = attribute_id\n",
    "        return label_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_train.head()\n",
    "batch_size = 32\n",
    "train_image = df_train[\"image_id\"]\n",
    "#target = df_train.drop(['id', 'attribute_ids'],axis=1)\n",
    "target = df_train[\"category\"]\n",
    "X_train, X_val, y_train, y_val = train_test_split(train_image, target, random_state=42, test_size=0.1)\n",
    "\n",
    "test_image = df_test[\"image_id\"]\n",
    "test_target = df_test[\"category\"]\n",
    "#test_target = df_test.drop(['id', 'attribute_ids'],axis=1)\n",
    "\n",
    "train_set = ImageLabelDataset(df_data=X_train, prediction=y_train, folder=\"train\", transformtype=\"train\")\n",
    "val_set = ImageLabelDataset(df_data=X_val, prediction=y_val, folder=\"train\", transformtype=\"test\")\n",
    "predict_set = ImageLabelDataset(df_data=test_image, prediction=test_target, folder=\"test\", transformtype=\"test\")\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=4)\n",
    "val_loader = torch.utils.data.DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "test_val_loader = torch.utils.data.DataLoader(val_set, batch_size=1, shuffle=False, num_workers=0)\n",
    "test_loader = torch.utils.data.DataLoader(predict_set, batch_size=1, num_workers=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "n_output=1\n",
    "# Number of Epochs\n",
    "num_epochs = 20\n",
    "# Learning Rate\n",
    "learning_rate = 0.01\n",
    "# Model parameters\n",
    "\n",
    "# Show stats for every n number of batches\n",
    "show_every_n_batches = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_rnn(model, batch_size, optimizer, criterion, n_epochs, show_every_n_batches=100):\n",
    "    train_loss = 0\n",
    "    valid_loss = 0\n",
    "    batch_losses = []\n",
    "    val_batch_losses = []\n",
    "    valid_loss_min = np.Inf\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    previousLoss = np.Inf\n",
    "    minLoss = np.Inf\n",
    "\n",
    "    print(\"Training for %d epoch(s)...\" % n_epochs)\n",
    "    for epoch_i in range(1, n_epochs + 1):\n",
    "        model.train()\n",
    "        for batch_idx, (data, target) in enumerate(train_loader):\n",
    "            #print(batch_idx + len(train_loader.dataset))\n",
    "            if use_cuda:\n",
    "                data, target = data.cuda(), target.cuda()\n",
    "            # clear the gradients of all optimized variables\n",
    "            target = np.squeeze(target)\n",
    "            #target = target.view(1,-1)\n",
    "            target= target.long()\n",
    "            #print(np.squeeze(target))\n",
    "            #targs = target.view(-1).long()\n",
    "            optimizer.zero_grad()\n",
    "            # forward pass: Train compute predicted outputs by passing inputs to the model\n",
    "            output = model.forward(data)\n",
    "            output, _ = output\n",
    "            #print(output.squeeze(1))\n",
    "            #print(target)\n",
    "            target = torch.max(target, 1)[1] + 1\n",
    "            #_, preds = torch.max(output.data, 1)\n",
    "            #loss = criterion(output, torch.max(target, 1)[1])\n",
    "            loss = criterion(output, target)\n",
    "            # backward pass: compute gradient of the loss with respect to model parameters\n",
    "            loss.backward()\n",
    "            # perform a single optimization step (parameter update)\n",
    "            optimizer.step()\n",
    "            #print(loss)\n",
    "            # update training loss\n",
    "            ## find the loss and update the model parameters accordingly       \n",
    "            train_loss += loss.item()\n",
    "            #print(loss.item())\n",
    "            \n",
    "        model.eval()\n",
    "        for batch_idx, (data, target) in enumerate(val_loader):\n",
    "            # move to GPU\n",
    "            if use_cuda:\n",
    "                data, target = data.cuda(), target.cuda()\n",
    "            ## update the average validation loss\n",
    "            # forward pass: compute predicted outputs by passing inputs to the model\n",
    "            target = np.squeeze(target)\n",
    "            target= target.long()\n",
    "            #target = target.view(1,-1)\n",
    "            target = torch.max(target, 1)[1] + 1\n",
    "            output = model.forward(data)\n",
    "            #print(output)\n",
    "            #print(target)\n",
    "            #output, _ = output\n",
    "            # calculate the Val batch loss\n",
    "            #loss = criterion(output, torch.max(target, 1)[1])\n",
    "            loss = criterion(output, target)\n",
    "            # update average validation loss \n",
    "            #val_batch_losses.append(loss.item())\n",
    "            #valid_loss += ((1 / (batch_idx + 1)) * (loss.item() - valid_loss))\n",
    "            valid_loss += loss.item()\n",
    "            #print(loss.item())\n",
    "        \n",
    "        valid_loss = valid_loss/len(val_loader.dataset)\n",
    "        train_loss = train_loss/len(train_loader.dataset)\n",
    "        \n",
    "        # print training/validation statistics \n",
    "        if epoch_i%show_every_n_batches == 0:\n",
    "            print('Epoch: {} \\tTraining Loss: {:.6f} \\tValidation Loss: {:.6f}'.format(epoch_i, train_loss, valid_loss))\n",
    "        \n",
    "            ## TODO: save the model if validation loss has decreased\n",
    "            # save model if validation loss has decreased\n",
    "            if valid_loss < valid_loss_min:\n",
    "                print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
    "                valid_loss_min,\n",
    "                valid_loss))\n",
    "                with open('trained_rnn_new', 'wb') as pickle_file:\n",
    "                    torch.save(model.state_dict(), 'save/trained_rnn_new')\n",
    "                valid_loss_min = valid_loss\n",
    "                train_loss = 0\n",
    "                valid_loss = 0\n",
    "                #batch_losses = []\n",
    "                #val_batch_losses = []\n",
    "  \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inception3(\n",
      "  (Conv2d_1a_3x3): BasicConv2d(\n",
      "    (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "    (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (Conv2d_2a_3x3): BasicConv2d(\n",
      "    (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (Conv2d_2b_3x3): BasicConv2d(\n",
      "    (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (Conv2d_3b_1x1): BasicConv2d(\n",
      "    (conv): Conv2d(64, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(80, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (Conv2d_4a_3x3): BasicConv2d(\n",
      "    (conv): Conv2d(80, 192, kernel_size=(3, 3), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (Mixed_5b): InceptionA(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_1): BasicConv2d(\n",
      "      (conv): Conv2d(192, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_2): BasicConv2d(\n",
      "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_5c): InceptionA(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_1): BasicConv2d(\n",
      "      (conv): Conv2d(256, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_2): BasicConv2d(\n",
      "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_5d): InceptionA(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_1): BasicConv2d(\n",
      "      (conv): Conv2d(288, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch5x5_2): BasicConv2d(\n",
      "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6a): InceptionB(\n",
      "    (branch3x3): BasicConv2d(\n",
      "      (conv): Conv2d(288, 384, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6b): InceptionC(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d(\n",
      "      (conv): Conv2d(128, 128, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d(\n",
      "      (conv): Conv2d(128, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(128, 128, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(128, 128, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d(\n",
      "      (conv): Conv2d(128, 128, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d(\n",
      "      (conv): Conv2d(128, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6c): InceptionC(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d(\n",
      "      (conv): Conv2d(160, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d(\n",
      "      (conv): Conv2d(160, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6d): InceptionC(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d(\n",
      "      (conv): Conv2d(160, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d(\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d(\n",
      "      (conv): Conv2d(160, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6e): InceptionC(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (AuxLogits): InceptionAux(\n",
      "    (conv0): BasicConv2d(\n",
      "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (conv1): BasicConv2d(\n",
      "      (conv): Conv2d(128, 768, kernel_size=(5, 5), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(768, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (fc): Linear(in_features=768, out_features=1000, bias=True)\n",
      "  )\n",
      "  (Mixed_7a): InceptionD(\n",
      "    (branch3x3_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_2): BasicConv2d(\n",
      "      (conv): Conv2d(192, 320, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7x3_1): BasicConv2d(\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7x3_2): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7x3_3): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch7x7x3_4): BasicConv2d(\n",
      "      (conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_7b): InceptionE(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(1280, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_1): BasicConv2d(\n",
      "      (conv): Conv2d(1280, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_2a): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_2b): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(1280, 448, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(448, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(448, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3a): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3b): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(1280, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_7c): InceptionE(\n",
      "    (branch1x1): BasicConv2d(\n",
      "      (conv): Conv2d(2048, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_1): BasicConv2d(\n",
      "      (conv): Conv2d(2048, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_2a): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3_2b): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d(\n",
      "      (conv): Conv2d(2048, 448, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(448, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d(\n",
      "      (conv): Conv2d(448, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3a): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch3x3dbl_3b): BasicConv2d(\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d(\n",
      "      (conv): Conv2d(2048, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=2048, out_features=1000, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model_transfer = models.inception_v3(pretrained=True)\n",
    "print(model_transfer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for 20 epoch(s)...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/koushik/.conda/envs/deep-learning/lib/python3.7/site-packages/torch/nn/functional.py:2016: UserWarning: Using a target size (torch.Size([32])) that is different to the input size (torch.Size([32, 102])) is deprecated. Please ensure they have the same size.\n",
      "  \"Please ensure they have the same size.\".format(target.size(), input.size()))\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Target and input must have the same number of elements. target nelement (32) != input nelement (3264)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-a319c4b66abc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m \u001b[0mtrained_rnn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_rnn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_transfer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_scratch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion_scratch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_epochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshow_every_n_batches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-7-3152a4cceaf4>\u001b[0m in \u001b[0;36mtrain_rnn\u001b[0;34m(model, batch_size, optimizer, criterion, n_epochs, show_every_n_batches)\u001b[0m\n\u001b[1;32m     33\u001b[0m             \u001b[0;31m#_, preds = torch.max(output.data, 1)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m             \u001b[0;31m#loss = criterion(output, torch.max(target, 1)[1])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m             \u001b[0;31m# backward pass: compute gradient of the loss with respect to model parameters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m             \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deep-learning/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deep-learning/lib/python3.7/site-packages/torch/nn/modules/loss.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, target)\u001b[0m\n\u001b[1;32m    502\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mweak_script_method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinary_cross_entropy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/deep-learning/lib/python3.7/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mbinary_cross_entropy\u001b[0;34m(input, target, weight, size_average, reduce, reduction)\u001b[0m\n\u001b[1;32m   2017\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2018\u001b[0m         raise ValueError(\"Target and input must have the same number of elements. target nelement ({}) \"\n\u001b[0;32m-> 2019\u001b[0;31m                          \"!= input nelement ({})\".format(target.numel(), input.numel()))\n\u001b[0m\u001b[1;32m   2020\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2021\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mweight\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Target and input must have the same number of elements. target nelement (32) != input nelement (3264)"
     ]
    }
   ],
   "source": [
    "# Freeze training for all \"features\" layers\n",
    "#for param in model_transfer.features.parameters():\n",
    "    #param.requires_grad = False\n",
    "\n",
    "## Freezing all layers\n",
    "for params in model_transfer.parameters():\n",
    "    params.requires_grad = False\n",
    "\n",
    "## Freezing the first few layers. Here I am freezing the first 7 layers \n",
    "        \n",
    "custom_model = nn.Sequential(nn.Linear(2048, 1024), \n",
    "                  nn.PReLU(),\n",
    "                  nn.Dropout(p=0.25), \n",
    "                  nn.Linear(1024, 102)\n",
    "                 )\n",
    "\n",
    "if use_cuda:\n",
    "    custom_model = custom_model.to(device)\n",
    "    \n",
    "model_transfer.fc = custom_model\n",
    "#model_transfer.classifier = custom_model\n",
    "decay_rate = learning_rate / num_epochs\n",
    "\n",
    "#model_transfer.load_state_dict(torch.load('save/trained_rnn_new'))\n",
    "\n",
    "if use_cuda:\n",
    "    model_transfer = model_transfer.cuda()\n",
    "\n",
    "# print(model_transfer)\n",
    "\n",
    "# specify loss function\n",
    "#criterion_scratch = nn.CrossEntropyLoss()\n",
    "criterion_scratch = nn.BCELoss()\n",
    "#criterion_scratch = nn.BCELoss(reduction=\"mean\").to('cuda:0')\n",
    "\n",
    "# specify optimizer\n",
    "optimizer_scratch = optim.SGD(model_transfer.parameters(), lr=learning_rate, momentum=0.9, weight_decay=decay_rate)\n",
    "#optimizer_scratch = optim.Adam(model_transfer.classifier.parameters(), lr=learning_rate)\n",
    "#optimizer_scratch = optim.SGD(list(filter(lambda p: p.requires_grad, model_transfer.parameters())), lr=learning_rate, momentum=0.9)\n",
    "\n",
    "\n",
    "trained_rnn = train_rnn(model_transfer, batch_size, optimizer_scratch, criterion_scratch, num_epochs, show_every_n_batches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_transfer.load_state_dict(torch.load('save/trained_rnn_new'))\n",
    "model_transfer.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "preds = []\n",
    "targetpred = []\n",
    "for batch_idx, (data, target) in enumerate(test_val_loader):\n",
    "    # move to GPU\n",
    "    #print(np.squeeze(target.detach().cpu().numpy()))\n",
    "    if use_cuda:\n",
    "        data, target = data.cuda(), target.cuda()\n",
    "    ## update the average validation loss\n",
    "    # forward pass: compute predicted outputs by passing inputs to the model\n",
    "    #target= target.long()\n",
    "    output = model_transfer.forward(data)\n",
    "    output = np.squeeze(output.detach().cpu().numpy())\n",
    "    target = np.squeeze(target.detach().cpu().numpy())\n",
    "    #print(torch.max(target, 0))\n",
    "    #print(np.argmax(target))\n",
    "    #print(np.argmax(output))\n",
    "    #print(torch.max(output, 1)[1].detach().cpu().numpy())\n",
    "    #values, indices = output.max(1)    \n",
    "    output = np.argmax(output)\n",
    "    target = np.argmax(target)\n",
    "    #print(tpr)\n",
    "    preds.append(int(output))\n",
    "    targetpred.append(int(target) + 1)\n",
    "    #print(pr.detach().cpu())\n",
    "print(preds[:100])\n",
    "print(targetpred[:100])\n",
    "r2_score(targetpred, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = []\n",
    "np.set_printoptions(threshold=100)\n",
    "submission = pd.read_csv('sample_submission.csv')\n",
    "for batch_idx, (data, target) in enumerate(test_loader):\n",
    "    # move to GPU\n",
    "    #print(np.squeeze(target.detach().cpu().numpy()))\n",
    "    if use_cuda:\n",
    "        data, target = data.cuda(), target.cuda()\n",
    "    ## update the average validation loss\n",
    "    # forward pass: compute predicted outputs by passing inputs to the model\n",
    "    output = model_transfer.forward(data)\n",
    "    #print(target)\n",
    "    #print(output)\n",
    "    #print(torch.max(output, 1)[1].detach().cpu().numpy())\n",
    "    #values, indices = output.max(1)    \n",
    "    pr = torch.max(output, 1)[1].detach().cpu().numpy()\n",
    "    preds.append(int(pr[0]) + 1)\n",
    "    #for i in pr:\n",
    "        #preds.append(i)\n",
    "    #print(batch_idx)\n",
    "    #print(pr)\n",
    "    #print(x.argsort()[-6:][::-1])\n",
    "    #print(pr.argsort()[-6:][::-1])\n",
    "    #print(values)\n",
    "    # calculate the batch loss\n",
    "submission[\"category\"] = preds\n",
    "print(submission.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}