005_Transfer_Learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transfer Learning Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "While some transfer learning models did have greater evaluation metrics, mainly recall, on average. No models outperformed the custom 2x2C1D with Dropout model from 004_Modeling_Local.ipynb."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Libraries to Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.style.use('fivethirtyeight')\n",
    "\n",
    "from PIL import Image\n",
    "import keras\n",
    "from tensorflow.keras.preprocessing import image, image_dataset_from_directory\n",
    "from tensorflow.keras import models, layers, optimizers\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.keras.layers import BatchNormalization\n",
    "\n",
    "import os\n",
    "import gc\n",
    "from timeit import default_timer as timer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Image Directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dir = '/Users/arencarpenter/Desktop/Detecting_ALL_with_CNN/Data/training/'\n",
    "test_dir = '/Users/arencarpenter/Desktop/Detecting_ALL_with_CNN/Data/testing/'\n",
    "val_dir = '/Users/arencarpenter/Desktop/Detecting_ALL_with_CNN/Data/validation/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create flow_from_directory objects for Keras model. Using a downsized dataset with class balance for local model running."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1346 images belonging to 2 classes.\n",
      "Found 593 images belonging to 2 classes.\n",
      "Found 947 images belonging to 2 classes.\n"
     ]
    }
   ],
   "source": [
    "train_g = image.ImageDataGenerator(rescale = 1/255).flow_from_directory(train_dir,\n",
    "                                                                  target_size = (256,256), \n",
    "                                                                  color_mode='rgb',\n",
    "                                                                  class_mode='binary')\n",
    "val_g = image.ImageDataGenerator(rescale = 1/255).flow_from_directory(val_dir,\n",
    "                                                                target_size = (256,256), \n",
    "                                                                color_mode='rgb',\n",
    "                                                                class_mode='binary')\n",
    "test_g = image.ImageDataGenerator(rescale = 1/255).flow_from_directory(test_dir,\n",
    "                                                                target_size = (256,256), \n",
    "                                                                color_mode='rgb',\n",
    "                                                                class_mode='binary')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Xception has 22,910,480 trainable parameters, and a top-5 accuracy of 0.945."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Xception -- 5 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
      "83689472/83683744 [==============================] - 3s 0us/step\n"
     ]
    }
   ],
   "source": [
    "base_model = keras.applications.Xception(\n",
    "    weights='imagenet',  # Load weights pre-trained on ImageNet.\n",
    "    input_shape=(450, 450, 3),\n",
    "    include_top=False)  # Do not include the ImageNet classifier at the top.\n",
    "base_model.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(450, 450, 3))\n",
    "x = base_model(inputs, training=False)\n",
    "\n",
    "# Convert features of shape `base_model.output_shape[1:]` to vectors\n",
    "x = keras.layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "# A Dense classifier with a single unit (binary classification)\n",
    "outputs = keras.layers.Dense(1, activation ='sigmoid')(x)\n",
    "model = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 140s 3s/step - loss: 0.6371 - accuracy: 0.6612 - recall: 0.6657 - val_loss: 0.5866 - val_accuracy: 0.7285 - val_recall: 0.7381\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 157s 4s/step - loss: 0.5689 - accuracy: 0.7348 - recall: 0.7504 - val_loss: 0.5515 - val_accuracy: 0.7369 - val_recall: 0.7517\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 149s 3s/step - loss: 0.5432 - accuracy: 0.7422 - recall: 0.7221 - val_loss: 0.5315 - val_accuracy: 0.7521 - val_recall: 0.7789\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 147s 3s/step - loss: 0.5259 - accuracy: 0.7600 - recall: 0.7340 - val_loss: 0.5498 - val_accuracy: 0.7167 - val_recall: 0.8946\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 145s 3s/step - loss: 0.5078 - accuracy: 0.7615 - recall: 0.7563 - val_loss: 0.5155 - val_accuracy: 0.7589 - val_recall: 0.7891\n",
      "Total Time Elapsed:  12  minutes  34  seconds\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "start = timer()\n",
    "model.fit(train_g, \n",
    "            epochs=5, \n",
    "            validation_data=val_g, \n",
    "            workers = 7)\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 65s 2s/step - loss: 0.9548 - accuracy: 0.4340 - recall: 0.2685\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Xception -- 15 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = keras.applications.Xception(\n",
    "    weights='imagenet',  # Load weights pre-trained on ImageNet.\n",
    "    input_shape=(450, 450, 3),\n",
    "    include_top=False)  # Do not include the ImageNet classifier at the top.\n",
    "base_model.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(450, 450, 3))\n",
    "x = base_model(inputs, training=False)\n",
    "\n",
    "# Convert features of shape `base_model.output_shape[1:]` to vectors\n",
    "x = keras.layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "# A Dense classifier with a single unit (binary classification)\n",
    "outputs = keras.layers.Dense(1, activation ='sigmoid')(x)\n",
    "model = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "43/43 [==============================] - 142s 3s/step - loss: 0.6473 - accuracy: 0.6367 - recall: 0.6865 - val_loss: 0.6001 - val_accuracy: 0.7403 - val_recall: 0.5782\n",
      "Epoch 2/15\n",
      "43/43 [==============================] - 152s 4s/step - loss: 0.5757 - accuracy: 0.7318 - recall: 0.6909 - val_loss: 0.5587 - val_accuracy: 0.7302 - val_recall: 0.7959\n",
      "Epoch 3/15\n",
      "43/43 [==============================] - 150s 3s/step - loss: 0.5401 - accuracy: 0.7511 - recall: 0.7251 - val_loss: 0.5347 - val_accuracy: 0.7504 - val_recall: 0.7823\n",
      "Epoch 4/15\n",
      "43/43 [==============================] - 148s 3s/step - loss: 0.5240 - accuracy: 0.7541 - recall: 0.7400 - val_loss: 0.5265 - val_accuracy: 0.7521 - val_recall: 0.7891\n",
      "Epoch 5/15\n",
      "43/43 [==============================] - 148s 3s/step - loss: 0.5080 - accuracy: 0.7689 - recall: 0.7578 - val_loss: 0.5233 - val_accuracy: 0.7504 - val_recall: 0.8163\n",
      "Epoch 6/15\n",
      "43/43 [==============================] - 147s 3s/step - loss: 0.5003 - accuracy: 0.7704 - recall: 0.7593 - val_loss: 0.5128 - val_accuracy: 0.7622 - val_recall: 0.7891\n",
      "Epoch 7/15\n",
      "43/43 [==============================] - 147s 3s/step - loss: 0.4926 - accuracy: 0.7741 - recall: 0.7637 - val_loss: 0.5315 - val_accuracy: 0.7555 - val_recall: 0.8878\n",
      "Epoch 8/15\n",
      "43/43 [==============================] - 147s 3s/step - loss: 0.4891 - accuracy: 0.7704 - recall: 0.7786 - val_loss: 0.5226 - val_accuracy: 0.7555 - val_recall: 0.8537\n",
      "Epoch 9/15\n",
      "43/43 [==============================] - 148s 3s/step - loss: 0.4814 - accuracy: 0.7749 - recall: 0.7682 - val_loss: 0.5084 - val_accuracy: 0.7656 - val_recall: 0.8197\n",
      "Epoch 10/15\n",
      "43/43 [==============================] - 146s 3s/step - loss: 0.4740 - accuracy: 0.7875 - recall: 0.7697 - val_loss: 0.5090 - val_accuracy: 0.7622 - val_recall: 0.8197\n",
      "Epoch 11/15\n",
      "43/43 [==============================] - 146s 3s/step - loss: 0.4684 - accuracy: 0.7845 - recall: 0.7890 - val_loss: 0.5075 - val_accuracy: 0.7774 - val_recall: 0.7347\n",
      "Epoch 12/15\n",
      "43/43 [==============================] - 145s 3s/step - loss: 0.4645 - accuracy: 0.7905 - recall: 0.7682 - val_loss: 0.5084 - val_accuracy: 0.7639 - val_recall: 0.8265\n",
      "Epoch 13/15\n",
      "43/43 [==============================] - 144s 3s/step - loss: 0.4617 - accuracy: 0.7883 - recall: 0.7771 - val_loss: 0.5145 - val_accuracy: 0.7605 - val_recall: 0.8401\n",
      "Epoch 14/15\n",
      "43/43 [==============================] - 143s 3s/step - loss: 0.4579 - accuracy: 0.7853 - recall: 0.7875 - val_loss: 0.4932 - val_accuracy: 0.7926 - val_recall: 0.8129\n",
      "Epoch 15/15\n",
      "43/43 [==============================] - 143s 3s/step - loss: 0.4537 - accuracy: 0.7942 - recall: 0.7890 - val_loss: 0.4922 - val_accuracy: 0.7892 - val_recall: 0.7891\n",
      "Total Time Elapsed:  37  minutes  18  seconds\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "start = timer()\n",
    "model.fit(train_g, \n",
    "            epochs=15, \n",
    "            validation_data=val_g, \n",
    "            workers = 7)\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 68s 2s/step - loss: 1.0511 - accuracy: 0.4636 - recall: 0.2778\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "VGG16 has 138,357,544 trainable parameters, and a top-5 accuracy of 0.901."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VGG16 -- 5 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = keras.applications.VGG16(\n",
    "    weights='imagenet',  # Load weights pre-trained on ImageNet.\n",
    "    input_shape=(450, 450, 3),\n",
    "    include_top=False)  # Do not include the ImageNet classifier at the top.\n",
    "base_model.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(450, 450, 3))\n",
    "x = base_model(inputs, training=False)\n",
    "\n",
    "# Convert features of shape `base_model.output_shape[1:]` to vectors\n",
    "x = keras.layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "# A Dense classifier with a single unit (binary classification)\n",
    "outputs = keras.layers.Dense(1, activation ='sigmoid')(x)\n",
    "model = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 322s 7s/step - loss: 0.6880 - accuracy: 0.5572 - recall: 0.3314 - val_loss: 0.6756 - val_accuracy: 0.7150 - val_recall: 0.7721\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 319s 7s/step - loss: 0.6673 - accuracy: 0.6709 - recall: 0.6256 - val_loss: 0.6589 - val_accuracy: 0.7184 - val_recall: 0.8333\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 316s 7s/step - loss: 0.6510 - accuracy: 0.7065 - recall: 0.6642 - val_loss: 0.6443 - val_accuracy: 0.7234 - val_recall: 0.7925\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 335s 8s/step - loss: 0.6362 - accuracy: 0.7065 - recall: 0.7073 - val_loss: 0.6331 - val_accuracy: 0.7302 - val_recall: 0.7619\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 312s 7s/step - loss: 0.6255 - accuracy: 0.7147 - recall: 0.7132 - val_loss: 0.6225 - val_accuracy: 0.7319 - val_recall: 0.7619\n",
      "Total Time Elapsed:  27  minutes  12  seconds\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "start = timer()\n",
    "model.fit(train_g, \n",
    "            epochs=5, \n",
    "            validation_data=val_g, \n",
    "            workers = 7)\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 148s 5s/step - loss: 0.7729 - accuracy: 0.4234 - recall: 0.2238\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VGG16 -- 15 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = keras.applications.VGG16(\n",
    "    weights='imagenet',  # Load weights pre-trained on ImageNet.\n",
    "    input_shape=(450, 450, 3),\n",
    "    include_top=False)  # Do not include the ImageNet classifier at the top.\n",
    "base_model.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(450, 450, 3))\n",
    "x = base_model(inputs, training=False)\n",
    "\n",
    "# Convert features of shape `base_model.output_shape[1:]` to vectors\n",
    "x = keras.layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "# A Dense classifier with a single unit (binary classification)\n",
    "outputs = keras.layers.Dense(1, activation ='sigmoid')(x)\n",
    "model = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "43/43 [==============================] - 309s 7s/step - loss: 0.6808 - accuracy: 0.6263 - recall: 0.7831 - val_loss: 0.6708 - val_accuracy: 0.6071 - val_recall: 0.8878\n",
      "Epoch 2/15\n",
      "43/43 [==============================] - 312s 7s/step - loss: 0.6611 - accuracy: 0.6471 - recall: 0.7385 - val_loss: 0.6565 - val_accuracy: 0.6779 - val_recall: 0.8095\n",
      "Epoch 3/15\n",
      "43/43 [==============================] - 313s 7s/step - loss: 0.6473 - accuracy: 0.6857 - recall: 0.7207 - val_loss: 0.6440 - val_accuracy: 0.7133 - val_recall: 0.7483\n",
      "Epoch 4/15\n",
      "43/43 [==============================] - 314s 7s/step - loss: 0.6364 - accuracy: 0.7058 - recall: 0.6374 - val_loss: 0.6365 - val_accuracy: 0.6813 - val_recall: 0.8197\n",
      "Epoch 5/15\n",
      "43/43 [==============================] - 312s 7s/step - loss: 0.6262 - accuracy: 0.7036 - recall: 0.7207 - val_loss: 0.6291 - val_accuracy: 0.6728 - val_recall: 0.8265\n",
      "Epoch 6/15\n",
      "43/43 [==============================] - 319s 7s/step - loss: 0.6147 - accuracy: 0.7043 - recall: 0.7236 - val_loss: 0.6166 - val_accuracy: 0.7218 - val_recall: 0.7211\n",
      "Epoch 7/15\n",
      "43/43 [==============================] - 318s 7s/step - loss: 0.6077 - accuracy: 0.7199 - recall: 0.7117 - val_loss: 0.6101 - val_accuracy: 0.7386 - val_recall: 0.7177\n",
      "Epoch 8/15\n",
      "43/43 [==============================] - 316s 7s/step - loss: 0.6013 - accuracy: 0.7259 - recall: 0.6939 - val_loss: 0.6045 - val_accuracy: 0.7167 - val_recall: 0.7857\n",
      "Epoch 9/15\n",
      "43/43 [==============================] - 316s 7s/step - loss: 0.5942 - accuracy: 0.7169 - recall: 0.7117 - val_loss: 0.5981 - val_accuracy: 0.7167 - val_recall: 0.7857\n",
      "Epoch 10/15\n",
      "43/43 [==============================] - 317s 7s/step - loss: 0.5870 - accuracy: 0.7221 - recall: 0.7474 - val_loss: 0.5922 - val_accuracy: 0.7352 - val_recall: 0.7279\n",
      "Epoch 11/15\n",
      "43/43 [==============================] - 316s 7s/step - loss: 0.5837 - accuracy: 0.7221 - recall: 0.7147 - val_loss: 0.5904 - val_accuracy: 0.7099 - val_recall: 0.8163\n",
      "Epoch 12/15\n",
      "43/43 [==============================] - 318s 7s/step - loss: 0.5790 - accuracy: 0.7236 - recall: 0.7221 - val_loss: 0.5903 - val_accuracy: 0.6981 - val_recall: 0.8401\n",
      "Epoch 13/15\n",
      "43/43 [==============================] - 317s 7s/step - loss: 0.5761 - accuracy: 0.7117 - recall: 0.7548 - val_loss: 0.5792 - val_accuracy: 0.7285 - val_recall: 0.7619\n",
      "Epoch 14/15\n",
      "43/43 [==============================] - 317s 7s/step - loss: 0.5717 - accuracy: 0.7266 - recall: 0.7177 - val_loss: 0.5763 - val_accuracy: 0.7302 - val_recall: 0.8095\n",
      "Epoch 15/15\n",
      "43/43 [==============================] - 318s 7s/step - loss: 0.5668 - accuracy: 0.7244 - recall: 0.7281 - val_loss: 0.5778 - val_accuracy: 0.7099 - val_recall: 0.8333\n",
      "Total Time Elapsed:  80  minutes  11  seconds\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "start = timer()\n",
    "model.fit(train_g, \n",
    "            epochs=15, \n",
    "            validation_data=val_g, \n",
    "            workers = 7)\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 151s 5s/step - loss: 0.8086 - accuracy: 0.4615 - recall: 0.3395\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "InceptionResNetV2 has 55,873,736 trainable parameters, and a top-5 accuracy of 0.953."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## InceptionResNetV2 -- 5 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/inception_resnet_v2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
      "219062272/219055592 [==============================] - 8s 0us/step\n"
     ]
    }
   ],
   "source": [
    "base_model = keras.applications.InceptionResNetV2(\n",
    "    weights='imagenet',  # Load weights pre-trained on ImageNet.\n",
    "    input_shape=(450, 450, 3),\n",
    "    include_top=False)  # Do not include the ImageNet classifier at the top.\n",
    "base_model.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(450, 450, 3))\n",
    "x = base_model(inputs, training=False)\n",
    "\n",
    "# Convert features of shape `base_model.output_shape[1:]` to vectors\n",
    "x = keras.layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "# A Dense classifier with a single unit (binary classification)\n",
    "outputs = keras.layers.Dense(1, activation ='sigmoid')(x)\n",
    "model = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 156s 4s/step - loss: 0.6612 - accuracy: 0.6040 - recall: 0.6270 - val_loss: 0.6085 - val_accuracy: 0.6155 - val_recall: 0.9184\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 155s 4s/step - loss: 0.6047 - accuracy: 0.6932 - recall: 0.6909 - val_loss: 0.5432 - val_accuracy: 0.7774 - val_recall: 0.8027\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 154s 4s/step - loss: 0.5666 - accuracy: 0.7266 - recall: 0.7192 - val_loss: 0.5370 - val_accuracy: 0.8061 - val_recall: 0.6837\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 154s 4s/step - loss: 0.5538 - accuracy: 0.7221 - recall: 0.6954 - val_loss: 0.5089 - val_accuracy: 0.7555 - val_recall: 0.8571\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 157s 4s/step - loss: 0.5404 - accuracy: 0.7340 - recall: 0.7251 - val_loss: 0.4899 - val_accuracy: 0.8128 - val_recall: 0.8027\n",
      "Total Time Elapsed:  13  minutes  17  seconds\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "start = timer()\n",
    "model.fit(train_g, \n",
    "            epochs=5, \n",
    "            validation_data=val_g, \n",
    "            workers = 7)\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 74s 2s/step - loss: 0.8767 - accuracy: 0.4657 - recall: 0.2963\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## InceptionResNetV2 -- 15 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = keras.applications.InceptionResNetV2(\n",
    "    weights='imagenet',  # Load weights pre-trained on ImageNet.\n",
    "    input_shape=(450, 450, 3),\n",
    "    include_top=False)  # Do not include the ImageNet classifier at the top.\n",
    "base_model.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(450, 450, 3))\n",
    "x = base_model(inputs, training=False)\n",
    "\n",
    "# Convert features of shape `base_model.output_shape[1:]` to vectors\n",
    "x = keras.layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "# A Dense classifier with a single unit (binary classification)\n",
    "outputs = keras.layers.Dense(1, activation ='sigmoid')(x)\n",
    "model = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "43/43 [==============================] - 163s 4s/step - loss: 0.6693 - accuracy: 0.6129 - recall: 0.5795 - val_loss: 0.6048 - val_accuracy: 0.6374 - val_recall: 0.9252\n",
      "Epoch 2/15\n",
      "43/43 [==============================] - 158s 4s/step - loss: 0.5991 - accuracy: 0.7013 - recall: 0.6969 - val_loss: 0.5703 - val_accuracy: 0.7437 - val_recall: 0.5408\n",
      "Epoch 3/15\n",
      "43/43 [==============================] - 158s 4s/step - loss: 0.5762 - accuracy: 0.6961 - recall: 0.6999 - val_loss: 0.5371 - val_accuracy: 0.7285 - val_recall: 0.9048\n",
      "Epoch 4/15\n",
      "43/43 [==============================] - 159s 4s/step - loss: 0.5621 - accuracy: 0.7296 - recall: 0.7251 - val_loss: 0.5015 - val_accuracy: 0.8061 - val_recall: 0.7823\n",
      "Epoch 5/15\n",
      "43/43 [==============================] - 158s 4s/step - loss: 0.5442 - accuracy: 0.7325 - recall: 0.7325 - val_loss: 0.4957 - val_accuracy: 0.8094 - val_recall: 0.7619\n",
      "Epoch 6/15\n",
      "43/43 [==============================] - 159s 4s/step - loss: 0.5264 - accuracy: 0.7370 - recall: 0.7103 - val_loss: 0.5052 - val_accuracy: 0.7454 - val_recall: 0.8776\n",
      "Epoch 7/15\n",
      "43/43 [==============================] - 158s 4s/step - loss: 0.5266 - accuracy: 0.7415 - recall: 0.7221 - val_loss: 0.4750 - val_accuracy: 0.8061 - val_recall: 0.7959\n",
      "Epoch 8/15\n",
      "43/43 [==============================] - 158s 4s/step - loss: 0.5167 - accuracy: 0.7585 - recall: 0.7266 - val_loss: 0.4876 - val_accuracy: 0.8145 - val_recall: 0.7109\n",
      "Epoch 9/15\n",
      "43/43 [==============================] - 157s 4s/step - loss: 0.5192 - accuracy: 0.7481 - recall: 0.7325 - val_loss: 0.4700 - val_accuracy: 0.8061 - val_recall: 0.7823\n",
      "Epoch 10/15\n",
      "43/43 [==============================] - 161s 4s/step - loss: 0.5041 - accuracy: 0.7600 - recall: 0.7311 - val_loss: 0.4691 - val_accuracy: 0.7875 - val_recall: 0.8435\n",
      "Epoch 11/15\n",
      "43/43 [==============================] - 158s 4s/step - loss: 0.5094 - accuracy: 0.7511 - recall: 0.7429 - val_loss: 0.4656 - val_accuracy: 0.7875 - val_recall: 0.8299\n",
      "Epoch 12/15\n",
      "43/43 [==============================] - 158s 4s/step - loss: 0.5013 - accuracy: 0.7533 - recall: 0.7370 - val_loss: 0.4751 - val_accuracy: 0.8179 - val_recall: 0.7755\n",
      "Epoch 13/15\n",
      "43/43 [==============================] - 158s 4s/step - loss: 0.5005 - accuracy: 0.7667 - recall: 0.7533 - val_loss: 0.4858 - val_accuracy: 0.7504 - val_recall: 0.8776\n",
      "Epoch 14/15\n",
      "43/43 [==============================] - 158s 4s/step - loss: 0.4901 - accuracy: 0.7571 - recall: 0.7325 - val_loss: 0.5468 - val_accuracy: 0.7555 - val_recall: 0.5306\n",
      "Epoch 15/15\n",
      "43/43 [==============================] - 158s 4s/step - loss: 0.4955 - accuracy: 0.7675 - recall: 0.7311 - val_loss: 0.4597 - val_accuracy: 0.8162 - val_recall: 0.8231\n",
      "Total Time Elapsed:  40  minutes  31  seconds\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "start = timer()\n",
    "model.fit(train_g, \n",
    "            epochs=15, \n",
    "            validation_data=val_g, \n",
    "            workers = 7)\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 74s 2s/step - loss: 0.8598 - accuracy: 0.4879 - recall: 0.3534\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MobileNetV2 has 3,538,984 trainable parameters, and a top-5 accuracy of 0.901."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MobileNetV2 -- 5 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n",
      "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/mobilenet_v2/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_224_no_top.h5\n",
      "9412608/9406464 [==============================] - 0s 0us/step\n"
     ]
    }
   ],
   "source": [
    "base_model = keras.applications.MobileNetV2(\n",
    "    weights='imagenet',  # Load weights pre-trained on ImageNet.\n",
    "    input_shape=(450, 450, 3),\n",
    "    include_top=False)  # Do not include the ImageNet classifier at the top.\n",
    "base_model.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(450, 450, 3))\n",
    "x = base_model(inputs, training=False)\n",
    "\n",
    "# Convert features of shape `base_model.output_shape[1:]` to vectors\n",
    "x = keras.layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "# A Dense classifier with a single unit (binary classification)\n",
    "outputs = keras.layers.Dense(1, activation ='sigmoid')(x)\n",
    "model = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 43s 994ms/step - loss: 0.6419 - accuracy: 0.6308 - recall: 0.5884 - val_loss: 0.5471 - val_accuracy: 0.7740 - val_recall: 0.6395\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 41s 965ms/step - loss: 0.5499 - accuracy: 0.7422 - recall: 0.7489 - val_loss: 0.5306 - val_accuracy: 0.7723 - val_recall: 0.5816\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 41s 952ms/step - loss: 0.5255 - accuracy: 0.7548 - recall: 0.7370 - val_loss: 0.4717 - val_accuracy: 0.7875 - val_recall: 0.8503\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 41s 942ms/step - loss: 0.5164 - accuracy: 0.7533 - recall: 0.7504 - val_loss: 0.4737 - val_accuracy: 0.7656 - val_recall: 0.8741\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 40s 935ms/step - loss: 0.5104 - accuracy: 0.7533 - recall: 0.7593 - val_loss: 0.4574 - val_accuracy: 0.8027 - val_recall: 0.8639\n",
      "Total Time Elapsed:  3  minutes  33  seconds\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "start = timer()\n",
    "model.fit(train_g, \n",
    "            epochs=5, \n",
    "            validation_data=val_g, \n",
    "            workers = 7)\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 18s 615ms/step - loss: 0.8800 - accuracy: 0.4921 - recall: 0.3395\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MobileNetV2 -- 15 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n"
     ]
    }
   ],
   "source": [
    "base_model = keras.applications.MobileNetV2(\n",
    "    weights='imagenet',  # Load weights pre-trained on ImageNet.\n",
    "    input_shape=(450, 450, 3),\n",
    "    include_top=False)  # Do not include the ImageNet classifier at the top.\n",
    "base_model.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(450, 450, 3))\n",
    "x = base_model(inputs, training=False)\n",
    "\n",
    "# Convert features of shape `base_model.output_shape[1:]` to vectors\n",
    "x = keras.layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "# A Dense classifier with a single unit (binary classification)\n",
    "outputs = keras.layers.Dense(1, activation ='sigmoid')(x)\n",
    "model = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "43/43 [==============================] - 40s 932ms/step - loss: 0.6082 - accuracy: 0.6917 - recall: 0.7013 - val_loss: 0.5377 - val_accuracy: 0.7218 - val_recall: 0.8299\n",
      "Epoch 2/15\n",
      "43/43 [==============================] - 41s 965ms/step - loss: 0.5452 - accuracy: 0.7422 - recall: 0.7385 - val_loss: 0.4932 - val_accuracy: 0.7909 - val_recall: 0.7925\n",
      "Epoch 3/15\n",
      "43/43 [==============================] - 41s 946ms/step - loss: 0.5331 - accuracy: 0.7474 - recall: 0.7504 - val_loss: 0.4797 - val_accuracy: 0.7858 - val_recall: 0.7959\n",
      "Epoch 4/15\n",
      "43/43 [==============================] - 41s 943ms/step - loss: 0.5201 - accuracy: 0.7600 - recall: 0.7459 - val_loss: 0.5375 - val_accuracy: 0.7167 - val_recall: 0.9082\n",
      "Epoch 5/15\n",
      "43/43 [==============================] - 41s 947ms/step - loss: 0.5128 - accuracy: 0.7600 - recall: 0.7474 - val_loss: 0.5184 - val_accuracy: 0.7201 - val_recall: 0.8946\n",
      "Epoch 6/15\n",
      "43/43 [==============================] - 40s 941ms/step - loss: 0.5130 - accuracy: 0.7689 - recall: 0.7355 - val_loss: 0.4899 - val_accuracy: 0.7420 - val_recall: 0.8673\n",
      "Epoch 7/15\n",
      "43/43 [==============================] - 40s 940ms/step - loss: 0.5012 - accuracy: 0.7727 - recall: 0.7519 - val_loss: 0.4722 - val_accuracy: 0.7622 - val_recall: 0.8503\n",
      "Epoch 8/15\n",
      "43/43 [==============================] - 41s 956ms/step - loss: 0.4987 - accuracy: 0.7689 - recall: 0.7533 - val_loss: 0.4791 - val_accuracy: 0.7538 - val_recall: 0.8571\n",
      "Epoch 9/15\n",
      "43/43 [==============================] - 40s 940ms/step - loss: 0.4895 - accuracy: 0.7749 - recall: 0.7444 - val_loss: 0.4565 - val_accuracy: 0.7976 - val_recall: 0.8265\n",
      "Epoch 10/15\n",
      "43/43 [==============================] - 41s 945ms/step - loss: 0.4874 - accuracy: 0.7779 - recall: 0.7459 - val_loss: 0.4533 - val_accuracy: 0.7976 - val_recall: 0.8061\n",
      "Epoch 11/15\n",
      "43/43 [==============================] - 40s 935ms/step - loss: 0.4810 - accuracy: 0.7808 - recall: 0.7489 - val_loss: 0.4621 - val_accuracy: 0.7858 - val_recall: 0.8469\n",
      "Epoch 12/15\n",
      "43/43 [==============================] - 41s 947ms/step - loss: 0.4845 - accuracy: 0.7771 - recall: 0.7444 - val_loss: 0.5552 - val_accuracy: 0.7049 - val_recall: 0.9388\n",
      "Epoch 13/15\n",
      "43/43 [==============================] - 41s 946ms/step - loss: 0.4777 - accuracy: 0.7756 - recall: 0.7682 - val_loss: 0.4634 - val_accuracy: 0.7875 - val_recall: 0.8401\n",
      "Epoch 14/15\n",
      "43/43 [==============================] - 41s 944ms/step - loss: 0.4726 - accuracy: 0.7890 - recall: 0.7667 - val_loss: 0.4874 - val_accuracy: 0.7572 - val_recall: 0.8741\n",
      "Epoch 15/15\n",
      "43/43 [==============================] - 41s 948ms/step - loss: 0.4691 - accuracy: 0.7808 - recall: 0.7548 - val_loss: 0.4602 - val_accuracy: 0.8061 - val_recall: 0.7415\n",
      "Total Time Elapsed:  10  minutes  26  seconds\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "start = timer()\n",
    "model.fit(train_g, \n",
    "            epochs=15, \n",
    "            validation_data=val_g, \n",
    "            workers = 7)\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 19s 617ms/step - loss: 1.2006 - accuracy: 0.4013 - recall: 0.1543\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ResNet50 has 25,636,712 trainable parameters, and a top-5 accuracy of 0.921."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ResNet50 -- 5 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
      "94773248/94765736 [==============================] - 4s 0us/step\n"
     ]
    }
   ],
   "source": [
    "base_model = keras.applications.ResNet50(\n",
    "    weights='imagenet',  # Load weights pre-trained on ImageNet.\n",
    "    input_shape=(450, 450, 3),\n",
    "    include_top=False)  # Do not include the ImageNet classifier at the top.\n",
    "base_model.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(450, 450, 3))\n",
    "x = base_model(inputs, training=False)\n",
    "\n",
    "# Convert features of shape `base_model.output_shape[1:]` to vectors\n",
    "x = keras.layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "# A Dense classifier with a single unit (binary classification)\n",
    "outputs = keras.layers.Dense(1, activation ='sigmoid')(x)\n",
    "model = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.7095 - accuracy: 0.5134 - recall: 0.6196 - val_loss: 0.6939 - val_accuracy: 0.4975 - val_recall: 1.0000\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.7012 - accuracy: 0.5052 - recall: 0.5141 - val_loss: 0.6919 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6989 - accuracy: 0.5156 - recall: 0.5022 - val_loss: 0.6916 - val_accuracy: 0.4958 - val_recall: 1.0000\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 122s 3s/step - loss: 0.6945 - accuracy: 0.5275 - recall: 0.5305 - val_loss: 0.6962 - val_accuracy: 0.4958 - val_recall: 1.0000\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6959 - accuracy: 0.5401 - recall: 0.5156 - val_loss: 0.6856 - val_accuracy: 0.5042 - val_recall: 0.9796\n",
      "Total Time Elapsed:  10  minutes  19  seconds\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "start = timer()\n",
    "model.fit(train_g, \n",
    "            epochs=5, \n",
    "            validation_data=val_g, \n",
    "            workers = 7)\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 58s 2s/step - loss: 0.6854 - accuracy: 0.6431 - recall: 0.9120\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ResNet50 -- 15 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = keras.applications.ResNet50(\n",
    "    weights='imagenet',  # Load weights pre-trained on ImageNet.\n",
    "    input_shape=(450, 450, 3),\n",
    "    include_top=False)  # Do not include the ImageNet classifier at the top.\n",
    "base_model.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(450, 450, 3))\n",
    "x = base_model(inputs, training=False)\n",
    "\n",
    "# Convert features of shape `base_model.output_shape[1:]` to vectors\n",
    "x = keras.layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "# A Dense classifier with a single unit (binary classification)\n",
    "outputs = keras.layers.Dense(1, activation ='sigmoid')(x)\n",
    "model = keras.Model(inputs, outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "43/43 [==============================] - 122s 3s/step - loss: 0.7055 - accuracy: 0.5067 - recall: 0.4368 - val_loss: 0.6933 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Epoch 2/15\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6992 - accuracy: 0.4926 - recall: 0.5736 - val_loss: 0.6928 - val_accuracy: 0.4958 - val_recall: 1.0000\n",
      "Epoch 3/15\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.7001 - accuracy: 0.4978 - recall: 0.4428 - val_loss: 0.6867 - val_accuracy: 0.6796 - val_recall: 0.8946\n",
      "Epoch 4/15\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6886 - accuracy: 0.5267 - recall: 0.5349 - val_loss: 0.7092 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Epoch 5/15\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6897 - accuracy: 0.5401 - recall: 0.5706 - val_loss: 0.6827 - val_accuracy: 0.6054 - val_recall: 0.2109\n",
      "Epoch 6/15\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6845 - accuracy: 0.5461 - recall: 0.5453 - val_loss: 0.6850 - val_accuracy: 0.4958 - val_recall: 1.0000\n",
      "Epoch 7/15\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6845 - accuracy: 0.5394 - recall: 0.5617 - val_loss: 0.6795 - val_accuracy: 0.5599 - val_recall: 0.9592\n",
      "Epoch 8/15\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6822 - accuracy: 0.5617 - recall: 0.4770 - val_loss: 0.6856 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Epoch 9/15\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6799 - accuracy: 0.5899 - recall: 0.5201 - val_loss: 0.6807 - val_accuracy: 0.4958 - val_recall: 1.0000\n",
      "Epoch 10/15\n",
      "43/43 [==============================] - 122s 3s/step - loss: 0.6784 - accuracy: 0.5854 - recall: 0.5736 - val_loss: 0.6746 - val_accuracy: 0.5363 - val_recall: 0.9660\n",
      "Epoch 11/15\n",
      "43/43 [==============================] - 123s 3s/step - loss: 0.6752 - accuracy: 0.6189 - recall: 0.6300 - val_loss: 0.6771 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Epoch 12/15\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6761 - accuracy: 0.5661 - recall: 0.5438 - val_loss: 0.6683 - val_accuracy: 0.7605 - val_recall: 0.7823\n",
      "Epoch 13/15\n",
      "43/43 [==============================] - 121s 3s/step - loss: 0.6732 - accuracy: 0.6040 - recall: 0.5914 - val_loss: 0.6758 - val_accuracy: 0.4975 - val_recall: 1.0000\n",
      "Epoch 14/15\n",
      "43/43 [==============================] - 122s 3s/step - loss: 0.6738 - accuracy: 0.5936 - recall: 0.6256 - val_loss: 0.6656 - val_accuracy: 0.6712 - val_recall: 0.3707\n",
      "Epoch 15/15\n",
      "43/43 [==============================] - 122s 3s/step - loss: 0.6678 - accuracy: 0.6144 - recall: 0.6107 - val_loss: 0.6644 - val_accuracy: 0.6712 - val_recall: 0.9388\n",
      "Total Time Elapsed:  30  minutes  56  seconds\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "start = timer()\n",
    "model.fit(train_g, \n",
    "            epochs=15, \n",
    "            validation_data=val_g, \n",
    "            workers = 7)\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 57s 2s/step - loss: 0.6835 - accuracy: 0.5502 - recall: 0.6065\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "While some transfer learning models did have greater evaluation metrics, mainly recall, on average. No models outperformed the custom 2x2C1D with Dropout model from 004_Modeling_Local.ipynb."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}