teacher_kernel.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "substantial-quantity",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:02.825291Z",
          "iopub.status.busy": "2021-05-02T00:28:02.824184Z",
          "iopub.status.idle": "2021-05-02T00:28:26.920629Z",
          "shell.execute_reply": "2021-05-02T00:28:26.919843Z"
        },
        "id": "substantial-quantity",
        "papermill": {
          "duration": 24.143231,
          "end_time": "2021-05-02T00:28:26.920784",
          "exception": false,
          "start_time": "2021-05-02T00:28:02.777553",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "!pip install --upgrade pip >> /dev/null\n",
        "!pip install -U --pre efficientnet >> /dev/null\n",
        "!pip install tensorflow-addons\n",
        "\n",
        "import efficientnet.tfkeras as efn\n",
        "import tensorflow_addons as tfa"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "hollywood-explanation",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:29.757035Z",
          "iopub.status.busy": "2021-05-02T00:28:29.756357Z",
          "iopub.status.idle": "2021-05-02T00:28:30.069153Z",
          "shell.execute_reply": "2021-05-02T00:28:30.068564Z"
        },
        "id": "hollywood-explanation",
        "papermill": {
          "duration": 0.385621,
          "end_time": "2021-05-02T00:28:30.069357",
          "exception": false,
          "start_time": "2021-05-02T00:28:29.683736",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import gc # garbage collection\n",
        "from kaggle_datasets import KaggleDatasets\n",
        "import tensorflow as tf, re, math\n",
        "import tensorflow.keras.backend as K\n",
        "from tensorflow.python.keras.utils import losses_utils\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from sklearn.utils import class_weight\n",
        "import matplotlib.pyplot as plt\n",
        "import pickle\n",
        "import random as r\n",
        "import cv2, os"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "adequate-cycle",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:30.212936Z",
          "iopub.status.busy": "2021-05-02T00:28:30.211866Z",
          "iopub.status.idle": "2021-05-02T00:28:30.214953Z",
          "shell.execute_reply": "2021-05-02T00:28:30.214343Z"
        },
        "id": "adequate-cycle",
        "papermill": {
          "duration": 0.078657,
          "end_time": "2021-05-02T00:28:30.215092",
          "exception": false,
          "start_time": "2021-05-02T00:28:30.136435",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "DEVICE = \"TPU\" # or \"GPU\"\n",
        "SEED = 42 # USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\n",
        "FOLDS = 5 # NUMBER OF FOLDS. USE 3, 5, OR 15 \n",
        "IMG_SIZES = [256] * FOLDS\n",
        "BATCH_SIZES = [32] * FOLDS\n",
        "EPOCHS = [40] * FOLDS\n",
        "EFF_NETS = [5] * FOLDS\n",
        "WGTS = [1 / FOLDS] * FOLDS # WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\n",
        "TTA = 81 # TEST TIME AUGMENTATION FACTOR\n",
        "\n",
        "def seed_all(seed):\n",
        "    \n",
        "    ''' A function to seed everything for getting reproducible results. '''\n",
        "    r.seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    os.environ['TF_DETERMINISTIC_OPS'] = str(seed)\n",
        "    os.environ['TF_KERAS'] = str(seed)\n",
        "    tf.random.set_seed(seed)\n",
        "    \n",
        "seed_all(SEED)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "4AxRPRmDLF4o",
      "metadata": {
        "id": "4AxRPRmDLF4o"
      },
      "source": [
        "##### BUILT on top of Chris Drotte's public kernel from [here](https://www.kaggle.com/code/cdeotte/triple-stratified-kfold-with-tfrecords)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "about-nudist",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:30.492693Z",
          "iopub.status.busy": "2021-05-02T00:28:30.492003Z",
          "iopub.status.idle": "2021-05-02T00:28:36.298951Z",
          "shell.execute_reply": "2021-05-02T00:28:36.299770Z"
        },
        "id": "about-nudist",
        "papermill": {
          "duration": 5.877391,
          "end_time": "2021-05-02T00:28:36.300041",
          "exception": false,
          "start_time": "2021-05-02T00:28:30.422650",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "if DEVICE == \"TPU\":\n",
        "    print(\"connecting to TPU...\")\n",
        "    try:\n",
        "        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
        "        print('Running on TPU ', tpu.master())\n",
        "    except ValueError:\n",
        "        print(\"Could not connect to TPU\")\n",
        "        tpu = None\n",
        "\n",
        "    if tpu:\n",
        "        try:\n",
        "            print(\"initializing  TPU ...\")\n",
        "            tf.config.experimental_connect_to_cluster(tpu)\n",
        "            tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
        "            print(\"TPU initialized\")\n",
        "        except _:\n",
        "            print(\"failed to initialize TPU\")\n",
        "    else:\n",
        "        DEVICE = \"GPU\"\n",
        "\n",
        "if DEVICE != \"TPU\":\n",
        "    print(\"Using default strategy for CPU and single GPU\")\n",
        "    strategy = tf.distribute.get_strategy()\n",
        "\n",
        "if DEVICE == \"GPU\":\n",
        "    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n",
        "\n",
        "AUTO     = tf.data.experimental.AUTOTUNE\n",
        "REPLICAS = strategy.num_replicas_in_sync\n",
        "print(f'REPLICAS: {REPLICAS}')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "7sC3SUWOLK6T",
      "metadata": {
        "id": "7sC3SUWOLK6T"
      },
      "source": [
        "## Methodology"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "alone-crash",
      "metadata": {
        "_kg_hide-output": true,
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:27.009521Z",
          "iopub.status.busy": "2021-05-02T00:28:27.008808Z",
          "iopub.status.idle": "2021-05-02T00:28:29.615338Z",
          "shell.execute_reply": "2021-05-02T00:28:29.615868Z"
        },
        "id": "alone-crash",
        "papermill": {
          "duration": 2.65861,
          "end_time": "2021-05-02T00:28:29.616052",
          "exception": false,
          "start_time": "2021-05-02T00:28:26.957442",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "import cv2, pandas as pd, matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "train = pd.read_csv('/kaggle/input/isic2020-tfrec-256x256-inpainted2/train_final2.csv')\n",
        "print('Examples WITH Melanoma')\n",
        "imgs = train.loc[train.target==1].loc[train.tfrecord==0].sample(10).image_name.values\n",
        "plt.figure(figsize=(20,8))\n",
        "\n",
        "for i,k in enumerate(imgs):\n",
        "    img = cv2.imread('/kaggle/input/isic2020-256x256-jpg/stratified_jpg_256/train0/%s.jpg'%k)\n",
        "    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n",
        "    plt.subplot(2,5,i+1); plt.axis('off')\n",
        "    plt.imshow(img)\n",
        "plt.show()\n",
        "\n",
        "print('Examples WITHOUT Melanoma')\n",
        "imgs = train.loc[train.target==0].loc[train.tfrecord==0].sample(10).image_name.values\n",
        "plt.figure(figsize=(20,8))\n",
        "for i,k in enumerate(imgs):\n",
        "    img = cv2.imread('/kaggle/input/isic2020-256x256-jpg/stratified_jpg_256/train0/%s.jpg'%k)\n",
        "    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n",
        "    plt.subplot(2,5,i+1); plt.axis('off')\n",
        "    plt.imshow(img)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "silver-russell",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:36.452566Z",
          "iopub.status.busy": "2021-05-02T00:28:36.451801Z",
          "iopub.status.idle": "2021-05-02T00:28:40.051104Z",
          "shell.execute_reply": "2021-05-02T00:28:40.050356Z"
        },
        "id": "silver-russell",
        "papermill": {
          "duration": 3.682519,
          "end_time": "2021-05-02T00:28:40.051282",
          "exception": false,
          "start_time": "2021-05-02T00:28:36.368763",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "GCS_PATH  = [None]*FOLDS\n",
        "for i,k in enumerate(IMG_SIZES):\n",
        "    GCS_PATH[i] = KaggleDatasets().get_gcs_path('isic2020-tfrec-256x256-inpainted2')\n",
        "\n",
        "files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/results/train*.tfrec')))\n",
        "files_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/results/test*.tfrec')))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "pregnant-sterling",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:40.191213Z",
          "iopub.status.busy": "2021-05-02T00:28:40.190537Z",
          "iopub.status.idle": "2021-05-02T00:28:40.193485Z",
          "shell.execute_reply": "2021-05-02T00:28:40.193942Z"
        },
        "id": "pregnant-sterling",
        "papermill": {
          "duration": 0.075294,
          "end_time": "2021-05-02T00:28:40.194130",
          "exception": false,
          "start_time": "2021-05-02T00:28:40.118836",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "# ['gs://kds-8117557969f16711066aa655c5e580d57b1430b7f4eaeebfc39ed426',\n",
        "#  'gs://kds-8117557969f16711066aa655c5e580d57b1430b7f4eaeebfc39ed426',\n",
        "#  'gs://kds-8117557969f16711066aa655c5e580d57b1430b7f4eaeebfc39ed426',\n",
        "#  'gs://kds-8117557969f16711066aa655c5e580d57b1430b7f4eaeebfc39ed426',\n",
        "#  'gs://kds-8117557969f16711066aa655c5e580d57b1430b7f4eaeebfc39ed426']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "variable-serial",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:40.634433Z",
          "iopub.status.busy": "2021-05-02T00:28:40.633415Z",
          "iopub.status.idle": "2021-05-02T00:28:40.636402Z",
          "shell.execute_reply": "2021-05-02T00:28:40.635757Z"
        },
        "id": "variable-serial",
        "papermill": {
          "duration": 0.077853,
          "end_time": "2021-05-02T00:28:40.636548",
          "exception": false,
          "start_time": "2021-05-02T00:28:40.558695",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "ROT_ = 180.0\n",
        "SHR_ = 2.0\n",
        "HZOOM_ = 8.0\n",
        "WZOOM_ = 8.0\n",
        "HSHIFT_ = 8.0\n",
        "WSHIFT_ = 8.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "proud-hartford",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:40.792649Z",
          "iopub.status.busy": "2021-05-02T00:28:40.787519Z",
          "iopub.status.idle": "2021-05-02T00:28:40.795837Z",
          "shell.execute_reply": "2021-05-02T00:28:40.795070Z"
        },
        "id": "proud-hartford",
        "papermill": {
          "duration": 0.091231,
          "end_time": "2021-05-02T00:28:40.796004",
          "exception": false,
          "start_time": "2021-05-02T00:28:40.704773",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n",
        "    # returns 3x3 transformmatrix which transforms indicies\n",
        "        \n",
        "    # CONVERT DEGREES TO RADIANS\n",
        "    rotation = math.pi * rotation / 180.\n",
        "    shear    = math.pi * shear    / 180.\n",
        "\n",
        "    def get_3x3_mat(lst):\n",
        "        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n",
        "    \n",
        "    # ROTATION MATRIX\n",
        "    c1   = tf.math.cos(rotation)\n",
        "    s1   = tf.math.sin(rotation)\n",
        "    one  = tf.constant([1],dtype='float32')\n",
        "    zero = tf.constant([0],dtype='float32')\n",
        "    \n",
        "    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n",
        "                                   -s1,  c1,   zero, \n",
        "                                   zero, zero, one])    \n",
        "    # SHEAR MATRIX\n",
        "    c2 = tf.math.cos(shear)\n",
        "    s2 = tf.math.sin(shear)    \n",
        "    \n",
        "    shear_matrix = get_3x3_mat([one,  s2,   zero, \n",
        "                                zero, c2,   zero, \n",
        "                                zero, zero, one])        \n",
        "    # ZOOM MATRIX\n",
        "    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n",
        "                               zero,            one/width_zoom, zero, \n",
        "                               zero,            zero,           one])    \n",
        "    # SHIFT MATRIX\n",
        "    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n",
        "                                zero, one,  width_shift, \n",
        "                                zero, zero, one])\n",
        "    \n",
        "    return K.dot(K.dot(rotation_matrix, shear_matrix), \n",
        "                 K.dot(zoom_matrix,     shift_matrix))\n",
        "\n",
        "\n",
        "def transform(image, DIM=256):    \n",
        "    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n",
        "    # output - image randomly rotated, sheared, zoomed, and shifted\n",
        "    XDIM = DIM%2 #fix for size 331\n",
        "    \n",
        "    rot = ROT_ * tf.random.normal([1], dtype='float32')\n",
        "    shr = SHR_ * tf.random.normal([1], dtype='float32') \n",
        "    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n",
        "    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n",
        "    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n",
        "    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n",
        "\n",
        "    # GET TRANSFORMATION MATRIX\n",
        "    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n",
        "\n",
        "    # LIST DESTINATION PIXEL INDICES\n",
        "    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n",
        "    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n",
        "    z   = tf.ones([DIM*DIM], dtype='int32')\n",
        "    idx = tf.stack( [x,y,z] )\n",
        "    \n",
        "    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n",
        "    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n",
        "    idx2 = K.cast(idx2, dtype='int32')\n",
        "    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n",
        "    \n",
        "    # FIND ORIGIN PIXEL VALUES           \n",
        "    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n",
        "    d    = tf.gather_nd(image, tf.transpose(idx3))\n",
        "    return tf.reshape(d,[DIM, DIM,3])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "hired-conversion",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:41.249528Z",
          "iopub.status.busy": "2021-05-02T00:28:41.248431Z",
          "iopub.status.idle": "2021-05-02T00:28:41.251207Z",
          "shell.execute_reply": "2021-05-02T00:28:41.251738Z"
        },
        "id": "hired-conversion",
        "papermill": {
          "duration": 0.079515,
          "end_time": "2021-05-02T00:28:41.251941",
          "exception": false,
          "start_time": "2021-05-02T00:28:41.172426",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "def microscopicCutOut(img):\n",
        "#     if r.random() <= 0.5:\n",
        "        Circle = cv2.circle(\n",
        "                                (np.ones(img.shape) * 255).astype(np.uint8),\n",
        "                                (img.shape[0]//2, img.shape[1]//2),\n",
        "                                r.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15),\n",
        "                                (0, 0, 0),\n",
        "                                -1\n",
        "                   )\n",
        "\n",
        "        mask = Circle - 255\n",
        "        img = tf.math.multiply(img, mask)\n",
        "\n",
        "        return img"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "level-taylor",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:40.938149Z",
          "iopub.status.busy": "2021-05-02T00:28:40.937442Z",
          "iopub.status.idle": "2021-05-02T00:28:40.942285Z",
          "shell.execute_reply": "2021-05-02T00:28:40.943017Z"
        },
        "id": "level-taylor",
        "papermill": {
          "duration": 0.078379,
          "end_time": "2021-05-02T00:28:40.943239",
          "exception": false,
          "start_time": "2021-05-02T00:28:40.864860",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "CFG = dict(\n",
        "    sprinkles_mode    = 'normal',\n",
        "    sprinkles_prob    =   1, # probability to spawn a box (between 0-1)\n",
        "    num_holes         =   10, # number of square patches to drop\n",
        "    side_length       =   12   #square size\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "phantom-broad",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:41.085700Z",
          "iopub.status.busy": "2021-05-02T00:28:41.084948Z",
          "iopub.status.idle": "2021-05-02T00:28:41.102897Z",
          "shell.execute_reply": "2021-05-02T00:28:41.102337Z"
        },
        "id": "phantom-broad",
        "papermill": {
          "duration": 0.089541,
          "end_time": "2021-05-02T00:28:41.103067",
          "exception": false,
          "start_time": "2021-05-02T00:28:41.013526",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "# based on: https://www.kaggle.com/benboren/tfrecord-progressive-sprinkles\n",
        "def make_mask(num_holes,side_length,rows, cols, num_channels):\n",
        "        '''Builds the mask for all sprinkles.'''\n",
        "        row_range = tf.tile(tf.range(rows)[..., tf.newaxis], [1, num_holes])\n",
        "        col_range = tf.tile(tf.range(cols)[..., tf.newaxis], [1, num_holes])\n",
        "        r_idx = tf.random.uniform([num_holes], minval=0, maxval=rows-1,\n",
        "                                  dtype=tf.int32)\n",
        "        c_idx = tf.random.uniform([num_holes], minval=0, maxval=cols-1,\n",
        "                                  dtype=tf.int32)\n",
        "        r1 = tf.clip_by_value(r_idx - side_length // 2, 0, rows)\n",
        "        r2 = tf.clip_by_value(r_idx + side_length // 2, 0, rows)\n",
        "        c1 = tf.clip_by_value(c_idx - side_length // 2, 0, cols)\n",
        "        c2 = tf.clip_by_value(c_idx + side_length // 2, 0, cols)\n",
        "        row_mask = (row_range > r1) & (row_range < r2)\n",
        "        col_mask = (col_range > c1) & (col_range < c2)\n",
        "\n",
        "        # Combine masks into one layer and duplicate over channels.\n",
        "        mask = row_mask[:, tf.newaxis] & col_mask\n",
        "        mask = tf.reduce_any(mask, axis=-1)\n",
        "        mask = mask[..., tf.newaxis]\n",
        "        mask = tf.tile(mask, [1, 1, num_channels])\n",
        "        return mask\n",
        "\n",
        "\n",
        "def sprinkles(image, cfg = CFG): \n",
        "    \n",
        "    '''Applies all sprinkles.'''\n",
        "    \n",
        "    num_holes = cfg['num_holes']\n",
        "    side_length = cfg['side_length']\n",
        "    mode = cfg['sprinkles_mode']\n",
        "    PROBABILITY = cfg['sprinkles_prob']\n",
        "    \n",
        "    RandProb = tf.cast( tf.random.uniform([],0,1) < PROBABILITY, tf.int32)\n",
        "    if (RandProb == 0)|(num_holes == 0): return image\n",
        "    \n",
        "    img_shape = tf.shape(image)\n",
        "    if mode is 'normal':\n",
        "        rejected = tf.zeros_like(image)\n",
        "    elif mode is 'salt_pepper':\n",
        "        num_holes = num_holes // 2\n",
        "        rejected_high = tf.ones_like(image)\n",
        "        rejected_low = tf.zeros_like(image)\n",
        "    elif mode is 'gaussian':\n",
        "        rejected = tf.random.normal(img_shape, dtype=tf.float32)\n",
        "    else:\n",
        "        raise ValueError(f'Unknown mode \"{mode}\" given.')\n",
        "        \n",
        "    rows = img_shape[0]\n",
        "    cols = img_shape[1]\n",
        "    num_channels = img_shape[-1]\n",
        "    if mode is 'salt_pepper':\n",
        "        mask1 = make_mask(num_holes,side_length,rows, cols, num_channels)\n",
        "        mask2 = make_mask(num_holes,side_length,rows, cols, num_channels)\n",
        "        filtered_image = tf.where(mask1, rejected_high, image)\n",
        "        filtered_image = tf.where(mask2, rejected_low, filtered_image)\n",
        "    else:\n",
        "        mask = make_mask(num_holes,side_length,rows, cols, num_channels)\n",
        "        filtered_image = tf.where(mask, rejected, image)\n",
        "    return filtered_image"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "lonely-process",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:41.398729Z",
          "iopub.status.busy": "2021-05-02T00:28:41.398011Z",
          "iopub.status.idle": "2021-05-02T00:28:43.212102Z",
          "shell.execute_reply": "2021-05-02T00:28:43.212654Z"
        },
        "id": "lonely-process",
        "papermill": {
          "duration": 1.888542,
          "end_time": "2021-05-02T00:28:43.212844",
          "exception": false,
          "start_time": "2021-05-02T00:28:41.324302",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "imgs = train.loc[train.target==1].loc[train.tfrecord==0].sample(10).image_name.values\n",
        "plt.figure(figsize=(20,8))\n",
        "\n",
        "for i,k in enumerate(imgs):\n",
        "    imgpath = ('/kaggle/input/isic2020-256x256-jpg/stratified_jpg_256/train0/%s.jpg'%k)\n",
        "    with open(imgpath, \"rb\") as local_file:\n",
        "        img = local_file.read()\n",
        "    img = tf.image.decode_jpeg(img, channels=3)\n",
        "    img = tf.cast(img, tf.float32) / 255.0\n",
        "    img = transform(img,DIM=256)\n",
        "    img = tf.image.random_flip_left_right(img)\n",
        "    img = tf.image.random_flip_up_down(img)\n",
        "    img = tf.image.random_saturation(img, 0.7, 1.3)\n",
        "    img = tf.image.random_contrast(img, 0.8, 1.2)\n",
        "    img = tf.image.random_brightness(img, 0.1)\n",
        "    img = sprinkles(img)\n",
        "    img = microscopicCutOut(img)\n",
        "    img = tf.reshape(img, [256, 256, 3])\n",
        "    plt.imshow(img)\n",
        "    plt.subplot(2,5,i+1); plt.axis('off')\n",
        "    plt.imshow(img)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "emotional-delaware",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:43.415934Z",
          "iopub.status.busy": "2021-05-02T00:28:43.414927Z",
          "iopub.status.idle": "2021-05-02T00:28:43.418176Z",
          "shell.execute_reply": "2021-05-02T00:28:43.417624Z"
        },
        "id": "emotional-delaware",
        "papermill": {
          "duration": 0.114267,
          "end_time": "2021-05-02T00:28:43.418349",
          "exception": false,
          "start_time": "2021-05-02T00:28:43.304082",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "def read_labeled_tfrecord(example):\n",
        "    tfrec_format = {\n",
        "        'image'                        : tf.io.FixedLenFeature([], tf.string),\n",
        "        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n",
        "        'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n",
        "        'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n",
        "        'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n",
        "        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n",
        "        'diagnosis'                    : tf.io.FixedLenFeature([], tf.int64),\n",
        "        'target'                       : tf.io.FixedLenFeature([], tf.int64),\n",
        "        'width'                        : tf.io.FixedLenFeature([], tf.int64),\n",
        "        'height'                       : tf.io.FixedLenFeature([], tf.int64)\n",
        "    }\n",
        "\n",
        "    example = tf.io.parse_single_example(example, tfrec_format)\n",
        "    label = tf.cast(example['target'], tf.float32)\n",
        "    return example['image'], label\n",
        "\n",
        "\n",
        "def read_unlabeled_tfrecord(example, return_image_name):\n",
        "    tfrec_format = {\n",
        "        'image'                        : tf.io.FixedLenFeature([], tf.string),\n",
        "        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n",
        "        'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n",
        "        'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n",
        "        'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n",
        "        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n",
        "        'width'                        : tf.io.FixedLenFeature([], tf.int64),\n",
        "        'height'                       : tf.io.FixedLenFeature([], tf.int64)\n",
        "        \n",
        "    }\n",
        "    example = tf.io.parse_single_example(example, tfrec_format)\n",
        "    return example['image'], example['image_name'] if return_image_name else 0\n",
        "\n",
        " \n",
        "def prepare_image(img, augment=True, dim=256):   \n",
        "    img = tf.image.decode_jpeg(img, channels=3)\n",
        "    img = tf.cast(img, tf.float32) / 255.0\n",
        "#     img = tf.image.per_image_standardization(img)    \n",
        "  \n",
        "    if augment:\n",
        "        img = transform(img,DIM=dim)\n",
        "        img = tf.image.random_flip_left_right(img)\n",
        "        img = tf.image.random_flip_up_down(img)\n",
        "        img = tf.image.random_saturation(img, 0.7, 1.3)\n",
        "        img = tf.image.random_contrast(img, 0.8, 1.2)\n",
        "        img = tf.image.random_brightness(img, 0.1)\n",
        "        img = sprinkles(img)\n",
        "        img = microscopicCutOut(img)\n",
        "\n",
        "    img = tf.reshape(img, [dim, dim, 3])\n",
        "    return img\n",
        "\n",
        "def count_data_items(filenames):\n",
        "    # the number of data items is written in the name of the .tfrec files\n",
        "    n = [int(re.compile(r\"-([0-9]*)\\_inpaint2.\").search(filename).group(1)) for filename in filenames]\n",
        "    return np.sum(n)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "current-socket",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:43.613065Z",
          "iopub.status.busy": "2021-05-02T00:28:43.612337Z",
          "iopub.status.idle": "2021-05-02T00:28:43.616217Z",
          "shell.execute_reply": "2021-05-02T00:28:43.615570Z"
        },
        "id": "current-socket",
        "papermill": {
          "duration": 0.10587,
          "end_time": "2021-05-02T00:28:43.616395",
          "exception": false,
          "start_time": "2021-05-02T00:28:43.510525",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "def get_dataset(files, augment = False, shuffle = False, repeat = False, \n",
        "                labeled=True, return_image_names=True, batch_size=16, dim=256):\n",
        "    \n",
        "    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n",
        "    ds = ds.cache()\n",
        "    \n",
        "    if repeat:\n",
        "        ds = ds.repeat()\n",
        "    \n",
        "    if shuffle: \n",
        "        ds = ds.shuffle(1024*8)\n",
        "        opt = tf.data.Options()\n",
        "        opt.experimental_deterministic = False\n",
        "        ds = ds.with_options(opt)\n",
        "        \n",
        "    if labeled: \n",
        "        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n",
        "    else:\n",
        "        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n",
        "                    num_parallel_calls=AUTO)      \n",
        "    \n",
        "    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, dim=dim), \n",
        "                                               imgname_or_label), \n",
        "                num_parallel_calls=AUTO)\n",
        "    \n",
        "    ds = ds.batch(batch_size * REPLICAS)\n",
        "    ds = ds.prefetch(AUTO)\n",
        "    return ds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "matched-imagination",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:43.804177Z",
          "iopub.status.busy": "2021-05-02T00:28:43.803067Z",
          "iopub.status.idle": "2021-05-02T00:28:43.806529Z",
          "shell.execute_reply": "2021-05-02T00:28:43.805653Z"
        },
        "id": "matched-imagination",
        "papermill": {
          "duration": 0.099678,
          "end_time": "2021-05-02T00:28:43.806689",
          "exception": false,
          "start_time": "2021-05-02T00:28:43.707011",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "from tensorflow.keras.applications.densenet import DenseNet121\n",
        "from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "structured-baking",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:44.047023Z",
          "iopub.status.busy": "2021-05-02T00:28:44.020549Z",
          "iopub.status.idle": "2021-05-02T00:28:44.058377Z",
          "shell.execute_reply": "2021-05-02T00:28:44.057765Z"
        },
        "id": "structured-baking",
        "jupyter": {
          "source_hidden": true
        },
        "papermill": {
          "duration": 0.161556,
          "end_time": "2021-05-02T00:28:44.058536",
          "exception": false,
          "start_time": "2021-05-02T00:28:43.896980",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "#helper functions to create teacher models\n",
        "\n",
        "#create InceptionResNet model\n",
        "def get_InceptionResNetV2(dim=128):\n",
        "    with strategy.scope():\n",
        "        inception_res = InceptionResNetV2(\n",
        "            input_shape = (dim, dim, 3),\n",
        "            weights = 'imagenet',\n",
        "            include_top = False\n",
        "        )\n",
        "        #make trainable so we can fine-tune\n",
        "        inception_res.trainable = True\n",
        "#         print(inception_res.summary())\n",
        "        model = tf.keras.Sequential([\n",
        "            inception_res,\n",
        "            tf.keras.layers.GlobalAveragePooling2D(),\n",
        "            tf.keras.layers.Dense(1, activation = 'sigmoid')\n",
        "        ])\n",
        "    opti = tfa.optimizers.RectifiedAdam(lr=0.00032, total_steps=10000,\n",
        "                               warmup_proportion=0.1, min_lr=1e-09)\n",
        "    loss = tfa.losses.SigmoidFocalCrossEntropy(alpha=0.90,gamma=2.0,\n",
        "                                     reduction=tf.keras.losses.Reduction.AUTO)\n",
        "    \n",
        "    METRICS = [\n",
        "      tf.keras.metrics.TruePositives(name='TP'),\n",
        "      tf.keras.metrics.FalsePositives(name='FP'),\n",
        "      tf.keras.metrics.TrueNegatives(name='TN'),\n",
        "      tf.keras.metrics.FalseNegatives(name='FN'), \n",
        "      tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n",
        "      tf.keras.metrics.Precision(name='precision'),\n",
        "      tf.keras.metrics.Recall(name='SEN/Recall'),\n",
        "      tf.keras.metrics.AUC(name='AUC'),\n",
        "    ]\n",
        "    model.compile(optimizer = opti,loss = loss,metrics=METRICS)\n",
        "    return model\n",
        "\n",
        "\n",
        "def get_DenseNet121(dim=128):\n",
        "    with strategy.scope():\n",
        "        dnet = DenseNet121(\n",
        "            input_shape = (dim, dim, 3),\n",
        "            weights = 'imagenet',\n",
        "            include_top = False\n",
        "        )\n",
        "        #make trainable so we can fine-tune\n",
        "        dnet.trainable = True\n",
        "#         print(dnet.summary())\n",
        "        model = tf.keras.Sequential([\n",
        "            dnet,\n",
        "            tf.keras.layers.GlobalAveragePooling2D(),\n",
        "            tf.keras.layers.Dense(1, activation = 'sigmoid')\n",
        "        ])\n",
        "    opt = tfa.optimizers.RectifiedAdam(lr=0.00032, total_steps=10000,\n",
        "                           warmup_proportion=0.1, min_lr=1e-7)\n",
        "    opti = tfa.optimizers.Lookahead(opt, sync_period=5, slow_step_size=0.8)\n",
        "    loss = tfa.losses.SigmoidFocalCrossEntropy(alpha=0.90,gamma=2.0,\n",
        "                                     reduction=tf.keras.losses.Reduction.AUTO)\n",
        "    METRICS = [\n",
        "        tf.keras.metrics.TruePositives(name='TP'),\n",
        "        tf.keras.metrics.FalseNegatives(name='FN'),\n",
        "        tf.keras.metrics.TrueNegatives(name='TN'),\n",
        "        tf.keras.metrics.FalsePositives(name='FP'),\n",
        "        tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n",
        "        tf.keras.metrics.Precision(name='precision'),\n",
        "        tf.keras.metrics.Recall(name='SEN/Recall'),\n",
        "        tfa.metrics.F1Score(name='f1score',num_classes=2, average= 'micro',threshold=0.5),\n",
        "        tf.keras.metrics.AUC(name='AUC'),\n",
        "    ]\n",
        "    model.compile(optimizer=opti, loss=loss, metrics=METRICS)\n",
        "#     model.summary()\n",
        "    \n",
        "    return model\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cross-ontario",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:45.643444Z",
          "iopub.status.busy": "2021-05-02T00:28:45.642368Z",
          "iopub.status.idle": "2021-05-02T00:28:45.658189Z",
          "shell.execute_reply": "2021-05-02T00:28:45.657624Z"
        },
        "id": "cross-ontario",
        "papermill": {
          "duration": 0.111347,
          "end_time": "2021-05-02T00:28:45.658359",
          "exception": false,
          "start_time": "2021-05-02T00:28:45.547012",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n",
        "def get_EFF_NET(dim=128, ef=0, output_bias=None):\n",
        "    if output_bias is not None:\n",
        "        output_bias = tf.keras.initializers.Constant(output_bias)\n",
        "    \n",
        "    inp = tf.keras.layers.Input(shape=(dim,dim,3), name='inp')\n",
        "    base = EFNS[ef](input_shape=(dim,dim,3),weights='noisy-student',include_top=False)\n",
        "    base.trainable = True\n",
        "    x = base(inp)\n",
        "    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n",
        "    x = tf.keras.layers.Dense(1,activation='sigmoid', bias_initializer=output_bias)(x)\n",
        "    \n",
        "    model = tf.keras.Model(inputs=[inp],outputs=[x])\n",
        "    opti = tfa.optimizers.RectifiedAdam(lr=0.00032, total_steps=10000,\n",
        "                               warmup_proportion=0.1, min_lr=1e-7)\n",
        "    loss = tfa.losses.SigmoidFocalCrossEntropy(alpha=0.90,gamma=2.0,\n",
        "                                    reduction=losses_utils.ReductionV2.NONE)\n",
        "\n",
        "    METRICS = [\n",
        "        tf.keras.metrics.TruePositives(name='TP'),\n",
        "        tf.keras.metrics.FalseNegatives(name='FN'),\n",
        "        tf.keras.metrics.TrueNegatives(name='TN'),\n",
        "        tf.keras.metrics.FalsePositives(name='FP'),\n",
        "        tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n",
        "        tf.keras.metrics.Precision(name='precision'),\n",
        "        tf.keras.metrics.Recall(name='SEN/Recall'),\n",
        "        tfa.metrics.F1Score(name='f1score',num_classes=2, average= 'micro',threshold=0.5),\n",
        "        tf.keras.metrics.AUC(name='AUC')\n",
        "    ]\n",
        "    model.compile(optimizer=opti, loss=loss, metrics=METRICS)\n",
        "#     model.summary()\n",
        "    \n",
        "    return model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "least-organ",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:45.852614Z",
          "iopub.status.busy": "2021-05-02T00:28:45.851593Z",
          "iopub.status.idle": "2021-05-02T00:28:45.854843Z",
          "shell.execute_reply": "2021-05-02T00:28:45.854156Z"
        },
        "id": "least-organ",
        "jupyter": {
          "source_hidden": true
        },
        "papermill": {
          "duration": 0.104818,
          "end_time": "2021-05-02T00:28:45.854994",
          "exception": false,
          "start_time": "2021-05-02T00:28:45.750176",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "def get_lr_callback(batch_size=8):    \n",
        "    lr_callback = tf.keras.callbacks.ReduceLROnPlateau(\n",
        "        monitor='val_AUC', factor=0.3, patience=2, verbose=2,\n",
        "        mode='max', min_delta=0.001, min_lr=0.000000001\n",
        "    )\n",
        "\n",
        "    return lr_callback\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "identified-martial",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T00:28:46.768196Z",
          "iopub.status.busy": "2021-05-02T00:28:46.745412Z",
          "iopub.status.idle": "2021-05-02T04:09:43.623051Z",
          "shell.execute_reply": "2021-05-02T04:09:43.624743Z"
        },
        "id": "identified-martial",
        "papermill": {
          "duration": 13257.012155,
          "end_time": "2021-05-02T04:09:43.624982",
          "exception": false,
          "start_time": "2021-05-02T00:28:46.612827",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "VERBOSE = 1\n",
        "DISPLAY_PLOT = True\n",
        "\n",
        "skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)\n",
        "\n",
        "oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []\n",
        "preds = np.zeros((count_data_items(files_test),1))\n",
        "preds1 = np.zeros((count_data_items(files_test),1))\n",
        "pred_foldWise = np.asarray([preds]*FOLDS)\n",
        "\n",
        "for fold,(idxT,idxV) in enumerate(skf.split(np.arange(15))):    \n",
        "    if DEVICE=='TPU':\n",
        "        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "    print('#'*25); print('#### FOLD',fold+1)\n",
        "    print('#### Image Size %i and batch_size %i'%\n",
        "          (IMG_SIZES[fold], BATCH_SIZES[fold]*REPLICAS))\n",
        "\n",
        "    # CREATE TRAIN AND VALIDATION SUBSETS\n",
        "    files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/_train%.2i*.tfrec'%x for x in idxT])\n",
        "\n",
        "    np.random.shuffle(files_train); print('#'*25)\n",
        "    files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/_train%.2i*.tfrec'%x for x in idxV])\n",
        "    files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/_test*.tfrec')))\n",
        "\n",
        "    # SAVE BEST MODEL EACH FOLD\n",
        "    sv = tf.keras.callbacks.ModelCheckpoint(\n",
        "        'fold-%i.h5'%fold, monitor='val_AUC', verbose=0, save_best_only=True,\n",
        "        save_weights_only=True, mode='max', save_freq='epoch')\n",
        "    \n",
        "    # early stopping with 5 patience\n",
        "    early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_AUC', mode = 'max', patience = 5, \n",
        "                          verbose = 2, min_delta = 0.0001, restore_best_weights = True)\n",
        "\n",
        "    a = count_data_items(files_train); b=count_data_items(files_valid);\n",
        "    print('AUGMENTED TRAIN SIZE: ', a, ' * ', TTA, ' = ', a*TTA)\n",
        "    print('AUGMENTED VALID SIZE: ', b, ' * ', TTA, ' = ', b*TTA)\n",
        "    \n",
        "    # BUILD MODEL\n",
        "    K.clear_session()\n",
        "    with strategy.scope():\n",
        "        # model = get_InceptionResNetV2(dim=IMG_SIZES[fold])\n",
        "        # model = get_DenseNet121(dim=IMG_SIZES[fold])\n",
        "        model = get_EFF_NET(dim=IMG_SIZES[fold], ef=EFF_NETS[fold], output_bias = None)\n",
        "   \n",
        "    # TRAIN\n",
        "    print('Training...')\n",
        "    history = model.fit(\n",
        "        get_dataset(files_train, augment=True, shuffle=True, repeat=True, \n",
        "                    dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),\n",
        "        epochs = EPOCHS[fold],\n",
        "        callbacks = [sv, early_stopping, get_lr_callback(BATCH_SIZES[fold])], \n",
        "        steps_per_epoch = count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,\n",
        "        validation_data = get_dataset(files_valid,augment=False,shuffle=False, repeat=False,\n",
        "                                      dim=IMG_SIZES[fold]),\n",
        "        verbose = VERBOSE,\n",
        "    )\n",
        "\n",
        "    print('Loading best model...')\n",
        "    model.load_weights('fold-%i.h5'%fold)\n",
        "    \n",
        "    # PREDICT OOF USING TTA\n",
        "    print('Predicting OOF with TTA...')\n",
        "    ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,\n",
        "            repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n",
        "    ct_valid = (count_data_items(files_valid)); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS\n",
        "    pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]\n",
        "    \n",
        "    oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1))\n",
        "    \n",
        "    # GET OOF TARGETS AND NAMES\n",
        "    ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n",
        "            labeled=True, return_image_names=True)\n",
        "    oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n",
        "    oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n",
        "    ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n",
        "                labeled=False, return_image_names=True)\n",
        "    oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n",
        "    \n",
        "    auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n",
        "    oof_val.append(np.max( history.history['val_AUC'] ))\n",
        "    print('#### FOLD %i OOF AUC without TTA = %.4f, with TTA = %.4f'%(fold+1,oof_val[-1],auc))\n",
        "\n",
        "    # PREDICT TEST with TTA\n",
        "    print('Predicting Test with TTA...')\n",
        "    ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,\n",
        "            repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n",
        "    ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS\n",
        "    pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]\n",
        "    tmp_pred = np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1)\n",
        "    preds[:,0] += tmp_pred * WGTS[fold]\n",
        "    \n",
        "    # PREDICT TEST without TTA\n",
        "    print('Predicting Test without TTA...')\n",
        "    ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n",
        "            repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n",
        "    ct_test = count_data_items(files_test); STEPS = 1 * ct_test/BATCH_SIZES[fold]/4/REPLICAS\n",
        "    pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:1*ct_test,]\n",
        "    tmp_pred1 = np.mean(pred.reshape((ct_test,1),order='F'),axis=1)\n",
        "    preds1[:,0] += tmp_pred1 * WGTS[fold]\n",
        "\n",
        "    pred_foldWise[fold][:,0] += tmp_pred\n",
        "    \n",
        "    hist=dict(zip(list(history.history.keys()), np.array(list(history.history.values()))))\n",
        "    pickle.dump(hist, open(\"history_fold-%i.p\"%(fold+1), \"wb\"))\n",
        "    # PLOT TRAINING\n",
        "    if DISPLAY_PLOT:\n",
        "        plt.figure(figsize=(15,5))\n",
        "        plt.plot(np.arange(EPOCHS[fold]),history.history['AUC'],'-o',label='Train AUC',color='#ff7f0e')\n",
        "        plt.plot(np.arange(EPOCHS[fold]),history.history['val_AUC'],'-o',label='Val AUC',color='#1f77b4')\n",
        "        x = np.argmax( history.history['val_AUC'] ); y = np.max( history.history['val_AUC'] )\n",
        "        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n",
        "        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.4f'%y,size=14)\n",
        "        plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n",
        "        plt.legend(loc=2)\n",
        "        plt2 = plt.gca().twinx()\n",
        "        plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n",
        "        plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='#1f77b4')\n",
        "        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n",
        "        ydist = plt.ylim()[1] - plt.ylim()[0]\n",
        "        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n",
        "        plt.ylabel('Loss',size=14)\n",
        "        plt.title('FOLD %i - Image Size %i, EFFNET B%i'% (fold+1,IMG_SIZES[fold],EFF_NETS[fold]),size=18)\n",
        "        plt.legend(loc=3)\n",
        "        plt.show()\n",
        "\n",
        "    del model; z = gc.collect()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "intense-salvation",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T04:11:13.773098Z",
          "iopub.status.busy": "2021-05-02T04:11:13.772405Z",
          "iopub.status.idle": "2021-05-02T04:11:14.042273Z",
          "shell.execute_reply": "2021-05-02T04:11:14.041550Z"
        },
        "id": "intense-salvation",
        "outputId": "5b577dfc-ae7b-41f1-8853-efa2ae407408",
        "papermill": {
          "duration": 9.257307,
          "end_time": "2021-05-02T04:11:14.042422",
          "exception": false,
          "start_time": "2021-05-02T04:11:04.785115",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overall OOF AUC with TTA = 0.9604\n"
          ]
        }
      ],
      "source": [
        "# COMPUTE OVERALL OOF AUC\n",
        "oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);\n",
        "names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)\n",
        "auc = roc_auc_score(true,oof)\n",
        "print('Overall OOF AUC with TTA = %.4f'%auc)\n",
        "\n",
        "# SAVE OOF TO DISK\n",
        "df_oof = pd.DataFrame(dict(\n",
        "    image_name = names, target=true, pred = oof, fold=folds))\n",
        "df_oof.to_csv('oof_Distilled_EFF_NETB2.csv',index=False)\n",
        "# df_oof.head()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "optional-feeling",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T04:11:31.901703Z",
          "iopub.status.busy": "2021-05-02T04:11:31.900965Z",
          "iopub.status.idle": "2021-05-02T04:11:54.253657Z",
          "shell.execute_reply": "2021-05-02T04:11:54.253035Z"
        },
        "id": "optional-feeling",
        "papermill": {
          "duration": 31.332478,
          "end_time": "2021-05-02T04:11:54.253844",
          "exception": false,
          "start_time": "2021-05-02T04:11:22.921366",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],\n",
        "                 labeled=False, return_image_names=True)\n",
        "\n",
        "image_names = np.array([img_name.numpy().decode(\"utf-8\")\n",
        "                        for img, img_name in iter(ds.unbatch())])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "framed-screening",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T04:12:12.164385Z",
          "iopub.status.busy": "2021-05-02T04:12:12.163346Z",
          "iopub.status.idle": "2021-05-02T04:12:12.262949Z",
          "shell.execute_reply": "2021-05-02T04:12:12.262418Z"
        },
        "id": "framed-screening",
        "papermill": {
          "duration": 9.070857,
          "end_time": "2021-05-02T04:12:12.263093",
          "exception": false,
          "start_time": "2021-05-02T04:12:03.192236",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "submission = pd.DataFrame(dict(image_name=image_names, target=preds1[:,0]))\n",
        "submission = submission.sort_values('image_name')\n",
        "submission.to_csv('without_TTA_submission_augmented_Distilled_EFF_NETB2.csv', index=False)\n",
        "submission.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "sought-doctor",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T04:12:30.158815Z",
          "iopub.status.busy": "2021-05-02T04:12:30.143947Z",
          "iopub.status.idle": "2021-05-02T04:12:30.483845Z",
          "shell.execute_reply": "2021-05-02T04:12:30.483304Z"
        },
        "id": "sought-doctor",
        "papermill": {
          "duration": 9.253784,
          "end_time": "2021-05-02T04:12:30.483984",
          "exception": false,
          "start_time": "2021-05-02T04:12:21.230200",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "plt.hist(submission.target,bins=100)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "parliamentary-county",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T04:12:48.441232Z",
          "iopub.status.busy": "2021-05-02T04:12:48.440590Z",
          "iopub.status.idle": "2021-05-02T04:12:48.518646Z",
          "shell.execute_reply": "2021-05-02T04:12:48.519293Z"
        },
        "id": "parliamentary-county",
        "papermill": {
          "duration": 9.035724,
          "end_time": "2021-05-02T04:12:48.519490",
          "exception": false,
          "start_time": "2021-05-02T04:12:39.483766",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))\n",
        "submission = submission.sort_values('image_name')\n",
        "submission.to_csv('with_TTA_submission_augmented_Distilled_EFF_NETB2.csv', index=False)\n",
        "submission.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "enabling-sponsorship",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T04:13:06.474799Z",
          "iopub.status.busy": "2021-05-02T04:13:06.474187Z",
          "iopub.status.idle": "2021-05-02T04:13:06.478742Z",
          "shell.execute_reply": "2021-05-02T04:13:06.479335Z"
        },
        "id": "enabling-sponsorship",
        "outputId": "5643b46e-4e36-469f-ac33-382026684a2a",
        "papermill": {
          "duration": 8.98424,
          "end_time": "2021-05-02T04:13:06.479507",
          "exception": false,
          "start_time": "2021-05-02T04:12:57.495267",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "10982"
            ]
          },
          "execution_count": 43,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(submission)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "higher-privacy",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T04:13:24.417599Z",
          "iopub.status.busy": "2021-05-02T04:13:24.416757Z",
          "iopub.status.idle": "2021-05-02T04:13:24.730986Z",
          "shell.execute_reply": "2021-05-02T04:13:24.731522Z"
        },
        "id": "higher-privacy",
        "papermill": {
          "duration": 9.233834,
          "end_time": "2021-05-02T04:13:24.731718",
          "exception": false,
          "start_time": "2021-05-02T04:13:15.497884",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "plt.hist(submission.target,bins=100)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "capital-collector",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-05-02T04:13:42.660680Z",
          "iopub.status.busy": "2021-05-02T04:13:42.659775Z",
          "iopub.status.idle": "2021-05-02T04:13:42.667102Z",
          "shell.execute_reply": "2021-05-02T04:13:42.667642Z"
        },
        "id": "capital-collector",
        "papermill": {
          "duration": 8.976312,
          "end_time": "2021-05-02T04:13:42.667841",
          "exception": false,
          "start_time": "2021-05-02T04:13:33.691529",
          "status": "completed"
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "np.save('foldWisePredictions_Distilled_EFF_NETB2.npy', pred_foldWise)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.10"
    },
    "papermill": {
      "default_parameters": {},
      "duration": 13579.226183,
      "end_time": "2021-05-02T04:14:12.410100",
      "environment_variables": {},
      "exception": null,
      "input_path": "__notebook__.ipynb",
      "output_path": "__notebook__.ipynb",
      "parameters": {},
      "start_time": "2021-05-02T00:27:53.183917",
      "version": "2.3.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}