model_without_skip_layers_global_pool_appended_june24.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "model_without_skip_layers-global pool appended june24.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/RajashreeDahal4/SUNRGBD-Segmentation/blob/main/model_without_skip_layers_global_pool_appended_june24.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_kTMpcSd1YrH",
        "outputId": "6cc5ba3c-f626-4723-990a-3d7abc1d1faf"
      },
      "source": [
        "!pip install tensorflow==2.4.1\n",
        "import tensorflow\n",
        "print(tensorflow.__version__)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow==2.4.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/70/dc/e8c5e7983866fa4ef3fd619faa35f660b95b01a2ab62b3884f038ccab542/tensorflow-2.4.1-cp37-cp37m-manylinux2010_x86_64.whl (394.3MB)\n",
            "\u001b[K     |████████████████████████████████| 394.3MB 44kB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (3.7.4.3)\n",
            "Collecting tensorflow-estimator<2.5.0,>=2.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/7e/622d9849abf3afb81e482ffc170758742e392ee129ce1540611199a59237/tensorflow_estimator-2.4.0-py2.py3-none-any.whl (462kB)\n",
            "\u001b[K     |████████████████████████████████| 471kB 40.2MB/s \n",
            "\u001b[?25hCollecting h5py~=2.10.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3f/c0/abde58b837e066bca19a3f7332d9d0493521d7dd6b48248451a9e3fe2214/h5py-2.10.0-cp37-cp37m-manylinux1_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 43.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.12.1)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (3.12.4)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (0.12.0)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.12)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.19.5)\n",
            "Requirement already satisfied: tensorboard~=2.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (2.5.0)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (0.2.0)\n",
            "Collecting grpcio~=1.32.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/06/54/1c8be62beafe7fb1548d2968e518ca040556b46b0275399d4f3186c56d79/grpcio-1.32.0-cp37-cp37m-manylinux2014_x86_64.whl (3.8MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8MB 37.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.15.0)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (0.36.2)\n",
            "Collecting gast==0.3.3\n",
            "  Downloading https://files.pythonhosted.org/packages/d6/84/759f5dd23fec8ba71952d97bcc7e2c9d7d63bdc582421f3cd4be845f0c98/gast-0.3.3-py2.py3-none-any.whl\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.1.2)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.1.0)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (3.3.0)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.1) (1.6.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.9.2->tensorflow==2.4.1) (57.0.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (1.0.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (0.4.4)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (2.23.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (0.6.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (1.8.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (1.31.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (3.3.4)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.1) (1.3.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1) (1.24.3)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1) (4.7.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1) (4.2.2)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.1) (4.5.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.1) (3.1.1)\n",
            "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.7/dist-packages (from rsa<5,>=3.1.4; python_version >= \"3.6\"->google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1) (0.4.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.1) (3.4.1)\n",
            "Installing collected packages: tensorflow-estimator, h5py, grpcio, gast, tensorflow\n",
            "  Found existing installation: tensorflow-estimator 2.5.0\n",
            "    Uninstalling tensorflow-estimator-2.5.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.5.0\n",
            "  Found existing installation: h5py 3.1.0\n",
            "    Uninstalling h5py-3.1.0:\n",
            "      Successfully uninstalled h5py-3.1.0\n",
            "  Found existing installation: grpcio 1.34.1\n",
            "    Uninstalling grpcio-1.34.1:\n",
            "      Successfully uninstalled grpcio-1.34.1\n",
            "  Found existing installation: gast 0.4.0\n",
            "    Uninstalling gast-0.4.0:\n",
            "      Successfully uninstalled gast-0.4.0\n",
            "  Found existing installation: tensorflow 2.5.0\n",
            "    Uninstalling tensorflow-2.5.0:\n",
            "      Successfully uninstalled tensorflow-2.5.0\n",
            "Successfully installed gast-0.3.3 grpcio-1.32.0 h5py-2.10.0 tensorflow-2.4.1 tensorflow-estimator-2.4.0\n",
            "2.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qNPRr_cubX4U",
        "outputId": "b7723396-958b-49d4-b500-6afc0032b902"
      },
      "source": [
        "import tensorflow\n",
        "print(tensorflow.__version__)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qPSbYPZyCIvP",
        "outputId": "b5939b5c-d0d6-46b6-f51a-20d64f653689"
      },
      "source": [
        "# model.summary()\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXMEt7jkCaVW"
      },
      "source": [
        "import matplotlib.pyplot as plt \n",
        "import numpy as np \n",
        "import cv2 as cv \n",
        "from PIL import Image, ImageDraw\n",
        "import json \n",
        "import os \n",
        "import glob \n",
        "import re \n",
        "from tqdm import tqdm\n",
        "import tensorflow as tf \n",
        "from tensorflow import keras \n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.layers import Dense, Conv2D, Reshape, Concatenate,MaxPooling2D, AveragePooling2D,Input, BatchNormalization, Activation, UpSampling2D, Concatenate, LeakyReLU,Reshape\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import GlobalAveragePooling2D,Multiply,SpatialDropout2D\n",
        "from tensorflow.keras.optimizers import Adam \n",
        "#using tensorflow backend 16 so keras backend still 32bytes so no keras to be used\n",
        "# from keras.layers.merge import concatenate,Concatenate\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from sklearn.model_selection import train_test_split\n",
        "import gc\n",
        "import psutil"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8jYMHKtEL-oK",
        "outputId": "2d4f28e1-8777-46ad-da5f-4ac36c52101b"
      },
      "source": [
        "tf.keras.mixed_precision.experimental.set_policy('mixed_float16')\n",
        "# # tf.compat.v1.disable_eager_execution()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Mixed precision compatibility check (mixed_float16): OK\n",
            "Your GPU will likely run quickly with dtype policy mixed_float16 as it has compute capability of at least 7.0. Your GPU: Tesla T4, compute capability 7.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0Jl9K0TuK0kZ",
        "outputId": "286526a0-e90f-4c51-cf33-d7f3b802cfa1"
      },
      "source": [
        "category = \"\"\"wall, floor, cabinet, bed, chair, sofa, table, door, window, bookshelf, picture, counter, blinds, desk, shelves, curtain, dresser\n",
        ", pillow, mirror, floor mat, clothes, ceiling, books, fridge, tv, paper, towel, shower curtain, box, whiteboard, person, nightstand, toilet, sink\n",
        ", lamp, bathtub, bag\"\"\"\n",
        "category = category.split(r', ')\n",
        "category = [i.replace('\\n','') for i in category]\n",
        "category_1 = ['background']\n",
        "for categ in category:category_1.append(categ) \n",
        "print(category_1)\n",
        "category_dict = {k:category_1.index(k) for k in category_1}\n",
        "print(category_dict)\n",
        "reverse_map={i:k for k,i in category_dict.items()}\n",
        "color_list=['gray','red','green','#FFFF00','#8c564b','#4B8BBE','#306998','#FFE873','#FFD43B','#646464','#5a0000','#003a27','#C0C0C0','#808080','#800000','#808000','#00FF00','#00FFFF','#008080','#000080','#FF00FF','#800080','#CD5C5C','#F08080','#FA8072','#E9967A','#FFA07A','#DC143C','#FF7F50','#FFD700','#ffffe0','#bdb76b','#228b22','#B0E0E6','#4169e1','#f0ffff','#d2691e','#BC8F8F']\n",
        "class_color_map={k:color_list[i] for i,k in enumerate(reverse_map.keys())}"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['background', 'wall', 'floor', 'cabinet', 'bed', 'chair', 'sofa', 'table', 'door', 'window', 'bookshelf', 'picture', 'counter', 'blinds', 'desk', 'shelves', 'curtain', 'dresser', 'pillow', 'mirror', 'floor mat', 'clothes', 'ceiling', 'books', 'fridge', 'tv', 'paper', 'towel', 'shower curtain', 'box', 'whiteboard', 'person', 'nightstand', 'toilet', 'sink', 'lamp', 'bathtub', 'bag']\n",
            "{'background': 0, 'wall': 1, 'floor': 2, 'cabinet': 3, 'bed': 4, 'chair': 5, 'sofa': 6, 'table': 7, 'door': 8, 'window': 9, 'bookshelf': 10, 'picture': 11, 'counter': 12, 'blinds': 13, 'desk': 14, 'shelves': 15, 'curtain': 16, 'dresser': 17, 'pillow': 18, 'mirror': 19, 'floor mat': 20, 'clothes': 21, 'ceiling': 22, 'books': 23, 'fridge': 24, 'tv': 25, 'paper': 26, 'towel': 27, 'shower curtain': 28, 'box': 29, 'whiteboard': 30, 'person': 31, 'nightstand': 32, 'toilet': 33, 'sink': 34, 'lamp': 35, 'bathtub': 36, 'bag': 37}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gH44_RJvK9TL",
        "outputId": "6878eec3-bec9-448f-816c-e4f5af45d9f1"
      },
      "source": [
        "AUTO = tf.data.experimental.AUTOTUNE\n",
        "Image_width = 640 \n",
        "Image_height = 480 \n",
        "no_classes = len(category_dict)\n",
        "def load_data(path, split=0.2):\n",
        "    images = sorted(glob.glob(os.path.join(path, \"image/*\")))\n",
        "    print(len(images))\n",
        "    masks = sorted(glob.glob(os.path.join(path, \"mask2/*\")))\n",
        "    print(len(masks))\n",
        "    total_size = 7990\n",
        "    test_size = 1990\n",
        "    valid_size=1000\n",
        "    training_x, test_x = train_test_split(images[:7990], test_size=test_size, random_state=42)\n",
        "    training_y, test_y = train_test_split(masks[:7990], test_size=test_size, random_state=42)\n",
        "    train_x, valid_x = train_test_split(training_x, test_size=valid_size, random_state=42)\n",
        "    train_y, valid_y = train_test_split(training_y, test_size=valid_size, random_state=42)\n",
        "    return (train_x, train_y), (valid_x, valid_y),(test_x, test_y)\n",
        "(train_x,train_y),(valid_x,valid_y),(test_x, test_y)=load_data('/content/drive/MyDrive/depth')\n",
        "train_size=len(train_x)\n",
        "valid_size=len(valid_x)\n",
        "\n",
        "def read_image(path):\n",
        "    path = path.decode()\n",
        "    x = cv.imread(path, cv.IMREAD_COLOR)\n",
        "    x = cv.resize(x,(640,480),interpolation=cv.INTER_AREA)\n",
        "    x = x/255.0\n",
        "    x=tf.cast(x,dtype=tf.float32)\n",
        "    return x\n",
        "\n",
        "def read_mask(path):\n",
        "    path = path.decode()\n",
        "    x = cv.imread(path, cv.IMREAD_GRAYSCALE)\n",
        "    x = cv.resize(x,(640,480),interpolation=cv.INTER_NEAREST)\n",
        "  \n",
        "    x = to_categorical(x,num_classes=no_classes)\n",
        "    x = x[:,:,:]\n",
        "    # x = np.expand_dims(x, axis=-1)\n",
        "    # x=np.concatenate([x,x],axis=-1)\n",
        "    x=tf.cast(x,dtype=tf.float32)\n",
        "    return x\n",
        "# read_mask('/content/drive/MyDrive/depth/mask100/mask1.png')\n",
        "def parser(x,y):\n",
        "    def _parse(x,y):\n",
        "        x=read_image(x)\n",
        "        y=read_mask(y)\n",
        "        return x,y\n",
        "    x,y = tf.numpy_function(_parse, [x,y], [tf.float32,tf.float32])\n",
        "    x.set_shape([Image_height, Image_width, 3])\n",
        "    y.set_shape([Image_height, Image_width, no_classes])\n",
        "    return x,y\n",
        "    \n",
        "def tf_dataset(x, y, batch):\n",
        "    dataset = tf.data.Dataset.from_tensor_slices((x, y))\n",
        "    dataset = dataset.map(parser)\n",
        "    # dataset = dataset.cache()\n",
        "    dataset = dataset.repeat()\n",
        "    dataset = dataset.batch(batch)\n",
        "    dataset = dataset.prefetch(AUTO)\n",
        "    return dataset\n",
        "train_dataset=tf_dataset(train_x,train_y,batch=16)\n",
        "valid_dataset=tf_dataset(valid_x,valid_y,batch=16)\n",
        "valid_dataset"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "8033\n",
            "8033\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<PrefetchDataset shapes: ((None, 480, 640, 3), (None, 480, 640, 38)), types: (tf.float32, tf.float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JlufJvHAlU4M",
        "outputId": "01991f26-1fd0-47b8-aae1-85460d7acf3d"
      },
      "source": [
        "print(test_x[:100])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['/content/drive/MyDrive/depth/image/image5457.jpg', '/content/drive/MyDrive/depth/image/image5917.jpg', '/content/drive/MyDrive/depth/image/image7738.jpg', '/content/drive/MyDrive/depth/image/image7847.jpg', '/content/drive/MyDrive/depth/image/image3387.jpg', '/content/drive/MyDrive/depth/image/image516.jpg', '/content/drive/MyDrive/depth/image/image1978.jpg', '/content/drive/MyDrive/depth/image/image4452.jpg', '/content/drive/MyDrive/depth/image/image470.jpg', '/content/drive/MyDrive/depth/image/image2603.jpg', '/content/drive/MyDrive/depth/image/image3895.jpg', '/content/drive/MyDrive/depth/image/image5776.jpg', '/content/drive/MyDrive/depth/image/image164.jpg', '/content/drive/MyDrive/depth/image/image7588.jpg', '/content/drive/MyDrive/depth/image/image5720.jpg', '/content/drive/MyDrive/depth/image/image2827.jpg', '/content/drive/MyDrive/depth/image/image1903.jpg', '/content/drive/MyDrive/depth/image/image5203.jpg', '/content/drive/MyDrive/depth/image/image2773.jpg', '/content/drive/MyDrive/depth/image/image1729.jpg', '/content/drive/MyDrive/depth/image/image2831.jpg', '/content/drive/MyDrive/depth/image/image4291.jpg', '/content/drive/MyDrive/depth/image/image1213.jpg', '/content/drive/MyDrive/depth/image/image7546.jpg', '/content/drive/MyDrive/depth/image/image249.jpg', '/content/drive/MyDrive/depth/image/image429.jpg', '/content/drive/MyDrive/depth/image/image2278.jpg', '/content/drive/MyDrive/depth/image/image7784.jpg', '/content/drive/MyDrive/depth/image/image5716.jpg', '/content/drive/MyDrive/depth/image/image3109.jpg', '/content/drive/MyDrive/depth/image/image4577.jpg', '/content/drive/MyDrive/depth/image/image2260.jpg', '/content/drive/MyDrive/depth/image/image1661.jpg', '/content/drive/MyDrive/depth/image/image2164.jpg', '/content/drive/MyDrive/depth/image/image48.jpg', '/content/drive/MyDrive/depth/image/image2817.jpg', '/content/drive/MyDrive/depth/image/image2400.jpg', '/content/drive/MyDrive/depth/image/image1285.jpg', '/content/drive/MyDrive/depth/image/image1954.jpg', '/content/drive/MyDrive/depth/image/image3078.jpg', '/content/drive/MyDrive/depth/image/image1923.jpg', '/content/drive/MyDrive/depth/image/image141.jpg', '/content/drive/MyDrive/depth/image/image1235.jpg', '/content/drive/MyDrive/depth/image/image7963.jpg', '/content/drive/MyDrive/depth/image/image3071.jpg', '/content/drive/MyDrive/depth/image/image7259.jpg', '/content/drive/MyDrive/depth/image/image3418.jpg', '/content/drive/MyDrive/depth/image/image1199.jpg', '/content/drive/MyDrive/depth/image/image7328.jpg', '/content/drive/MyDrive/depth/image/image4750.jpg', '/content/drive/MyDrive/depth/image/image1286.jpg', '/content/drive/MyDrive/depth/image/image7629.jpg', '/content/drive/MyDrive/depth/image/image4930.jpg', '/content/drive/MyDrive/depth/image/image1907.jpg', '/content/drive/MyDrive/depth/image/image2053.jpg', '/content/drive/MyDrive/depth/image/image1465.jpg', '/content/drive/MyDrive/depth/image/image4490.jpg', '/content/drive/MyDrive/depth/image/image4884.jpg', '/content/drive/MyDrive/depth/image/image7101.jpg', '/content/drive/MyDrive/depth/image/image949.jpg', '/content/drive/MyDrive/depth/image/image889.jpg', '/content/drive/MyDrive/depth/image/image7615.jpg', '/content/drive/MyDrive/depth/image/image6905.jpg', '/content/drive/MyDrive/depth/image/image4475.jpg', '/content/drive/MyDrive/depth/image/image7420.jpg', '/content/drive/MyDrive/depth/image/image2072.jpg', '/content/drive/MyDrive/depth/image/image2401.jpg', '/content/drive/MyDrive/depth/image/image4131.jpg', '/content/drive/MyDrive/depth/image/image5407.jpg', '/content/drive/MyDrive/depth/image/image3231.jpg', '/content/drive/MyDrive/depth/image/image4194.jpg', '/content/drive/MyDrive/depth/image/image2947.jpg', '/content/drive/MyDrive/depth/image/image4459.jpg', '/content/drive/MyDrive/depth/image/image2558.jpg', '/content/drive/MyDrive/depth/image/image4706.jpg', '/content/drive/MyDrive/depth/image/image7327.jpg', '/content/drive/MyDrive/depth/image/image1921.jpg', '/content/drive/MyDrive/depth/image/image5978.jpg', '/content/drive/MyDrive/depth/image/image2330.jpg', '/content/drive/MyDrive/depth/image/image7428.jpg', '/content/drive/MyDrive/depth/image/image4878.jpg', '/content/drive/MyDrive/depth/image/image5983.jpg', '/content/drive/MyDrive/depth/image/image3002.jpg', '/content/drive/MyDrive/depth/image/image2780.jpg', '/content/drive/MyDrive/depth/image/image7681.jpg', '/content/drive/MyDrive/depth/image/image7383.jpg', '/content/drive/MyDrive/depth/image/image2913.jpg', '/content/drive/MyDrive/depth/image/image3217.jpg', '/content/drive/MyDrive/depth/image/image7280.jpg', '/content/drive/MyDrive/depth/image/image1312.jpg', '/content/drive/MyDrive/depth/image/image6673.jpg', '/content/drive/MyDrive/depth/image/image452.jpg', '/content/drive/MyDrive/depth/image/image3972.jpg', '/content/drive/MyDrive/depth/image/image3340.jpg', '/content/drive/MyDrive/depth/image/image5157.jpg', '/content/drive/MyDrive/depth/image/image3790.jpg', '/content/drive/MyDrive/depth/image/image6024.jpg', '/content/drive/MyDrive/depth/image/image3771.jpg', '/content/drive/MyDrive/depth/image/image5392.jpg', '/content/drive/MyDrive/depth/image/image7628.jpg']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jy-st-KI11bj"
      },
      "source": [
        "from tensorflow.keras.layers import Lambda"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pOTZSsCjlUf4"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MGHr-V6pFCFw"
      },
      "source": [
        "#changes \n",
        "#Added a extra convolution in the encoder part lets look if it makes improvement or not \n",
        "def squeeze_excite_block(inputs, ratio=8):\n",
        "    init = inputs\n",
        "    filters = init.shape[-1]\n",
        "    se_shape = (1, 1, filters)\n",
        "    se = GlobalAveragePooling2D()(inputs)\n",
        "    se = Reshape(se_shape)(se)\n",
        "    se = Dense(filters // ratio, activation='relu', kernel_initializer='he_normal', use_bias=False)(se)\n",
        "    se = Dense(filters, activation='sigmoid', kernel_initializer='he_normal', use_bias=False)(se)\n",
        "    se = Multiply()([init, se])\n",
        "    return se\n",
        "    \n",
        "def encoder_conv_block(inputs,filters):\n",
        "  x = Conv2D(filters, (3, 3), padding=\"same\",kernel_initializer='he_normal')(inputs)\n",
        "  x = SpatialDropout2D(0.1)(x)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = LeakyReLU(alpha=0.001)(x)\n",
        "  x = Conv2D(filters, (3, 3), padding=\"same\",kernel_initializer='he_normal')(x)\n",
        "  x = SpatialDropout2D(0.1)(x)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = LeakyReLU(alpha=0.001)(x)\n",
        "#   x =  squeeze_excite_block(x)\n",
        "  return x \n",
        "def decoder_conv_block(inputs,filters1,filters2):\n",
        "  x = Conv2D(filters1, (3, 3), padding=\"same\",kernel_initializer='he_normal')(inputs)\n",
        "  x = SpatialDropout2D(0.1)(x)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = LeakyReLU(alpha=0.001)(x)\n",
        "  x = Conv2D(filters2, (3, 3), padding=\"same\",kernel_initializer='he_normal')(x)\n",
        "  x = SpatialDropout2D(0.1)(x)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = LeakyReLU(alpha=0.001)(x)\n",
        "  return x\n",
        "\n",
        "def encoder(inputs):\n",
        "  num_filters = [32,64, 128,256,256]\n",
        "  x= inputs\n",
        "  for i,filters  in enumerate(num_filters):\n",
        "    x = encoder_conv_block(x,filters)\n",
        "    x = MaxPooling2D((2, 2))(x)\n",
        "  return x \n",
        "def decoder(x):\n",
        "  num_filters = [32,64, 128,256,256,512]\n",
        "  num_filters.reverse()\n",
        "  for i in range(len(num_filters)-1):\n",
        "    x = decoder_conv_block(x,num_filters[i],num_filters[i+1])\n",
        "    x = UpSampling2D((2,2))(x)\n",
        "    \n",
        "  return x\n",
        "def output_block(inputs,classes):\n",
        "    x = Conv2D(classes,(3, 3), padding=\"same\")(inputs)\n",
        "    x=Conv2D(classes,(3,3),padding=\"same\")(x)\n",
        "    x_size=x.shape\n",
        "    MyGlobalAverage2D = Lambda(lambda y: tf.reduce_mean(y, axis=(1,2), keepdims=True), name='GlobalAverage2D' )(x)\n",
        "    # out=GlobalAveragePooling2D(data_format='channels_last')(x)\n",
        "    out=UpSampling2D((x_size[1],x_size[2]))(MyGlobalAverage2D)\n",
        "    result=Concatenate()([x,out])\n",
        "    result=Conv2D(classes,(1,1),padding='same')(result)\n",
        "    return result\n",
        "def main_model(shape):\n",
        "  inputs = Input(shape)\n",
        "  encoded = encoder(inputs)\n",
        "  decoded = decoder(encoded)\n",
        "  output = output_block(decoded,classes=38)\n",
        "  model = Model(inputs,output)\n",
        "  return model\n",
        "\n",
        "model = main_model((480,640,3))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s5OPWbZNKuPM",
        "outputId": "11db5de7-32b3-4332-b785-b3c5ae9dfab9"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 480, 640, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d (Conv2D)                 (None, 480, 640, 32) 896         input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d (SpatialDropo (None, 480, 640, 32) 0           conv2d[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization (BatchNorma (None, 480, 640, 32) 128         spatial_dropout2d[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu (LeakyReLU)         (None, 480, 640, 32) 0           batch_normalization[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 480, 640, 32) 9248        leaky_re_lu[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_1 (SpatialDro (None, 480, 640, 32) 0           conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 480, 640, 32) 128         spatial_dropout2d_1[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_1 (LeakyReLU)       (None, 480, 640, 32) 0           batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D)    (None, 240, 320, 32) 0           leaky_re_lu_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 240, 320, 64) 18496       max_pooling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_2 (SpatialDro (None, 240, 320, 64) 0           conv2d_2[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 240, 320, 64) 256         spatial_dropout2d_2[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_2 (LeakyReLU)       (None, 240, 320, 64) 0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 240, 320, 64) 36928       leaky_re_lu_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_3 (SpatialDro (None, 240, 320, 64) 0           conv2d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 240, 320, 64) 256         spatial_dropout2d_3[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_3 (LeakyReLU)       (None, 240, 320, 64) 0           batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2D)  (None, 120, 160, 64) 0           leaky_re_lu_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 120, 160, 128 73856       max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_4 (SpatialDro (None, 120, 160, 128 0           conv2d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 120, 160, 128 512         spatial_dropout2d_4[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_4 (LeakyReLU)       (None, 120, 160, 128 0           batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_5 (Conv2D)               (None, 120, 160, 128 147584      leaky_re_lu_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_5 (SpatialDro (None, 120, 160, 128 0           conv2d_5[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_5 (BatchNor (None, 120, 160, 128 512         spatial_dropout2d_5[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_5 (LeakyReLU)       (None, 120, 160, 128 0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2D)  (None, 60, 80, 128)  0           leaky_re_lu_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_6 (Conv2D)               (None, 60, 80, 256)  295168      max_pooling2d_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_6 (SpatialDro (None, 60, 80, 256)  0           conv2d_6[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_6 (BatchNor (None, 60, 80, 256)  1024        spatial_dropout2d_6[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_6 (LeakyReLU)       (None, 60, 80, 256)  0           batch_normalization_6[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_7 (Conv2D)               (None, 60, 80, 256)  590080      leaky_re_lu_6[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_7 (SpatialDro (None, 60, 80, 256)  0           conv2d_7[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_7 (BatchNor (None, 60, 80, 256)  1024        spatial_dropout2d_7[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_7 (LeakyReLU)       (None, 60, 80, 256)  0           batch_normalization_7[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_3 (MaxPooling2D)  (None, 30, 40, 256)  0           leaky_re_lu_7[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_8 (Conv2D)               (None, 30, 40, 256)  590080      max_pooling2d_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_8 (SpatialDro (None, 30, 40, 256)  0           conv2d_8[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_8 (BatchNor (None, 30, 40, 256)  1024        spatial_dropout2d_8[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_8 (LeakyReLU)       (None, 30, 40, 256)  0           batch_normalization_8[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_9 (Conv2D)               (None, 30, 40, 256)  590080      leaky_re_lu_8[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_9 (SpatialDro (None, 30, 40, 256)  0           conv2d_9[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_9 (BatchNor (None, 30, 40, 256)  1024        spatial_dropout2d_9[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_9 (LeakyReLU)       (None, 30, 40, 256)  0           batch_normalization_9[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_4 (MaxPooling2D)  (None, 15, 20, 256)  0           leaky_re_lu_9[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_10 (Conv2D)              (None, 15, 20, 512)  1180160     max_pooling2d_4[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_10 (SpatialDr (None, 15, 20, 512)  0           conv2d_10[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_10 (BatchNo (None, 15, 20, 512)  2048        spatial_dropout2d_10[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_10 (LeakyReLU)      (None, 15, 20, 512)  0           batch_normalization_10[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_11 (Conv2D)              (None, 15, 20, 256)  1179904     leaky_re_lu_10[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_11 (SpatialDr (None, 15, 20, 256)  0           conv2d_11[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_11 (BatchNo (None, 15, 20, 256)  1024        spatial_dropout2d_11[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_11 (LeakyReLU)      (None, 15, 20, 256)  0           batch_normalization_11[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "up_sampling2d (UpSampling2D)    (None, 30, 40, 256)  0           leaky_re_lu_11[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_12 (Conv2D)              (None, 30, 40, 256)  590080      up_sampling2d[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_12 (SpatialDr (None, 30, 40, 256)  0           conv2d_12[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_12 (BatchNo (None, 30, 40, 256)  1024        spatial_dropout2d_12[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_12 (LeakyReLU)      (None, 30, 40, 256)  0           batch_normalization_12[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_13 (Conv2D)              (None, 30, 40, 256)  590080      leaky_re_lu_12[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_13 (SpatialDr (None, 30, 40, 256)  0           conv2d_13[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_13 (BatchNo (None, 30, 40, 256)  1024        spatial_dropout2d_13[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_13 (LeakyReLU)      (None, 30, 40, 256)  0           batch_normalization_13[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "up_sampling2d_1 (UpSampling2D)  (None, 60, 80, 256)  0           leaky_re_lu_13[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_14 (Conv2D)              (None, 60, 80, 256)  590080      up_sampling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_14 (SpatialDr (None, 60, 80, 256)  0           conv2d_14[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_14 (BatchNo (None, 60, 80, 256)  1024        spatial_dropout2d_14[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_14 (LeakyReLU)      (None, 60, 80, 256)  0           batch_normalization_14[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_15 (Conv2D)              (None, 60, 80, 128)  295040      leaky_re_lu_14[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_15 (SpatialDr (None, 60, 80, 128)  0           conv2d_15[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_15 (BatchNo (None, 60, 80, 128)  512         spatial_dropout2d_15[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_15 (LeakyReLU)      (None, 60, 80, 128)  0           batch_normalization_15[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "up_sampling2d_2 (UpSampling2D)  (None, 120, 160, 128 0           leaky_re_lu_15[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_16 (Conv2D)              (None, 120, 160, 128 147584      up_sampling2d_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_16 (SpatialDr (None, 120, 160, 128 0           conv2d_16[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_16 (BatchNo (None, 120, 160, 128 512         spatial_dropout2d_16[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_16 (LeakyReLU)      (None, 120, 160, 128 0           batch_normalization_16[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_17 (Conv2D)              (None, 120, 160, 64) 73792       leaky_re_lu_16[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_17 (SpatialDr (None, 120, 160, 64) 0           conv2d_17[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_17 (BatchNo (None, 120, 160, 64) 256         spatial_dropout2d_17[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_17 (LeakyReLU)      (None, 120, 160, 64) 0           batch_normalization_17[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "up_sampling2d_3 (UpSampling2D)  (None, 240, 320, 64) 0           leaky_re_lu_17[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_18 (Conv2D)              (None, 240, 320, 64) 36928       up_sampling2d_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_18 (SpatialDr (None, 240, 320, 64) 0           conv2d_18[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_18 (BatchNo (None, 240, 320, 64) 256         spatial_dropout2d_18[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_18 (LeakyReLU)      (None, 240, 320, 64) 0           batch_normalization_18[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_19 (Conv2D)              (None, 240, 320, 32) 18464       leaky_re_lu_18[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout2d_19 (SpatialDr (None, 240, 320, 32) 0           conv2d_19[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_19 (BatchNo (None, 240, 320, 32) 128         spatial_dropout2d_19[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_19 (LeakyReLU)      (None, 240, 320, 32) 0           batch_normalization_19[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "up_sampling2d_4 (UpSampling2D)  (None, 480, 640, 32) 0           leaky_re_lu_19[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_20 (Conv2D)              (None, 480, 640, 38) 10982       up_sampling2d_4[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_21 (Conv2D)              (None, 480, 640, 38) 13034       conv2d_20[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "GlobalAverage2D (Lambda)        (None, 1, 1, 38)     0           conv2d_21[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "up_sampling2d_5 (UpSampling2D)  (None, 480, 640, 38) 0           GlobalAverage2D[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 480, 640, 76) 0           conv2d_21[0][0]                  \n",
            "                                                                 up_sampling2d_5[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_22 (Conv2D)              (None, 480, 640, 38) 2926        concatenate[0][0]                \n",
            "==================================================================================================\n",
            "Total params: 7,095,166\n",
            "Trainable params: 7,088,318\n",
            "Non-trainable params: 6,848\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gaSnZSXvyIj0"
      },
      "source": [
        "#june 18 5000 with an additional conv layer ..faster but not more accurate i guess"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RzHtdx6wNevn"
      },
      "source": [
        "#loss function \n",
        "\n",
        "import gc \n",
        "from tensorflow.keras.losses import binary_crossentropy\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "from tensorflow.keras.callbacks import CSVLogger ,LearningRateScheduler,Callback\n",
        "# callbacks = [  \n",
        "#     EarlyStopping(monitor='val_custom_metrics', patience=3),\n",
        "#     ModelCheckpoint(filepath, monitor='val_custom_metrics',verbose=1,mode='max',save_best_only=True),\n",
        "# ]\n",
        "# loss = tf.keras.losses.categorical_crossentropy(from_logits=True)\n",
        "def binary_loss(y_true, y_pred):\n",
        "    loss = binary_crossentropy(y_true, y_pred,from_logits=True)\n",
        "    loss_value = tf.math.reduce_mean(loss,axis=[0])\n",
        "    return loss_value\n",
        "# loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True, )\n",
        "# def cross_entropy_loss(y_true,y_pred):\n",
        "#     return loss(y_true,y_pred)\n",
        "def cross_entropy_loss(y_true,y_pred):\n",
        "    loss = tf.nn.softmax_cross_entropy_with_logits(\n",
        "    y_true, y_pred, axis=-1, name=None)\n",
        "    return loss\n",
        "\n",
        "def custom_metrics(y_true,y_pred,smooth=1):\n",
        "    y_pred = tf.math.argmax(y_pred, axis=-1)\n",
        "    # y_pred = tf.expand_dims(y_pred,axis=-1)\n",
        "    y_pred = tf.one_hot(y_pred,38)\n",
        "    y_true = y_true[:,:,:,1:38]\n",
        "    y_pred= y_pred[:,:,:,1:38]\n",
        "    intersection  = tf.math.reduce_sum((y_true*y_pred),axis = [1,2,3])\n",
        "    union = (tf.math.reduce_sum(y_true,axis=[1,2,3])+tf.math.reduce_sum(y_pred,axis = [1,2,3])) - intersection\n",
        "    return tf.math.reduce_mean((intersection+smooth)/(union+smooth),axis=0)\n",
        "smooth = 1\n",
        "def dice_coef(y_true, y_pred):\n",
        "    y_true = tf.keras.layers.Flatten()(y_true)\n",
        "    y_pred = tf.keras.layers.Flatten()(y_pred)\n",
        "    intersection = tf.reduce_sum(y_true * y_pred)\n",
        "    return (2. * intersection + smooth) / (tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) + smooth)\n",
        "model=tf.keras.models.load_model('/content/drive/MyDrive/training-daily/June25/global_pool',custom_objects={'cross_entropy_loss': cross_entropy_loss ,'dice_coef': dice_coef,'custom_metrics':custom_metrics})\n",
        "model.compile(loss = cross_entropy_loss,optimizer=Adam(learning_rate =.001),metrics=[custom_metrics])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uzAcZF20EWZt"
      },
      "source": [
        "#31 percent accuracy with squeeze in encoder than overfits fast\n",
        "for both decoder and encoder the accuracy was same as well so it may be bad to use before the downsampling\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cOu4GKVVOpis",
        "outputId": "7fcf0344-8988-4f0e-d394-bf0f326ac7d7"
      },
      "source": [
        "epoch=30\n",
        "# model.save('/content/drive/MyDrive/new_architecture/model_normal-june22/',save_format='tf')\n",
        "t_steps=train_size//16\n",
        "v_steps=valid_size//16\n",
        "val_custom_metrics = 0.28\n",
        "for i in range(epoch):\n",
        "  print('epoch =',i)\n",
        "  model.fit(train_dataset,epochs= 1,steps_per_epoch=t_steps)\n",
        "  loss = model.evaluate(valid_dataset,steps=v_steps)\n",
        "  if loss[1]>=val_custom_metrics:\n",
        "    val_custom_metrics = loss[1]\n",
        "    #epoch10 samma matra save vayeko thiyo yo chai\n",
        "    model.save('/content/drive/MyDrive/training-daily/June25/global_pool',save_format='tf')\n",
        "  tf.keras.backend.clear_session()\n",
        "  gc.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch = 0\n",
            "312/312 [==============================] - 3889s 12s/step - loss: 2.2943 - custom_metrics: 0.1302\n",
            "62/62 [==============================] - 583s 9s/step - loss: 2.2867 - custom_metrics: 0.2195\n",
            "epoch = 1\n",
            "312/312 [==============================] - 1281s 4s/step - loss: 1.9601 - custom_metrics: 0.2150\n",
            "62/62 [==============================] - 85s 1s/step - loss: 1.9729 - custom_metrics: 0.2538\n",
            "epoch = 2\n",
            "312/312 [==============================] - 1282s 4s/step - loss: 1.9124 - custom_metrics: 0.2344\n",
            "62/62 [==============================] - 85s 1s/step - loss: 1.9174 - custom_metrics: 0.2491\n",
            "epoch = 3\n",
            "312/312 [==============================] - 1271s 4s/step - loss: 1.8781 - custom_metrics: 0.2431\n",
            "62/62 [==============================] - 84s 1s/step - loss: 1.8523 - custom_metrics: 0.2737\n",
            "epoch = 4\n",
            "312/312 [==============================] - 1272s 4s/step - loss: 1.8426 - custom_metrics: 0.2517\n",
            "62/62 [==============================] - 86s 1s/step - loss: 1.8942 - custom_metrics: 0.2728\n",
            "epoch = 5\n",
            "312/312 [==============================] - 1265s 4s/step - loss: 1.8134 - custom_metrics: 0.2586\n",
            "62/62 [==============================] - 85s 1s/step - loss: 1.7787 - custom_metrics: 0.2918\n",
            "epoch = 6\n",
            "312/312 [==============================] - 1265s 4s/step - loss: 1.7922 - custom_metrics: 0.2643\n",
            "62/62 [==============================] - 84s 1s/step - loss: 1.8157 - custom_metrics: 0.2960\n",
            "epoch = 7\n",
            "312/312 [==============================] - 1258s 4s/step - loss: 1.7717 - custom_metrics: 0.2694\n",
            "62/62 [==============================] - 84s 1s/step - loss: 1.7471 - custom_metrics: 0.3095\n",
            "epoch = 8\n",
            "312/312 [==============================] - 1285s 4s/step - loss: 1.7485 - custom_metrics: 0.2758\n",
            "62/62 [==============================] - 85s 1s/step - loss: 1.7486 - custom_metrics: 0.3063\n",
            "epoch = 9\n",
            "312/312 [==============================] - 1255s 4s/step - loss: 1.7338 - custom_metrics: 0.2794\n",
            "62/62 [==============================] - 85s 1s/step - loss: 1.7221 - custom_metrics: 0.3158\n",
            "epoch = 10\n",
            "312/312 [==============================] - 1262s 4s/step - loss: 1.7138 - custom_metrics: 0.2822\n",
            "62/62 [==============================] - 85s 1s/step - loss: 1.7148 - custom_metrics: 0.3099\n",
            "epoch = 11\n",
            "175/312 [===============>..............] - ETA: 9:12 - loss: 1.7053 - custom_metrics: 0.2848"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q8GwH6gcPnRZ"
      },
      "source": [
        "# model=tf.keras.models.load_model('/content/drive/MyDrive/new_architecture/model_normal-june22/',custom_objects={'cross_entropy_loss': cross_entropy_loss ,'dice_coef': dice_coef,'custom_metrics':custom_metrics})\n",
        "def read_image(path):\n",
        "    x = cv.imread(path, cv.IMREAD_COLOR)\n",
        "    x = cv.resize(x,(640,480),interpolation=cv.INTER_AREA)\n",
        "    x = x/255.0\n",
        "    x=tf.cast(x,dtype=tf.float32)\n",
        "    return x\n",
        "\n",
        "def read_mask(path):\n",
        "    x = cv.imread(path, cv.IMREAD_GRAYSCALE)\n",
        "    x = cv.resize(x,(640,480),interpolation=cv.INTER_AREA)\n",
        "    x = to_categorical(x,num_classes=no_classes+1)\n",
        "    x = x[:,:,:]\n",
        "    # x = np.expand_dims(x, axis=-1)\n",
        "    # x=np.concatenate([x,x],axis=-1)\n",
        "    x=tf.cast(x,dtype=tf.float32)\n",
        "    return x\n",
        "def custom_metrics(y_true,y_pred,smooth=1):\n",
        "    y_pred = tf.math.argmax(y_pred, axis=-1)\n",
        "    # y_pred = tf.expand_dims(y_pred,axis=-1)\n",
        "    y_pred = tf.one_hot(y_pred,38)\n",
        "    y_true = y_true[:,:,:,1:38]\n",
        "    y_pred= y_pred[:,:,:,1:38]\n",
        "    intersection  = tf.math.reduce_sum((y_true*y_pred),axis = [1,2,3])\n",
        "    union = tf.math.reduce_sum(y_true,axis=[1,2,3])+tf.math.reduce_sum(y_pred,axis = [1,2,3]) - intersection \n",
        "    return tf.math.reduce_mean((intersection+smooth)/(union+smooth),axis=0)\n",
        "def load_data(images,masks):\n",
        "    length = 1990\n",
        "    iou=0\n",
        "    for i,(image,mask) in enumerate(zip(images,masks)):\n",
        "    \n",
        "      img = read_image(image)\n",
        "      mas = read_mask(mask)\n",
        "      img = np.expand_dims(img,0)\n",
        "      mas = np.expand_dims(mas,0)\n",
        "      prediction = model.predict(img)\n",
        "      iou += custom_metrics(mas,prediction)\n",
        "      print(i,iou/i)\n",
        "    return iou/length\n",
        "no_classes=37\n",
        "print(len(test_x))\n",
        "x=load_data(test_x,test_y)\n",
        "print('*****************************************************************************************************************')\n",
        "x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sXH9XLBMM2bB"
      },
      "source": [
        "model2 = main_model2(shape,no_classes)\n",
        "model2=tf.keras.models.load_model(filepath2,custom_objects={'cross_entropy_loss':cross_entropy_loss,'dice_coef':dice_coef,'custom_metrics':custom_metrics})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0-gQ6TBDMf4B"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nZsGANPHM1T5"
      },
      "source": [
        "import matplotlib\n",
        "cmap = matplotlib.colors.ListedColormap(color_list)\n",
        "def plot(image,numpy,prediction):\n",
        "    plt.figure(figsize=(14,14))\n",
        "    plt.subplot(131)\n",
        "    plt.imshow(np.squeeze(prediction)+1,cmap=cmap,vmin=1, vmax=len(color_list))\n",
        "    plt.subplot(132)\n",
        "    plt.imshow(image)\n",
        "    plt.subplot(133)\n",
        "    plt.imshow(numpy,cmap='gray')\n",
        "    print(prediction)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "irZ0RNFNVg5q"
      },
      "source": [
        "import matplotlib\n",
        "def plot(image,numpy,prediction):\n",
        "    plt.figure(figsize=(14,14))\n",
        "    plt.subplot(131)\n",
        "    plt.imshow(np.squeeze(prediction),cmap=plt.get_cmap('gray'))\n",
        "    plt.subplot(132)\n",
        "    plt.imshow(image)\n",
        "    plt.subplot(133)\n",
        "    plt.imshow(numpy,cmap='gray')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XiDguJ9C-6Pg"
      },
      "source": [
        "def prediction(path):\n",
        "    image = cv.imread(path,cv.IMREAD_COLOR)\n",
        "    x = cv.resize(image,(640,480),interpolation=cv.INTER_NEAREST)\n",
        "    x = x/255.0\n",
        "    x=np.expand_dims(x,0)\n",
        "    y = model.predict(x)\n",
        "    print(y.shape)\n",
        "    numpy = np.zeros((y.shape[1],y.shape[2],y.shape[3]))\n",
        "    print(numpy.shape)\n",
        "    numpy[:,:,:] = y \n",
        "    prediction= numpy.argmax(axis=-1)\n",
        "    print(prediction.shape)\n",
        "    numpy= tf.one_hot(prediction,38)\n",
        "    plot(image,numpy[:,:,2],prediction)\n",
        "    return numpy\n",
        "path = '/content/drive/MyDrive/depth/image/image1.jpg'\n",
        "prediction1=prediction(path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tb2AVBOiBwYB"
      },
      "source": [
        "for i in range(prediction1.shape[2]):\n",
        "    # for j in range(prediction1.shape[1]):\n",
        "    #     # if prediction1[i,j]>2:\n",
        "    (prediction1.shape)\n",
        "    print(prediction1[])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IZIgQWQFHcr7"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}