Run_Benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e6f5c6a0",
   "metadata": {},
   "source": [
    "## This file will runs experiments on ImageNet Dataset\n",
    "\n",
    "- this file will \n",
    "   - find segmentation hyperparamers for all 150 images \n",
    "   - generate expalantions for all the images in data folder and will save the results as csv file \n",
    "<br><br>\n",
    "- you may run the full benchmark on all the 150 images which will take more than one day to run, or run a simplified version on a single image which will take less time.\n",
    "\n",
    "This code was tested on two windows laptop and a mac-os laptop with following specifications:\n",
    "\n",
    "```\n",
    "3.1. Dell G5      (Corei7 12 CPUs 2.20Ghz, 16GB RAM, NVidia 1060 GPU 6GB)\n",
    "3.2. Santech XN2  (Corei9 20 CPUs 2.60Ghz, 16GB RAM, NVidia 4070 GPU 8GB)\n",
    "3.3. Macbook Pro  (M1, 16GB RAM)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d991a68f",
   "metadata": {},
   "source": [
    "### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a0c5dd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import importlib\n",
    "import utils as ut\n",
    "from datetime import date,datetime\n",
    "from IPython.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:98% !important; }</style>\"))    # Stretch Notebook Width to 98% size of the Screen\n",
    "# It will use the modified code of lime downloaded from https://anonymous.4open.science/r/lime-stratified-BDE5/\n",
    "from tensorflow.keras.applications.resnet50 import preprocess_input\n",
    "from lime_stratified.lime import lime_image"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e41377b",
   "metadata": {},
   "source": [
    "### Setting Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0f2e492",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get Current Working Directory and joining subfolders and subfiles path\n",
    "Main_dir =   os.getcwd()\n",
    "DS_path =  os.path.join(Main_dir, \"data\")\n",
    "result_folder = os.path.join(Main_dir, \"result\")\n",
    "json_file    =  os.path.join(DS_path,\"imagenet_class_index.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33fac9ec",
   "metadata": {},
   "source": [
    "## BlackBox Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b258b2d3",
   "metadata": {},
   "source": [
    "Load BlackBox Model, here ResNet50 Model is loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c79f3377",
   "metadata": {},
   "outputs": [],
   "source": [
    "def bb_predict(imgs):\n",
    "    # On some platform, you will need model.predict(..) instead of model(..)\n",
    "    return model.predict(preprocess_input(imgs), verbose=False)\n",
    "#     return model(preprocess_input(imgs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f855679b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'ResNet50'\n",
    "model = ut.load_model(model_name)\n",
    "# getting ImageNet class names\n",
    "class_names = ut.get_ImageNet_ClassLabels(json_file) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f623da5",
   "metadata": {},
   "outputs": [],
   "source": [
    "now = datetime.now()\n",
    "print(\"Example Started\\t\\t:\\t\\t\\t\", now.strftime(\"%d/%m/%Y %H:%M:%S\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "315ec98a",
   "metadata": {},
   "source": [
    "# Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5eae7c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#######################################################################################################\n",
    "## SEGMENTATION PARAMETERS\n",
    "seg_algo                    =  'quickshift'\n",
    "segs_list                   =  [50,100,150,200]           # Target No of Segments in Image Explanation\n",
    "# These hyperparameters can be used to create LIME Image Explanations\n",
    "hide_color                  =  [None]\n",
    "use_stratification          =  [False,True]\n",
    "num_samples                 =  1000    # samples in the synthetic neighbourhood needed to compute every explanation\n",
    "batch_size                  =  500     # batch size to compute the synthetic neighbourhood \n",
    "# Top Labels: Upto how much labels the explanation is needed, repeat_exp: Upto how much Explanation is needed to be \n",
    "# to get an average explanation to remove randomness \n",
    "top_labels                  =  2      # labels for the   \n",
    "repeat_exp                  =  10     # aeverage of 10 explanations for the same image to reduce randomness \n",
    "#  Results of Experiments\n",
    "plot_prediction             =  True # Set it to True if plots for prediction are needed to be plot, Default: False\n",
    "plot_segments               =  True # Set it to True if plots for segments are needed to be plot, Default: False\n",
    "plot_explanation            =  True # Set it to True if plots for explanations are needed to be plot, Default: False\n",
    "plot_classification_score   =  True # Set it to True if plots for classification_score are needed to be plot, Default: False\n",
    "plot_heatmap                =  True # Set it to True if plots for plot_heatmap are needed to be plot, Default: False\n",
    "plot_image_mask             =  True # Set it to True if plots for plot_image_mask are needed to be plot, Default: False\n",
    "save_explanations_as_plot   =  True # Set it to True if plots for explanations are needed to be saved also, Default: False\n",
    "\n",
    "###################    PATHS for Saving results      ##########################\n",
    "sub_results__ = os.path.join(result_folder,str(segs_list))\n",
    "sub_results_ = os.path.join(sub_results__,str(num_samples))\n",
    "_ = ut.check_folders(sub_results__)\n",
    "############################### \n",
    "create_segments_file        =  True ### IF SEGMENT FILE Needs to be regenerated, SET 'compute_segments' as True\n",
    "create_data_file            =  True  ### IF SEGMENT FILE IS LOADED AND COMPUTE EXPERIMENTS ARE REQUIRED, SET 'compute_experiments' as True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a96c251",
   "metadata": {},
   "source": [
    "### Full Benchmark\n",
    "Compute the benchmark by using these pararmeters on the 150 images "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fc42ea9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# files                       =  range(0,150,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7e3bb67",
   "metadata": {},
   "source": [
    "### Simplified Benchmark\n",
    "\n",
    "Use these pararmeters on the single on a image "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5eb65dee",
   "metadata": {},
   "outputs": [],
   "source": [
    "files                       =  range(124,125,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7311db1d",
   "metadata": {},
   "source": [
    "### Create Segments 0-100, 100-200, 200-300\n",
    "Identify the max_dist hyperparamer needed to get exactly the desired amount of segments in of images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96a123b6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_seg = ut.segmentation_module(create_segments_file,files,DS_path,result_folder,model,segs_list,seg_algo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1fc9222",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_seg"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4a0ff33",
   "metadata": {},
   "source": [
    "### Run the Experiments and Save the CSV File\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a999145c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_data = ut.explanation_module(create_data_file,files,df_seg,DS_path,sub_results_,result_folder,segs_list,\n",
    "                                 model,model_name,class_names,save_explanations_as_plot,use_stratification,\n",
    "                                 plot_prediction,plot_segments,plot_heatmap,plot_image_mask,hide_color,\n",
    "                                 num_samples,repeat_exp,top_labels,batch_size,lime_image,bb_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40b79658",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47a44133",
   "metadata": {},
   "outputs": [],
   "source": [
    "now = datetime.now()\n",
    "print(\"Example Started\\t\\t:\\t\\t\\t\", now.strftime(\"%d/%m/%Y %H:%M:%S\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a30a14c9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}