ICLR2022.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "name": "ICLR2022",
   "provenance": [],
   "collapsed_sections": [
    "rtfq0M7ZcuYC",
    "Te0vKnQFC_5T",
    "r6SyKLjUhYxW",
    "snXG5pNZ-pWJ",
    "EeJkizdECj-m",
    "h47L2X2wff1g",
    "PXbs8GnAFDEB",
    "WpFxfPwkwysB",
    "Klv0t3XZFPGH",
    "kEZBVrkOHOcW",
    "QTfoEr6QOu3J",
    "37PNJNVHL2lN"
   ],
   "machine_shape": "hm"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "language_info": {
   "name": "python"
  },
  "accelerator": "GPU"
 },
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rtfq0M7ZcuYC"
   },
   "source": [
    "#**INITIALIZATIONS**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Te0vKnQFC_5T"
   },
   "source": [
    "##**IMPORTS**"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "53cmHd80cTl_"
   },
   "source": [
    "# --- INSTALLS\n",
    "! pip install ftfy regex tqdm\n",
    "! pip install git+https://github.com/openai/CLIP.git\n",
    "! pip install jellyfish\n",
    "\n",
    "# --- IMPORTS\n",
    "import torchvision, jellyfish, warnings, scipy.io, zipfile, gensim, urllib, torch, numpy, json, math, clip, os\n",
    "from scipy.stats import normaltest, shapiro, mannwhitneyu\n",
    "from PIL import ImageFont, ImageDraw, Image\n",
    "from IPython.display import clear_output\n",
    "import matplotlib.patches as mpatches\n",
    "from matplotlib import font_manager\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.nn.functional as F\n",
    "from google.colab import drive\n",
    "import statistics\n",
    "\n",
    "# --- ENVIRONMENT\n",
    "drive.mount('/content/drive')\n",
    "path = \"drive/MyDrive/Stage_2021/\"\n",
    "if not os.path.exists(path) : os.mkdir(path)\n",
    "if not os.path.exists(path+\"FONTS\") : os.mkdir(path+\"FONTS\")\n",
    "if not os.path.exists(path+\"DATA\") : os.mkdir(path+\"DATA\")\n",
    "if not os.path.exists(path+\"DATASETS\") : os.mkdir(path+\"DATASETS\")\n",
    "if not os.path.exists(path+\"DATASETS/LABELS\") : os.mkdir(path+\"DATASETS/LABELS\")\n",
    "\n",
    "# --- DOWNLOAD FONTS\n",
    "urllib.request.urlretrieve (\"https://www.font-police.com/classique/sans-serif/arial.ttf\", path+\"FONTS/arial.ttf\")\n",
    "url = \"https://fr.ffonts.net/Lexend-Deca-Regular.font.zip\"\n",
    "urllib.request.urlretrieve (url, path+\"FONTS/lexend_deca.zip\")\n",
    "with zipfile.ZipFile(path+\"FONTS/lexend_deca.zip\", 'r') as zip_file:\n",
    "    with open(path+\"FONTS/lexend_deca.ttf\", 'wb') as f:\n",
    "        f.write(zip_file.read('lexenddeca/LexendDeca-Regular.ttf'))\n",
    "\n",
    "font_dirs  = [path+\"FONTS\"]\n",
    "font_files = font_manager.findSystemFonts(fontpaths=font_dirs)\n",
    "for font_file in font_files:\n",
    "    font_manager.fontManager.addfont(font_file)\n",
    "\n",
    "# --- SET UP MATPLOTLIB\n",
    "plt.rcParams['font.family'] = 'Lexend Deca'\n",
    "plt.rc('font', size=8)                     # controls default text sizes\n",
    "plt.rc('axes', titlesize=15)               # fontsize of the axes title\n",
    "plt.rc('axes', labelsize=15)               # fontsize of the x and y labels\n",
    "plt.rc('xtick', labelsize=9)               # fontsize of the tick labels\n",
    "plt.rc('ytick', labelsize=9)               # fontsize of the tick labels\n",
    "plt.rc('legend', fontsize=15)              # legend fontsize\n",
    "plt.rc('figure', titlesize=12)             # fontsize of the figure title\n",
    "\n",
    "clear_output()\n",
    "warnings.filterwarnings(\"ignore\")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "r6SyKLjUhYxW"
   },
   "source": [
    "##**DATASETS**"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "6YLFzfgoJ4Uy"
   },
   "source": [
    "# ----- STIMULIS DATASETS\n",
    "if not os.path.exists(path+\"DATASETS/dataset1.mat\"):  urllib.request.urlretrieve (\"http://wednesday.csail.mit.edu/MEG1_MEG_Clear_Data/visual_stimuli.mat\", path+\"DATASETS/dataset1.mat\")\n",
    "if not os.path.exists(path+\"DATASETS/dataset2.mat\"):  urllib.request.urlretrieve (\"http://userpage.fu-berlin.de/rmcichy/Khaligh_Razavi_et_al_2018JoCN/118_visual_stimuli.mat\", path+\"DATASETS/dataset2.mat\")\n",
    "if not os.path.exists(path+\"DATASETS/dataset3.mat\"):\n",
    "  urllib.request.urlretrieve (\"http://wednesday.csail.mit.edu/fusion_rep/stimulus/156ImageStimuliSet.zip\", path+\"DATASETS/dataset3.zip\")\n",
    "  with zipfile.ZipFile(path+\"DATASETS/dataset3.zip\", 'r') as zip_file:\n",
    "    with open(path+\"DATASETS/dataset3.mat\", 'wb') as f:\n",
    "        f.write(zip_file.read('156ImageStimuliSet/visual_stimuli156.mat'))"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "u9d0nyjZ0dXz"
   },
   "source": [
    "s_92  = scipy.io.loadmat(path+'/DATASETS/dataset1.mat')    # i : LABEL - HUMAN/NONHUMAN - FACE/BODY - ANIMATE/INANIMATE - NATURAL/ARTIFICIAL - PIXELS\n",
    "s_118 = scipy.io.loadmat(path+'/DATASETS/dataset2.mat')    # i : LABEL - PIXELS - ANIMATE - SMALL - MEDIUM - LARGE\n",
    "s_156 = scipy.io.loadmat(path+'/DATASETS/dataset3.mat')    # i : PIXELS - TWINSET - ('animals', 'objects', 'scenes', 'people', or 'faces')\n",
    "\n",
    "images = []\n",
    "for i in range(len(s_118['visual_stimuli'][0])):\n",
    "  images.append(s_118['visual_stimuli'][0][i][1])\n",
    "for i in range(len(s_156['visual_stimuli156'][0])):\n",
    "  images.append(s_156['visual_stimuli156'][0][i][0])"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "snXG5pNZ-pWJ"
   },
   "source": [
    "##**LABELS**\n",
    "\n",
    "* Super from MS-COCO, Basics from MS-COCO & CIFAR-100\n",
    "* Remove 'person' from basics\n",
    "* Replace multi-words\n",
    "* Remove 's' duplicates"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "NyVwG5W3__uK"
   },
   "source": [
    "# ----- LOAD/CREATE LABELS\n",
    "if os.path.exists(path+\"/DATASETS/LABELS/labels.pt\") : \n",
    "  labels                     = torch.load(path+\"/DATASETS/LABELS/labels.pt\")\n",
    "  super_labels, basic_labels, hierarchy = labels[\"SUPERORDINATES\"], labels[\"BASICS\"], labels[\"CLUSTERING\"]\n",
    "else:\n",
    "  super_labels = set()\n",
    "  basic_labels = set()\n",
    "\n",
    "  # --- ADDING LABELS FROM MS-COCO\n",
    "  if not os.path.exists(\"/drive/MyDrive/Stage_2021/DATASETS/LABELS/annotations.json\"):\n",
    "    urllib.request.urlretrieve (\"http://images.cocodataset.org/annotations/annotations_trainval2017.zip\", path+\"DATASETS/LABELS/mscoco_annotations.zip\")\n",
    "    with zipfile.ZipFile(path+\"DATASETS/LABELS/mscoco_annotations.zip\", 'r') as zip_file:\n",
    "      with open(path+\"DATASETS/LABELS/annotations.json\", 'wb') as f:\n",
    "        f.write(zip_file.read('annotations/instances_val2017.json'))  \n",
    "  with open(path+\"DATASETS/LABELS/annotations.json\") as an_file :\n",
    "    annotations_coco = json.load(an_file)\n",
    "    for labels in annotations_coco[\"categories\"]:\n",
    "      super_labels.add(labels[\"supercategory\"])\n",
    "      basic_labels.add(labels[\"name\"])\n",
    "\n",
    "  # --- MANUALY ADDING LABELS FROM CIFAR-100\n",
    "  basic_labels.remove(\"person\") # \"person\" from ms-coco should not be a basic label, as CIFAR-100 has \"man\",\"woman\",\"boy\",\"girl\",\"baby\"\n",
    "  cifar100 = torchvision.datasets.CIFAR100(root=path+\"/DATASETS/LABELS\", train=False, download=True) \n",
    "  basic_labels.update(cifar100.classes)\n",
    "\n",
    "  # --- REMOVING MULTI-WORDS AND \"s\" DUPLICATES\n",
    "  remove_list = []\n",
    "\n",
    "  for label in basic_labels:\n",
    "    if label+\"s\" in basic_labels:       remove_list.append(label+\"s\")\n",
    "    elif \" \" in label or \"_\" in label:  remove_list.append(label)\n",
    "\n",
    "  for remove_label in remove_list:\n",
    "    basic_labels.remove(remove_label)\n",
    "\n",
    "  # --- HIERARCHY \n",
    "\n",
    "  hierarchy = { super:set() for super in sorted(list(super_labels)) }\n",
    "\n",
    "  cifar_manual_clustering = {\n",
    "      \"outdoor\":    [\"bridge\",\"castle\",\"cloud\",\"forest\",\"house\",\"mountain\",\"orchid\",\"plain\",\"poppy\",\"road\",\"rose\",\"sea\",\"skyscraper\",\"sunflower\",\"tulip\"],\n",
    "      \"animal\":     [\"bear\",\"beaver\",\"bee\",\"beetle\",\"butterfly\",\"camel\",\"caterpillar\",\"cattle\",\"chimpanzee\",\"cockroach\",\"crab\",\"crocodile\",\"dinosaur\",\"dolphin\",\"elephant\",\"flatfish\",\"fox\",\"hamster\",\"kangaroo\",\"leopard\",\"lion\",\"lizard\",\"lobster\",\"mouse\",\"otter\",\"porcupine\",\"possum\",\"rabbit\",\"raccoon\",\"ray\",\"seal\",\"shark\",\"shrew\",\"skunk\",\"snail\",\"snake\",\"spider\",\"squirrel\",\"tiger\",\"trout\",\"turtle\",\"whale\",\"wolf\",\"worm\"],\n",
    "      \"vehicle\":    [\"bicyle\",\"bus\",\"motorcycle\",\"rocket\",\"streetcar\",\"tank\",\"tractor\",\"train\"],\n",
    "      \"kitchen\":    [\"bottle\",\"bowl\",\"can\",\"cup\",\"plate\"],\n",
    "      \"electronic\": [\"clock\",\"keyboard\",\"lamp\",\"telephone\",\"television\"],\n",
    "      \"person\":     [\"baby\",\"boy\",\"girl\",\"man\",\"woman\"],\n",
    "      \"food\":       [\"apple\",\"mushroom\",\"orange\",\"pear\"],\n",
    "      \"furniture\":  [\"bed\",\"chair\",\"couch\",\"table\",\"wardrobe\"]\n",
    "  }\n",
    "\n",
    "  for basic_label in basic_labels :\n",
    "    basic_in_hierarchy = False \n",
    "\n",
    "    for labels in annotations_coco[\"categories\"]:\n",
    "      if basic_label == labels[\"name\"] :\n",
    "        hierarchy[labels[\"supercategory\"]].add(basic_label)\n",
    "        basic_in_hierarchy = True \n",
    "\n",
    "    if not basic_in_hierarchy :\n",
    "      for category in cifar_manual_clustering :\n",
    "        if basic_label in cifar_manual_clustering[category]:\n",
    "          hierarchy[category].add(basic_label)\n",
    "          basic_in_hierarchy = True \n",
    "\n",
    "    assert(basic_in_hierarchy)\n",
    "\n",
    "  # --- SAVE LABELS\n",
    "  super_labels = list(hierarchy.keys())\n",
    "  basic_labels = []\n",
    "\n",
    "  for key in hierarchy:\n",
    "    for basic_label in hierarchy[key]:\n",
    "      basic_labels.append(basic_label)\n",
    "\n",
    "  torch.save({\"SUPERORDINATES\":super_labels, \"BASICS\":basic_labels, \"CLUSTERING\":hierarchy},path+\"/DATASETS/LABELS/labels.pt\")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EeJkizdECj-m"
   },
   "source": [
    "##**CLIP MODEL**"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "mgDfBcL1OkPj"
   },
   "source": [
    "device            = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "model, preprocess = clip.load(\"ViT-B/32\", device=device, jit=False)\n",
    "#model, preprocess = clip.load(\"RN50x16\", device=device, jit=False)\n",
    "#model, preprocess = clip.load(\"RN101\", device=device, jit=False)\n",
    "#model, preprocess = clip.load(\"ViT-B/16\", device=device, jit=False)\n",
    "#model, preprocess = clip.load(\"RN50x4\", device=device, jit=False)\n",
    "#model, preprocess = clip.load(\"RN50\", device=device, jit=False)\n",
    "\n",
    "text_super = clip.tokenize([\"a photo of a \"+label for label in super_labels]).to(device)\n",
    "text_basic = clip.tokenize([\"a photo of a \"+label for label in basic_labels]).to(device)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "h47L2X2wff1g"
   },
   "source": [
    "# **Classification for word-superimposed images**\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "U-nAnuJg1H93"
   },
   "source": [
    "## **UTILS**"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "y9MXVKluhYRI"
   },
   "source": [
    "def get_stimuli(i,preprocess=None,word=None,raw=False):\n",
    "  img = Image.fromarray(images[i])\n",
    "  img.show()\n",
    "  # --- ADD WORD TO STIMULI\n",
    "  if None is not word :\n",
    "    img_width, img_height   = img.size\n",
    "    fontsize                = math.floor(img_width/6)\n",
    "    font                    = ImageFont.truetype(path+\"/FONTS/arial.ttf\", fontsize)\n",
    "    text_width, text_height = font.getsize(word)\n",
    "    x,y                     = img_width/2  - text_width/2, img_height/2 - text_height/2\n",
    "    draw                    = ImageDraw.Draw(img)\n",
    "    draw.text((x,y),word,(255,0,0),font=font,stroke_width=math.ceil(fontsize/20),stroke_fill=(255,255,255))\n",
    "\n",
    "  # --- APPLY PREPROCESSING\n",
    "  if None is not preprocess : return preprocess(img)\n",
    "  elif not raw :              return torch.tensor(numpy.array(img))\n",
    "  else :                      return img\n",
    "\n",
    "def get_stimulis(a,b,preprocess=None,word=None):\n",
    "    return torch.cat(([get_stimuli(i,preprocess=preprocess,word=word).unsqueeze(0) for i in range(a,b)]))"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "WxaZQkoDN0-B"
   },
   "source": [
    "def compute_original_preds(batch_size=128):\n",
    "  data_path    = path+\"/DATA/\"+model_name+\"_\"+\"context\"+str(contexts.index(context))+\"_original_preds.pt\"\n",
    "  dataset_size = 274\n",
    "\n",
    "  # ----- CHECKING IF DATA ALREADY EXISTS -------------------------------------------\n",
    "  if os.path.exists(data_path): original_predictions = torch.load(data_path,map_location=device)\n",
    "  else:                         \n",
    "    original_predictions = {}\n",
    "    # ---- PROCESSING DATA IN MINI-BATCHES --------------------------------\n",
    "    for i in range(0,math.ceil(dataset_size/batch_size)):\n",
    "      a,b    = i*batch_size, min(dataset_size,i*batch_size + batch_size)\n",
    "      batch  = get_stimulis(a,b,preprocess).to(device)\n",
    "\n",
    "      # --- BASIC PREDICTIONS ------------------------------------------\n",
    "      with torch.no_grad():\n",
    "          logits_per_image_basic, _  = model(batch, text_basic)\n",
    "          probs_basic                = logits_per_image_basic.softmax(dim=-1)\n",
    "          _, preds_basic             = torch.max(probs_basic,1)\n",
    "\n",
    "      # --- SUPERORDINATE PREDICTIONS ----------------------------------\n",
    "      with torch.no_grad():\n",
    "          logits_per_image_super, _  = model(batch, text_super)\n",
    "          probs_super                = logits_per_image_super.softmax(dim=-1)\n",
    "          _, preds_super             = torch.max(probs_super,1)\n",
    "          \n",
    "      # --- SAVING PREDICTIONS -------------------------------------------\n",
    "      for j in range(a,b):\n",
    "        result = {\"Superordinate\":{},\"Basic\":{}}\n",
    "\n",
    "        result[\"Superordinate\"][\"Prediction\"]   = preds_super[j-a]\n",
    "        result[\"Superordinate\"][\"Logits\"]       = logits_per_image_super[j-a]\n",
    "        result[\"Superordinate\"][\"Probas\"]       = probs_super[j-a]\n",
    "\n",
    "        result[\"Basic\"][\"Prediction\"]           = preds_basic[j-a]\n",
    "        result[\"Basic\"][\"Logits\"]               = logits_per_image_basic[j-a]\n",
    "        result[\"Basic\"][\"Probas\"]               = probs_basic[j-a]\n",
    "\n",
    "        original_predictions[j] = result\n",
    "        \n",
    "    torch.save(original_predictions,data_path)\n",
    "  return original_predictions"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "87nD4ZiVnBW9"
   },
   "source": [
    "def get_original_preds(i,display=False):\n",
    "  data = original_predictions[i]\n",
    "\n",
    "  super_pred = data[\"Superordinate\"][\"Prediction\"]\n",
    "  super_logit = data[\"Superordinate\"][\"Logits\"][super_pred]\n",
    "  super_proba = data[\"Superordinate\"][\"Probas\"][super_pred]\n",
    "\n",
    "  basic_pred = data[\"Basic\"][\"Prediction\"]\n",
    "  basic_logit = data[\"Basic\"][\"Logits\"][basic_pred]\n",
    "  basic_proba = data[\"Basic\"][\"Probas\"][basic_pred]\n",
    "\n",
    "  if display :\n",
    "    stimuli = get_stimuli(i)\n",
    "    plt.imshow(stimuli.cpu())\n",
    "    plt.show()\n",
    "    print(f\"SUPERORDINATE LABEL \\t: {super_labels[super_pred]} | {super_proba*100}%\")\n",
    "    print(f\"BASIC LABEL \\t\\t: {basic_labels[basic_pred]} | {basic_proba*100}%\")\n",
    "\n",
    "  return data"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "UTtZ-Wzppc9n"
   },
   "source": [
    "def compute_new_preds(batch_size=128):\n",
    "  data_path    = path+\"/DATA/\"+model_name+\"_\"+\"context\"+str(contexts.index(context))+\"_wordsAdd_preds.pt\"\n",
    "  dataset_size = 274\n",
    "  words        = list(set(super_labels) | set(basic_labels))\n",
    "\n",
    "  print(len(words))\n",
    "\n",
    "  # ----- CHECKING IF DATA ALREADY EXISTS -------------------------------------------\n",
    "  if os.path.exists(data_path): wordsAdd_predictions = torch.load(data_path,map_location=device)\n",
    "  else:                         wordsAdd_predictions = {}\n",
    "  start = len(wordsAdd_predictions.keys())  # Start at where we're at\n",
    "\n",
    "  # ----- COLLECTING THE PREDICTIONS FOR EACH WORD-IMAGE PAIRS -------------------\n",
    "  for w in range(start,len(words)) :\n",
    "    word                       = words[w]\n",
    "    wordsAdd_predictions[word] = []\n",
    "    clear_output()\n",
    "    print(word)\n",
    "    print(f\"{w+1}/{len(words)}\")\n",
    "\n",
    "    # ---- PROCESSING DATA IN MINI-BATCHES --------------------------------\n",
    "    for i in range(0,math.ceil(dataset_size/batch_size)):\n",
    "      a,b    = i*batch_size, min(dataset_size,i*batch_size + batch_size)\n",
    "      batch = get_stimulis(a,b,preprocess,word).to(device)\n",
    "\n",
    "      # --- BASIC PREDICTIONS ------------------------------------------\n",
    "      with torch.no_grad():\n",
    "          logits_per_image_basic, _  = model(batch, text_basic)\n",
    "          probs_basic          = logits_per_image_basic.softmax(dim=-1)\n",
    "          _, preds_basic       = torch.max(probs_basic,1)\n",
    "\n",
    "      # --- SUPERORDINATE PREDICTIONS ----------------------------------\n",
    "      with torch.no_grad():\n",
    "          logits_per_image_super, _  = model(batch, text_super)\n",
    "          probs_super          = logits_per_image_super.softmax(dim=-1)\n",
    "          _, preds_super       = torch.max(probs_super,1)\n",
    "\n",
    "      # --- SAVING PREDICTIONS -------------------------------------------\n",
    "      for j in range(a,b):\n",
    "        result = {\"Superordinate\":{},\"Basic\":{}}\n",
    "\n",
    "        result[\"Superordinate\"][\"Prediction\"]           = preds_super[j-a]\n",
    "        result[\"Superordinate\"][\"Logits\"]               = logits_per_image_super[j-a]\n",
    "        result[\"Superordinate\"][\"Probas\"]               = probs_super[j-a]\n",
    "\n",
    "        result[\"Basic\"][\"Prediction\"]                   = preds_basic[j-a]\n",
    "        result[\"Basic\"][\"Logits\"]                       = logits_per_image_basic[j-a]\n",
    "        result[\"Basic\"][\"Probas\"]                       = probs_basic[j-a]\n",
    "\n",
    "        wordsAdd_predictions[word].append(result)\n",
    "\n",
    "  torch.save(wordsAdd_predictions,data_path)\n",
    "  return wordsAdd_predictions"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "fedGJxWT1Xi6"
   },
   "source": [
    "def get_wordAdd_preds(i,word,display=False):\n",
    "\n",
    "  data    = wordsAdd_predictions[word][i]\n",
    "  og_data = get_original_preds(i)\n",
    "\n",
    "  if display :\n",
    "    og_super_pred     = og_data[\"Superordinate\"][\"Prediction\"]\n",
    "    og_super_logit    = og_data[\"Superordinate\"][\"Logits\"][og_super_pred]\n",
    "    og_super_proba    = og_data[\"Superordinate\"][\"Probas\"][og_super_pred]\n",
    "\n",
    "    super_pred     = data[\"Superordinate\"][\"Prediction\"]\n",
    "    super_logit    = data[\"Superordinate\"][\"Logits\"][super_pred]\n",
    "    super_proba    = data[\"Superordinate\"][\"Probas\"][super_pred]\n",
    "    \n",
    "    og_super_newLogit = data[\"Superordinate\"][\"Logits\"][og_super_pred]\n",
    "    og_super_newProba = data[\"Superordinate\"][\"Probas\"][og_super_pred]\n",
    "\n",
    "    # --------------------------------------------------------------\n",
    "\n",
    "    og_basic_pred     = og_data[\"Basic\"][\"Prediction\"]\n",
    "    og_basic_logit    = og_data[\"Basic\"][\"Logits\"][og_basic_pred]\n",
    "    og_basic_proba    = og_data[\"Basic\"][\"Probas\"][og_basic_pred]\n",
    "\n",
    "    basic_pred     = data[\"Basic\"][\"Prediction\"]\n",
    "    basic_logit    = data[\"Basic\"][\"Logits\"][basic_pred]\n",
    "    basic_proba    = data[\"Basic\"][\"Probas\"][basic_pred]\n",
    "    \n",
    "    og_basic_newLogit = data[\"Basic\"][\"Logits\"][og_basic_pred]\n",
    "    og_basic_newProba = data[\"Basic\"][\"Probas\"][og_basic_pred]\n",
    "\n",
    "    stimuli = get_stimuli(i,preprocess=None,word=word)\n",
    "    plt.imshow(stimuli.cpu())\n",
    "    plt.show()\n",
    "\n",
    "    print(f\"SUPERORDINATE \\t NEW \\t\\t: {super_labels[super_pred]} | {super_proba*100}%\")\n",
    "    print(f\"\\t\\t ORIGINAL \\t: {super_labels[og_super_pred]} | {og_super_proba*100}% --> {og_super_newProba*100}%\")\n",
    "    print(\"----------------------------------------------------------------------------------------------------------------------------------------\")\n",
    "    print(f\"BASIC \\t\\t NEW \\t\\t: {basic_labels[basic_pred]} | {basic_proba*100}%\")\n",
    "    print(f\"\\t\\t ORIGINAL \\t: {basic_labels[og_basic_pred]} | {og_basic_proba*100}% --> {og_basic_newProba*100}%\")\n",
    "\n",
    "  return data"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "0QSCaQtM2cZR"
   },
   "source": [
    "if not os.path.exists(path+\"DATASETS/GoogleNews-vectors-negative300.bin\"):  \n",
    "  import gzip\n",
    "  urllib.request.urlretrieve (\"https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz\", path+\"DATASETS/GoogleNews-vectors-negative300.bin.gz\")\n",
    "  with gzip.open(path+\"DATASETS/GoogleNews-vectors-negative300.bin.gz\", 'r') as zip_ref:\n",
    "    with open(path+\"DATASETS/GoogleNews-vectors-negative300.bin\", 'wb') as f:\n",
    "      f.write(zip_ref.read())\n",
    "model_w2v = gensim.models.KeyedVectors.load_word2vec_format(path+\"DATASETS/GoogleNews-vectors-negative300.bin\", binary=True)  \n",
    "\n",
    "def semantic_similarity_w2v(w1,w2):\n",
    "  f1, f2 = torch.tensor(model_w2v[w1]), torch.tensor(model_w2v[w2])\n",
    "  return torch.nn.CosineSimilarity(dim=0)(f1,f2).item()"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "2yKLDTjY1Qky"
   },
   "source": [
    "def get_switching_rate():\n",
    "  miss_rates = {\"Superordinate\":{},\"Basic\":{}}\n",
    "  miss_rates[\"Superordinate\"][\"Superordinate\"] = 0.0\n",
    "  miss_rates[\"Basic\"][\"Superordinate\"]         = 0.0\n",
    "  miss_rates[\"Superordinate\"][\"Basic\"]         = 0.0\n",
    "  miss_rates[\"Basic\"][\"Basic\"]                 = 0.0\n",
    "\n",
    "  # ----- SUPERORDINATE WA ------------------------------------------------------------------------------------------------------------------------\n",
    "  counted_basic = 0\n",
    "  counted_super = 0\n",
    "  for wa in super_labels:\n",
    "    for i in range(0,274):\n",
    "      original_preds, new_preds = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "      if (wa != super_labels[original_preds[\"Superordinate\"][\"Prediction\"]]):\n",
    "        miss_rates[\"Superordinate\"][\"Superordinate\"] += (original_preds[\"Superordinate\"][\"Prediction\"].item() != new_preds[\"Superordinate\"][\"Prediction\"].item())\n",
    "        miss_rates[\"Basic\"][\"Superordinate\"]         += (original_preds[\"Basic\"][\"Prediction\"].item()         != new_preds[\"Basic\"][\"Prediction\"].item())\n",
    "        counted_super += 1\n",
    "      \n",
    "  miss_rates[\"Superordinate\"][\"Superordinate\"] /= counted_super\n",
    "  miss_rates[\"Basic\"][\"Superordinate\"]         /= counted_super\n",
    "\n",
    "  # ----- BASIC WA --------------------------------------------------------------------------------------------------------------------------------\n",
    "  counted_basic = 0\n",
    "  for wa in basic_labels:\n",
    "    for i in range(0,274):\n",
    "      original_preds, new_preds = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "      if (wa != basic_labels[original_preds[\"Basic\"][\"Prediction\"]]):\n",
    "        miss_rates[\"Basic\"][\"Basic\"]         += (original_preds[\"Basic\"][\"Prediction\"].item()         != new_preds[\"Basic\"][\"Prediction\"].item())\n",
    "        miss_rates[\"Superordinate\"][\"Basic\"] += (original_preds[\"Superordinate\"][\"Prediction\"].item() != new_preds[\"Superordinate\"][\"Prediction\"].item())\n",
    "        counted_basic += 1\n",
    "        \n",
    "  miss_rates[\"Superordinate\"][\"Basic\"] /= counted_basic\n",
    "  miss_rates[\"Basic\"][\"Basic\"]         /= counted_basic\n",
    "\n",
    "  return miss_rates"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "1Wc1ynrb7eeA"
   },
   "source": [
    "def get_word_correlation_references(metric):\n",
    "  references = {\"Superordinate\":{},\"Basic\":{}}\n",
    "  references[\"Superordinate\"][\"Superordinate\"] = []\n",
    "  references[\"Basic\"][\"Superordinate\"]         = []\n",
    "  references[\"Superordinate\"][\"Basic\"]         = []\n",
    "  references[\"Basic\"][\"Basic\"]                 = [] \n",
    "\n",
    "  for pred_category in [\"Basic\",\"Superordinate\"]:\n",
    "    for wa_category in [\"Basic\",\"Superordinate\"]:\n",
    "      added_words = (super_labels if wa_category==\"Superordinate\" else basic_labels)\n",
    "      labels      = (super_labels if pred_category==\"Superordinate\" else basic_labels)\n",
    "\n",
    "      similarities   = []\n",
    "      for wa in added_words:\n",
    "        for i in range(0,274):\n",
    "          original_data   = get_original_preds(i)\n",
    "          original_pred   = original_data[pred_category][\"Prediction\"]\n",
    "          original_label  = labels[original_pred]\n",
    "\n",
    "          #print(wa)\n",
    "          similarities.append(metric(wa,original_label))\n",
    "      \n",
    "      references[pred_category][wa_category] = similarities\n",
    "\n",
    "  return references"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "QEHnT2PjnMdU"
   },
   "source": [
    "def get_word_correlation_references_nonswitchonly(metric):\n",
    "  references = {\"Superordinate\":{},\"Basic\":{}}\n",
    "  references[\"Superordinate\"][\"Superordinate\"] = []\n",
    "  references[\"Basic\"][\"Superordinate\"]         = []\n",
    "  references[\"Superordinate\"][\"Basic\"]         = []\n",
    "  references[\"Basic\"][\"Basic\"]                 = [] \n",
    "\n",
    "  for pred_category in [\"Basic\",\"Superordinate\"]:\n",
    "    for wa_category in [\"Basic\",\"Superordinate\"]:\n",
    "      added_words = (super_labels if wa_category==\"Superordinate\" else basic_labels)\n",
    "      labels      = (super_labels if pred_category==\"Superordinate\" else basic_labels)\n",
    "\n",
    "      similarities   = []\n",
    "      for wa in added_words:\n",
    "        for i in range(0,274):\n",
    "          original_data, new_data   = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "          original_pred, new_pred   = original_data[pred_category][\"Prediction\"], new_data[pred_category][\"Prediction\"]\n",
    "          original_label, new_label = labels[original_pred], labels[new_pred]\n",
    "\n",
    "          if original_pred == new_pred:\n",
    "            similarities.append(metric(wa,original_label))\n",
    "\n",
    "      references[pred_category][wa_category] = similarities\n",
    "\n",
    "\n",
    "  return references"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "8MHER04v81-g"
   },
   "source": [
    "def get_OAC(metric):\n",
    "  references = {\"Superordinate\":{},\"Basic\":{}}\n",
    "  references[\"Superordinate\"][\"Superordinate\"] = []\n",
    "  references[\"Basic\"][\"Superordinate\"]         = []\n",
    "  references[\"Superordinate\"][\"Basic\"]         = []\n",
    "  references[\"Basic\"][\"Basic\"]                 = [] \n",
    "\n",
    "  for pred_category in [\"Basic\",\"Superordinate\"]:\n",
    "    for wa_category in [\"Basic\",\"Superordinate\"]:\n",
    "      added_words = (super_labels if wa_category==\"Superordinate\" else basic_labels)\n",
    "      labels      = (super_labels if pred_category==\"Superordinate\" else basic_labels)\n",
    "\n",
    "      similarities   = []\n",
    "      for wa in added_words:\n",
    "        for i in range(0,274):\n",
    "          original_data, new_data   = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "          original_pred, new_pred   = original_data[pred_category][\"Prediction\"], new_data[pred_category][\"Prediction\"]\n",
    "          original_label, new_label = labels[original_pred], labels[new_pred]\n",
    "\n",
    "          if original_pred != new_pred:\n",
    "            similarities.append(metric(wa,original_label))\n",
    "\n",
    "      references[pred_category][wa_category] = similarities\n",
    "\n",
    "\n",
    "  return references"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "r9-OBXiH9Ntk"
   },
   "source": [
    "def get_NAC(metric):\n",
    "  references = {\"Superordinate\":{},\"Basic\":{}}\n",
    "  references[\"Superordinate\"][\"Superordinate\"] = []\n",
    "  references[\"Basic\"][\"Superordinate\"]         = []\n",
    "  references[\"Superordinate\"][\"Basic\"]         = []\n",
    "  references[\"Basic\"][\"Basic\"]                 = [] \n",
    "\n",
    "  for pred_category in [\"Basic\",\"Superordinate\"]:\n",
    "    for wa_category in [\"Basic\",\"Superordinate\"]:\n",
    "      added_words = (super_labels if wa_category==\"Superordinate\" else basic_labels)\n",
    "      labels      = (super_labels if pred_category==\"Superordinate\" else basic_labels)\n",
    "\n",
    "      similarities   = []\n",
    "      for wa in added_words:\n",
    "        for i in range(0,274):\n",
    "          original_data, new_data   = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "          original_pred, new_pred   = original_data[pred_category][\"Prediction\"], new_data[pred_category][\"Prediction\"]\n",
    "          original_label, new_label = labels[original_pred], labels[new_pred]\n",
    "\n",
    "          if original_pred != new_pred:\n",
    "            similarities.append(metric(wa,new_label))\n",
    "\n",
    "      references[pred_category][wa_category] = similarities\n",
    "\n",
    "  return references"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "AS4bbGhtB7al"
   },
   "source": [
    "def get_probabilities_references():\n",
    "  references = {\"Basic\":[],\"Superordinate\":[]}\n",
    "\n",
    "  for i in range(0,274):\n",
    "    original_preds         = get_original_preds(i)\n",
    "    super_pred, basic_pred = original_preds[\"Superordinate\"][\"Prediction\"], original_preds[\"Basic\"][\"Prediction\"]\n",
    "    references[\"Superordinate\"].append(original_preds[\"Superordinate\"][\"Probas\"][super_pred].item())\n",
    "    references[\"Basic\"].append(original_preds[\"Basic\"][\"Probas\"][basic_pred].item())\n",
    "\n",
    "  return references"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "is5JSVd3l1i-"
   },
   "source": [
    "def get_probabilities_references_nonswitched():\n",
    "  original_probs = {\"Superordinate\":{},\"Basic\":{}}\n",
    "  original_probs[\"Superordinate\"][\"Superordinate\"] = []\n",
    "  original_probs[\"Basic\"][\"Superordinate\"]         = []\n",
    "  original_probs[\"Superordinate\"][\"Basic\"]         = []\n",
    "  original_probs[\"Basic\"][\"Basic\"]                 = []  \n",
    "\n",
    "  # ----- SUPERORDINATE WA -------------------------------------------------------------------------------------------------\n",
    "  for wa in super_labels:\n",
    "    for i in range(0,274):\n",
    "      original_preds, new_preds = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "\n",
    "      # --- SUPERORDINATE PRED\n",
    "      originalPred, newPred = original_preds[\"Superordinate\"][\"Prediction\"], new_preds[\"Superordinate\"][\"Prediction\"]\n",
    "      if originalPred == newPred :\n",
    "        original_probs[\"Superordinate\"][\"Superordinate\"].append(original_preds[\"Superordinate\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "      # --- BASIC PRED\n",
    "      originalPred, newPred = original_preds[\"Basic\"][\"Prediction\"], new_preds[\"Basic\"][\"Prediction\"]\n",
    "      if originalPred == newPred :\n",
    "        original_probs[\"Basic\"][\"Superordinate\"].append(original_preds[\"Basic\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "  # ----- BASIC WA --------------------------------------------------------------------------------------------------------\n",
    "  for wa in basic_labels:\n",
    "    for i in range(0,274):\n",
    "      original_preds, new_preds = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "\n",
    "      # --- SUPERORDINATE PRED\n",
    "      originalPred, newPred = original_preds[\"Superordinate\"][\"Prediction\"], new_preds[\"Superordinate\"][\"Prediction\"]\n",
    "      if originalPred == newPred :\n",
    "        original_probs[\"Superordinate\"][\"Basic\"].append(original_preds[\"Superordinate\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "      # --- BASIC PRED\n",
    "      originalPred, newPred = original_preds[\"Basic\"][\"Prediction\"], new_preds[\"Basic\"][\"Prediction\"]\n",
    "      if originalPred == newPred :\n",
    "        original_probs[\"Basic\"][\"Basic\"].append(original_preds[\"Basic\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "\n",
    "  return original_probs"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "aGucvyYfE0QW"
   },
   "source": [
    "def get_COM_original():\n",
    "  original_probs = {\"Superordinate\":{},\"Basic\":{}}\n",
    "  original_probs[\"Superordinate\"][\"Superordinate\"] = []\n",
    "  original_probs[\"Basic\"][\"Superordinate\"]         = []\n",
    "  original_probs[\"Superordinate\"][\"Basic\"]         = []\n",
    "  original_probs[\"Basic\"][\"Basic\"]                 = []  \n",
    "\n",
    "  # ----- SUPERORDINATE WA -------------------------------------------------------------------------------------------------\n",
    "  for wa in super_labels:\n",
    "    for i in range(0,274):\n",
    "      original_preds, new_preds = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "\n",
    "      # --- SUPERORDINATE PRED\n",
    "      originalPred, newPred = original_preds[\"Superordinate\"][\"Prediction\"], new_preds[\"Superordinate\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        original_probs[\"Superordinate\"][\"Superordinate\"].append(original_preds[\"Superordinate\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "      # --- BASIC PRED\n",
    "      originalPred, newPred = original_preds[\"Basic\"][\"Prediction\"], new_preds[\"Basic\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        original_probs[\"Basic\"][\"Superordinate\"].append(original_preds[\"Basic\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "  # ----- BASIC WA --------------------------------------------------------------------------------------------------------\n",
    "  for wa in basic_labels:\n",
    "    for i in range(0,274):\n",
    "      original_preds, new_preds = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "\n",
    "      # --- SUPERORDINATE PRED\n",
    "      originalPred, newPred = original_preds[\"Superordinate\"][\"Prediction\"], new_preds[\"Superordinate\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        original_probs[\"Superordinate\"][\"Basic\"].append(original_preds[\"Superordinate\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "      # --- BASIC PRED\n",
    "      originalPred, newPred = original_preds[\"Basic\"][\"Prediction\"], new_preds[\"Basic\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        original_probs[\"Basic\"][\"Basic\"].append(original_preds[\"Basic\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "\n",
    "  return original_probs"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "QSZYPvaUGSTj"
   },
   "source": [
    "def get_COM_new():\n",
    "  new_probs = {\"Superordinate\":{},\"Basic\":{}}\n",
    "  new_probs[\"Superordinate\"][\"Superordinate\"] = []\n",
    "  new_probs[\"Basic\"][\"Superordinate\"]         = []\n",
    "  new_probs[\"Superordinate\"][\"Basic\"]         = []\n",
    "  new_probs[\"Basic\"][\"Basic\"]                 = []  \n",
    "\n",
    "  # ----- SUPERORDINATE WA -------------------------------------------------------------------------------------------------\n",
    "  for wa in super_labels:\n",
    "    for i in range(0,274):\n",
    "      original_preds, new_preds = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "\n",
    "      # --- SUPERORDINATE PRED\n",
    "      originalPred, newPred = original_preds[\"Superordinate\"][\"Prediction\"], new_preds[\"Superordinate\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        new_probs[\"Superordinate\"][\"Superordinate\"].append(new_preds[\"Superordinate\"][\"Probas\"][newPred].item())\n",
    "\n",
    "      # --- BASIC PRED\n",
    "      originalPred, newPred = original_preds[\"Basic\"][\"Prediction\"], new_preds[\"Basic\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        new_probs[\"Basic\"][\"Superordinate\"].append(new_preds[\"Basic\"][\"Probas\"][newPred].item())\n",
    "\n",
    "  # ----- BASIC WA --------------------------------------------------------------------------------------------------------\n",
    "  for wa in basic_labels:\n",
    "    for i in range(0,274):\n",
    "      original_preds, new_preds = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "\n",
    "      # --- SUPERORDINATE PRED\n",
    "      originalPred, newPred = original_preds[\"Superordinate\"][\"Prediction\"], new_preds[\"Superordinate\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        new_probs[\"Superordinate\"][\"Basic\"].append(new_preds[\"Superordinate\"][\"Probas\"][newPred].item())\n",
    "\n",
    "      # --- BASIC PRED\n",
    "      originalPred, newPred = original_preds[\"Basic\"][\"Prediction\"], new_preds[\"Basic\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        new_probs[\"Basic\"][\"Basic\"].append(new_preds[\"Basic\"][\"Probas\"][newPred].item())\n",
    "\n",
    "\n",
    "  return new_probs"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "-az7EU2Zmxg3"
   },
   "source": [
    "def get_COM_neworiginal():\n",
    "  new_probs = {\"Superordinate\":{},\"Basic\":{}}\n",
    "  new_probs[\"Superordinate\"][\"Superordinate\"] = []\n",
    "  new_probs[\"Basic\"][\"Superordinate\"]         = []\n",
    "  new_probs[\"Superordinate\"][\"Basic\"]         = []\n",
    "  new_probs[\"Basic\"][\"Basic\"]                 = []  \n",
    "\n",
    "  # ----- SUPERORDINATE WA -------------------------------------------------------------------------------------------------\n",
    "  for wa in super_labels:\n",
    "    for i in range(0,274):\n",
    "      original_preds, new_preds = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "\n",
    "      # --- SUPERORDINATE PRED\n",
    "      originalPred, newPred = original_preds[\"Superordinate\"][\"Prediction\"], new_preds[\"Superordinate\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        new_probs[\"Superordinate\"][\"Superordinate\"].append(new_preds[\"Superordinate\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "      # --- BASIC PRED\n",
    "      originalPred, newPred = original_preds[\"Basic\"][\"Prediction\"], new_preds[\"Basic\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        new_probs[\"Basic\"][\"Superordinate\"].append(new_preds[\"Basic\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "  # ----- BASIC WA --------------------------------------------------------------------------------------------------------\n",
    "  for wa in basic_labels:\n",
    "    for i in range(0,274):\n",
    "      original_preds, new_preds = get_original_preds(i), get_wordAdd_preds(i,wa)\n",
    "\n",
    "      # --- SUPERORDINATE PRED\n",
    "      originalPred, newPred = original_preds[\"Superordinate\"][\"Prediction\"], new_preds[\"Superordinate\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        new_probs[\"Superordinate\"][\"Basic\"].append(new_preds[\"Superordinate\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "      # --- BASIC PRED\n",
    "      originalPred, newPred = original_preds[\"Basic\"][\"Prediction\"], new_preds[\"Basic\"][\"Prediction\"]\n",
    "      if originalPred != newPred :\n",
    "        new_probs[\"Basic\"][\"Basic\"].append(new_preds[\"Basic\"][\"Probas\"][originalPred].item())\n",
    "\n",
    "\n",
    "  return new_probs"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "C7xEheNBNYCd"
   },
   "source": [
    "def display_test_boxplots(REF,O,N,testCategory,distNames):\n",
    "  fig, axs = plt.subplots(2, 2, figsize=(12,10))\n",
    "  i,j = 0,0\n",
    "  for pred_category in [\"Superordinate\", \"Basic\"]:\n",
    "    for wa_category in [\"Superordinate\", \"Basic\"]:\n",
    "      box_ref  = display_boxes(axs[i][j],[REF[pred_category][wa_category] if testCategory!=\"PROBABILITY\" else REF[pred_category], [], []],[distNames[0],distNames[1],distNames[2]],\"\",testCategory,\"\",(-0.1,1.1))\n",
    "      box_oac  = display_boxes(axs[i][j],[[], O[pred_category][wa_category],[]],[distNames[0],distNames[1],distNames[2]],\"\",testCategory,\"\",(-0.1,1.1))\n",
    "      box_nac  = display_boxes(axs[i][j],[[], [],N[pred_category][wa_category]],[distNames[0],distNames[1],distNames[2]],\"\",testCategory,\"\",(-0.1,1.1))\n",
    "      axs[i][j].set_title(f\"{pred_category.upper()} LABELS | {wa_category.upper()} wa\")\n",
    "\n",
    "      set_box_colors(box_ref,[\"white\",\"black\",\"black\",\"black\"])\n",
    "      set_box_colors(box_oac,[\"black\",\"green\",\"green\",\"green\"])\n",
    "      set_box_colors(box_nac,[\"black\",\"red\",\"red\",\"red\"])\n",
    "      j+=1 \n",
    "    i+=1\n",
    "    j=0\n",
    "\n",
    "  fig.subplots_adjust(wspace=0.25, hspace=0.3)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "BKaliGTWy0NI"
   },
   "source": [
    "def display_test_boxplots_twoconds(REF,O,testCategory,distNames):\n",
    "  #I changed the order of panels (pred_category and wa_category)to make consistency in the paper \n",
    "  fig, axs = plt.subplots(2, 2, figsize=(12,10))\n",
    "  i,j = 0,0\n",
    "  for wa_category in [\"Superordinate\", \"Basic\"]:\n",
    "    for pred_category in [\"Superordinate\", \"Basic\"]:\n",
    "      #exclude the same category between images and words\n",
    "      tmp = REF[pred_category][wa_category] if testCategory!=\"PROBABILITY\" else REF[pred_category]\n",
    "      tmp = [i for i in tmp if i != 1.0]\n",
    "      tmp = [tmp,[]]\n",
    "      box_ref  = display_boxes_forpaper(axs[i][j],tmp,[distNames[0],distNames[1]],\"\",testCategory,\"\",(-0.1,1.1))\n",
    "      box_oac  = display_boxes_forpaper(axs[i][j],[[], O[pred_category][wa_category]],[distNames[0],distNames[1]],\"\",testCategory,\"\",(-0.1,1.1))\n",
    "      #axs[i][j].set_title(f\"{pred_category.upper()} LABELS | {wa_category.upper()} ew\")\n",
    "\n",
    "      #save_csv_two(REF[pred_category][wa_category] if testCategory!=\"PROBABILITY\" else REF[pred_category],O[pred_category][wa_category],pred_category,wa_category,testCategory)\n",
    "\n",
    "      set_box_colors(box_ref,[\"white\",\"black\",\"black\",\"black\"])\n",
    "      set_box_colors(box_oac,[\"black\",\"red\",\"red\",\"red\"])\n",
    "      j+=1 \n",
    "    i+=1\n",
    "    j=0\n",
    "\n",
    "  fig.subplots_adjust(wspace=0.25, hspace=0.3)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "fUjhWjixhrF5"
   },
   "source": [
    "def display_test_boxplots_fourconds(REF,O,ON,N,testCategory,distNames):\n",
    "  #I changed the order of panels (pred_category and wa_category)to make consistency in the paper \n",
    "  fig, axs = plt.subplots(2, 2, figsize=(24,10))\n",
    "  i,j = 0,0\n",
    "  for wa_category in [\"Superordinate\", \"Basic\"]:\n",
    "    for pred_category in [\"Superordinate\", \"Basic\"]:\n",
    "      #exclude the same category between images and words\n",
    "      tmp = [REF[pred_category][wa_category],[],[],[]]\n",
    "      #tmp = [i for i in tmp if i != 1.0]\n",
    "      #tmp = [tmp,[]]\n",
    "      box_ref  = display_boxes_forpaper(axs[i][j],tmp,[distNames[0],distNames[1],distNames[2],distNames[3]],\"\",testCategory,\"\",(-0.1,1.1))\n",
    "      box_switchedoriginal  = display_boxes_forpaper(axs[i][j],[[], O[pred_category][wa_category],[],[]],[distNames[0],distNames[1],distNames[2],distNames[3]],\"\",testCategory,\"\",(-0.1,1.1))\n",
    "      box_oridinalnew  = display_boxes_forpaper(axs[i][j],[[],[], ON[pred_category][wa_category],[]],[distNames[0],distNames[1],distNames[2],distNames[3]],\"\",testCategory,\"\",(-0.1,1.1))\n",
    "      box_newlabel  = display_boxes_forpaper(axs[i][j],[[],[],[], N[pred_category][wa_category]],[distNames[0],distNames[1],distNames[2],distNames[3]],\"\",testCategory,\"\",(-0.1,1.1))\n",
    "      #axs[i][j].set_title(f\"{pred_category.upper()} LABELS | {wa_category.upper()} ew\")\n",
    "\n",
    "      #save_csv_four(REF[pred_category][wa_category],O[pred_category][wa_category],ON[pred_category][wa_category],N[pred_category][wa_category],pred_category,wa_category,testCategory)\n",
    "\n",
    "      set_box_colors(box_ref,[\"white\",\"black\",\"black\",\"black\"])\n",
    "      set_box_colors(box_switchedoriginal,[\"black\",\"red\",\"red\",\"red\"])\n",
    "      set_box_colors(box_oridinalnew,[\"black\",\"red\",\"red\",\"red\"])\n",
    "      set_box_colors(box_newlabel,[\"black\",\"blue\",\"blue\",\"blue\"])\n",
    "      j+=1 \n",
    "    i+=1\n",
    "    j=0\n",
    "\n",
    "  fig.subplots_adjust(wspace=0.25, hspace=0.3)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "A7RAflhzNOZI"
   },
   "source": [
    "def save_csv_four(REF,O,ON,N,pred_category,wa_category,testtCategory):\n",
    "    import pandas as pd\n",
    "    df = pd.DataFrame(REF,\n",
    "                  columns=['unswitch'])\n",
    "    df['labelswitched'] = O\n",
    "    df['original_label'] = ON\n",
    "    df['new_label'] = N\n",
    "    df.to_csv(path+testCategory+pred_category+wa_category+'.csv')\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "I_YCCHShOzm2"
   },
   "source": [
    "def save_csv_two(REF,O,pred_category,wa_category,testtCategory):\n",
    "    import pandas as pd\n",
    "    df = pd.DataFrame(REF,\n",
    "                  columns=['unswitch'])\n",
    "    df['labelswitched'] = O\n",
    "    df.to_csv(path+testCategory+pred_category+wa_category+'.csv')"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "WkDdIWvj8JGR"
   },
   "source": [
    "def display_boxes_forpaper(ax,data,tickLabels,xlabel,ylabel,title,yscale):\n",
    "  boxes = ax.boxplot(data,showfliers=True,patch_artist=True,widths=0.5)\n",
    "  ax.set_ylabel(ylabel,labelpad=20)\n",
    "  ax.set_xlabel(xlabel,labelpad=20)\n",
    "  ax.set_xticklabels(tickLabels,fontsize=20)\n",
    "  ax.set_yticks([0.0,0.2,0.4,0.6,0.8,1.0])\n",
    "  ax.set_yticklabels(['0.0','0.2','0.4','0.6','0.8','1.0'],fontsize=18)\n",
    "  ax.set_title(title)\n",
    "  ax.set_ylim(yscale)\n",
    "  #ax.spines['top'].set_visible(False)\n",
    "  #ax.spines['right'].set_visible(False)\n",
    "  #plt.tight_layout()\n",
    "\n",
    "  return boxes"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "jlojJv68OA3-"
   },
   "source": [
    "def display_test_histograms(REF,O,N,testCategory,distNames,bins):\n",
    "  fig, axs = plt.subplots(2, 2, figsize=(15,10))\n",
    "  i,j = 0,0\n",
    "  for pred_category in [\"Superordinate\", \"Basic\"]:\n",
    "    for wa_category in [\"Superordinate\", \"Basic\"]:\n",
    "      axs[i][j].set_title(f\"{pred_category.upper()} LABELS | {wa_category.upper()} wa\")\n",
    "      axs[i][j].set_xlabel(testCategory)\n",
    "      axs[i][j].hist(REF[pred_category][wa_category] if testCategory!=\"PROBABILITY\" else REF[pred_category],bins=bins,color=\"black\",alpha = 1,density=True,label=distNames[0])\n",
    "      axs[i][j].hist(O[pred_category][wa_category],bins=bins,color=\"green\",alpha = 0.7,density=True,label=distNames[1])\n",
    "      axs[i][j].hist(N[pred_category][wa_category],bins=bins,color=\"red\",alpha = 0.7,density=True,label=distNames[2])\n",
    "      axs[i][j].set_ylabel(\"%\")\n",
    "\n",
    "      if (i,j) == (1,1):\n",
    "        handles, labels = axs[i][j].get_legend_handles_labels()\n",
    "      j+=1 \n",
    "    i+=1\n",
    "    j=0\n",
    "\n",
    "  fig.legend(handles, labels, loc = (0.91, 0.828))\n",
    "  fig.subplots_adjust(wspace=0.3, hspace=0.3)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "c5AaV1V18UtW"
   },
   "source": [
    "def dist_comparison(REF,O,N,distNames,probas=False):\n",
    "  for pred_category in [\"Superordinate\",\"Basic\"]:\n",
    "    for wa_category in [\"Superordinate\",\"Basic\"]:\n",
    "\n",
    "      print(\"\\n---\")\n",
    "      print(f\"{pred_category.upper()} LABELS | {wa_category.upper()} wa\")\n",
    "\n",
    "      # --- REF / O\n",
    "      u,p = mannwhitneyu(REF[pred_category][wa_category] if not probas else REF[pred_category],O[pred_category][wa_category],alternative=\"two-sided\")\n",
    "      m1, m2 = statistics.median(REF[pred_category][wa_category] if not probas else REF[pred_category]), statistics.median(O[pred_category][wa_category])\n",
    "      res = \"\"\n",
    "      if p < 0.05:\n",
    "        res = distNames[0]+\" STATISTICALLY DIFFERENT FROM \"+distNames[1]+\" | \"\n",
    "        if m1 < m2   : res += distNames[0]+\" < \"+distNames[1]\n",
    "        elif m1 > m2 : res += distNames[0]+\" > \"+distNames[1]\n",
    "      else:\n",
    "        res = distNames[0]+\" NOT SIGNIFICANTLY DIFFERENT FROM \"+distNames[1]\n",
    "      print(res+\" | u = \"+str(u)+\",\\t\\t p = \"+str(p))\n",
    "\n",
    "      # --- REF / N\n",
    "      u,p = mannwhitneyu(REF[pred_category][wa_category] if not probas else REF[pred_category],N[pred_category][wa_category],alternative=\"two-sided\")\n",
    "      m1, m2 = statistics.median(REF[pred_category][wa_category] if not probas else REF[pred_category]), statistics.median(N[pred_category][wa_category])\n",
    "      res = \"\"\n",
    "      if p < 0.05:\n",
    "        res = distNames[0]+\" STATISTICALLY DIFFERENT FROM \"+distNames[2]+\" | \"\n",
    "        if m1 < m2   : res += distNames[0]+\" < \"+distNames[2]\n",
    "        elif m1 > m2 : res += distNames[0]+\" > \"+distNames[2]\n",
    "      else:\n",
    "        res = distNames[0]+\" NOT SIGNIFICANTLY DIFFERENT FROM \"+distNames[2]\n",
    "      print(res+\" | u = \"+str(u)+\",\\t\\t p = \"+str(p))\n",
    "\n",
    "\n",
    "      # --- O / N\n",
    "      u,p = mannwhitneyu(O[pred_category][wa_category],N[pred_category][wa_category],alternative=\"two-sided\")\n",
    "      m1, m2 = statistics.median(O[pred_category][wa_category]), statistics.median(N[pred_category][wa_category])\n",
    "      res = \"\"\n",
    "      if p < 0.05:\n",
    "        res = distNames[1]+\" STATISTICALLY DIFFERENT FROM \"+distNames[2]+\" | \"\n",
    "        if m1 < m2   : res += distNames[1]+\" < \"+distNames[2]\n",
    "        elif m1 > m2 : res += distNames[1]+\" > \"+distNames[2]\n",
    "      else:\n",
    "        res = distNames[1]+\" NOT SIGNIFICANTLY DIFFERENT FROM \"+distNames[2]\n",
    "      print(res+\" | u = \"+str(u)+\",\\t\\t p = \"+str(p))\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "nskprBejNx6o"
   },
   "source": [
    "def visual_normalityCheck(REF):\n",
    "  fig, axs = plt.subplots(2, 2, figsize=(15,10))\n",
    "  i,j = 0,0\n",
    "\n",
    "  for pred_category in [\"Superordinate\", \"Basic\"]:\n",
    "    for wa_category in [\"Superordinate\", \"Basic\"]:\n",
    "      scipy.stats.probplot(REF[pred_category][wa_category] if testCategory!=\"PROBABILITY\" else REF[pred_category], dist=\"norm\", plot=axs[i][j])\n",
    "      axs[i][j].set_title(f\"{pred_category.upper()} LABELS | {wa_category.upper()} wa\")\n",
    "\n",
    "      j+=1\n",
    "    i+=1\n",
    "    j=0\n",
    "\n",
    "  fig.subplots_adjust(wspace=0.3, hspace=0.3)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Lhyj0_M8OhPi"
   },
   "source": [
    "def numerical_normalityCheck(REF,method):\n",
    "  print(\"USING \"+(\"Shapiro-Wilk Test\" if method==\"Shapiro\" else \"D’Agostino’s K² Test\"))\n",
    "  for pred_category in [\"Superordinate\", \"Basic\"]:\n",
    "    for wa_category in [\"Superordinate\", \"Basic\"]:\n",
    "      print(\"---\")\n",
    "      print(f\"REF FOR : {pred_category.upper()} LABELS | {wa_category.upper()} wa\")\n",
    "      if method==\"Shapiro\":\n",
    "        _, p1 = shapiro(REF[pred_category][wa_category] if testCategory!=\"PROBABILITY\" else REF[pred_category])\n",
    "        print((\"  NOT NORMAL \" if p1 < 0.05 else (\"NORMAL     \")) + \"\\t p = \" + str(p1))\n",
    "      else:\n",
    "        _, p2 = normaltest(REF[pred_category][wa_category] if testCategory!=\"PROBABILITY\" else REF[pred_category])\n",
    "        print((\"  NOT NORMAL \" if p2 < 0.05 else (\"NORMAL     \")) + \"\\t p = \" + str(p2))"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "8IOF8UekYzQ8"
   },
   "source": [
    "def display_boxes(ax,data,tickLabels,xlabel,ylabel,title,yscale):\n",
    "  boxes = ax.boxplot(data,showfliers=False,patch_artist=True)\n",
    "  ax.set_ylabel(ylabel,labelpad=15)\n",
    "  ax.set_xlabel(xlabel,labelpad=15)\n",
    "  ax.set_xticklabels(tickLabels,fontsize=15)\n",
    "  ax.set_title(title)\n",
    "  ax.set_ylim(yscale)\n",
    "  ax.spines['top'].set_visible(False)\n",
    "  ax.spines['right'].set_visible(False)\n",
    "  plt.tight_layout()\n",
    "\n",
    "  return boxes"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Wk57PQw0gf_J"
   },
   "source": [
    "def set_box_colors(box,colors):\n",
    "  for element in box['medians']:\n",
    "      element.set_color(colors[0])\n",
    "      element.set_linewidth(2)\n",
    "  for element in box['boxes']:\n",
    "      element.set_facecolor(colors[1])\n",
    "      element.set_linewidth(2)\n",
    "  for element in box['whiskers']:\n",
    "      element.set_linewidth(2)\n",
    "  for element in box['caps']:\n",
    "      element.set_linewidth(2)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uIHhIonQ1MXU"
   },
   "source": [
    "## **BENCHMARK TEST**"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "D3iKHYsTkouk"
   },
   "source": [
    "# ----- INITIALIZATIONS -------------------------------------------------------------------------------------------------\n",
    "model_name    = \"clip\"           # Name of the model\n",
    "model         = model            # Model\n",
    "preprocess_fn = preprocess       # Preprocessing to be applied on raw images\n",
    "tokenize_fn   = clip.tokenize    # Tokenize function\n",
    "images        = images           # Dataset of test images\n",
    "labels        = hierarchy        # Basic labels clustered by superordinate category\n",
    "contexts      = [\"a photo of a \"]\n",
    "dataset_size = 274\n",
    "# ------------------------------------------------------------------------------------------------------------------------"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "nomKdDio_CLW",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "297a59f1-56a4-45b3-d9bd-385d2a5f166b"
   },
   "source": [
    "  if not os.path.exists(path+\"/DATA/\"+model_name+\"_benchmark_results.pt\"):\n",
    "    results = {context:{} for context in contexts}\n",
    "    super_labels = [super_label for super_label in labels]\n",
    "    basic_labels = sorted(sum([[basic_label for basic_label in labels[super_label]] for super_label in labels],[]))\n",
    "\n",
    "    print(f\"SUPERORDINATE CATEGORIES :\\t {super_labels}\")\n",
    "    print(f\"BASIC CATEGORIES :\\t\\t {basic_labels}\")\n",
    "\n",
    "    # -----------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "    for context in contexts :\n",
    "      print(f\"CONTEXT : {context}[...]\")\n",
    "      text_super = tokenize_fn([context+label for label in super_labels]).to(device)\n",
    "      text_basic = tokenize_fn([context+label for label in basic_labels]).to(device)\n",
    "\n",
    "      # ----- COLLECTING THE DATA -------------------------------------------------------------------------------------------------\n",
    "      original_predictions = compute_original_preds(batch_size=128)\n",
    "      wordsAdd_predictions = compute_new_preds(batch_size=128)\n",
    "      # ----- calculate the task switching rate --------------------------------------------------------------------------------\n",
    "      switching_rate = get_switching_rate()\n",
    "\n",
    "      # ----- semantic/spelling similarity between original prediction & added-word (fig3)\n",
    "      nonswitch_semantic = get_word_correlation_references_nonswitchonly(semantic_similarity_w2v)\n",
    "      nonswitch_spelling = get_word_correlation_references_nonswitchonly(jellyfish.jaro_winkler_similarity)\n",
    "\n",
    "      switch_semantic = get_OAC(semantic_similarity_w2v)\n",
    "      switch_spelling = get_OAC(jellyfish.jaro_winkler_similarity)\n",
    "\n",
    "      # ----- Prediction confidence (figure e1)------\n",
    "      switch_proba = get_COM_original()\n",
    "      switch_new_proba_original = get_COM_new()\n",
    "      nonswitch_proba = get_probabilities_references_nonswitched()\n",
    "      switch_new_proba_new = get_COM_neworiginal()\n",
    "\n",
    "      results[context] = [switching_rate, nonswitch_semantic, nonswitch_spelling, switch_semantic, switch_spelling,\n",
    "                          switch_proba,switch_new_proba_original,nonswitch_proba,switch_new_proba_new]\n",
    "\n",
    "    torch.save(results,path+\"/DATA/\"+model_name+\"_benchmark_results.pt\")\n",
    "  # -----------------------------------------------------------"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "bowl\n",
      "150/150\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PXbs8GnAFDEB"
   },
   "source": [
    "## **RESULTS**"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ETHMRsUEGXDJ"
   },
   "source": [
    "model_name = \"clip\"\n",
    "images     = images             # Dataset of test images\n",
    "labels     = hierarchy        # Basic labels clustered by superordinate category\n",
    "contexts   = [\"a photo of a \"]\n",
    "\n",
    "super_labels = [super_label for super_label in labels]\n",
    "basic_labels = sorted(sum([[basic_label for basic_label in labels[super_label]] for super_label in labels],[]))"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "qjrI7HDUnjBb"
   },
   "source": [
    "results = torch.load(path+\"/DATA/\"+model_name+\"_benchmark_results.pt\")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "80eJfChfFWpc"
   },
   "source": [
    "### **CONTEXT SELECTION**"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Ky0h2xrCnsiB",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "c366e4ad-3bbe-42ba-a101-0ee3f54b93ce"
   },
   "source": [
    "context = contexts[0]\n",
    "switching_rate, nonswitch_semantic, nonswitch_spelling, switch_semantic, switch_spelling,\\\n",
    "switch_proba,switch_new_proba_original,nonswitch_proba,switch_new_proba_new = results[context]\n",
    "print(f\"CHOOSEN CONTEXT : {context}[...]\")"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "CHOOSEN CONTEXT : a photo of a [...]\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3oOy-KI8n-Ys"
   },
   "source": [
    "### **MANUAL CHECK OF PREDICTIONS** "
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "0JBFD0-xGMaB"
   },
   "source": [
    "original_predictions = torch.load(path+\"/DATA/\"+model_name+\"_\"+\"context\"+str(contexts.index(context))+\"_original_preds.pt\")\n",
    "wordsAdd_predictions =      torch.load(path+\"/DATA/\"+model_name+\"_\"+\"context\"+str(contexts.index(context))+\"_wordsAdd_preds.pt\")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "_ = get_original_preds(15,True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "_= get_wordAdd_preds(173, \"electronic\", True)\n",
    "_= get_wordAdd_preds(173, \"laptop\", True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BVGXyOgrxkbX"
   },
   "source": [
    "### **LABEL SWITCHING RATE**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "print(switching_rate)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### **SEMANTIC SIMILARITY**"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "testCategory = \"SEMANTIC SIMILARITY\"\n",
    "display_test_boxplots_twoconds(nonswitch_semantic, switch_semantic, testCategory, [\"Unswitched\", \"Label-switched\"])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### **SPELLING SIMILARITY**"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "testCategory = \"SPELLING SIMILARITY\"\n",
    "display_test_boxplots_twoconds(nonswitch_spelling, switch_spelling, testCategory, [\"Unswitched\", \"Label-switched\"])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### **PROBABILITIES OF PREDICTIONS**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "testCategory = \"PROBABILITY\"\n",
    "display_test_boxplots_fourconds(nonswitch_proba,switch_proba, switch_new_proba_new, switch_new_proba_original,testCategory,[\"Unswitched\",\"Label-switched\",\"Original label\",\"New label\"])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# **Representational similarity analysis**\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##**INITIALIZATION**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# initialization\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "model, preprocess = clip.load(\"ViT-B/32\", device=device, jit=False)\n",
    "s_156 = scipy.io.loadmat(path+'DATASETS/dataset3.mat')\n",
    "images = [s_156['visual_stimuli156'][0][i][0] for i in range(156)]\n",
    "images_torch = get_stimulis(0,156, preprocess=preprocess, word=None).to(device)\n",
    "\n",
    "model_name = \"clip\"\n",
    "contexts   = [\"a photo of a \"]\n",
    "labels  = torch.load(path+\"DATASETS/LABELS/labels.pt\")\n",
    "super_labels, basic_labels, hierarchy = labels[\"SUPERORDINATES\"], labels[\"BASICS\"], labels[\"CLUSTERING\"]\n",
    "context = contexts[0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##**UTILS**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "###**ORDER STIMULIS BY LABELS**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "###**DISPLAY ORDERED STIMULIS**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def get_ticks_and_labels(ordered_stimulis):\n",
    "\n",
    "  labels = list(ordered_stimulis.keys())\n",
    "\n",
    "  # GET DELIMITATION TICKS\n",
    "  ticks = []\n",
    "  for label in ordered_stimulis:\n",
    "    if len(ticks) > 0 :\n",
    "      ticks.append(ticks[len(ticks)-1] + len(ordered_stimulis[label]) )\n",
    "    else :\n",
    "      ticks.append(0)\n",
    "\n",
    "      if len(ordered_stimulis[label])-1 > 0:\n",
    "        ticks.append(len(ordered_stimulis[label])-1)\n",
    "\n",
    "  # GET CENTER TICKS\n",
    "  centers = []\n",
    "  for i in range(len(ticks)-1): centers.append(math.floor((ticks[i] + ticks[i+1])/2))\n",
    "\n",
    "  if len(centers) == len(labels)-1:\n",
    "    centers.append(centers[len(centers)-1] + 1)\n",
    "\n",
    "  # SET LABELS\n",
    "  centerLabels = labels\n",
    "  tickLabels  = [\"\"] * len(ticks)\n",
    "\n",
    "  # CORRECT CENTERS\n",
    "\n",
    "  remove_centers = []\n",
    "  for i in range(len(centers)):\n",
    "    if ticks[i] == centers[i] : remove_centers.append(i)\n",
    "\n",
    "  for i in range(len(centers)-1,-1,-1):\n",
    "    if i in remove_centers :\n",
    "      centers.pop(i)\n",
    "      tickLabels[i] = centerLabels[i]\n",
    "      centerLabels.pop(i)\n",
    "\n",
    "\n",
    "  return ticks, centers, tickLabels, centerLabels"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "AAa7y6gjdcRa"
   },
   "source": [
    "def display_img_with_ordered_labels(img, ordered_labels_x, ordered_labels_y, colorbar=False, size=(5,5), ratio=(1,1), min=0, max=1, xlabel=\"\", ylabel=\"\", title=\"\", title_cbar=\"\"):\n",
    "  \n",
    "  x_ticks, x_centers, x_tickLabels, x_centerLabels = get_ticks_and_labels(ordered_labels_x)\n",
    "  if None is ordered_labels_y : y_ticks, y_centers, y_tickLabels, y_centerLabels = x_ticks, x_centers, x_tickLabels, x_centerLabels\n",
    "  else : y_ticks, y_centers, y_tickLabels, y_centerLabels = get_ticks_and_labels(ordered_labels_y)\n",
    "\n",
    "  # --- DISPLAY IMAGE ---\n",
    "  fig, ax = plt.subplots(1, 1, figsize=size)\n",
    "  hm      = ax.imshow(img, cmap='Spectral', interpolation='nearest',vmin=min,vmax=max)\n",
    "  ax.set_aspect(float(ratio[0]) / float(ratio[1]))\n",
    "  if colorbar:\n",
    "    cbar = fig.colorbar(hm)\n",
    "    cbar.set_label(title_cbar,rotation=270,labelpad=30)\n",
    "\n",
    "  ax.set_xticks(      x_centers,      minor=True)\n",
    "  ax.set_xticklabels( x_centerLabels, minor=True,   rotation=90)\n",
    "  ax.set_xticks(      x_ticks,        minor=False)\n",
    "  ax.set_xticklabels( x_tickLabels,   minor=False,  rotation=90)\n",
    "  ax.set_yticks(      y_centers,      minor=True)\n",
    "  ax.set_yticklabels( y_centerLabels, minor=True,   rotation=0)\n",
    "  ax.set_yticks(      y_ticks,        minor=False)\n",
    "  ax.set_yticklabels( y_tickLabels,   minor=False,  rotation=0)\n",
    "  ax.tick_params(axis=u'both', which=u'minor',length=0)\n",
    "  ax.tick_params(axis=u'both', which=u'major',length=10)\n",
    "  ax.set_xlabel(xlabel, labelpad=30, fontsize=18)\n",
    "  ax.set_ylabel(ylabel, labelpad=30, fontsize=18)\n",
    "  ax.set_title(title)\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QTfoEr6QOu3J"
   },
   "source": [
    "##**REPRESENTATIONS FOR FEEDFORWARD MODELS**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import torchvision.models as models\n",
    "\n",
    "vgg19_bn         = models.vgg19_bn(pretrained=True)\n",
    "resnet152        = models.resnet18(pretrained=True)\n",
    "\n",
    "models = {\n",
    "    \"VGG19 - Batch Normalization\": vgg19_bn,\n",
    "    \"ResNet152\":                   resnet152\n",
    "}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "c-1XpDfjRZy6"
   },
   "source": [
    "reps = None\n",
    "def hook_fn(module, input, output):\n",
    "    global reps\n",
    "    reps = input[0]"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "k3-ntWBHWu8i"
   },
   "source": [
    "def get_RDM(model, images):\n",
    "\n",
    "  if   hasattr(model, 'classifier') : hook = model.classifier[-1].register_forward_hook(hook_fn)\n",
    "  elif hasattr(model, 'fc') :         hook = model.fc.register_forward_hook(hook_fn)\n",
    "  else: assert(False)\n",
    "\n",
    "  model.to(device)\n",
    "  model.eval()\n",
    "  with torch.no_grad():\n",
    "    _ = model(images)\n",
    "\n",
    "  similarities = []\n",
    "\n",
    "  for i in range(reps.size(0)):\n",
    "    similarities.append([])\n",
    "    for j in range(reps.size(0)):\n",
    "          similarities[len(similarities)-1].append( torch.nn.CosineSimilarity(dim=0)(reps[i],reps[j]).item() )\n",
    "\n",
    "  return similarities"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for key in models :\n",
    "  print(key)\n",
    "  similarities = get_RDM(models[key], images_torch)\n",
    "  ordered_labels = {\"animal\":[None]*28, \"plant\":[None]*14, \"food\":[None]*16,\"indoor\":[None]*22,\"outdoor\":[None]*20,\"human body\":[None]*24,\"human face\":[None]*32}\n",
    "  display_img_with_ordered_labels(similarities, ordered_labels, None, True, (8,8))\n",
    "  plt.show()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "37PNJNVHL2lN"
   },
   "source": [
    "##**REPRESENTATIONS CLIP**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "similarities = []\n",
    "features = torch.empty(0, 512).to(device)\n",
    "\n",
    "dataset_size = 156\n",
    "batch_size = 32\n",
    "\n",
    "with torch.no_grad():\n",
    "    features = model.encode_image(images_torch)\n",
    "\n",
    "for i in range(features.size(0)):\n",
    "    similarities.append([])\n",
    "    for j in range(features.size(0)):\n",
    "        similarities[len(similarities) - 1].append(torch.nn.CosineSimilarity(dim=0)(features[i], features[j]).item())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "ordered_labels = {\"animal\": [None] * 28, \"plant\": [None] * 14, \"food\": [None] * 16, \"indoor\": [None] * 22,\n",
    "                  \"outdoor\": [None] * 20, \"human body\": [None] * 24, \"human face\": [None] * 32}\n",
    "\n",
    "display_img_with_ordered_labels(similarities, ordered_labels, None, True, (8, 8))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "word = 'animal'\n",
    "images_words = get_stimulis(0, 156, preprocess, word=word).to(device)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "similarities = []\n",
    "features = torch.empty(0, 512).to(device)\n",
    "\n",
    "with torch.no_grad():\n",
    "    features = model.encode_image(images_words)\n",
    "\n",
    "    print(features.size())\n",
    "\n",
    "for i in range(features.size(0)):\n",
    "    similarities.append([])\n",
    "    for j in range(features.size(0)):\n",
    "        similarities[len(similarities) - 1].append(torch.nn.CosineSimilarity(dim=0)(features[i], features[j]).item())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "ordered_labels = {\"animal\": [None] * 28, \"plant\": [None] * 14, \"food\": [None] * 16, \"indoor\": [None] * 22,\n",
    "                  \"outdoor\": [None] * 20, \"human body\": [None] * 24, \"human face\": [None] * 32}\n",
    "display_img_with_ordered_labels(similarities, ordered_labels, None, True, (8, 8))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ]
}