demo_rsa.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "import torchvision.models as models\n",
    "import scipy.io\n",
    "import torch, clip\n",
    "from UTILS.utils import *"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "# initialization\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "path = ''\n",
    "model, preprocess = clip.load(\"ViT-B/32\", device=device, jit=False)\n",
    "s_156 = scipy.io.loadmat(path+'DATASETS/dataset3.mat')\n",
    "images = [s_156['visual_stimuli156'][0][i][0] for i in range(156)]\n",
    "images_torch = get_stimulis(0,156, images, preprocess=preprocess, word=None).to(device)\n",
    "\n",
    "model_name = \"clip\"\n",
    "contexts   = [\"a photo of a \"]\n",
    "labels  = torch.load(path+\"DATASETS/LABELS/labels.pt\")\n",
    "super_labels, basic_labels, hierarchy = labels[\"SUPERORDINATES\"], labels[\"BASICS\"], labels[\"CLUSTERING\"]\n",
    "context = contexts[0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##**REPRESENTATIONS FOR FEEDFORWARD MODELS**"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "vgg19_bn = models.vgg19_bn(pretrained=True)\n",
    "resnet152 = models.resnet18(pretrained=True)\n",
    "\n",
    "models = {\n",
    "    \"VGG19 - Batch Normalization\": vgg19_bn,\n",
    "    \"ResNet152\": resnet152\n",
    "}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "reps = None\n",
    "def hook_fn(module, input, output):\n",
    "    global reps\n",
    "    reps = input[0]\n",
    "\n",
    "\n",
    "images = torch.cat(\n",
    "    [preprocess(Image.fromarray(s_156['visual_stimuli156'][0][i][0])).unsqueeze(0) for i in range(156)]).to(device)\n",
    "\n",
    "\n",
    "def get_RDM(model, images):\n",
    "    if hasattr(model, 'classifier'):\n",
    "        hook = model.classifier[-1].register_forward_hook(hook_fn)\n",
    "    elif hasattr(model, 'fc'):\n",
    "        hook = model.fc.register_forward_hook(hook_fn)\n",
    "    else:\n",
    "        assert (False)\n",
    "\n",
    "    model.to(device)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        _ = model(images)\n",
    "\n",
    "    similarities = []\n",
    "\n",
    "    for i in range(reps.size(0)):\n",
    "        similarities.append([])\n",
    "        for j in range(reps.size(0)):\n",
    "            similarities[len(similarities) - 1].append(torch.nn.CosineSimilarity(dim=0)(reps[i], reps[j]).item())\n",
    "\n",
    "    return similarities"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for key in models:\n",
    "    print(key)\n",
    "    similarities = get_RDM(models[key], images_torch)\n",
    "    ordered_labels = {\"animal\": [None] * 28, \"plant\": [None] * 14, \"food\": [None] * 16, \"indoor\": [None] * 22,\n",
    "                      \"outdoor\": [None] * 20, \"human body\": [None] * 24, \"human face\": [None] * 32}\n",
    "    display_img_with_ordered_labels(similarities, ordered_labels, None, True, (8, 8))\n",
    "    plt.show()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##**REPRESENTATIONS CLIP**"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "similarities = []\n",
    "features = torch.empty(0, 512).to(device)\n",
    "\n",
    "dataset_size = 156\n",
    "batch_size = 32\n",
    "\n",
    "with torch.no_grad():\n",
    "    features = model.encode_image(images_torch)\n",
    "\n",
    "for i in range(features.size(0)):\n",
    "    similarities.append([])\n",
    "    for j in range(features.size(0)):\n",
    "        similarities[len(similarities) - 1].append(torch.nn.CosineSimilarity(dim=0)(features[i], features[j]).item())\n",
    "ordered_labels = {\"animal\": [None] * 28, \"plant\": [None] * 14, \"food\": [None] * 16, \"indoor\": [None] * 22,\n",
    "                  \"outdoor\": [None] * 20, \"human body\": [None] * 24, \"human face\": [None] * 32}\n",
    "\n",
    "display_img_with_ordered_labels(similarities, ordered_labels, None, True, (8, 8))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##**REPRESENTATION OF WORD-SUPERIMPOSED IMAGES ON CLIP**"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "word = 'animal'\n",
    "images_words = get_stimulis(0, 156, preprocess, word=word).to(device)\n",
    "similarities = []\n",
    "features = torch.empty(0, 512).to(device)\n",
    "\n",
    "\n",
    "with torch.no_grad():\n",
    "    features = model.encode_image(images_words)\n",
    "\n",
    "for i in range(features.size(0)):\n",
    "    similarities.append([])\n",
    "    for j in range(features.size(0)):\n",
    "        similarities[len(similarities) - 1].append(torch.nn.CosineSimilarity(dim=0)(features[i], features[j]).item())\n",
    "ordered_labels = {\"animal\": [None] * 28, \"plant\": [None] * 14, \"food\": [None] * 16, \"indoor\": [None] * 22,\n",
    "                  \"outdoor\": [None] * 20, \"human body\": [None] * 24, \"human face\": [None] * 32}\n",
    "display_img_with_ordered_labels(similarities, ordered_labels, None, True, (8, 8))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}