Pytorch dataset

notebooks/responsibleaidashboard/vision/responsibleaidashboard-standard-pytorch-model-dataset.ipynb

@@ -0,0 +1,172 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73d012de",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import torch\n",
+    "import torchvision\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import torch.nn as nn\n",
+    "import torch.nn.functional as F\n",
+    "import pickle\n",
+    "\n",
+    "from fastai.vision.all import *\n",
+    "\n",
+    "from PIL import Image\n",
+    "from torchvision.transforms import ToTensor\n",
+    "from responsibleai_vision.common.constants import ImageColumns\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "BATCH_SIZE = 4\n",
+    "DATASET_NAME = \"CIFAR\"\n",
+    "\n",
+    "dataset = torchvision.datasets.CIFAR10(root=\"data\", train=False, download=True, transform=ToTensor())\n",
+    "data_loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
+    "\n",
+    "def convert_torch_model(model, df, train=False):\n",
+    "    dls = ImageDataLoaders.from_df(df, path='./')\n",
+    "    learn = Learner(dls, model)\n",
+    "    \n",
+    "    if train:\n",
+    "        xb,yb = learn.dls.one_batch()\n",
+    "        init_loss = learn.loss_func(learn.model(xb), yb)\n",
+    "        learn.fit_one_cycle(10)\n",
+    "        xb,yb = learn.dls.one_batch()\n",
+    "        final_loss = learn.loss_func(learn.model(xb), yb)\n",
+    "        print(final_loss)\n",
+    "    \n",
+    "    return learn\n",
+    "    \n",
+    "        \n",
+    "def convert_torch_data(data_loader):\n",
+    "    \n",
+    "    data = pd.DataFrame(columns=[ImageColumns.IMAGE.value,\n",
+    "                                 ImageColumns.LABEL.value])\n",
+    "\n",
+    "    cnts = {}\n",
+    "    data_path = \"data/\" + DATASET_NAME + \"/\" \n",
+    "    \n",
+    "    for batch in tqdm(data_loader):\n",
+    "        \n",
+    "        images, labels = batch\n",
+    "        for i in range(BATCH_SIZE):\n",
+    "            img, label = images[i], labels[i]\n",
+    "            label = str(label.numpy())\n",
+    "            img = img.numpy()\n",
+    "            img = np.transpose(img, (1, 2, 0))\n",
+    "            img = Image.fromarray((img * 255).astype(np.uint8))\n",
+    "\n",
+    "            if label not in cnts:\n",
+    "                os.makedirs(data_path + label, exist_ok=True)\n",
+    "                cnts[label] = 1\n",
+    "            else:\n",
+    "                cnts[label] += 1\n",
+    "\n",
+    "            img_path = data_path + label + \"/class_\" + label + \"_img_\" + str(cnts[label]) + \".jpeg\"\n",
+    "            img.save(img_path)\n",
+    "            item = pd.DataFrame(data={ImageColumns.IMAGE.value: [img_path], \n",
+    "                                      ImageColumns.LABEL.value: [label]})\n",
+    "            data = pd.concat([data, item], ignore_index=True)\n",
+    "                \n",
+    "    return data\n",
+    "\n",
+    "data = convert_torch_data(data_loader)\n",
+    "\n",
+    "test_data = data\n",
+    "class_names = data[ImageColumns.LABEL.value].unique()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "87572a47",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "device = 'cuda'\n",
+    "\n",
+    "class CNN(nn.Module):\n",
+    "    def __init__(self):\n",
+    "        super().__init__()\n",
+    "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
+    "        self.pool = nn.MaxPool2d(2, 2)\n",
+    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
+    "        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
+    "        self.fc2 = nn.Linear(120, 84)\n",
+    "        self.fc3 = nn.Linear(84, 10)\n",
+    "\n",
+    "    def forward(self, x):\n",
+    "        x = self.pool(F.relu(self.conv1(x)))\n",
+    "        x = self.pool(F.relu(self.conv2(x)))\n",
+    "        x = torch.flatten(x, 1)\n",
+    "        x = F.relu(self.fc1(x))\n",
+    "        x = F.relu(self.fc2(x))\n",
+    "        x = self.fc3(x)\n",
+    "        return x\n",
+    "\n",
+    "model = CNN().to(device)\n",
+    "\n",
+    "model = convert_torch_model(model, data, train=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5b9184ef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from raiwidgets import ResponsibleAIDashboard\n",
+    "from responsibleai_vision import ModelTask, RAIVisionInsights\n",
+    "\n",
+    "rai_insights = RAIVisionInsights(model, data.sample(10, random_state=42),\n",
+    "                                \"label\", task_type=ModelTask.IMAGE_CLASSIFICATION,\n",
+    "                                classes=class_names)\n",
+    "rai_insights.explainer.add()\n",
+    "rai_insights.error_analysis.add()\n",
+    "rai_insights.compute()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4cb92832",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "ResponsibleAIDashboard(rai_insights)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}