sysdemo/system_demonstration.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "0f9c14ea",
   "metadata": {},
   "source": [
    "\n",
    "## Description\n",
    "This notebook builds on the outputs created beforehand by running the demo.py script and additional OCR information.\n",
    "For convenience, we add the outputs of DSG and additional an OCR text file for one sample in this directory."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f48524cf",
   "metadata": {},
   "source": [
    "For use of relative paths, please make sure you are in the base directory (`DSG`) of this repository. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2b12061",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "cwd = os.path.basename(os.getcwd())\n",
    "print('current dir: {}'.format(cwd))\n",
    "if cwd != 'DSG' and cwd == 'sysdemo':\n",
    "    os.chdir('..')\n",
    "    print('changed dir to: {}'.format(os.getcwd()))\n",
    "#assert cwd == 'DSG'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72083d33",
   "metadata": {},
   "outputs": [],
   "source": [
    "from xmlrpc.client import MAXINT\n",
    "import torch\n",
    "import os\n",
    "from PIL import Image\n",
    "from pathlib import Path\n",
    "\n",
    "from segmentationsg.data import add_dataset_config, register_datasets\n",
    "from segmentationsg.modeling.roi_heads.scenegraph_head import add_scenegraph_config\n",
    "from detectron2.config import get_cfg\n",
    "\n",
    "import glob\n",
    "def setup_cfg(config_file, output_dir, confidence_threshold, config_list):\n",
    "    # load config from file and command-line arguments\n",
    "    cfg = get_cfg()\n",
    "    add_dataset_config(cfg)\n",
    "    add_scenegraph_config(cfg)\n",
    "    assert(cfg.MODEL.ROI_SCENEGRAPH_HEAD.MODE in ['predcls', 'sgls', 'sgdet']) , \"Mode {} not supported\".format(cfg.MODEL.ROI_SCENEGRaGraph.MODE)\n",
    "    cfg.merge_from_file(config_file)\n",
    "    #cfg.merge_from_list(args.opts)\n",
    "    cfg.merge_from_list(config_list)\n",
    "    # Set score_threshold for builtin models\n",
    "    cfg.MODEL.RETINANET.SCORE_THRESH_TEST = confidence_threshold\n",
    "    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = confidence_threshold\n",
    "    cfg.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH = confidence_threshold\n",
    "\n",
    "    register_datasets(cfg)\n",
    "    #default_setup(cfg, args)\n",
    "    cfg.freeze()\n",
    "    #print(cfg)\n",
    "    return cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab6c19b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure the datasets and checkpoints folders that you downloaded from the google drive have been extracted to the correct folder \n",
    "# dsg specific commands should be executed from the base folder\n",
    "\n",
    "#path to config file\n",
    "config_file = './configs/sgg_end2end_EP.yaml'\n",
    "#path to model checkpoint \n",
    "config_list = [\"MODEL.WEIGHTS\", './checkpoints/DSG_E2E_eperiodica/dsg_e2e_eperiodica_checkpoint.pth']\n",
    "confidence_threshold = 0.5\n",
    "output_dir = './sysdemo/'\n",
    "\n",
    "#important that this is executed from the base folder and that checkpoints and configs are in there\n",
    "cfg = setup_cfg(config_file, output_dir, confidence_threshold, config_list)\n",
    "\n",
    "from segmentationsg.utils.visualizer import SGVisualizer\n",
    "from detectron2.data import MetadataCatalog\n",
    "from detectron2.utils.visualizer import ColorMode\n",
    "from detectron2.data.detection_utils import read_image\n",
    "\n",
    "\n",
    "\n",
    "metadata = MetadataCatalog.get(\n",
    "    cfg.DATASETS.TEST[0] if len(cfg.DATASETS.TEST) else \"__unused\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b164d029",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define class mapping list that we use for postprocessing and hocr file creation\n",
    "class_mapping_list = metadata.thing_classes\n",
    "print(class_mapping_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9f7b404",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we run our example demonstration on the image chr-001_1974_013_0199 from the eperiodica dataset\n",
    "raw_tensor_path = os.path.join(output_dir, \"chr-001_1974_013_0199.pt\")\n",
    "ocr_file_path = os.path.join(output_dir, \"chr-001_1974_013_0199.txt\")\n",
    "image_path = os.path.join(output_dir, \"chr-001_1974_013_0199.jpg\")\n",
    "image_instances_path = os.path.join(output_dir, \"chr-001_1974_013_0199_instances.jpg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb3525c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(ocr_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32323bab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# image that we look at\n",
    "im = Image.open(image_path)\n",
    "im.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32e11bfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# detected instances\n",
    "im_instances = Image.open(image_instances_path)\n",
    "im_instances.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2b52231",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tensor that contains the predicted bounding boxes, the predicted classes/categories, and relationships + score\n",
    "# for each instance pair\n",
    "raw_tensor = torch.load(raw_tensor_path)\n",
    "#raw_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a389118e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# postprocessed the raw_tensor, output is a tensor with a tree structure for parentof relations\n",
    "from segmentationsg.utils import postprocessing\n",
    "tensor_before_postprocessing, postprocessed_tensor = postprocessing.postprocess_raw_tensor(raw_tensor, class_mapping_list)\n",
    "\n",
    "#postprocessed_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b923637",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create hocr file from postprocessed tensor\n",
    "from segmentationsg.utils.makehocr import create_hocr\n",
    "#output_folder = \"./\"\n",
    "filename_hocr = os.path.basename(ocr_file_path.replace(\".txt\", \".hocr\"))\n",
    "root_hocr = create_hocr(postprocessed_tensor, ocr_file_path, class_mapping_list, output_dir, filename_hocr)\n",
    "output_path_hocr = os.path.join(output_dir, filename_hocr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60b62736",
   "metadata": {},
   "outputs": [],
   "source": [
    "# some example queries using lxml\n",
    "from lxml import etree as ET\n",
    "root_hocr = ET.parse(output_path_hocr)\n",
    "root_hocr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d713cae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get rows that are children of tabular\n",
    "rows = root_hocr.xpath('//div[@dgg_class=\"tabular\"]/*/div[@dgg_class=\"row\"]')\n",
    "print(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0f838fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the node that contains the word \"Schriftsteller\"\n",
    "contains_schriftsteller = root_hocr.xpath('//span[text()=\"Schriftsteller\"]/..')\n",
    "\n",
    "#important to note is that .xpath returns a list of elements that match the path, even if there's only 1 node that matches it\n",
    "#this means that we have to access contains_schriftsteller[0]\n",
    "\n",
    "print(contains_schriftsteller)\n",
    "print()\n",
    "\n",
    "print(contains_schriftsteller[0].attrib)\n",
    "print()\n",
    "\n",
    "# print all words contained in the row that contains the word \"Schrifsteller\"\n",
    "for word in contains_schriftsteller[0].iterchildren():\n",
    "    print(word.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "005aef4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#find all headings\n",
    "headings = root_hocr.xpath('//div[@dgg_class=\"header\"]')\n",
    "\n",
    "#print the first 3 heading ids and their textual contents\n",
    "for heading in headings[:3]:\n",
    "    print(\"heading id: \"+heading.attrib[\"dgg_id\"])\n",
    "    for word in heading:\n",
    "        print(word.text)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd338ab5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#find the textblock that follows a heading that contains the word \"Biographie\"\n",
    "from segmentationsg.utils.makehocr import followedby\n",
    "textblock_after_biographie = followedby('//div[@dgg_class=\"header\"]/span[text()=\"Biographie\"]/..', '//div[@dgg_class=\"contentblock\"]', root_hocr)\n",
    "print(textblock_after_biographie[0].attrib)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3548c232",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print the first 15 words contained in that textblock\n",
    "for word in list(textblock_after_biographie[0].iterchildren())[:15]:\n",
    "    print(word.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cc264b3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}