specialized_form_parser.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L0J138mj7p1s"
   },
   "source": [
    "# Document AI Form Parser Visualizer\n",
    "This notebook shows you how to analyze a pdf using the Google Cloud DocumentAI API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_v0XtSwn7fmN",
    "outputId": "373b9379-f6ac-451a-e428-ad9219fb31d0"
   },
   "outputs": [],
   "source": [
    "# Install necessary Python libraries\n",
    "!pip install google-cloud-documentai\n",
    "!pip install wand\n",
    "!pip install pillo\n",
    "!pip install tabulate\n",
    "\n",
    "!apt-get update\n",
    "!apt-get install libmagickwand-dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Y8eO6Kcp7v2x"
   },
   "outputs": [],
   "source": [
    "from google.cloud import documentai_v1beta3 as documentai\n",
    "from wand.image import Image as WImage\n",
    "from PIL import Image, ImageDraw\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "from tabulate import tabulate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "q83GOFsUHlYA"
   },
   "source": [
    "## Download our sample pdf from GCS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0iBwYfBtHHBr"
   },
   "outputs": [],
   "source": [
    "PDF_URI = \"gs://cesummit_workshop_data/invoice.pdf\" #@param {type: \"string\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "LiEgCOrHIL7Q",
    "outputId": "edbe5adb-40cc-44bd-abd4-4dc9b615224f"
   },
   "outputs": [],
   "source": [
    "# Download the doc\n",
    "!gsutil cp $PDF_URI ./invoice.pdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set your processor variables "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "k3c1mTa6IOk3"
   },
   "outputs": [],
   "source": [
    "PROJECT_ID = \"YOUR_PROJECT_ID_HERE\"\n",
    "LOCATION = \"us\"  # Format is 'us' or 'eu'\n",
    "PROCESSOR_ID = \"PROCESSOR_ID\"  # Create processor in Cloud Console"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code calls the synchronous API and parses the form fields and values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hO3yJpDoJ3Zf"
   },
   "outputs": [],
   "source": [
    "def process_document_sample():\n",
    "    # Instantiates a client\n",
    "    client = documentai.DocumentProcessorServiceClient()\n",
    "\n",
    "    # The full resource name of the processor, e.g.:\n",
    "    # projects/project-id/locations/location/processor/processor-id\n",
    "    # You must create new processors in the Cloud Console first\n",
    "    name = f\"projects/{PROJECT_ID}/locations/{LOCATION}/processors/{PROCESSOR_ID}\"\n",
    "\n",
    "    with open('invoice.pdf', \"rb\") as image:\n",
    "        image_content = image.read()\n",
    "\n",
    "    # Read the file into memory\n",
    "    document = {\"content\": image_content, \"mime_type\": \"application/pdf\"}\n",
    "\n",
    "    # Configure the process request\n",
    "    request = {\"name\": name, \"document\": document}\n",
    "\n",
    "    # Recognizes text entities in the PDF document\n",
    "    result = client.process_document(request=request)\n",
    "    document = result.document\n",
    "    entities = document.entities\n",
    "    print(\"Document processing complete.\\n\\n\")\n",
    "    \n",
    "    if result.human_review_operation:\n",
    "        print (\"Triggered HITL operation: {}\".format(result.human_review_operation))\n",
    "\n",
    "    # For a full list of Document object attributes, please reference this page: https://googleapis.dev/python/documentai/latest/_modules/google/cloud/documentai_v1beta3/types/document.html#Document  \n",
    "\n",
    "    types = []\n",
    "    values = []\n",
    "    confidence = []\n",
    "    \n",
    "    # Grab each key/value pair and their corresponding confidence scores.\n",
    "    for entity in entities:\n",
    "        types.append(entity.type_)\n",
    "        values.append(entity.mention_text)\n",
    "        confidence.append(round(entity.confidence,4))\n",
    "        \n",
    "    # Create a Pandas Dataframe to print the values in tabular format. \n",
    "    df = pd.DataFrame({'Type': types, 'Value': values, 'Confidence': confidence}) \n",
    "    print(tabulate(df, headers='keys', tablefmt='psql'))\n",
    "\n",
    "    return document\n",
    "\n",
    "\n",
    "def get_text(doc_element: dict, document: dict):\n",
    "    \"\"\"\n",
    "    Document AI identifies form fields by their offsets\n",
    "    in document text. This function converts offsets\n",
    "    to text snippets.\n",
    "    \"\"\"\n",
    "    response = \"\"\n",
    "    # If a text segment spans several lines, it will\n",
    "    # be stored in different text segments.\n",
    "    for segment in doc_element.text_anchor.text_segments:\n",
    "        start_index = (\n",
    "            int(segment.start_index)\n",
    "            if segment in doc_element.text_anchor.text_segments\n",
    "            else 0\n",
    "        )\n",
    "        end_index = int(segment.end_index)\n",
    "        response += document.text[start_index:end_index]\n",
    "    return response\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = process_document_sample()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rjxTY06qnFnU"
   },
   "source": [
    "## Draw the bounding boxes\n",
    "We will now download the pdf above a jpg and use the spatial data to mark our values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HjfbWczcnE_V"
   },
   "outputs": [],
   "source": [
    "JPG_URI = \"gs://cesummit_workshop_data/invoice.jpg\" #@param {type: \"string\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download the doc\n",
    "!gsutil cp $JPG_URI ./invoice.jpg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NyHpYBN8_45g"
   },
   "outputs": [],
   "source": [
    "im = Image.open('invoice.jpg')\n",
    "draw = ImageDraw.Draw(im)\n",
    "for entity in doc.entities:\n",
    "    # Draw the bounding box around the entities\n",
    "    vertices = []\n",
    "    for vertex in entity.page_anchor.page_refs[0].bounding_poly.normalized_vertices:\n",
    "        vertices.append({'x': vertex.x * im.size[0], 'y': vertex.y * im.size[1]})\n",
    "    draw.polygon([\n",
    "        vertices[0]['x'], vertices[0]['y'],\n",
    "        vertices[1]['x'], vertices[1]['y'],\n",
    "        vertices[2]['x'], vertices[2]['y'],\n",
    "        vertices[3]['x'], vertices[3]['y']], outline='blue')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Kt6jzmisRefT"
   },
   "outputs": [],
   "source": [
    "im"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Human in the loop (HITL) Operation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Only complete this section if a HITL Operation is triggered**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lro_id = \"HITL Operation\" # ex. projects/660199673046/locations/us/operations/174674963333130330"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = documentai.DocumentProcessorServiceClient()\n",
    "operation = client._transport.operations_client.get_operation(lro_id)\n",
    "if operation.done:\n",
    "    print(\"HITL location: {} \".format(str(operation.response.value)[5:-1]))\n",
    "else:\n",
    "    print('Waiting on human review.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!gsutil cp \"HITL_LOCATION\" response.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"response.json\", \"r\") as file:\n",
    "    import json\n",
    "    entities = {}\n",
    "    data = json.load(file)\n",
    "    for entity in data['entities']:\n",
    "        if 'mentionText' in entity:\n",
    "            entities[entity['type']] = entity['mentionText']\n",
    "        else:\n",
    "            entities[entity['type']] = \"\"\n",
    "    \n",
    "    for t in entities:\n",
    "        print(\"{} : {}\\n \".format(t, entities[t]))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "LendingAI Bouding Boxes v3.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "environment": {
   "name": "tf2-2-3-gpu.2-3.m59",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-2-3-gpu.2-3:m59"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}