20231219_japanese_stable_clip.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aicuai/GenAI-Steam/blob/main/20231219_japanese_stable_clip.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yxKn0STrLO3F"
      },
      "source": [
        "# Japanese Stable CLIP Demo by AICU\n",
        "This is a demo for text retrieval using [Japanese Stable CLIP](https://huggingface.co/stabilityai/japanese-stable-clip-vit-l-16) from [Stability AI](https://stability.ai/).\n",
        "\n",
        "- Blog: https://ja.stability.ai/blog/japanese-stable-clip\n",
        "- Twitter: https://twitter.com/StabilityAI_JP\n",
        "- Discord: https://discord.com/invite/StableJP\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K92bGOlkLNC5"
      },
      "outputs": [],
      "source": [
        "#@title Setup\n",
        "!nvidia-smi\n",
        "!pip install ftfy regex tqdm gradio transformers sentencepiece 'accelerate>=0.12.0' 'bitsandbytes>=0.31.5'\n",
        "# download samples image for demo\n",
        "!wget https://upload.wikimedia.org/wikipedia/commons/thumb/2/29/JAPANPOST-DSC00250.JPG/500px-JAPANPOST-DSC00250.JPG -O sample1.png\n",
        "!wget https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Search_and_rescue_at_Unosumai%2C_Kamaishi%2C_-17_Mar._2011_a.jpg/500px-Search_and_rescue_at_Unosumai%2C_Kamaishi%2C_-17_Mar._2011_a.jpg -O sample2.png\n",
        "!wget https://upload.wikimedia.org/wikipedia/commons/thumb/6/60/Policeman_at_Tokyo.jpg/500px-Policeman_at_Tokyo.jpg -O sample3.png"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uNNRyH3WLw84"
      },
      "outputs": [],
      "source": [
        "# @title Login HuggingFace\n",
        "# @markdown [Put HuggingFrace User Access Tokens here](https://huggingface.co/settings/tokens)\n",
        "# @markdown  StabilityAI 社の HuggingFace で [JSCLIP のライセンス](https://huggingface.co/stabilityai/japanese-stable-clip-vit-l-16) に合意するのを忘れずに。\n",
        "HUGGINGFACE_TOKEN = ''  # @param {type: \"string\"}\n",
        "# !huggingface-cli login\n",
        "!huggingface-cli login --token $HUGGINGFACE_TOKEN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mKC-h58pLyOu"
      },
      "outputs": [],
      "source": [
        "#@title Load Japanese Stable CLIP\n",
        "from typing import Union, List\n",
        "import ftfy, html, re, io\n",
        "import requests\n",
        "from PIL import Image\n",
        "import torch\n",
        "from transformers import AutoModel, AutoTokenizer, AutoImageProcessor, BatchFeature\n",
        "\n",
        "\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "model_path = \"stabilityai/japanese-stable-clip-vit-l-16\"\n",
        "model = AutoModel.from_pretrained(model_path, trust_remote_code=True).eval().to(device)\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
        "processor = AutoImageProcessor.from_pretrained(model_path)\n",
        "\n",
        "\n",
        "# taken from https://github.com/mlfoundations/open_clip/blob/main/src/open_clip/tokenizer.py#L65C8-L65C8\n",
        "def basic_clean(text):\n",
        "    text = ftfy.fix_text(text)\n",
        "    text = html.unescape(html.unescape(text))\n",
        "    return text.strip()\n",
        "\n",
        "\n",
        "def whitespace_clean(text):\n",
        "    text = re.sub(r\"\\s+\", \" \", text)\n",
        "    text = text.strip()\n",
        "    return text\n",
        "\n",
        "\n",
        "def tokenize(\n",
        "    texts: Union[str, List[str]],\n",
        "    max_seq_len: int = 77,\n",
        "):\n",
        "    \"\"\"\n",
        "    This is a function that have the original clip's code has.\n",
        "    https://github.com/openai/CLIP/blob/main/clip/clip.py#L195\n",
        "    \"\"\"\n",
        "    if isinstance(texts, str):\n",
        "        texts = [texts]\n",
        "    texts = [whitespace_clean(basic_clean(text)) for text in texts]\n",
        "\n",
        "    inputs = tokenizer(\n",
        "        texts,\n",
        "        max_length=max_seq_len - 1,\n",
        "        padding=\"max_length\",\n",
        "        truncation=True,\n",
        "        add_special_tokens=False,\n",
        "    )\n",
        "    # add bos token at first place\n",
        "    input_ids = [[tokenizer.bos_token_id] + ids for ids in inputs[\"input_ids\"]]\n",
        "    attention_mask = [[1] + am for am in inputs[\"attention_mask\"]]\n",
        "    position_ids = [list(range(0, len(input_ids[0])))] * len(texts)\n",
        "\n",
        "    return BatchFeature(\n",
        "        {\n",
        "            \"input_ids\": torch.tensor(input_ids, dtype=torch.long),\n",
        "            \"attention_mask\": torch.tensor(attention_mask, dtype=torch.long),\n",
        "            \"position_ids\": torch.tensor(position_ids, dtype=torch.long),\n",
        "        }\n",
        "    )\n",
        "\n",
        "\n",
        "def compute_text_embeddings(text):\n",
        "  if isinstance(text, str):\n",
        "    text = [text]\n",
        "  text = tokenize(texts=text)\n",
        "  text_features = model.get_text_features(**text.to(device))\n",
        "  text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)\n",
        "  del text\n",
        "  return text_features.cpu().detach()\n",
        "\n",
        "def compute_image_embeddings(image):\n",
        "  image = processor(images=image, return_tensors=\"pt\").to(device)\n",
        "  image_features = model.get_image_features(**image)\n",
        "  image_features = image_features / image_features.norm(p=2, dim=-1, keepdim=True)\n",
        "  del image\n",
        "  return image_features.cpu().detach()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u2biq4NwMPMR"
      },
      "outputs": [],
      "source": [
        "#@title Prepare for the demo\n",
        "#@markdown Please feel free to change `categories` for your usage.\n",
        "categories = [\n",
        "    \"配達員\",\"子供\",\"消防士\",\"救急隊員\",\"先生\",\n",
        "    \"スポーツ選手\", \"警察官\",\n",
        "    \"バイク\", \"タクシー\", \"自転車\",\n",
        "    \"味玉子\",\"チャーシュー\", \"海苔\", \"寿司\",\n",
        "    \"ポスター\",\"ロゴマーク\", \"写真\",\"アニメ\",\n",
        "    \"男の子\",\"女の子\",\n",
        "    \"日本語\", \"英語\",\"スライド\",\"チャート\",\n",
        "    \"ニュース\", \"事故\", \"YouTube\"\n",
        "]\n",
        "\n",
        "# pre-compute text embeddings\n",
        "text_embeds = compute_text_embeddings(categories)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MPX7dCw1MAof"
      },
      "outputs": [],
      "source": [
        "# @title Launch the demo\n",
        "import gradio as gr\n",
        "\n",
        "num_categories = len(categories)\n",
        "TOP_K = 5\n",
        "\n",
        "\n",
        "def inference_fn(img):\n",
        "  image_embeds = compute_image_embeddings(img)\n",
        "  similarity = (100.0 * image_embeds @ text_embeds.T).softmax(dim=-1)\n",
        "  similarity = similarity[0].numpy().tolist()\n",
        "  output_dict = {categories[i]: float(similarity[i]) for i in range(num_categories)}\n",
        "  del image_embeds\n",
        "  return output_dict\n",
        "\n",
        "\n",
        "with gr.Blocks() as demo:\n",
        "    gr.Markdown(\"# Japanese Stable CLIP Demo\")\n",
        "    gr.Markdown(\n",
        "        \"\"\"[Japanese Stable CLIP](https://huggingface.co/stabilityai/japanese-stable-clip-vit-l-16) is a [CLIP](https://arxiv.org/abs/2103.00020) model by [Stability AI](https://ja.stability.ai/).\n",
        "                - Blog: https://ja.stability.ai/blog/japanese-stable-clip\n",
        "                - Twitter: https://twitter.com/StabilityAI_JP\n",
        "                - Discord: https://discord.com/invite/StableJP\"\"\"\n",
        "    )\n",
        "    with gr.Row():\n",
        "      with gr.Column():\n",
        "        inp = gr.Image(type=\"pil\")\n",
        "      with gr.Column():\n",
        "        out = gr.Label(num_top_classes=TOP_K)\n",
        "\n",
        "    btn = gr.Button(\"Run\")\n",
        "    btn.click(fn=inference_fn, inputs=inp, outputs=out)\n",
        "    examples = gr.Examples(\n",
        "        examples=[\n",
        "            # https://ja.wikipedia.org/wiki/%E9%83%B5%E4%BE%BF\n",
        "            \"sample1.png\",\n",
        "            # https://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E6%B6%88%E9%98%B2\n",
        "            \"sample2.png\",\n",
        "            # https://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC%E3%81%AE%E8%AD%A6%E5%AF%9F%E5%AE%98\n",
        "            \"sample3.png\",\n",
        "        ],\n",
        "        inputs=inp\n",
        "    )\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    demo.launch(debug=True, share=True)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AwpqfcojI51R"
      },
      "source": [
        "### Credit\n",
        "\n",
        "* The mail carrier photograph was taken by Ryouta0411 and is provided under the Creative Commons Attribution-Share Alike 3.0 Unported License.\n",
        "  * Source: https://commons.wikimedia.org/wiki/File:JAPANPOST-DSC00250.JPG\n",
        "  * License: https://creativecommons.org/licenses/by-sa/3.0/\n",
        "* The firefighter photograph was taken by Master Sgt. Jeremy Lock and is in the public domain.\n",
        "  * Source: https://commons.wikimedia.org/wiki/File:Search_and_rescue_at_Unosumai,_Kamaishi,_-17_Mar._2011_a.jpg\n",
        "* The police officer photograph was taken by Spaz Tacular and is provided under the Creative Commons Attribution 2.0 Generic License.\n",
        "  * Source: https://commons.wikimedia.org/wiki/File:Policeman_at_Tokyo.jpg\n",
        "  * License: https://creativecommons.org/licenses/by/2.0/deed.en"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "V100",
      "private_outputs": true,
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}