image_captioning.ipynb

{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-18T08:30:09.894694Z",
     "start_time": "2024-06-18T08:29:51.227670Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# lets create a image captioning model using transformer\n",
    "\n",
    "from transformers import VisionEncoderDecoderModel, ViTFeatureExtractor, AutoTokenizer\n",
    "from PIL import Image\n",
    "import torch\n"
   ],
   "id": "df61950c01f605cd",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-18T08:39:59.016284Z",
     "start_time": "2024-06-18T08:30:12.100368Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    "# Load pre-trained model\n",
    "model = VisionEncoderDecoderModel.from_pretrained(\"nlpconnect/vit-gpt2-image-captioning\") \n",
    "feature_extractor = ViTFeatureExtractor.from_pretrained(\"nlpconnect/vit-gpt2-image-captioning\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"nlpconnect/vit-gpt2-image-captioning\")\n",
    "\n"
   ],
   "id": "dd7bbde9ed26dd4a",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "config.json:   0%|          | 0.00/4.61k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "e170aeff9df540d5845a4c3dfdacd33f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\shivam\\PycharmProjects\\machine_learning_full_stack\\Deep_learning\\newenv\\Lib\\site-packages\\huggingface_hub\\file_download.py:157: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\shivam\\.cache\\huggingface\\hub\\models--nlpconnect--vit-gpt2-image-captioning. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "pytorch_model.bin:   0%|          | 0.00/982M [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "f1ab5706b65a4f9f8c732802391c5c11"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "preprocessor_config.json:   0%|          | 0.00/228 [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "d27e49cb3f6d4e6bb03aeef648186ae9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\shivam\\PycharmProjects\\machine_learning_full_stack\\Deep_learning\\newenv\\Lib\\site-packages\\transformers\\models\\vit\\feature_extraction_vit.py:28: FutureWarning: The class ViTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please use ViTImageProcessor instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/241 [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "1c55bd6bcc944333ac2ed9652da5cd40"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "vocab.json:   0%|          | 0.00/798k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "7a6484183fa54bd8a456abd52ecfcb0e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "02fb1733638946729ce1220b318dccc9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "fb0ecc5e2b7c427c9aaa7b476f131e41"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/120 [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "e5775adf36f243899fe42c301d5bca93"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-18T08:40:14.336579Z",
     "start_time": "2024-06-18T08:40:14.319828Z"
    }
   },
   "cell_type": "code",
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "def generate_caption(image_path):\n",
    "    image = Image.open(image_path).convert(\"RGB\")\n",
    "    pixel_values = feature_extractor(images=image, return_tensors=\"pt\").pixel_values.to(device)\n",
    "    \n",
    "    # Generate captions\n",
    "    output_ids = model.generate(pixel_values, max_length=16, num_beams=4)\n",
    "    caption = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
    "    return caption\n",
    "\n"
   ],
   "id": "7841251b64d1078a",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-18T08:40:32.974699Z",
     "start_time": "2024-06-18T08:40:28.931367Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# Generate caption for an image\n",
    "generate_caption('datasets/cat.18.jpg')"
   ],
   "id": "b11838d754ff3585",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a cat that is laying down on a red blanket '"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (newenv)",
   "language": "python",
   "name": "newenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}