playgroud_for_all_tasks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"7\"\n",
    "# print(os.environ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import torch\n",
    "from PIL import Image\n",
    "from transformers import AutoProcessor, AutoModelForCausalLM \n",
    "\n",
    "\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\"microsoft/Florence-2-large-ft\", torch_dtype=torch_dtype, trust_remote_code=True).to(device)\n",
    "processor = AutoProcessor.from_pretrained(\"microsoft/Florence-2-large-ft\", trust_remote_code=True)\n",
    "\n",
    "url = \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg?download=true\"\n",
    "image = Image.open(requests.get(url, stream=True).raw)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x480 at 0x1474CE050210>\n"
     ]
    }
   ],
   "source": [
    "print(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def run_example(task_prompt, text_input=None):\n",
    "    if text_input is None:\n",
    "        prompt = task_prompt\n",
    "    else:\n",
    "        prompt = task_prompt + text_input\n",
    "    inputs = processor(text=prompt, images=image, return_tensors=\"pt\").to(device, torch_dtype)\n",
    "    print(f\"input_ids.shape:{inputs['input_ids'].shape}\")\n",
    "    print(f\"pixel_values.shape:{inputs['pixel_values'].shape}\")\n",
    "    print(f'texts input to model forward:{processor.tokenizer.decode(inputs[\"input_ids\"][0], skip_special_tokens=False)}')\n",
    "    generated_ids = model.generate(\n",
    "      input_ids=inputs[\"input_ids\"],\n",
    "      pixel_values=inputs[\"pixel_values\"],\n",
    "      max_new_tokens=1024,\n",
    "      num_beams=3\n",
    "    )\n",
    "    generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]\n",
    "    print(f\"generated_text: {generated_text}\")\n",
    "    parsed_answer = processor.post_process_generation(generated_text, task=task_prompt, image_size=(image.width, image.height))\n",
    "    print(parsed_answer)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Caption"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids.shape:torch.Size([1, 8])\n",
      "pixel_values.shape:torch.Size([1, 3, 768, 768])\n",
      "texts input to model forward:<s>What does the image describe?</s>\n",
      "generated_text: </s><s><s><s>A green car parked in front of a yellow building.</s>\n",
      "{'<CAPTION>': 'A green car parked in front of a yellow building.'}\n"
     ]
    }
   ],
   "source": [
    "prompt = \"<CAPTION>\"\n",
    "run_example(prompt)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Detailed Caption"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids.shape:torch.Size([1, 13])\n",
      "pixel_values.shape:torch.Size([1, 3, 768, 768])\n",
      "texts input to model forward:<s>Describe in detail what is shown in the image.</s>\n",
      "generated_text: </s><s>In this image we can see a car on the road. In the background there is a building with doors. At the top there is sky.</s>\n",
      "{'<DETAILED_CAPTION>': 'In this image we can see a car on the road. In the background there is a building with doors. At the top there is sky.'}\n"
     ]
    }
   ],
   "source": [
    "prompt = \"<DETAILED_CAPTION>\"\n",
    "run_example(prompt)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### More Detailed Caption"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids.shape:torch.Size([1, 14])\n",
      "pixel_values.shape:torch.Size([1, 3, 768, 768])\n",
      "texts input to model forward:<s>Describe with a paragraph what is shown in the image.</s>\n",
      "generated_text: </s><s>A light blue car is parked in front of a building. There are two brown doors on the building. The car has large black tires on it. </s>\n",
      "{'<MORE_DETAILED_CAPTION>': 'A light blue car is parked in front of a building. There are two brown doors on the building. The car has large black tires on it. '}\n"
     ]
    }
   ],
   "source": [
    "prompt = \"<MORE_DETAILED_CAPTION>\"\n",
    "run_example(prompt)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Caption to Phrase Grounding\n",
    "caption to phrase grounding task requires additional text input, i.e. caption.\n",
    "\n",
    "Caption to phrase grounding results format: {'<CAPTION_TO_PHRASE_GROUNDING>': {'bboxes': [[x1, y1, x2, y2], ...], 'labels': ['', '', ...]}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids.shape:torch.Size([1, 21])\n",
      "pixel_values.shape:torch.Size([1, 3, 768, 768])\n",
      "texts input to model forward:<s>Locate the phrases in the caption: A green car parked in front of a yellow building.</s>\n",
      "generated_text: </s><s>A green car<loc_54><loc_331><loc_912><loc_780>a yellow building<loc_2><loc_9><loc_998><loc_635></s>\n",
      "{'<CAPTION_TO_PHRASE_GROUNDING>': {'bboxes': [[34.880001068115234, 159.1199951171875, 584.0, 374.6399841308594], [1.5999999046325684, 4.559999942779541, 639.0399780273438, 305.03997802734375]], 'labels': ['A green car', 'a yellow building']}}\n"
     ]
    }
   ],
   "source": [
    "task_prompt = \"<CAPTION_TO_PHRASE_GROUNDING>\"\n",
    "results = run_example(task_prompt, text_input=\"A green car parked in front of a yellow building.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "254.20800000000003\n"
     ]
    }
   ],
   "source": [
    "print(331/1000 * 768)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Object Detection\n",
    "OD results format: {'<OD>': {'bboxes': [[x1, y1, x2, y2], ...], 'labels': ['label1', 'label2', ...]} }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids.shape:torch.Size([1, 13])\n",
      "pixel_values.shape:torch.Size([1, 3, 768, 768])\n",
      "texts input to model forward:<s>Locate the objects with category name in the image.</s>\n",
      "generated_text: </s><s><s><s>car<loc_53><loc_333><loc_933><loc_774>door<loc_710><loc_202><loc_905><loc_544>wheel<loc_707><loc_576><loc_865><loc_772><loc_147><loc_584><loc_306><loc_773></s>\n",
      "{'<OD>': {'bboxes': [[34.23999786376953, 160.0800018310547, 597.4400024414062, 371.7599792480469], [454.7200012207031, 97.19999694824219, 579.5199584960938, 261.3599853515625], [452.79998779296875, 276.7200012207031, 553.9199829101562, 370.79998779296875], [94.4000015258789, 280.55999755859375, 196.1599884033203, 371.2799987792969]], 'labels': ['car', 'door', 'wheel', 'wheel']}}\n"
     ]
    }
   ],
   "source": [
    "prompt = \"<OD>\"\n",
    "run_example(prompt)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dense Region Caption\n",
    "Dense region caption results format: {'<DENSE_REGION_CAPTION>' : {'bboxes': [[x1, y1, x2, y2], ...], 'labels': ['label1', 'label2', ...]} }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids.shape:torch.Size([1, 14])\n",
      "pixel_values.shape:torch.Size([1, 3, 768, 768])\n",
      "texts input to model forward:<s>Locate the objects in the image, with their descriptions.</s>\n",
      "generated_text: </s><s><s>turquoise Volkswagen Beetle<loc_53><loc_333><loc_933><loc_774>wheel<loc_708><loc_576><loc_864><loc_772><loc_148><loc_584><loc_305><loc_773></s>\n",
      "{'<DENSE_REGION_CAPTION>': {'bboxes': [[34.23999786376953, 160.0800018310547, 597.4400024414062, 371.7599792480469], [453.44000244140625, 276.7200012207031, 553.2799682617188, 370.79998779296875], [95.04000091552734, 280.55999755859375, 195.51998901367188, 371.2799987792969]], 'labels': ['turquoise Volkswagen Beetle', 'wheel', 'wheel']}}\n"
     ]
    }
   ],
   "source": [
    "prompt = \"<DENSE_REGION_CAPTION>\"\n",
    "run_example(prompt)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Region proposal\n",
    "Dense region caption results format: {'<REGION_PROPOSAL>': {'bboxes': [[x1, y1, x2, y2], ...], 'labels': ['', '', ...]}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids.shape:torch.Size([1, 11])\n",
      "pixel_values.shape:torch.Size([1, 3, 768, 768])\n",
      "texts input to model forward:<s>Locate the region proposals in the image.</s>\n",
      "generated_text: </s><s><s><s><loc_53><loc_333><loc_933><loc_774><loc_710><loc_202><loc_905><loc_544><loc_147><loc_584><loc_306><loc_773><loc_708><loc_576><loc_864><loc_772><loc_425><loc_504><loc_472><loc_514></s>\n",
      "{'<REGION_PROPOSAL>': {'bboxes': [[34.23999786376953, 160.0800018310547, 597.4400024414062, 371.7599792480469], [454.7200012207031, 97.19999694824219, 579.5199584960938, 261.3599853515625], [94.4000015258789, 280.55999755859375, 196.1599884033203, 371.2799987792969], [453.44000244140625, 276.7200012207031, 553.2799682617188, 370.79998779296875], [272.32000732421875, 242.1599884033203, 302.3999938964844, 246.95999145507812]], 'labels': ['', '', '', '', '']}}\n"
     ]
    }
   ],
   "source": [
    "prompt = \"<REGION_PROPOSAL>\"\n",
    "run_example(prompt)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OCR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids.shape:torch.Size([1, 10])\n",
      "pixel_values.shape:torch.Size([1, 3, 768, 768])\n",
      "texts input to model forward:<s>What is the text in the image?</s>\n",
      "generated_text: </s><s><s>00</s>\n",
      "{'<OCR>': '00'}\n"
     ]
    }
   ],
   "source": [
    "prompt = \"<OCR>\"\n",
    "run_example(prompt)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OCR with Region\n",
    "OCR with region output format: {'<OCR_WITH_REGION>': {'quad_boxes': [[x1, y1, x2, y2, x3, y3, x4, y4], ...], 'labels': ['text1', ...]}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids.shape:torch.Size([1, 13])\n",
      "pixel_values.shape:torch.Size([1, 3, 768, 768])\n",
      "texts input to model forward:<s>What is the text in the image, with regions?</s>\n",
      "generated_text: </s><s><s><s>00<loc_363><loc_208><loc_507><loc_208><loc_507><loc_328><loc_363><loc_328></s>\n",
      "{'<OCR_WITH_REGION>': {'quad_boxes': [[232.63999938964844, 100.07999420166016, 324.79998779296875, 100.07999420166016, 324.79998779296875, 157.67999267578125, 232.63999938964844, 157.67999267578125]], 'labels': ['</s>00']}}\n"
     ]
    }
   ],
   "source": [
    "prompt = \"<OCR_WITH_REGION>\"\n",
    "run_example(prompt)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note:**  \n",
    "I print(image) and get blow:\n",
    "<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x480 at 0x1474CE050210>\n",
    "\n",
    "It's resolution is 640*480.\n",
    "\n",
    "For the generated token <loc_363>, the post-process will handle it as a bounding box with the resolution of 640*480.\n",
    "print(363 / 1000 * 640 == 232.32) # which is very close to 232.63999938964844\n",
    "print(208 / 1000 * 480 == 99.84) # which is very close to 100.07999420166016\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "florence2-finetuning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}