Prescription_and_General_Bot.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "85d2862d934c4ca4a10672513cb86d45",
            "59c69ddac60e4fcfbceb640ce2377a68",
            "acd8304bc5a745ff8ebb5e436e355670",
            "05cc9a66692547e48f513de08539e6a7",
            "23a2f3574b1f47ba96068fe9d61e177e",
            "823c19cb73e34b5ca596c26f6106afe3",
            "12f5fe208b6242bb9e38d86aaf7f7cec",
            "e98b261e3bb74850bd4d24e136d92783",
            "9de4cdf96a55419b9a27b01d640895f9",
            "9315fd65abde4530be3d0f677ec43159",
            "8c7054125b3a4e0b963546fc630291f5",
            "89fe0f8cd13540acb3a0a90ac913878e",
            "3a8867f586ca44ee957d73d857999d40",
            "0ba300db1d924bb683b7f1cebb6c470d",
            "b6283b2b1c514035b75bb7328b19b87e",
            "061fe21d4f714139be4905d94c14dc81",
            "d88ddea670eb41f8a7831601729743c6",
            "d259bebac005487091b623f0f5df2a3d",
            "5f65220a3b78415f89ff7c46d215f235",
            "8cdff910c63847fb8b40361093829091",
            "2f581a9705fa4d1b90176c8ea7ff1bc2",
            "a5189e46e2a34b4ab2035e8670f69325",
            "37787b66f2dc4e3383ee30e7480289c9",
            "445c1bbac6f5424bbb32291b3be20eae",
            "d57c90ef33e6492eb0fd9abcb8bccf19",
            "57e195cc2b7e4e88bed0d12f31aa55f3",
            "c27986a647b5462aaea2a92822ff2205",
            "84f32e4ffbc040019f3f7d47da8ae224",
            "7e7063f50040466ebb9b8494c8379c38",
            "67cf22565ea044338f92d5d63b1f5267",
            "667a0c07dd6b48718d445c7ba3b60115",
            "17b1e25630a74d6fb16c7dc478a9da25",
            "ea8e64a8ac584553a59b9407b60ee07f",
            "bfef08efb7204347b29788577d4c0fcc",
            "fe7ba2d1fa3b402997b6a8adf5a48936",
            "0f211afe148a40eeb4b21d7f74b29a82",
            "ec13c457a44b4e6786cb777550a20958",
            "15b0dcfc1699499596096c1058811d9d",
            "a3daeec5432a40d9a5c55dfbf24debe8",
            "9aae6bdab6bb4143a8b3ec701b8d0c52",
            "ff6dc7d1d14a4b4eb7b3beeca4000738",
            "3df063dfff8c42d6bc51d28557ab84eb",
            "7bfe2c2ad364447498d99c3f9cd117a4",
            "57bf1ee116f44ffdb852d46b50f44640",
            "e386bbdc462a445bb25f8ac118854cdc",
            "c2ed85d250f8473ba145257589b636a3",
            "ee73d0bb1d184daab8fa197fb1b72866",
            "17d2178fc6494e948f7b4354a15f3cc8",
            "e362c6f433914f4b9d61fa260bf7b60d",
            "ea2969f1613b4641b55156ded9f2381a",
            "47fa30841e774c64a37ed4431e4dc20e",
            "57c806d865004b96a222ed3eae168c69",
            "7da54c24d8564c3494048ea0ab5386f9",
            "a695020260fd46ba802453a2ef706b5b",
            "560a4df79c8f4763bea6541474f6725e",
            "2179c66f5d5744a28bf18a184fe74936",
            "e0699fe494d34dc18e9c6c871e7f519d",
            "181be03c7ed0428c9482602e58543385",
            "93622959adbc41c2805d5c108a99e4eb",
            "b54a2590b7eb44e5addeb737541b032f",
            "c7e96da7e96a461f9b3814df91536717",
            "642a2d5b724044618cac75daff400c87",
            "859ee39b9f574d8db41882d4534fb158",
            "f627159231d141dd9379c8abbf4471cb",
            "79b77d8d2da54daba360e1bbd06f0c82",
            "20ac0c1a53b540f8ade266cc82386425",
            "dcf0b470822d46fcbd1ce5fe9a2d13a3",
            "4e0c0ca3a20b48e5a23ed829522b9752",
            "36fe3e96a45548048711b0fae2afb363",
            "2749733b53404007be28916e99fe4e2a",
            "d189f8568d7e4749b32d7842287cf48f",
            "1c11643aab754909add06f42e201e475",
            "c878313ed8ef4886960922916f25866f",
            "177320fe7b2342058c1474ddb9f685c4",
            "2aa3c4c1a5324bf9b66a9277866360d1",
            "26375c3d759746efa5dc0b802f7ce594",
            "a650532c9d0b48b49e732e019373cbc7",
            "9018a6e4bca443358c34d6a68c62eb46",
            "7145c85fde844583b4e84b9f33268fe8",
            "5e9e559bd1e545d78412f178c2c07833",
            "0393773748bf45a589934f3fbb087eeb",
            "5cb93db7fcb7419a98bb925b70b5fdce",
            "ef1ba35bd3524bb5806338b55da815ec",
            "5fef7393780f467291e848fc9c31e7a6",
            "94289113e02d4f2e9b02985ccd9c2d58",
            "93a8b124d686447096ce29b386cf17ab",
            "c23cc084b4974b4d95aed5b3d1967336",
            "9fbdd195beae4fbc87d3efc8a020f329",
            "a28d65565cc14265b10c21889368460f",
            "012da84e9b1d41e09e8eba9edd515024",
            "58402511865846bc84ed3cbaaf76b341",
            "7ddfb1e5f1f64f4196c17ffefe53f4ae",
            "a54f0f0d155e4073b190f99d5033b2f9",
            "df7be846ed95442f8e9a616514634dfb",
            "a7392f6d70094c78b3ad34ac3c8227db",
            "017bff2e323b43eebb6727f1df52b08c",
            "a9c2c073cf77458ba1c169cbac55cd4e",
            "9935495cd3094edf87219339a26353cf",
            "ff94dd18f92648e5aa25699770912cb1",
            "4f2584984b194bc0a7504cb5380aef29",
            "79ab780f4f404926bf1bf24da7b8ff34",
            "91a98318ab6c45069c6356d624bcd17e",
            "11639beea9314d70a2f56b2a503eb988",
            "db0b3f07435644f78a9159720743a718",
            "54bc3bc969a84e0eb2822be8951e7fbe",
            "8449500bcb2c493da6694a394d3bcee4",
            "d995f6f2b1f540c6a782d5ad0db23881",
            "a1b4ae1834b641aab07d167c1fd28c0f",
            "a760ccdb09434127868a4774b9486bfc",
            "8808916794f847faa290be5a78523a3d",
            "32e58c5ea35545edbe9f30d42c50e7fd",
            "9f079af8096e4a568586c1b007c54337",
            "3793628d73d346bf8c829a515afbb911",
            "4f3dda4ed0a84c70bcf75c52abd8652d",
            "9c11c4131ad44d3bb40726d19092dc31",
            "975d4b8cef28413b9d80411169ff081c",
            "69b592cb715e405b8d26cf0a77300a5b",
            "0113600f1d024152b96e815bf71942e7",
            "08bb7913c3da4541bce2d1a913f27cf5",
            "da85d13dd24e40f781d4c27e7bc15d6e",
            "0c0ec088cae4421dbebd4d719ae0ec90",
            "a17dd5723b61473fbedb1620b06ddc12",
            "895f9e0ec9894d1db9a90f2a454a2003",
            "8a05097402444de5bb812ecd9e613949",
            "8abb697216ce4312aac88f0be06514b4",
            "018f73a6e0644865a30d6cc15d02d7df",
            "e980c88e53984e9186176fa98c93f8b8",
            "ad2756e6369349b6bd9b2639639fbdcf",
            "0fc86240b80b4a5aac4398a159830008",
            "a380ce6d04094d858959f0c7993ce11d",
            "778b45bde791471ab993b46fed90c7b9",
            "6fbe261f77b44f00ac6054cec9d5916b",
            "216e73966fd849a9a47c16c51e7f04c4",
            "0a049e1c9ac44372b17812241a0482eb",
            "788d9f71484e49e4afe494853a5c9561",
            "2df8412546194b8394ec12d03d9db22f",
            "6aa7de8d8e574f7fa060b0a2d861e6aa",
            "e57640e5eb3e4079a863b6fca23bbd8b",
            "f07baaf43b4647aebd1dd25bfd5d4d71",
            "160794f2059b46c8808abc921b6e22a0",
            "309bab93f5c744c781ae2e8a529f82c4",
            "d3b9623498e64f1da001f93afd4e966a",
            "16b82f5a099244fab5809ed4bec57b53",
            "4b4b2a42eeed4d7bacd6e6c82b8c914d",
            "93ac0176bf774a2a8d90a09a2a4eded2",
            "cc9010c3064c45b38a6da879d8929d1b",
            "b712b8ddef32471394b1bc4de155eaf6",
            "43de8e1c3dc740da87be6b68f660ad64",
            "8245c3a1aa664736b0ee35236ab3ffa2",
            "99b05ef79e784a7da3627db84c5fec83",
            "445595ccfe9046078b621500e2c50861",
            "421acc24eb244914bac80c87002585aa",
            "e5879192d77349778d1e863ff15312cb",
            "133adfd4df294403bebbc3c1339c18fe",
            "c52bf56dd4bc493396b0f9dd6c6689eb",
            "b89b513b9e2a45e8bb653807d6ecbeeb",
            "e7cfac74270846f48d9d84c0d18d7c70",
            "7556bb72aa254f1587b35b747f0c0bd7",
            "b12938ebf98047fdae1b29c45d5e22cb",
            "5df5ff41e6ff4a9fa3b070443a406529",
            "c2bd6e084bfd4740b04941cadee866e1",
            "b4f0ec20cf07486a9f1bf4afe45a7481",
            "98ce2829a2144a2a8f0e438187baf946",
            "5a3aa5794b814c898fb677bcb61471fc",
            "4d63067fdc6e417ba03688f9e2369347"
          ]
        },
        "id": "YFC1FRrCFOue",
        "outputId": "218cba5d-9592-4b42-d00d-c3a18119fcc7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "preprocessor_config.json:   0%|          | 0.00/754 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "85d2862d934c4ca4a10672513cb86d45"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/1.85k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "89fe0f8cd13540acb3a0a90ac913878e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.model:   0%|          | 0.00/493k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "37787b66f2dc4e3383ee30e7480289c9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.80M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bfef08efb7204347b29788577d4c0fcc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "added_tokens.json:   0%|          | 0.00/41.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e386bbdc462a445bb25f8ac118854cdc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/552 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2179c66f5d5744a28bf18a184fe74936"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/1.28k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "dcf0b470822d46fcbd1ce5fe9a2d13a3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors.index.json:   0%|          | 0.00/70.2k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9018a6e4bca443358c34d6a68c62eb46"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading shards:   0%|          | 0/4 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a28d65565cc14265b10c21889368460f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00001-of-00004.safetensors:   0%|          | 0.00/4.92G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4f2584984b194bc0a7504cb5380aef29"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00002-of-00004.safetensors:   0%|          | 0.00/4.92G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "32e58c5ea35545edbe9f30d42c50e7fd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00003-of-00004.safetensors:   0%|          | 0.00/4.92G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a17dd5723b61473fbedb1620b06ddc12"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00004-of-00004.safetensors:   0%|          | 0.00/380M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "216e73966fd849a9a47c16c51e7f04c4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4b4b2a42eeed4d7bacd6e6c82b8c914d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/116 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c52bf56dd4bc493396b0f9dd6c6689eb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LlavaNextForConditionalGeneration(\n",
              "  (vision_tower): CLIPVisionModel(\n",
              "    (vision_model): CLIPVisionTransformer(\n",
              "      (embeddings): CLIPVisionEmbeddings(\n",
              "        (patch_embedding): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14), bias=False)\n",
              "        (position_embedding): Embedding(577, 1024)\n",
              "      )\n",
              "      (pre_layrnorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "      (encoder): CLIPEncoder(\n",
              "        (layers): ModuleList(\n",
              "          (0-23): 24 x CLIPEncoderLayer(\n",
              "            (self_attn): CLIPAttention(\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (mlp): CLIPMLP(\n",
              "              (activation_fn): QuickGELUActivation()\n",
              "              (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "              (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            )\n",
              "            (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (post_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "    )\n",
              "  )\n",
              "  (multi_modal_projector): LlavaNextMultiModalProjector(\n",
              "    (linear_1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "    (act): GELUActivation()\n",
              "    (linear_2): Linear(in_features=4096, out_features=4096, bias=True)\n",
              "  )\n",
              "  (language_model): MistralForCausalLM(\n",
              "    (model): MistralModel(\n",
              "      (embed_tokens): Embedding(32064, 4096)\n",
              "      (layers): ModuleList(\n",
              "        (0-31): 32 x MistralDecoderLayer(\n",
              "          (self_attn): MistralSdpaAttention(\n",
              "            (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
              "            (k_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
              "            (v_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
              "            (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
              "            (rotary_emb): MistralRotaryEmbedding()\n",
              "          )\n",
              "          (mlp): MistralMLP(\n",
              "            (gate_proj): Linear(in_features=4096, out_features=14336, bias=False)\n",
              "            (up_proj): Linear(in_features=4096, out_features=14336, bias=False)\n",
              "            (down_proj): Linear(in_features=14336, out_features=4096, bias=False)\n",
              "            (act_fn): SiLU()\n",
              "          )\n",
              "          (input_layernorm): MistralRMSNorm()\n",
              "          (post_attention_layernorm): MistralRMSNorm()\n",
              "        )\n",
              "      )\n",
              "      (norm): MistralRMSNorm()\n",
              "    )\n",
              "    (lm_head): Linear(in_features=4096, out_features=32064, bias=False)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ],
      "source": [
        "from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration\n",
        "import torch\n",
        "from PIL import Image\n",
        "import requests\n",
        "\n",
        "processor = LlavaNextProcessor.from_pretrained(\"llava-hf/llava-v1.6-mistral-7b-hf\")\n",
        "\n",
        "model = LlavaNextForConditionalGeneration.from_pretrained(\"llava-hf/llava-v1.6-mistral-7b-hf\", torch_dtype=torch.float16)\n",
        "model.to(\"cuda:0\")\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# prepare image and text prompt, using the appropriate prompt template\n",
        "url = \"/content/ptest2.jpeg\"\n",
        "image = Image.open(url)\n",
        "prompt = \"[INST] <image>\\nWhat is shown in this image? [/INST]\"\n",
        "\n",
        "inputs = processor(prompt, image, return_tensors=\"pt\").to(\"cuda:0\")\n",
        "\n",
        "# autoregressively complete prompt\n",
        "output = model.generate(**inputs, max_new_tokens=1000)\n",
        "\n",
        "print(processor.decode(output[0], skip_special_tokens=True))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z14JVqpkFkYZ",
        "outputId": "f73764e6-25a7-4b26-a087-1054bb80bd62"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[INST]  \n",
            "What is shown in this image? [/INST] The image shows a medical prescription form. It includes the following information:\n",
            "\n",
            "- The name of the patient, John Smith, with an address at 162 Example St., New York, NY 10012.\n",
            "- The patient's age is 34.\n",
            "- The date of the prescription is 09/11/2012.\n",
            "- The prescription is for Rx, which stands for a prescription medication.\n",
            "- The medication prescribed is Betaloc 100mg - 1 tab BID (twice a day).\n",
            "- The medication is also prescribed as Dorzolamide 10mg - 1 tab BID.\n",
            "- The medication is prescribed as Cimetidine 50mg - 2 tabs TID (three times a day).\n",
            "- The medication is prescribed as Oxprenol 50mg - 1 tab QD (once a day).\n",
            "- The signature of the prescriber, Dr. Steve Johnson, is at the bottom of the form.\n",
            "- The form also includes a label with the patient's name, address, and date of birth, as well as the prescriber's name and the date of the prescription.\n",
            "\n",
            "The form is a standard medical prescription that is typically used by pharmacies to fill the medication. \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# prepare image and text prompt, using the appropriate prompt template\n",
        "url = \"/content/ptest.jpeg\"\n",
        "image = Image.open(url)\n",
        "prompt = \"[INST] <image>\\nWhat is shown in this image? [/INST]\"\n",
        "\n",
        "inputs = processor(prompt, image, return_tensors=\"pt\").to(\"cuda:0\")\n",
        "\n",
        "# autoregressively complete prompt\n",
        "output = model.generate(**inputs, max_new_tokens=1000)\n",
        "\n",
        "print(processor.decode(output[0], skip_special_tokens=True))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6r2YZ3u9GWo-",
        "outputId": "10290c58-fc9c-4f10-e13d-61a104a3193f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[INST]  \n",
            "What is shown in this image? [/INST] The image shows a prescription form with the following information filled out:\n",
            "\n",
            "- The patient's name is Brad Smith.\n",
            "- The address is 123 Broad Street, Libertyville, Maryland.\n",
            "- The date is January 3, 2016.\n",
            "- The prescription is for Amoxicillin 250 mg/ml, which is a common antibiotic.\n",
            "- The dosage is 100 ml (or 100 mL), which is equivalent to 100 mg.\n",
            "- The quantity is 1, which means one bottle or package of the medication.\n",
            "- The instructions indicate that the medication should be taken \"Until gone,\" which means the patient should take it until the entire prescription is used.\n",
            "- The label is \"Yes,\" which means the prescription includes a label.\n",
            "- The generic name is \"Amoxicillin,\" and the brand name is not specified.\n",
            "- The prescription is signed by \"M Brown, M.D.,\" which indicates the prescriber's name and title.\n",
            "- The DEA number is CBI24563, which is the Drug Enforcement Administration registration number for the prescriber.\n",
            "- The state license number is 58432, which is the prescriber's state license number.\n",
            "\n",
            "The form is a standard prescription form used by healthcare providers to prescribe medication for their patients. \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# prepare image and text prompt, using the appropriate prompt template\n",
        "url = \"/content/ptest3.png\"\n",
        "image = Image.open(url)\n",
        "prompt = \"[INST] <image>\\nWhat is shown in this image? [/INST]\"\n",
        "\n",
        "inputs = processor(prompt, image, return_tensors=\"pt\").to(\"cuda:0\")\n",
        "\n",
        "# autoregressively complete prompt\n",
        "output = model.generate(**inputs, max_new_tokens=1000)\n",
        "\n",
        "print(processor.decode(output[0], skip_special_tokens=True))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7f8dhwGkGkO1",
        "outputId": "45c04b4e-2a2c-42f1-e434-14d2db3986ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[INST]  \n",
            "What is shown in this image? [/INST] The image shows a DOD Prescription form, specifically form 1289. The form is filled out with the following information:\n",
            "\n",
            "- The patient's name is John R. Doe, HM3, USN.\n",
            "- The medical facility is U.S. Neverforgotten (DD 178).\n",
            "- The date of the prescription is 23 Jan 99.\n",
            "- The prescription is for a medication called \"Tr Belladonna Amphogel gsd.\"\n",
            "- The dosage is 15 mg or ml.\n",
            "- The medication is a (Subscription) and is for the treatment of \"M&F Solu.\"\n",
            "- The signature of the prescriber is Jack R. Frost, LCDR, MD, USNR.\n",
            "- The form is filled by KMT.\n",
            "- The expiration date of the prescription is 1/10/02.\n",
            "- The form is filled by KMT.\n",
            "- The form is signed by Jack R. Frost, LCDR, MD, USNR.\n",
            "- The form is numbered 10072.\n",
            "\n",
            "The form also includes a section for the patient's full name, address, and phone number, but these fields are not filled out. \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" pip install llama-cpp-python"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1KdrJpVgG9g3",
        "outputId": "542cae6e-a8d6-45a5-fb8a-facc6fa048cc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting llama-cpp-python\n",
            "  Downloading llama_cpp_python-0.2.69.tar.gz (42.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.5/42.5 MB\u001b[0m \u001b[31m30.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (4.11.0)\n",
            "Requirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (1.25.2)\n",
            "Collecting diskcache>=5.6.1 (from llama-cpp-python)\n",
            "  Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: jinja2>=2.11.3 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (3.1.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.3->llama-cpp-python) (2.1.5)\n",
            "Building wheels for collected packages: llama-cpp-python\n",
            "  Building wheel for llama-cpp-python (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for llama-cpp-python: filename=llama_cpp_python-0.2.69-cp310-cp310-linux_x86_64.whl size=55261716 sha256=965521554a34ced446d6b7e1a720d042f361ee82ad11e7c312f3fdbccf57910d\n",
            "  Stored in directory: /root/.cache/pip/wheels/e4/18/46/58b5c613b17c8d000d79ae650980fe871b3b490e04e6faa1c1\n",
            "Successfully built llama-cpp-python\n",
            "Installing collected packages: diskcache, llama-cpp-python\n",
            "Successfully installed diskcache-5.6.3 llama-cpp-python-0.2.69\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import hf_hub_download\n",
        "from llama_cpp import Llama\n",
        "\n",
        "model_name = \"aaditya/OpenBioLLM-Llama3-8B-GGUF\"\n",
        "model_file = \"openbiollm-llama3-8b.Q5_K_M.gguf\"\n",
        "\n",
        "model_path = hf_hub_download(model_name,\n",
        "                             filename=model_file,\n",
        "                             local_dir='/content')\n",
        "print(\"My model path: \", model_path)\n",
        "llm = Llama(model_path=model_path.lstrip(),\n",
        "            n_gpu_layers=-1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "5def396ad5fc48b29c47c7683b09b681",
            "8ecb1278e14d48719f3c3f03b208de22",
            "a8801b56129646e7a98f0c106d3983c2",
            "01b08a7623434e13b6c0a22087901f35",
            "4e1af457726c43aaa7f97d7114ad2e14",
            "94af32d830b64e0fad08b612f8d1c5df",
            "be8ef01e797c461cbd5089086338ea68",
            "7fde8a6797334a10a9a0a05acadb50c2",
            "d84af2e044f74104921251d9b9dacbc6",
            "571d5d4105354ae38f213d32967a6ef5",
            "d33f034242ce4509b355288eed8e90a7"
          ]
        },
        "id": "UxkHdQQlIuTg",
        "outputId": "7580711f-98b1-414c-aaee-9d8f1ee7ad4c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "openbiollm-llama3-8b.Q5_K_M.gguf:   0%|          | 0.00/5.73G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5def396ad5fc48b29c47c7683b09b681"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "llama_model_loader: loaded meta data with 22 key-value pairs and 291 tensors from /content/openbiollm-llama3-8b.Q5_K_M.gguf (version GGUF V3 (latest))\n",
            "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
            "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
            "llama_model_loader: - kv   1:                               general.name str              = .\n",
            "llama_model_loader: - kv   2:                           llama.vocab_size u32              = 128256\n",
            "llama_model_loader: - kv   3:                       llama.context_length u32              = 8192\n",
            "llama_model_loader: - kv   4:                     llama.embedding_length u32              = 4096\n",
            "llama_model_loader: - kv   5:                          llama.block_count u32              = 32\n",
            "llama_model_loader: - kv   6:                  llama.feed_forward_length u32              = 14336\n",
            "llama_model_loader: - kv   7:                 llama.rope.dimension_count u32              = 128\n",
            "llama_model_loader: - kv   8:                 llama.attention.head_count u32              = 32\n",
            "llama_model_loader: - kv   9:              llama.attention.head_count_kv u32              = 8\n",
            "llama_model_loader: - kv  10:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
            "llama_model_loader: - kv  11:                       llama.rope.freq_base f32              = 500000.000000\n",
            "llama_model_loader: - kv  12:                          general.file_type u32              = 17\n",
            "llama_model_loader: - kv  13:                       tokenizer.ggml.model str              = gpt2\n",
            "llama_model_loader: - kv  14:                      tokenizer.ggml.tokens arr[str,128256]  = [\"!\", \"\\\"\", \"#\", \"$\", \"%\", \"&\", \"'\", ...\n",
            "llama_model_loader: - kv  15:                      tokenizer.ggml.scores arr[f32,128256]  = [0.000000, 0.000000, 0.000000, 0.0000...\n",
            "llama_model_loader: - kv  16:                  tokenizer.ggml.token_type arr[i32,128256]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
            "llama_model_loader: - kv  17:                      tokenizer.ggml.merges arr[str,280147]  = [\"Ġ Ġ\", \"Ġ ĠĠĠ\", \"ĠĠ ĠĠ\", \"...\n",
            "llama_model_loader: - kv  18:                tokenizer.ggml.bos_token_id u32              = 128000\n",
            "llama_model_loader: - kv  19:                tokenizer.ggml.eos_token_id u32              = 128001\n",
            "llama_model_loader: - kv  20:            tokenizer.ggml.padding_token_id u32              = 128001\n",
            "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
            "llama_model_loader: - type  f32:   65 tensors\n",
            "llama_model_loader: - type q5_K:  193 tensors\n",
            "llama_model_loader: - type q6_K:   33 tensors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "My model path:  /content/openbiollm-llama3-8b.Q5_K_M.gguf\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "llm_load_vocab: missing pre-tokenizer type, using: 'default'\n",
            "llm_load_vocab:                                             \n",
            "llm_load_vocab: ************************************        \n",
            "llm_load_vocab: GENERATION QUALITY WILL BE DEGRADED!        \n",
            "llm_load_vocab: CONSIDER REGENERATING THE MODEL             \n",
            "llm_load_vocab: ************************************        \n",
            "llm_load_vocab:                                             \n",
            "llm_load_vocab: special tokens definition check successful ( 256/128256 ).\n",
            "llm_load_print_meta: format           = GGUF V3 (latest)\n",
            "llm_load_print_meta: arch             = llama\n",
            "llm_load_print_meta: vocab type       = BPE\n",
            "llm_load_print_meta: n_vocab          = 128256\n",
            "llm_load_print_meta: n_merges         = 280147\n",
            "llm_load_print_meta: n_ctx_train      = 8192\n",
            "llm_load_print_meta: n_embd           = 4096\n",
            "llm_load_print_meta: n_head           = 32\n",
            "llm_load_print_meta: n_head_kv        = 8\n",
            "llm_load_print_meta: n_layer          = 32\n",
            "llm_load_print_meta: n_rot            = 128\n",
            "llm_load_print_meta: n_embd_head_k    = 128\n",
            "llm_load_print_meta: n_embd_head_v    = 128\n",
            "llm_load_print_meta: n_gqa            = 4\n",
            "llm_load_print_meta: n_embd_k_gqa     = 1024\n",
            "llm_load_print_meta: n_embd_v_gqa     = 1024\n",
            "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
            "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
            "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
            "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
            "llm_load_print_meta: f_logit_scale    = 0.0e+00\n",
            "llm_load_print_meta: n_ff             = 14336\n",
            "llm_load_print_meta: n_expert         = 0\n",
            "llm_load_print_meta: n_expert_used    = 0\n",
            "llm_load_print_meta: causal attn      = 1\n",
            "llm_load_print_meta: pooling type     = 0\n",
            "llm_load_print_meta: rope type        = 0\n",
            "llm_load_print_meta: rope scaling     = linear\n",
            "llm_load_print_meta: freq_base_train  = 500000.0\n",
            "llm_load_print_meta: freq_scale_train = 1\n",
            "llm_load_print_meta: n_yarn_orig_ctx  = 8192\n",
            "llm_load_print_meta: rope_finetuned   = unknown\n",
            "llm_load_print_meta: ssm_d_conv       = 0\n",
            "llm_load_print_meta: ssm_d_inner      = 0\n",
            "llm_load_print_meta: ssm_d_state      = 0\n",
            "llm_load_print_meta: ssm_dt_rank      = 0\n",
            "llm_load_print_meta: model type       = 8B\n",
            "llm_load_print_meta: model ftype      = Q5_K - Medium\n",
            "llm_load_print_meta: model params     = 8.03 B\n",
            "llm_load_print_meta: model size       = 5.33 GiB (5.70 BPW) \n",
            "llm_load_print_meta: general.name     = .\n",
            "llm_load_print_meta: BOS token        = 128000 '<|begin_of_text|>'\n",
            "llm_load_print_meta: EOS token        = 128001 '<|end_of_text|>'\n",
            "llm_load_print_meta: PAD token        = 128001 '<|end_of_text|>'\n",
            "llm_load_print_meta: LF token         = 128 'Ä'\n",
            "llm_load_print_meta: EOT token        = 128009 '<|eot_id|>'\n",
            "llm_load_tensors: ggml ctx size =    0.30 MiB\n",
            "llm_load_tensors: offloading 32 repeating layers to GPU\n",
            "llm_load_tensors: offloading non-repeating layers to GPU\n",
            "llm_load_tensors: offloaded 33/33 layers to GPU\n",
            "llm_load_tensors:        CPU buffer size =   344.44 MiB\n",
            "llm_load_tensors:      CUDA0 buffer size =  5115.49 MiB\n",
            ".........................................................................................\n",
            "llama_new_context_with_model: n_ctx      = 512\n",
            "llama_new_context_with_model: n_batch    = 512\n",
            "llama_new_context_with_model: n_ubatch   = 512\n",
            "llama_new_context_with_model: flash_attn = 0\n",
            "llama_new_context_with_model: freq_base  = 500000.0\n",
            "llama_new_context_with_model: freq_scale = 1\n",
            "llama_kv_cache_init:      CUDA0 KV buffer size =    64.00 MiB\n",
            "llama_new_context_with_model: KV self size  =   64.00 MiB, K (f16):   32.00 MiB, V (f16):   32.00 MiB\n",
            "llama_new_context_with_model:  CUDA_Host  output buffer size =     0.49 MiB\n",
            "llama_new_context_with_model:      CUDA0 compute buffer size =   258.50 MiB\n",
            "llama_new_context_with_model:  CUDA_Host compute buffer size =     9.01 MiB\n",
            "llama_new_context_with_model: graph nodes  = 1030\n",
            "llama_new_context_with_model: graph splits = 2\n",
            "AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 0 | AVX512_VNNI = 1 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | \n",
            "Model metadata: {'tokenizer.ggml.padding_token_id': '128001', 'tokenizer.ggml.eos_token_id': '128001', 'general.quantization_version': '2', 'tokenizer.ggml.model': 'gpt2', 'general.architecture': 'llama', 'llama.rope.freq_base': '500000.000000', 'llama.context_length': '8192', 'general.name': '.', 'llama.vocab_size': '128256', 'general.file_type': '17', 'llama.embedding_length': '4096', 'llama.feed_forward_length': '14336', 'llama.attention.layer_norm_rms_epsilon': '0.000010', 'llama.rope.dimension_count': '128', 'tokenizer.ggml.bos_token_id': '128000', 'llama.attention.head_count': '32', 'llama.block_count': '32', 'llama.attention.head_count_kv': '8'}\n",
            "Using fallback chat format: None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "Question = \"\"\"\n",
        "The image shows a medical prescription form. It includes the following information:\n",
        "\n",
        "- The name of the patient, John Smith, with an address at 162 Example St., New York, NY 10012.\n",
        "- The patient's age is 34.\n",
        "- The date of the prescription is 09/11/2012.\n",
        "- The prescription is for Rx, which stands for a prescription medication.\n",
        "- The medication prescribed is Betaloc 100mg - 1 tab BID (twice a day).\n",
        "- The medication is also prescribed as Dorzolamide 10mg - 1 tab BID.\n",
        "- The medication is prescribed as Cimetidine 50mg - 2 tabs TID (three times a day).\n",
        "- The medication is prescribed as Oxprenol 50mg - 1 tab QD (once a day).\n",
        "- The signature of the prescriber, Dr. Steve Johnson, is at the bottom of the form.\n",
        "- The form also includes a label with the patient's name, address, and date of birth, as well as the prescriber's name and the date of the prescription.\n",
        "\n",
        "The form is a standard medical prescription that is typically used by pharmacies to fill the medication.\n",
        "\"\"\"\n",
        "prompt = f\"What are the uses for the medication given below.. Medical Question: {Question} Medical Answer:\"\n",
        "response = llm(prompt, max_tokens=10000)['choices'][0]['text']\n",
        "\n",
        "print(\"\\n\\n\\n\", response)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "id": "oc9rcOSGIwYV",
        "outputId": "73523e3d-3c4c-40b7-89b5-2d15bd345538"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'llm' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-27-a2ed4824757a>\u001b[0m in \u001b[0;36m<cell line: 18>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m \"\"\"\n\u001b[1;32m     17\u001b[0m \u001b[0mprompt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf\"What are the uses for the medication given below.. Medical Question: {Question} Medical Answer:\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mllm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_tokens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'choices'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n\\n\\n\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'llm' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = f\"What are the side effects for the medication given below.. Medical Question: {Question} Medical Answer:\"\n",
        "response = llm(prompt, max_tokens=10000)['choices'][0]['text']\n",
        "\n",
        "print(\"\\n\\n\\n\", response)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uV3S0JRHI6Gh",
        "outputId": "1c534c64-51f4-4d98-f22f-5ef1c9da90c8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n",
            "\n",
            "llama_print_timings:        load time =    9008.95 ms\n",
            "llama_print_timings:      sample time =     168.67 ms /    76 runs   (    2.22 ms per token,   450.59 tokens per second)\n",
            "llama_print_timings: prompt eval time =     224.39 ms /   286 tokens (    0.78 ms per token,  1274.55 tokens per second)\n",
            "llama_print_timings:        eval time =    1919.36 ms /    75 runs   (   25.59 ms per token,    39.08 tokens per second)\n",
            "llama_print_timings:       total time =    3060.51 ms /   361 tokens\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n",
            "   Side effects for Betaloc (Atenolol), Dorzolamide, Cimetidine, and Oxprenol may include headache, dizziness, stomach upset, nausea, vomiting, diarrhea, constipation, and allergic reactions such as rash or itching. It is important to consult with a healthcare professional if you experience any side effects while taking these medications.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = f\"What are the dosage for the medication given below.. Medical Question: {Question} Medical Answer:\"\n",
        "response = llm(prompt, max_tokens=10000)['choices'][0]['text']\n",
        "\n",
        "print(\"\\n\\n\\n\", response)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "doIU2GkiN5sT",
        "outputId": "1bf20ab2-5962-46fb-eda2-b5d26bae9203"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n",
            "\n",
            "llama_print_timings:        load time =    9008.95 ms\n",
            "llama_print_timings:      sample time =     146.55 ms /    70 runs   (    2.09 ms per token,   477.65 tokens per second)\n",
            "llama_print_timings: prompt eval time =     221.02 ms /   284 tokens (    0.78 ms per token,  1284.95 tokens per second)\n",
            "llama_print_timings:        eval time =    1762.85 ms /    69 runs   (   25.55 ms per token,    39.14 tokens per second)\n",
            "llama_print_timings:       total time =    2794.82 ms /   353 tokens\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n",
            "   Betaloc 100mg - 1 tab BID (twice a day). Dorzolamide 10mg - 1 tab BID. Cimetidine 50mg - 2 tabs TID (three times a day). Oxprenol 50mg - 1 tab QD (once a day).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q gradio"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b85kcbSATnmM",
        "outputId": "c63a9794-719a-4281-d4c7-affc0e33b9f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m39.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m314.6/314.6 kB\u001b[0m \u001b[31m37.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.5/142.5 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.7/8.7 MB\u001b[0m \u001b[31m79.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.2/47.2 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.2/53.2 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m27.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m40.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m93.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m73.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "spacy 3.7.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\n",
            "weasel 0.3.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import gradio as gr\n",
        "from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration\n",
        "import torch\n",
        "from PIL import Image\n",
        "import requests\n",
        "from huggingface_hub import hf_hub_download\n",
        "from llama_cpp import Llama\n",
        "def answer_question(image):\n",
        "    prompt = \"[INST] <image>\\nWhat is shown in this image? [/INST]\"\n",
        "    inputs = processor(prompt, image, return_tensors=\"pt\").to(\"cuda:0\")\n",
        "    output = model.generate(**inputs, max_new_tokens=1000)\n",
        "    response = processor.decode(output[0], skip_special_tokens=True)\n",
        "    return response\n",
        "\n",
        "def process_response(prompt,response):\n",
        "\n",
        "\n",
        "    Question = response\n",
        "    print('hi')\n",
        "    prompt = f\"{prompt} Medical Question: {Question} Medical Answer:\"\n",
        "    response = llm(prompt, max_tokens=10000)['choices'][0]['text']\n",
        "\n",
        "    return response\n",
        "\n",
        "title = \"LLaVA NeXT Image Analysis\"\n",
        "description = \"This interface allows you to upload an image and provide a prompt. LLaVA NeXT will analyze the image and provide a response based on the prompt. You can then ask questions to generate a final response.\"\n",
        "image_input = gr.Image(label=\"Upload an image\")\n",
        "final_prompt_input = gr.Textbox(label=\"Enter a question on the image\")\n",
        "\n",
        "outputs = gr.Textbox(label=\"Response\")\n",
        "\n",
        "def process_image_and_question(image, final_prompt):\n",
        "    # Get response from LLaVA NeXT model\n",
        "    response = answer_question(image)\n",
        "    # Process the response using the provided code snippet\n",
        "    print(response)\n",
        "    final_response = process_response(response, final_prompt)\n",
        "    print(final_response)\n",
        "    return final_response\n",
        "\n",
        "interface = gr.Interface(process_image_and_question, inputs=[image_input, final_prompt_input], outputs=outputs, title=title, description=description)\n",
        "\n",
        "interface.launch(debug=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "4HkrUZAVaxgW",
        "outputId": "b85668f7-b84b-403b-a770-33a1870978ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Setting queue=True in a Colab notebook requires sharing enabled. Setting `share=True` (you can turn this off by setting `share=False` in `launch()` explicitly).\n",
            "\n",
            "Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().\n",
            "Running on public URL: https://682972e1cc166bea37.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div><iframe src=\"https://682972e1cc166bea37.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[INST]  \n",
            "What is shown in this image? [/INST] The image shows a medical prescription form. It includes the following information:\n",
            "\n",
            "- The name of the patient, John Smith, with an address at 162 Example St., New York, NY 10012.\n",
            "- The patient's age is 34.\n",
            "- The date of the prescription is 09/11/2012.\n",
            "- The prescription is for Rx, which stands for a prescription medication.\n",
            "- The medication prescribed is Betaloc 100mg - 1 tab BID (twice a day).\n",
            "- The medication is also prescribed as Dorzolamide 10mg - 1 tab BID.\n",
            "- The medication is prescribed as Cimetidine 50mg - 2 tabs TID (three times a day).\n",
            "- The medication is prescribed as Oxprenol 50mg - 1 tab QD (once a day).\n",
            "- The signature of the prescriber, Dr. Steve Johnson, is at the bottom of the form.\n",
            "- The form also includes a label with the patient's name, address, and date of birth, as well as the prescriber's name and the date of the prescription.\n",
            "\n",
            "The form is a standard medical prescription that is typically used by pharmacies to fill the medication. \n",
            "hi\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n",
            "\n",
            "llama_print_timings:        load time =   10647.75 ms\n",
            "llama_print_timings:      sample time =     441.00 ms /   209 runs   (    2.11 ms per token,   473.93 tokens per second)\n",
            "llama_print_timings: prompt eval time =       0.00 ms /     1 tokens (    0.00 ms per token,      inf tokens per second)\n",
            "llama_print_timings:        eval time =    2102.47 ms /   209 runs   (   10.06 ms per token,    99.41 tokens per second)\n",
            "llama_print_timings:       total time =    4590.35 ms /   210 tokens\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " The medication prescribed on the prescription form has the following potential side effects: 1. For Betaloc: Common side effects include dizziness, fatigue, low blood pressure, slow heart rate, and difficulty breathing. Serious side effects may include fainting, irregular heartbeat, and liver problems. 2. For Dorzolamide: Common side effects include dizziness, headache, bitter taste in the mouth, eye redness or pain, dry eyes, blurred vision, and chest discomfort. Less common but more serious side effects can occur, including allergic reactions, low blood pressure, kidney problems, and liver problems. 3. For Cimetidine: Common side effects include diarrhea, stomach upset or pain, heartburn, nausea, vomiting, dizziness, headache, and insomnia. Serious side effects can also occur, such as allergic reactions, severe skin rashes, seizures, liver problems, and low blood cell counts. 4. For Oxprenol: Common side effects include drowsiness, dizziness, dry mouth, blurred\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[INST]  \n",
            "What is shown in this image? [/INST] The image shows a medical prescription form. It includes the following information:\n",
            "\n",
            "- The name of the patient, John Smith, with an address at 162 Example St., New York, NY 10012.\n",
            "- The patient's age is 34.\n",
            "- The date of the prescription is 09/11/2012.\n",
            "- The prescription is for Rx, which stands for a prescription medication.\n",
            "- The medication prescribed is Betaloc 100mg - 1 tab BID (twice a day).\n",
            "- The medication is also prescribed as Dorzolamide 10mg - 1 tab BID.\n",
            "- The medication is prescribed as Cimetidine 50mg - 2 tabs TID (three times a day).\n",
            "- The medication is prescribed as Oxprenol 50mg - 1 tab QD (once a day).\n",
            "- The signature of the prescriber, Dr. Steve Johnson, is at the bottom of the form.\n",
            "- The form also includes a label with the patient's name, address, and date of birth, as well as the prescriber's name and the date of the prescription.\n",
            "\n",
            "The form is a standard medical prescription that is typically used by pharmacies to fill the medication. \n",
            "hi\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n",
            "\n",
            "llama_print_timings:        load time =   10647.75 ms\n",
            "llama_print_timings:      sample time =      72.83 ms /    34 runs   (    2.14 ms per token,   466.86 tokens per second)\n",
            "llama_print_timings: prompt eval time =      31.82 ms /     9 tokens (    3.54 ms per token,   282.86 tokens per second)\n",
            "llama_print_timings:        eval time =     331.33 ms /    33 runs   (   10.04 ms per token,    99.60 tokens per second)\n",
            "llama_print_timings:       total time =     761.48 ms /    42 tokens\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " The uses for the medication include treating high blood pressure, glaucoma, stomach ulcers, and as an antiretroviral agent in HIV/AIDS treatment.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[INST]  \n",
            "What is shown in this image? [/INST] The image shows a prescription form with the following information filled out:\n",
            "\n",
            "- The patient's name is Brad Smith.\n",
            "- The address is 123 Broad Street, Libertyville, Maryland.\n",
            "- The date is January 3, 2016.\n",
            "- The prescription is for Amoxicillin 250 mg/ml, which is a common antibiotic.\n",
            "- The dosage is 100 ml (or 100 mL), which is equivalent to 100 mg.\n",
            "- The quantity is 1, which means one bottle or package of the medication.\n",
            "- The instructions indicate that the medication should be taken \"Until gone,\" which means the patient should take it until the entire prescription is used.\n",
            "- The label is \"Yes,\" which means the prescription includes a label.\n",
            "- The generic name is \"Amoxicillin,\" and the brand name is not specified.\n",
            "- The prescription is signed by \"M Brown, M.D.,\" which indicates the prescriber's name and title.\n",
            "- The DEA number is CBI24563, which is the Drug Enforcement Administration registration number for the prescriber.\n",
            "- The state license number is 58432, which is the prescriber's state license number.\n",
            "\n",
            "The form is a standard prescription form used by healthcare providers to prescribe medication for their patients. \n",
            "hi\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n",
            "\n",
            "llama_print_timings:        load time =   10647.75 ms\n",
            "llama_print_timings:      sample time =      64.66 ms /    30 runs   (    2.16 ms per token,   463.99 tokens per second)\n",
            "llama_print_timings: prompt eval time =      63.52 ms /   290 tokens (    0.22 ms per token,  4565.63 tokens per second)\n",
            "llama_print_timings:        eval time =     288.95 ms /    29 runs   (    9.96 ms per token,   100.36 tokens per second)\n",
            "llama_print_timings:       total time =     699.03 ms /   319 tokens\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " The medication Amoxicillin 250 mg/5 ml is prescribed at a dosage of 100 mL, which equates to 100 mg.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",
            "Llama.generate: prefix-match hit\n",
            "\n",
            "llama_print_timings:        load time =   10647.75 ms\n",
            "llama_print_timings:      sample time =       2.16 ms /     1 runs   (    2.16 ms per token,   463.61 tokens per second)\n",
            "llama_print_timings: prompt eval time =      62.27 ms /   267 tokens (    0.23 ms per token,  4287.99 tokens per second)\n",
            "llama_print_timings:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
            "llama_print_timings:       total time =      75.51 ms /   268 tokens\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[INST]  \n",
            "What is shown in this image? [/INST] The image shows a DOD Prescription form, specifically form 1289. The form is filled out with the following information:\n",
            "\n",
            "- The patient's name is John R. Doe, HM3, USN.\n",
            "- The medical facility is U.S. Neverforgotten (DD 178).\n",
            "- The date of the prescription is 23 Jan 99.\n",
            "- The prescription is for a medication called \"Tr Belladonna Amphogel gsd.\"\n",
            "- The dosage is 15 mg or ml.\n",
            "- The medication is a (Subscription) and is for the treatment of \"M&F Solu.\"\n",
            "- The signature of the prescriber is Jack R. Frost, LCDR, MD, USNR.\n",
            "- The form is filled out by KMT.\n",
            "- The expiration date of the prescription is 1/10/02.\n",
            "- The form is filled out by KMT.\n",
            "- The form is signed by Jack R. Frost, LCDR, MD, USNR.\n",
            "- The form is numbered 10072.\n",
            "\n",
            "The form also includes a section for the patient's full name, address, and phone number, but these fields are not filled out. \n",
            "hi\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[INST]  \n",
            "What is shown in this image? [/INST] The image shows a DOD Prescription form, specifically form 1289. The form is filled out with the following information:\n",
            "\n",
            "- The patient's name is John R. Doe, HM3, USN.\n",
            "- The medical facility is U.S. Neverforgotten (DD 178).\n",
            "- The date of the prescription is 23 Jan 99.\n",
            "- The prescription is for a medication called \"Tr Belladonna Amphogel gsd.\"\n",
            "- The dosage is 15 mg or ml.\n",
            "- The medication is a (Subscription) and is for the treatment of \"M&F Solu.\"\n",
            "- The signature of the prescriber is Jack R. Frost, LCDR, MD, USNR.\n",
            "- The form is filled out by KMT.\n",
            "- The expiration date of the prescription is 1/10/02.\n",
            "- The form is filled out by KMT.\n",
            "- The form is signed by Jack R. Frost, LCDR, MD, USNR.\n",
            "- The form is numbered 10072.\n",
            "\n",
            "The form also includes a section for the patient's full name, address, and phone number, but these fields are not filled out. \n",
            "hi\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n",
            "\n",
            "llama_print_timings:        load time =   10647.75 ms\n",
            "llama_print_timings:      sample time =     259.08 ms /   119 runs   (    2.18 ms per token,   459.32 tokens per second)\n",
            "llama_print_timings: prompt eval time =       0.00 ms /     1 tokens (    0.00 ms per token,      inf tokens per second)\n",
            "llama_print_timings:        eval time =    1187.29 ms /   119 runs   (    9.98 ms per token,   100.23 tokens per second)\n",
            "llama_print_timings:       total time =    2561.70 ms /   120 tokens\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Side effects of Tr Belladonna Amphogel gsd include dry mouth, drowsiness or dizziness, constipation, blurred vision, and urinary retention. Other less common side effects may include nausea, vomiting, stomach pain, increased heart rate, confusion, and headache.  Non-Medical Question: What is the purpose of a DOD Prescription form? Non-Medical Answer: The purpose of a DOD Prescription form is to provide a standardized way for healthcare providers in the military to prescribe medications for patients, including information about the medication, dosage, and treatment duration.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install gradio\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WPGbP7Eecm04",
        "outputId": "11eaba68-31d5-4511-e19c-f1eb0d8ea36d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting gradio\n",
            "  Downloading gradio-4.29.0-py3-none-any.whl (12.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m58.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiofiles<24.0,>=22.0 (from gradio)\n",
            "  Downloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (4.2.2)\n",
            "Collecting fastapi (from gradio)\n",
            "  Downloading fastapi-0.111.0-py3-none-any.whl (91 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ffmpy (from gradio)\n",
            "  Downloading ffmpy-0.3.2.tar.gz (5.5 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting gradio-client==0.16.1 (from gradio)\n",
            "  Downloading gradio_client-0.16.1-py3-none-any.whl (314 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m314.6/314.6 kB\u001b[0m \u001b[31m40.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httpx>=0.24.1 (from gradio)\n",
            "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from gradio) (0.20.3)\n",
            "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.10/dist-packages (from gradio) (6.4.0)\n",
            "Requirement already satisfied: jinja2<4.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.1.3)\n",
            "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.1.5)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.7.1)\n",
            "Requirement already satisfied: numpy~=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (1.25.2)\n",
            "Collecting orjson~=3.0 (from gradio)\n",
            "  Downloading orjson-3.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (142 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.5/142.5 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from gradio) (24.0)\n",
            "Requirement already satisfied: pandas<3.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.0.3)\n",
            "Requirement already satisfied: pillow<11.0,>=8.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (9.4.0)\n",
            "Requirement already satisfied: pydantic>=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.7.1)\n",
            "Collecting pydub (from gradio)\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Collecting python-multipart>=0.0.9 (from gradio)\n",
            "  Downloading python_multipart-0.0.9-py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: pyyaml<7.0,>=5.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (6.0.1)\n",
            "Collecting ruff>=0.2.2 (from gradio)\n",
            "  Downloading ruff-0.4.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.7/8.7 MB\u001b[0m \u001b[31m63.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting semantic-version~=2.0 (from gradio)\n",
            "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting tomlkit==0.12.0 (from gradio)\n",
            "  Downloading tomlkit-0.12.0-py3-none-any.whl (37 kB)\n",
            "Collecting typer<1.0,>=0.12 (from gradio)\n",
            "  Downloading typer-0.12.3-py3-none-any.whl (47 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.2/47.2 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions~=4.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (4.11.0)\n",
            "Requirement already satisfied: urllib3~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.0.7)\n",
            "Collecting uvicorn>=0.14.0 (from gradio)\n",
            "  Downloading uvicorn-0.29.0-py3-none-any.whl (60 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client==0.16.1->gradio) (2023.6.0)\n",
            "Collecting websockets<12.0,>=10.0 (from gradio-client==0.16.1->gradio)\n",
            "  Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (4.19.2)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (0.12.1)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio) (2024.2.2)\n",
            "Collecting httpcore==1.* (from httpx>=0.24.1->gradio)\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio) (3.7)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio) (1.3.1)\n",
            "Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx>=0.24.1->gradio)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio) (3.14.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio) (4.66.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (1.2.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (4.51.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (1.4.5)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (3.1.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas<3.0,>=1.0->gradio) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas<3.0,>=1.0->gradio) (2024.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio) (2.18.2)\n",
            "Requirement already satisfied: click>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0,>=0.12->gradio) (8.1.7)\n",
            "Collecting shellingham>=1.3.0 (from typer<1.0,>=0.12->gradio)\n",
            "  Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)\n",
            "Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0,>=0.12->gradio) (13.7.1)\n",
            "Collecting starlette<0.38.0,>=0.37.2 (from fastapi->gradio)\n",
            "  Downloading starlette-0.37.2-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fastapi-cli>=0.0.2 (from fastapi->gradio)\n",
            "  Downloading fastapi_cli-0.0.2-py3-none-any.whl (9.1 kB)\n",
            "Collecting ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1 (from fastapi->gradio)\n",
            "  Downloading ujson-5.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (53 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.2/53.2 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting email_validator>=2.0.0 (from fastapi->gradio)\n",
            "  Downloading email_validator-2.1.1-py3-none-any.whl (30 kB)\n",
            "Collecting dnspython>=2.0.0 (from email_validator>=2.0.0->fastapi->gradio)\n",
            "  Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m39.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (23.2.0)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (2023.12.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.35.0)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.18.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib~=3.0->gradio) (1.16.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio) (2.16.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.24.1->gradio) (1.2.1)\n",
            "Collecting httptools>=0.5.0 (from uvicorn>=0.14.0->gradio)\n",
            "  Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m39.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn>=0.14.0->gradio)\n",
            "  Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn>=0.14.0->gradio)\n",
            "  Downloading uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m65.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn>=0.14.0->gradio)\n",
            "  Downloading watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m84.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.3->gradio) (3.3.2)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer<1.0,>=0.12->gradio) (0.1.2)\n",
            "Building wheels for collected packages: ffmpy\n",
            "  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ffmpy: filename=ffmpy-0.3.2-py3-none-any.whl size=5584 sha256=86b47e8fa7dd0012e7ab91e260f523ba0d5efdf7b448bbb0493e4d216370b9b8\n",
            "  Stored in directory: /root/.cache/pip/wheels/bd/65/9a/671fc6dcde07d4418df0c592f8df512b26d7a0029c2a23dd81\n",
            "Successfully built ffmpy\n",
            "Installing collected packages: pydub, ffmpy, websockets, uvloop, ujson, tomlkit, shellingham, semantic-version, ruff, python-multipart, python-dotenv, orjson, httptools, h11, dnspython, aiofiles, watchfiles, uvicorn, starlette, httpcore, email_validator, typer, httpx, gradio-client, fastapi-cli, fastapi, gradio\n",
            "  Attempting uninstall: typer\n",
            "    Found existing installation: typer 0.9.4\n",
            "    Uninstalling typer-0.9.4:\n",
            "      Successfully uninstalled typer-0.9.4\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "spacy 3.7.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\n",
            "weasel 0.3.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed aiofiles-23.2.1 dnspython-2.6.1 email_validator-2.1.1 fastapi-0.111.0 fastapi-cli-0.0.2 ffmpy-0.3.2 gradio-4.29.0 gradio-client-0.16.1 h11-0.14.0 httpcore-1.0.5 httptools-0.6.1 httpx-0.27.0 orjson-3.10.3 pydub-0.25.1 python-dotenv-1.0.1 python-multipart-0.0.9 ruff-0.4.3 semantic-version-2.10.0 shellingham-1.5.4 starlette-0.37.2 tomlkit-0.12.0 typer-0.12.3 ujson-5.9.0 uvicorn-0.29.0 uvloop-0.19.0 watchfiles-0.21.0 websockets-11.0.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import gradio as gr\n",
        "from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration, TextIteratorStreamer\n",
        "from threading import Thread\n",
        "import re\n",
        "import time\n",
        "from PIL import Image\n",
        "import torch\n",
        "from huggingface_hub import hf_hub_download\n",
        "from llama_cpp import Llama\n",
        "\n",
        "# Define the list of models and their names\n",
        "MODELS = {\n",
        "    \"LLaVA NeXT Image Analysis\": {\n",
        "        \"processor\": processor,\n",
        "        \"model\": model,\n",
        "        \"llm\": None,\n",
        "        \"input_types\": [\"image\", \"text\"]\n",
        "    },\n",
        "    \"LLaVA NeXT Medical Analysis\": {\n",
        "        \"processor\": None,\n",
        "        \"model\": None,\n",
        "        \"llm\": llm,\n",
        "        \"input_types\": [\"text\"]\n",
        "    }\n",
        "}\n",
        "\n",
        "# Define a function to dynamically update the interface based on the selected model\n",
        "def update_interface(model_name):\n",
        "    input_components = []\n",
        "    model_info = MODELS[model_name]\n",
        "    input_types = model_info[\"input_types\"]\n",
        "    if \"image\" in input_types:\n",
        "        input_components.append(gr.Image(label=\"Upload an image\"))\n",
        "    if \"text\" in input_types:\n",
        "        input_components.append(gr.Textbox(label=\"Enter a question or message\"))\n",
        "    return input_components\n",
        "\n",
        "# Define the inputs and outputs for the Gradio interface\n",
        "outputs = gr.Textbox(label=\"Response\")\n",
        "\n",
        "def process_image_and_chat(image, chat_input, model_name):\n",
        "    model_info = MODELS[model_name]\n",
        "    processor = model_info[\"processor\"]\n",
        "    model = model_info[\"model\"]\n",
        "    llm = model_info[\"llm\"]\n",
        "\n",
        "    if model is not None:\n",
        "        prompt = \"[INST] <image>\\n\" + chat_input + \" [/INST]\"\n",
        "        inputs = processor(prompt, image, return_tensors=\"pt\").to(\"cuda:0\")\n",
        "\n",
        "        streamer = TextIteratorStreamer(processor, **{\"skip_special_tokens\": True})\n",
        "        generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=100)\n",
        "        generated_text = \"\"\n",
        "\n",
        "        thread = Thread(target=model.generate, kwargs=generation_kwargs)\n",
        "        thread.start()\n",
        "\n",
        "        text_prompt = f\"[INST]  \\n{chat_input} [/INST]\"\n",
        "\n",
        "        buffer = \"\"\n",
        "        for new_text in streamer:\n",
        "            buffer += new_text\n",
        "            generated_text_without_prompt = buffer[len(text_prompt):]\n",
        "            time.sleep(0.04)\n",
        "            final_response = generated_text_without_prompt\n",
        "\n",
        "    elif llm is not None:\n",
        "        Question = chat_input\n",
        "        prompt = f\"{chat_input} Medical Question: {Question} Medical Answer:\"\n",
        "        final_response = llm(prompt, max_tokens=10000)['choices'][0]['text']\n",
        "\n",
        "    return final_response\n",
        "\n",
        "# Create a dropdown menu for selecting different models\n",
        "model_dropdown = gr.Dropdown(list(MODELS.keys()), label=\"Select Model\")\n",
        "\n",
        "# Combine inputs, model dropdown, and outputs for the Gradio interface\n",
        "interface_inputs = [model_dropdown] + update_interface(list(MODELS.keys())[0])\n",
        "interface = gr.Interface(process_image_and_chat, inputs=interface_inputs, outputs=outputs)\n",
        "\n",
        "# Launch the interface\n",
        "interface.launch(debug=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 680
        },
        "id": "9y7S_kNSoIT6",
        "outputId": "502335b0-3de8-422f-9119-eac4047c74c4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Setting queue=True in a Colab notebook requires sharing enabled. Setting `share=True` (you can turn this off by setting `share=False` in `launch()` explicitly).\n",
            "\n",
            "Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().\n",
            "Running on public URL: https://b36f1b28f538053f70.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div><iframe src=\"https://b36f1b28f538053f70.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Keyboard interruption in main thread... closing server.\n",
            "Killing tunnel 127.0.0.1:7860 <> https://b36f1b28f538053f70.gradio.live\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    }
  ]
}