example - data_jsonl file.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration\n",
    "## Install python packages\n",
    "* LangChain SDK and GCS Service SDK for python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Architecture\n",
    "![Architecture](./data/architecture.png)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install google-cloud-storage # google-cloud-storage: Google Cloud Storage 객체 관리\n",
    "%pip install --upgrade google-cloud-discoveryengine # google-cloud-discoveryengine: Google Cloud Discovery Engine 클라이언트\n",
    "%pip install --upgrade --quiet google-cloud-aiplatform # google-cloud-aiplatform: Google Cloud AI Platform 모델 관리\n",
    "%pip install --upgrade --quiet langchain langchain_community # langchain: LLM 기반 응용 프로그램 구축, langchain_community: LangChain 커뮤니티 확장 기능\n",
    "%pip install -U langchain-google-community # langchain-google-community: LangChain과 Google Cloud 제품 통합"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Authentication to access to the GCP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set the environment on GCP Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Google Cloud Storage client initialized\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from google.cloud import storage\n",
    "from dotenv import load_dotenv, set_key\n",
    "from google.oauth2 import service_account\n",
    "import google.oauth2.credentials\n",
    "\n",
    "# Load Google API Key from .env file\n",
    "env_path = '.env'\n",
    "load_dotenv(env_path)\n",
    "api_key = os.getenv(\"GOOGLE_API_KEY\")\n",
    "if not api_key:\n",
    "    raise ValueError(\"Failed to load API key. Please set GOOGLE_API_KEY in the .env file.\")\n",
    "\n",
    "# Service account key file path and project settings\n",
    "KEY_PATH = \"./pablo-test-425702-22d29fa73af8.json\"\n",
    "PROJECT_ID = \"pablo-test-425702\"\n",
    "GS_LOCATION = \"asia-northeast3\"\n",
    "\n",
    "# Initialize Google Cloud Storage client\n",
    "try:\n",
    "    storage_client = storage.Client.from_service_account_json(KEY_PATH)\n",
    "    print(\"Google Cloud Storage client initialized\")\n",
    "except Exception as e:\n",
    "    print(f\"Google Cloud Storage client error: {e}\")\n",
    "\n",
    "credentials = service_account.Credentials.from_service_account_file(\n",
    "    KEY_PATH, \n",
    "    scopes=['https://www.googleapis.com/auth/cloud-platform']\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Authentication to access to the GCP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upload data\n",
    "## Upload data in Cloud Storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File ./data/json_example_resume.json uploaded to example/json_example_resume.json!\n"
     ]
    }
   ],
   "source": [
    "from google.cloud import storage\n",
    "\n",
    "def upload_to_gcs_cloud_storage(bucket_name, source_file_name, destination_blob_name):\n",
    "    storage_client = storage.Client()\n",
    "    bucket = storage_client.bucket(bucket_name)\n",
    "    blob = bucket.blob(destination_blob_name)\n",
    "    blob.upload_from_filename(source_file_name)\n",
    "\n",
    "    print(f\"File {source_file_name} uploaded to {destination_blob_name}!\")\n",
    "\n",
    "bucket_name = 'example_test_bucket_rag'\n",
    "source_file_name = './data/json_example_resume.json'\n",
    "workspace_name = \"example/\"\n",
    "destination_blob_name = workspace_name + source_file_name.split('/')[-1]\n",
    "\n",
    "upload_to_gcs_cloud_storage(bucket_name, source_file_name, destination_blob_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create data store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for operation to complete: projects/151473909705/locations/global/collections/default_collection/operations/create-data-store-15406143751886910738\n",
      "name: \"projects/151473909705/locations/global/collections/default_collection/dataStores/example_resume_data\"\n",
      "display_name: \"example_resume_data\"\n",
      "industry_vertical: GENERIC\n",
      "solution_types: SOLUTION_TYPE_SEARCH\n",
      "default_schema_id: \"default_schema\"\n",
      "content_config: NO_CONTENT\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'projects/151473909705/locations/global/collections/default_collection/operations/create-data-store-15406143751886910738'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from google.api_core.client_options import ClientOptions\n",
    "from google.cloud import discoveryengine\n",
    "\n",
    "def create_data_store_sample(\n",
    "    project_id: str,\n",
    "    location: str,\n",
    "    data_store_id: str,\n",
    "    display_name: str\n",
    ") -> str:\n",
    "    \"\"\"\n",
    "    Create a data store in Google Cloud Discovery Engine.\n",
    "\n",
    "    Args:\n",
    "        project_id (str): Google Cloud project ID.\n",
    "        location (str): Location of the data store.\n",
    "        data_store_id (str): Unique ID for the data store.\n",
    "        display_name (str): Display name for the data store.\n",
    "\n",
    "    Returns:\n",
    "        str: Operation name.\n",
    "    \"\"\"\n",
    "    # Set client options based on location\n",
    "    client_options = (ClientOptions(api_endpoint=f\"{location}-discoveryengine.googleapis.com\"))\n",
    "\n",
    "    # Initialize the DataStoreServiceClient\n",
    "    client = discoveryengine.DataStoreServiceClient(client_options=client_options)\n",
    "    parent = client.collection_path(\n",
    "        project=project_id,\n",
    "        location=location,\n",
    "        collection=\"default_collection\",\n",
    "    )\n",
    "    \n",
    "    # Define the data store configuration\n",
    "    data_store = discoveryengine.DataStore(\n",
    "        display_name=display_name, \n",
    "        industry_vertical=discoveryengine.IndustryVertical.GENERIC,\n",
    "        solution_types=[discoveryengine.SolutionType.SOLUTION_TYPE_SEARCH],\n",
    "        content_config=discoveryengine.DataStore.ContentConfig.NO_CONTENT,\n",
    "    )\n",
    "    request = discoveryengine.CreateDataStoreRequest(\n",
    "        parent=parent,\n",
    "        data_store_id=data_store_id,\n",
    "        data_store=data_store,\n",
    "    )\n",
    "\n",
    "    operation = client.create_data_store(request=request)\n",
    "\n",
    "    print(f\"Waiting for operation to complete: {operation.operation.name}\")\n",
    "    response = operation.result()\n",
    "    \n",
    "    # Create the request to create a data store\n",
    "    metadata = discoveryengine.CreateDataStoreMetadata(operation.metadata)\n",
    "    print(response)\n",
    "    print(metadata)\n",
    "    return operation.operation.name\n",
    "\n",
    "\n",
    "DS_LOCATION = \"global\"\n",
    "DATA_STORE_ID = 'example_resume_data'\n",
    "DISPLAY_NAME = 'example_resume_data'\n",
    "create_data_store_sample(PROJECT_ID, DS_LOCATION, DATA_STORE_ID, DISPLAY_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import data to data store\n",
    "* store embedding data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for operation to complete: projects/151473909705/locations/global/collections/default_collection/dataStores/example_resume_data/branches/0/operations/import-documents-4555773986608185226\n",
      "error_config {\n",
      "  gcs_prefix: \"gs://151473909705_asia_northeast3_import_custom/errors4555773986608185234\"\n",
      "}\n",
      "\n",
      "create_time {\n",
      "  seconds: 1720762898\n",
      "  nanos: 621037000\n",
      "}\n",
      "update_time {\n",
      "  seconds: 1720762903\n",
      "  nanos: 905509000\n",
      "}\n",
      "success_count: 9\n",
      "total_count: 9\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'projects/151473909705/locations/global/collections/default_collection/dataStores/example_resume_data/branches/0/operations/import-documents-4555773986608185226'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from google.cloud import discoveryengine_v1beta as discoveryengine\n",
    "from google.api_core.client_options import ClientOptions\n",
    "from google.protobuf import field_mask_pb2\n",
    "\n",
    "def import_documents(\n",
    "    project_id: str,\n",
    "    location: str,\n",
    "    data_store_id: str,\n",
    "    gcs_uri: str\n",
    ") -> str:\n",
    "    \"\"\"\n",
    "    Import documents from Google Cloud Storage into Google Cloud Discovery Engine.\n",
    "\n",
    "    Args:\n",
    "        project_id (str): Google Cloud project ID.\n",
    "        location (str): Location of the data store.\n",
    "        data_store_id (str): ID of the data store.\n",
    "        gcs_uri (str): Google Cloud Storage URI of the documents to import.\n",
    "\n",
    "    Returns:\n",
    "        str: Operation name.\n",
    "    \"\"\"\n",
    "    \n",
    "    api_endpoint = f\"{location}-discoveryengine.googleapis.com\"\n",
    "    client_options = ClientOptions(api_endpoint=api_endpoint)\n",
    "    \n",
    "    client = discoveryengine.DocumentServiceClient(client_options=client_options)\n",
    "    parent = client.branch_path(\n",
    "        project=project_id,\n",
    "        location=location,\n",
    "        data_store=data_store_id,\n",
    "        branch=\"default_branch\",\n",
    "    )\n",
    "    \n",
    "    gcs_source = discoveryengine.GcsSource(\n",
    "        input_uris=[gcs_uri],\n",
    "        data_schema=\"custom\"\n",
    "    )\n",
    "    \n",
    "    request = discoveryengine.ImportDocumentsRequest(\n",
    "        parent=parent,\n",
    "        gcs_source=gcs_source,\n",
    "        reconciliation_mode=discoveryengine.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL,\n",
    "    )\n",
    "\n",
    "    operation = client.import_documents(request=request)\n",
    "    print(f\"Waiting for operation to complete: {operation.operation.name}\")\n",
    "    response = operation.result()\n",
    "    metadata = discoveryengine.ImportDocumentsMetadata(operation.metadata)\n",
    "\n",
    "    print(response)\n",
    "    print(metadata) \n",
    "\n",
    "    return operation.operation.name\n",
    "\n",
    "gcs_uri = 'gs://pablo_test_bucket1/pablo_example/json_example_resume.json'\n",
    "import_documents(PROJECT_ID, DS_LOCATION, DATA_STORE_ID, gcs_uri)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RAG with Vertex AI Search "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vertex AI initialization\n",
    "**Configure project information**\n",
    "* Model name : LLM model name : https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models\n",
    "    * Project Id : prodect id in GCP\n",
    "\n",
    "**Configure Vertex AI and access to the foundation model.**\n",
    "* Vertex AI initialization : aiplatform.init(..)\n",
    "    * https://cloud.google.com/python/docs/reference/aiplatform/latest#initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import vertexai\n",
    "from vertexai.generative_models import GenerativeModel, Part, Tool\n",
    "import vertexai.generative_models as generative_models\n",
    "from langchain.retrievers import GoogleVertexAIMultiTurnSearchRetriever\n",
    "from langchain_google_community import VertexAISearchRetriever\n",
    "from vertexai.preview.generative_models import grounding\n",
    "\n",
    "MODEL_NAME=\"gemini-1.5-flash\"\n",
    "VERTEX_AI_LOCATION=\"us-central1\"\n",
    "\n",
    "# Initalizate the current vertex AI execution environment.\n",
    "vertexai.init(project=PROJECT_ID, location=VERTEX_AI_LOCATION)\n",
    "model = GenerativeModel(MODEL_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve VertexAISearch "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.retrievers import GoogleVertexAIMultiTurnSearchRetriever\n",
    "from langchain_google_community import VertexAISearchRetriever\n",
    "from langchain_community.retrievers import (\n",
    "    GoogleVertexAIMultiTurnSearchRetriever,\n",
    "    GoogleVertexAISearchRetriever,\n",
    ")\n",
    "\n",
    "retriever = VertexAISearchRetriever(\n",
    "    project_id=PROJECT_ID,\n",
    "    location_id=DS_LOCATION,\n",
    "    data_store_id=DATA_STORE_ID,\n",
    "    max_documents=3,\n",
    "    max_extractive_answer_count=3,\n",
    "    max_extractive_segment_count=1,\n",
    "    query_expansion_condition=2,\n",
    "    engine_data_type = 1\n",
    ")\n",
    "\n",
    "query = \"김철수의 휴대전화 번호와 생일\"\n",
    "\n",
    "result = retriever.get_relevant_documents(query)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Convert Unicode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_unicode(data):\n",
    "    if isinstance(data, dict):\n",
    "        return {k: fix_unicode(v) for k, v in data.items()}\n",
    "    elif isinstance(data, list):\n",
    "        return [fix_unicode(v) for v in data]\n",
    "    elif isinstance(data, str):\n",
    "        return data.encode('latin1').decode('unicode_escape')\n",
    "    else:\n",
    "        return data\n",
    "context ={}\n",
    "\n",
    "for doc in result:\n",
    "    context[fix_unicode(doc.metadata['name'])] = fix_unicode(doc.page_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Request LLM using prompt and retriever data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt : \n",
      "  당신은 내 질문에 대해 답변하는 AI 어시스턴트입니다.\n",
      "  아래 Question 에 대해서 반드시 Context에 있는 개별 내용을 기반으로 간결하게 답변주세요.\n",
      "  \n",
      "  Context : {'projects/151473909705/locations/global/collections/default_collection/dataStores/example_resume_data/branches/0/documents/resume2': '{\"jsonData\": \"{\"name\":\"김철수\",\"birth_date\":\"1985-05-20\",\"contact\":{\"email\":\"kimchulsoo@example.com\",\"phone\":\"010-9876-5432\"},\"education\":[{\"school\":\"고려대학교\",\"degree\":\"경영학 학사\",\"year\":\"2008\"},{\"school\":\"고려대학교\",\"degree\":\"경영학 석사\",\"year\":\"2011\"}],\"experience\":[{\"company\":\"삼성전자\",\"role\":\"마케팅 매니저\",\"years\":\"2011-2017\",\"description\":\"글로벌 마케팅 전략 수립 및 실행\"},{\"company\":\"LG전자\",\"role\":\"프로덕트 매니저\",\"years\":\"2017-현재\",\"description\":\"신제품 기획 및 출시 관리\"}],\"skills\":[\"Marketing Strategy\",\"Product Management\",\"Business Development\"],\"languages\":[\"한국어\",\"영어\",\"일본어\"]}\", \"content\": {\"mimeType\": \"application/pdf\", \"uri\": \"gs://pablo_test_bucket1/pablo_example/111/resume_test2.pdf\"}}', 'projects/151473909705/locations/global/collections/default_collection/dataStores/example_resume_data/branches/0/documents/resume4': '{\"content\": {\"mimeType\": \"application/pdf\", \"uri\": \"gs://pablo_test_bucket1/pablo_example/111/resume_test4.pdf\"}, \"jsonData\": \"{\"name\":\"박민수\",\"birth_date\":\"1988-11-11\",\"contact\":{\"email\":\"parkminsoo@example.com\",\"phone\":\"010-2233-4455\"},\"education\":[{\"school\":\"한양대학교\",\"degree\":\"전기전자공학 학사\",\"year\":\"2011\"},{\"school\":\"한양대학교\",\"degree\":\"전기전자공학 석사\",\"year\":\"2013\"}],\"experience\":[{\"company\":\"삼성전자\",\"role\":\"하드웨어 엔지니어\",\"years\":\"2013-2018\",\"description\":\"반도체 설계 및 개발\"},{\"company\":\"SK 하이닉스\",\"role\":\"연구개발 매니저\",\"years\":\"2018-현재\",\"description\":\"신제품 연구 및 개발 관리\"}],\"skills\":[\"Semiconductor Design\",\"Hardware Engineering\",\"Project Management\"],\"languages\":[\"한국어\",\"영어\"]}\"}', 'projects/151473909705/locations/global/collections/default_collection/dataStores/example_resume_data/branches/0/documents/resume5': '{\"jsonData\": \"{\"name\":\"최영수\",\"birth_date\":\"1987-03-12\",\"contact\":{\"email\":\"choiyoungsoo@example.com\",\"phone\":\"010-3333-4444\"},\"education\":[{\"school\":\"한양대학교\",\"degree\":\"화학공학 학사\",\"year\":\"2009\"},{\"school\":\"한양대학교\",\"degree\":\"화학공학 석사\",\"year\":\"2012\"}],\"experience\":[{\"company\":\"LG화학\",\"role\":\"연구원\",\"years\":\"2012-2016\",\"description\":\"고분자 연구 및 개발\"},{\"company\":\"SK이노베이션\",\"role\":\"선임 연구원\",\"years\":\"2016-현재\",\"description\":\"신소재 개발 및 프로젝트 관리\"}],\"skills\":[\"Polymer Research\",\"Material Science\",\"Project Management\"],\"languages\":[\"한국어\",\"영어\"]}\", \"content\": {\"mimeType\": \"application/pdf\", \"uri\": \"gs://pablo_test_bucket1/pablo_example/111/resume_test5.pdf\"}}'}\n",
      "  Context Size : 3\n",
      "  Question : 김철수의 휴대전화 번호와 생일\n",
      "\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "김철수의 휴대전화 번호는 010-9876-5432이고 생일은 1985-05-20입니다. \n"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from langchain_google_vertexai.llms import VertexAI\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.chains import LLMChain\n",
    "from IPython.display import display, Markdown\n",
    "\n",
    "gemini_pro = VertexAI( model_name = MODEL_NAME,\n",
    "                  project=PROJECT_ID,\n",
    "                  location=VERTEX_AI_LOCATION,\n",
    "                  verbose=True,\n",
    "                  streaming=False,\n",
    "                  temperature = 0.2,\n",
    "                  top_p = 1,\n",
    "                  top_k = 20\n",
    "                 )\n",
    "prompt = PromptTemplate.from_template(\"\"\"\n",
    "  당신은 내 질문에 대해 답변하는 AI 어시스턴트입니다.\n",
    "  아래 Question 에 대해서 반드시 Context에 있는 개별 내용을 기반으로 간결하게 답변주세요.\n",
    "  \n",
    "  Context : {context}\n",
    "  Context Size : {len_context}\n",
    "  Question : {question}\n",
    "\n",
    "  \"\"\")\n",
    "\n",
    "prompt = prompt.format(context=context,\n",
    "                       question=query,\n",
    "                       len_context = len(context)\n",
    "                       )\n",
    "\n",
    "print(f\"Prompt : {prompt}\")\n",
    "response = gemini_pro.invoke(prompt)\n",
    "display(Markdown(response))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm_test2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}