changed rag input from summary to full text

karthiksoman · Sep 24, 2024 · 12a21f7 · 12a21f7
1 parent e8eb5db
commit 12a21f7
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 199 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,8 @@
+code/notebook/.ipynb_checkpoints/
+code/__pycache__/
+code/.ipynb_checkpoints/
+.virtual_documents/
+.gpt_config.env
 .DS_Store
 manuscript/
 # lambda-pipeline

diff --git a/code/notebook/.ipynb_checkpoints/pinecone_retrieval-checkpoint.ipynb b/code/notebook/.ipynb_checkpoints/pinecone_retrieval-checkpoint.ipynb
diff --git a/code/notebook/pinecone_retrieval.ipynb b/code/notebook/pinecone_retrieval.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 51,
+   "execution_count": 1,
    "id": "d9a114f4-c03a-49bf-a519-89db2c5bb3f1",
    "metadata": {},
    "outputs": [],
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 52,
+   "execution_count": 2,
    "id": "2dc22ebd-63d5-4b4c-a2dc-eabb627dae82",
    "metadata": {},
    "outputs": [],
@@ -37,7 +37,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 53,
+   "execution_count": 3,
    "id": "a1db1e41-63a4-4b80-9e72-ccdef603d73f",
    "metadata": {},
    "outputs": [],
@@ -55,7 +55,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 54,
+   "execution_count": 4,
    "id": "cadb5fb8-2ed1-4cbe-a124-c0dfba19594b",
    "metadata": {},
    "outputs": [],
@@ -69,7 +69,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 56,
+   "execution_count": 5,
    "id": "ba3dd1db-8385-4e36-abcc-79aae7cafcd0",
    "metadata": {},
    "outputs": [],
@@ -83,7 +83,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 66,
+   "execution_count": 6,
    "id": "2988ef50-0906-40f0-9228-ba56e671f22e",
    "metadata": {},
    "outputs": [],

diff --git a/lambda-pipeline/app.py b/lambda-pipeline/app.py
@@ -32,7 +32,7 @@ def get_model_response(input_text, temperature: float = 0.7):
         logging.error(f"Failed to get model response: {str(e)}")
         return None
 
-def get_rag_context(query, top_k=5):
+def get_rag_context(query, top_k=2):
     try:
         embed_model = OpenAIEmbedding(
             model='text-embedding-ada-002',
@@ -47,7 +47,8 @@ def get_rag_context(query, top_k=5):
             top_k=top_k,
             include_metadata=True
         )
-        extracted_context_summary = list(map(lambda x: json.loads(x.metadata['_node_content'])['metadata']['section_summary'], retrieved_doc.matches))
+        # extracted_context_summary = list(map(lambda x: json.loads(x.metadata['_node_content'])['metadata']['section_summary'], retrieved_doc.matches))
+        extracted_context_summary = list(map(lambda x: json.loads(x.metadata['_node_content'])['metadata']['text'], retrieved_doc.matches))
         provenance = list(map(lambda x: x.metadata['c_document_id'], retrieved_doc.matches))
         context = ''
         for i in range(top_k):