From 3e0064719de365dde023da266479f842724c27f8 Mon Sep 17 00:00:00 2001 From: Emma Pearce Date: Wed, 27 Nov 2024 11:41:28 +0000 Subject: [PATCH] Address Gagan's comments --- backend/promptfoo/intent_config.yaml | 8 ++--- .../src/prompts/templates/intent-system.j2 | 10 +++---- backend/src/supervisors/supervisor.py | 29 +++++++++---------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/backend/promptfoo/intent_config.yaml b/backend/promptfoo/intent_config.yaml index f3f695ad..353f5703 100644 --- a/backend/promptfoo/intent_config.yaml +++ b/backend/promptfoo/intent_config.yaml @@ -8,7 +8,7 @@ providers: prompts: file://promptfoo_test_runner.py:create_prompt tests: - - description: "questions directed towards the database look ups should have only 1 question -1" + - description: "questions directed towards the database lookups should have only 1 question -1" vars: system_prompt_template: "intent-system" user_prompt_template: "intent" @@ -30,7 +30,7 @@ tests: - type: javascript value: JSON.parse(output).questions.length === 0 - - description: "verify that the correct company name is determinded from the chat history" + - description: "verify that the correct company name is determined from the chat history" vars: system_prompt_template: "intent-system" user_prompt_template: "intent" @@ -51,11 +51,11 @@ tests: user_prompt_template: "intent" user_prompt_args: chat_history: [] - question: "Compare Ryanair emmissions to other companies in the industry" + question: "Compare Ryanair emissions to other companies in the industry" assert: - type: javascript value: JSON.parse(output).questions[0].includes("Ryanair") - type: llm-rubric value: The 1st item in the questions array contains a question about finding the emissions for Ryanair - type: llm-rubric - value: The 2nd item in the questions array contains a question about finding the emissions for companiesin the industry + value: The 2nd item in the questions array contains a question about finding the emissions for companies in the industry diff --git a/backend/src/prompts/templates/intent-system.j2 b/backend/src/prompts/templates/intent-system.j2 index d0c25db1..0f7cdf8f 100644 --- a/backend/src/prompts/templates/intent-system.j2 +++ b/backend/src/prompts/templates/intent-system.j2 @@ -1,10 +1,9 @@ -You are an expert in determinging the intent behind a user's question, breaking down complex questions into multiple simplier questions and forming stand alone questions based on context from chat history. +You are an expert in determining the intent behind a user's question, breaking down complex questions into multiple simpler questions and forming standalone questions based on context from chat history. You will be given a question and may also be provided with chat history which is to be used as context if provided. - -- First determin the user intent of the question by taking key points from the question and gaining context from the chat history. -- Second, if the question does not mentions csv, dataset or database, using the user intent try to split the question up into multiple questions with singular objectives. +- First determine the user intent of the question by taking key points from the question and gaining context from the chat history. +- Second, if the question does not mention csv, dataset or database, using the user intent try to split the question up into multiple questions with singular objectives. Output your result in the following json format: @@ -12,5 +11,4 @@ Output your result in the following json format: "question": "string of the original question", "user_intent": "string of the intent of the user's question", "questions": array of singular objective questions or if the question mentions csv, dataset or database an empty array -} - +} \ No newline at end of file diff --git a/backend/src/supervisors/supervisor.py b/backend/src/supervisors/supervisor.py index 5246c85d..9817bfeb 100644 --- a/backend/src/supervisors/supervisor.py +++ b/backend/src/supervisors/supervisor.py @@ -12,28 +12,25 @@ no_agent_response = "I am sorry, but I was unable to find an agent to solve this task" +async def process_question(question) -> None: + try: + (agent_name, answer, status) = await solve_task(question, get_scratchpad()) + update_scratchpad(agent_name, question, answer) + if status == "error": + raise Exception(answer) + except Exception as error: + update_scratchpad(error=str(error)) + + async def solve_all(intent_json) -> None: questions = intent_json["questions"] if len(questions) == 0: - try: - question = intent_json["question"] - (agent_name, answer, status) = await solve_task(question, get_scratchpad()) - update_scratchpad(agent_name, question, answer) - if status == "error": - raise Exception(answer) - except Exception as error: - update_scratchpad(error=str(error)) + question = intent_json["question"] + await process_question(question) else: for question in questions: - try: - (agent_name, answer, status) = await solve_task(question, get_scratchpad()) - update_scratchpad(agent_name, question, answer) - if status == "error": - logger.info("status == error") - raise Exception(answer) - except Exception as error: - update_scratchpad(error=str(error)) + await process_question(question) async def solve_task(task, scratchpad, attempt=0) -> Tuple[str, str, str]: