From 7ff3a6601ce44fe6686d7ee35ad15bf50ee838ee Mon Sep 17 00:00:00 2001
From: James Braza <jamesbraza@gmail.com>
Date: Fri, 10 Jan 2025 16:26:59 -0800
Subject: [PATCH] Updated evidence added in gather_evidence's
 ToolResponseMessage

---
 paperqa/agents/tools.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/paperqa/agents/tools.py b/paperqa/agents/tools.py
index fa86eef97..f22943e07 100644
--- a/paperqa/agents/tools.py
+++ b/paperqa/agents/tools.py
@@ -227,11 +227,12 @@ async def gather_evidence(self, question: str, state: EnvironmentState) -> str:
 
         logger.info(f"{self.TOOL_FN_NAME} starting for question {question!r}.")
         original_question = state.session.question
+        l1_all = l1_relevant = l0 = len(state.session.contexts)
+
         try:
             # Swap out the question with the more specific question
             # TODO: remove this swap, as it prevents us from supporting parallel calls
             state.session.question = question
-            l0 = len(state.session.contexts)
 
             # TODO: refactor answer out of this...
             state.session = await state.docs.aget_evidence(
@@ -244,7 +245,14 @@ async def gather_evidence(self, question: str, state: EnvironmentState) -> str:
                     f"{self.TOOL_FN_NAME}_aget_evidence"
                 ),
             )
-            l1 = len(state.session.contexts)
+            l1_all = len(state.session.contexts)
+            l1_relevant = len(
+                [
+                    c
+                    for c in state.session.contexts
+                    if c.score > state.RELEVANT_SCORE_CUTOFF
+                ]
+            )
         finally:
             state.session.question = original_question
 
@@ -275,7 +283,10 @@ async def gather_evidence(self, question: str, state: EnvironmentState) -> str:
                 )
             )
 
-        return f"Added {l1 - l0} pieces of evidence.{best_evidence}\n\n" + status
+        return (
+            f"Added {l1_all - l0} pieces of evidence, {l1_relevant - l0} of which were"
+            f" relevant.{best_evidence}\n\n" + status
+        )
 
 
 class GenerateAnswer(NamedTool):