feat(agents-api): Add some custom error messages to base_evaluate

agents-api/agents_api/activities/task_steps/base_evaluate.py

@@ -4,13 +4,33 @@
 from beartype import beartype
 from box import Box
 from openai import BaseModel
+from simpleeval import NameNotDefined
 from temporalio import activity
+from thefuzz import fuzz
 
 from ...common.storage_handler import auto_blob_store
 from ...env import testing
 from ..utils import get_evaluator
 
 
+class EvaluateError(Exception):
+    def __init__(self, error, expression, values):
+        error_message = error.message if hasattr(error, "message") else str(error)
+        message = error_message
+
+        # Catch a possible jinja template error
+        if "unhashable" in error_message and "{{" in expression:
+            message += "\nSuggestion: It seems like you used a jinja template, did you mean to use a python expression?"
+
+        # Catch a possible misspell in a variable name
+        if isinstance(error, NameNotDefined):
+            misspelledName = error_message.split("'")[1]
+            for variableName in values.keys():
+                if fuzz.ratio(variableName, misspelledName) >= 90.0:
+                    message += f"\nDid you mean '{variableName}' instead of '{misspelledName}'?"
+        super().__init__(message)
+
+
 @auto_blob_store
 @beartype
 async def base_evaluate(
@@ -47,22 +67,32 @@ async def base_evaluate(
 
     evaluator = get_evaluator(names=values, extra_functions=extra_lambdas)
 
+    chosen_expression = ""
+
     try:
         result = None
         match exprs:
             case str():
+                chosen_expression = exprs
                 result = evaluator.eval(exprs)
             case list():
-                result = [evaluator.eval(expr) for expr in exprs]
+                result = []
+                for expr in exprs:
+                    chosen_expression = expr
+                    result.append(evaluator.eval(expr))
             case dict() as d if all(
                 isinstance(v, dict) or isinstance(v, str) for v in d.values()
             ):
-                result = {
-                    k: {ik: evaluator.eval(iv) for ik, iv in v.items()}
-                    if isinstance(v, dict)
-                    else evaluator.eval(v)
-                    for k, v in d.items()
-                }
+                result = {}
+                for k, v in d.items():
+                    if isinstance(v, str):
+                        chosen_expression = v
+                        result[k] = evaluator.eval(v)
+                    else:
+                        result[k] = {}
+                        for k1, v1 in v.items():
+                            chosen_expression = v1
+                            result[k][k1] = evaluator.eval(v1)
             case _:
                 raise ValueError(f"Invalid expression: {exprs}")
 
@@ -71,7 +101,8 @@ async def base_evaluate(
     except BaseException as e:
         if activity.in_activity():
             activity.logger.error(f"Error in base_evaluate: {e}")
-        raise
+        newException = EvaluateError(e, chosen_expression, values)
+        raise newException from e
 
 
 # Note: This is here just for clarity. We could have just imported base_evaluate directly

agents-api/pyproject.toml

@@ -45,6 +45,7 @@ xxhash = "^3.5.0"
 spacy = "^3.8.2"
 en-core-web-sm = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl"}
 msgpack = "^1.1.0"
+thefuzz = "^0.22.1"
 gunicorn = "^23.0.0"
 uvloop = "^0.21.0"
 [tool.poetry.group.dev.dependencies]