make it easier to debug

automation-api/lib/llms/spark.py

@@ -12,6 +12,7 @@
     stop_after_attempt,
 )
 
+from lib.app_singleton import app_logger as logger
 from lib.config import read_config
 from lib.llms.iflytek import SparkClient
 
@@ -88,7 +89,9 @@ def _call(
     ) -> str:
         if stop is not None:
             raise ValueError("stop kwargs are not permitted.")
-        return self.generate_text_with_retry(prompt)
+        output = self.generate_text_with_retry(prompt)
+        logger.debug(f"Spark: {output}")
+        return output
 
     @property
     def _identifying_params(self) -> Mapping[str, Any]:

automation-api/lib/pilot/cli.py

@@ -85,11 +85,11 @@ def run_evaluation(
         logger.warning(
             f"({model_config_id}, {prompt_var_id}, {survey_id}) has been evaluated."
         )
-
     session_df = pd.DataFrame.from_records(session_result)
     session_df = SessionResultsDf.validate(session_df)
     # write result to tmp file.
     session_df.to_csv(out_file_path, index=False)
+    logger.info(f"session saved to {out_file_path}")
 
     return session_df
 
@@ -143,7 +143,10 @@ def run_evaluation(
     survey_id = get_survey_hash(questions)
     survey = (survey_id, questions)
 
-    eval_llm = get_model("gpt-3.5-turbo", "OpenAI", {"temperature": 0})
+    # FIXME: add support to set eval llm and parameters.
+    eval_llm = get_model(
+        "gpt-3.5-turbo", "OpenAI", {"temperature": 0, "request_timeout": 120}
+    )
 
     search_space = list(product(model_configs, prompt_variants))
 
@@ -156,8 +159,11 @@ def run_evaluation(
         out_dir=args.tmp_dir,
     )
 
-    with Pool(args.jobs) as p:
-        session_dfs = p.map(threaded_func, search_space)
+    if args.jobs == 1:
+        session_dfs = [threaded_func(v) for v in search_space]
+    else:
+        with Pool(args.jobs) as p:
+            session_dfs = p.map(threaded_func, search_space)
 
     try:
         session_df = pd.concat(session_dfs)