diff --git a/automation-api/yival_experiments/scripts/generate_experiment_config.py b/automation-api/yival_experiments/scripts/generate_experiment_config.py index 6bba630..0b30a58 100644 --- a/automation-api/yival_experiments/scripts/generate_experiment_config.py +++ b/automation-api/yival_experiments/scripts/generate_experiment_config.py @@ -35,7 +35,9 @@ def str_presenter(dumper, data): latest_experiment_path = current_script_path / "../experiment_latest.yaml" -def get_evaluators(ai_eval_sheet: AiEvalData, evaluator_model="gpt4"): +def get_evaluators( + ai_eval_sheet: AiEvalData, evaluator_model="gpt4" +) -> List[Dict[str, Any]]: metrics = get_metrics(ai_eval_sheet) res = list() @@ -113,6 +115,15 @@ def main(evaluator_model): # metrics config["evaluators"] = get_evaluators(sheet, evaluator_model=evaluator_model) + + # also append a simple evaluator + simple_evaluator = { + "evaluator_type": "individual", + "metric_calculators": [{"method": "AVERAGE"}], + "name": "simple_evaluator", + } + config["evaluators"].append(simple_evaluator) + # model configs and prompt variations model_configs = get_model_configs(sheet) model_ids = {model.model_id for model, model_config in model_configs}