diff --git a/prepare/metrics/rag.py b/prepare/metrics/rag.py index 8a1f3792c1..1cc3fc2c98 100644 --- a/prepare/metrics/rag.py +++ b/prepare/metrics/rag.py @@ -347,12 +347,24 @@ # metrics.rag.recall # metrics.rag.bert_recall -for axis, base_metric, main_score in [ - ("correctness", "token_overlap", "f1"), - ("correctness", "bert_score.deberta_large_mnli", "recall"), - ("correctness", "bert_score.deberta_v3_base_mnli_xnli_ml", "recall"), - ("faithfullness", "token_overlap", "precision"), +for axis, base_metric, main_score, new_metric in [ + ("correctness", "token_overlap", "f1", "answer_correctness.token_recall"), + ( + "correctness", + "bert_score.deberta_large_mnli", + "recall", + "answer_correctness.bert_score_recall", + ), + ( + "correctness", + "bert_score.deberta_v3_base_mnli_xnli_ml", + "recall", + "answer_correctness.bert_score_recall_ml", + ), + ("faithfullness", "token_overlap", "precision", "faithfulness.token_k_precision"), ]: + deprecated_path = f"metrics.rag.response_generation.{axis}.{base_metric}" + new_metric_path = f"metrics.rag.response_generation.{new_metric}" preprocess_steps = ( [ Copy(field="task_data/contexts", to_field="references"), @@ -379,10 +391,13 @@ ], metric=f"metrics.{base_metric}", prediction_type=str, + __deprecated_msg__=f"Metric {deprecated_path} is deprecated. Please use {new_metric_path} instead.", ) add_to_catalog( - metric, f"metrics.rag.response_generation.{axis}.{base_metric}", overwrite=True + metric, + f"metrics.rag.response_generation.{axis}.{base_metric}", + overwrite=True, ) # end to end diff --git a/src/unitxt/catalog/metrics/rag/response_generation/correctness/bert_score/deberta_large_mnli.json b/src/unitxt/catalog/metrics/rag/response_generation/correctness/bert_score/deberta_large_mnli.json index 7173a94093..83a1b1f89d 100644 --- a/src/unitxt/catalog/metrics/rag/response_generation/correctness/bert_score/deberta_large_mnli.json +++ b/src/unitxt/catalog/metrics/rag/response_generation/correctness/bert_score/deberta_large_mnli.json @@ -17,5 +17,6 @@ } ], "metric": "metrics.bert_score.deberta_large_mnli", - "prediction_type": "str" + "prediction_type": "str", + "__deprecated_msg__": "Metric metrics.rag.response_generation.correctness.bert_score.deberta_large_mnli is deprecated. Please use metrics.rag.response_generation.answer_correctness.bert_score_recall instead." } diff --git a/src/unitxt/catalog/metrics/rag/response_generation/correctness/bert_score/deberta_v3_base_mnli_xnli_ml.json b/src/unitxt/catalog/metrics/rag/response_generation/correctness/bert_score/deberta_v3_base_mnli_xnli_ml.json index 9394fb9c0b..97884f596c 100644 --- a/src/unitxt/catalog/metrics/rag/response_generation/correctness/bert_score/deberta_v3_base_mnli_xnli_ml.json +++ b/src/unitxt/catalog/metrics/rag/response_generation/correctness/bert_score/deberta_v3_base_mnli_xnli_ml.json @@ -17,5 +17,6 @@ } ], "metric": "metrics.bert_score.deberta_v3_base_mnli_xnli_ml", - "prediction_type": "str" + "prediction_type": "str", + "__deprecated_msg__": "Metric metrics.rag.response_generation.correctness.bert_score.deberta_v3_base_mnli_xnli_ml is deprecated. Please use metrics.rag.response_generation.answer_correctness.bert_score_recall_ml instead." } diff --git a/src/unitxt/catalog/metrics/rag/response_generation/correctness/token_overlap.json b/src/unitxt/catalog/metrics/rag/response_generation/correctness/token_overlap.json index 09282392e0..54689de4f3 100644 --- a/src/unitxt/catalog/metrics/rag/response_generation/correctness/token_overlap.json +++ b/src/unitxt/catalog/metrics/rag/response_generation/correctness/token_overlap.json @@ -17,5 +17,6 @@ } ], "metric": "metrics.token_overlap", - "prediction_type": "str" + "prediction_type": "str", + "__deprecated_msg__": "Metric metrics.rag.response_generation.correctness.token_overlap is deprecated. Please use metrics.rag.response_generation.answer_correctness.token_recall instead." } diff --git a/src/unitxt/catalog/metrics/rag/response_generation/faithfullness/token_overlap.json b/src/unitxt/catalog/metrics/rag/response_generation/faithfullness/token_overlap.json index b62cbaf4ac..84b3c36e2c 100644 --- a/src/unitxt/catalog/metrics/rag/response_generation/faithfullness/token_overlap.json +++ b/src/unitxt/catalog/metrics/rag/response_generation/faithfullness/token_overlap.json @@ -23,5 +23,6 @@ } ], "metric": "metrics.token_overlap", - "prediction_type": "str" + "prediction_type": "str", + "__deprecated_msg__": "Metric metrics.rag.response_generation.faithfullness.token_overlap is deprecated. Please use metrics.rag.response_generation.faithfulness.token_k_precision instead." }