From 6cf33563f70ee75711313b2aa114f98664d2188f Mon Sep 17 00:00:00 2001 From: Nathan Weinberg Date: Thu, 27 Jun 2024 11:54:32 -0400 Subject: [PATCH] Reverted from tuple to list, added lint error to pylint ignore Signed-off-by: Nathan Weinberg --- .pylintrc | 3 ++- src/instructlab/eval/mmlu.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.pylintrc b/.pylintrc index 64ef3d1..0959615 100644 --- a/.pylintrc +++ b/.pylintrc @@ -446,7 +446,8 @@ disable=raw-checker-failed, abstract-method, pointless-statement, wrong-import-order, - line-too-long + line-too-long, + dangerous-default-value # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/src/instructlab/eval/mmlu.py b/src/instructlab/eval/mmlu.py index 261dbbd..20b4375 100644 --- a/src/instructlab/eval/mmlu.py +++ b/src/instructlab/eval/mmlu.py @@ -11,7 +11,7 @@ # First Party from instructlab.eval.evaluator import Evaluator -MMLU_TASKS = ( +MMLU_TASKS = [ "mmlu_abstract_algebra", "mmlu_anatomy", "mmlu_astronomy", @@ -73,7 +73,7 @@ "mmlu_us_foreign_policy", "mmlu_virology", "mmlu_world_religions", -) +] class MMLUEvaluator(Evaluator): @@ -91,7 +91,7 @@ class MMLUEvaluator(Evaluator): def __init__( self, model_path, - tasks: tuple[str, ...] = MMLU_TASKS, + tasks: list[str] = MMLU_TASKS, model_dtype="bfloat16", few_shots: int = 2, batch_size: int = 5,