Skip to content

Commit

Permalink
Change ret value from single dict to multiple ret values
Browse files Browse the repository at this point in the history
Signed-off-by: Nathan Weinberg <[email protected]>
  • Loading branch information
nathan-weinberg committed Jun 17, 2024
1 parent 4e72d07 commit d9204eb
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 17 deletions.
17 changes: 4 additions & 13 deletions src/instructlab/eval/mmlu.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,7 @@ def __init__(
def run(self) -> dict:
individual_scores: dict[str, float] = {}
overall_score: float = 0.0
payload = {
"individual_scores": individual_scores,
"overall_score": overall_score,
}
return payload
return overall_score, individual_scores


class PR_MMLU_Evaluator(Evaluator):
Expand All @@ -39,8 +35,8 @@ class PR_MMLU_Evaluator(Evaluator):
Attributes:
sdg_path path where all the PR MMLU tasks are stored
task group name that is shared by all the PR MMLU tasks
few_shots number of examples
batch_size number of GPUs
few_shots number of examples
batch_size number of GPUs
"""

def __init__(
Expand All @@ -61,9 +57,4 @@ def run(self) -> dict:
individual_scores: dict[str, float] = {}
overall_score: float = 0.0
qa_pairs: list[tuple] = []
payload = {
"individual_scores": individual_scores,
"overall_score": overall_score,
"qa_pairs": qa_pairs,
}
return payload
return overall_score, individual_scores, qa_pairs
6 changes: 2 additions & 4 deletions src/instructlab/eval/mtbench.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@ def __init__(self, model_path, server_url: str) -> None:
def run(self) -> dict:
overall_score: float = 0.0
qa_pairs: list[tuple] = []
payload = {"overall_score": overall_score, "qa_pairs": qa_pairs}
return payload
return overall_score, qa_pairs


class PR_Bench_Evaluator(Evaluator):
Expand All @@ -40,5 +39,4 @@ def __init__(self, model_path, server_url: str, questions: str) -> None:
def run(self) -> dict:
overall_score = 0.0
qa_pairs: list[tuple] = []
payload = {"overall_score": overall_score, "qa_pairs": qa_pairs}
return payload
return overall_score, qa_pairs

0 comments on commit d9204eb

Please sign in to comment.