Initial skeleton for Evaluator classes and exceptions

src/instructlab/eval/evaluator.py

@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: Apache-2.0
+
+
+class Evaluator:
+    """
+    Parent class for Evaluators
+    """
+
+    def __init__(self, model) -> None:
+        self.model = model
+
+    def run(self) -> dict:
+        return {}

src/instructlab/eval/exceptions.py

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: Apache-2.0
+
+
+class EvalError(Exception):
+    """
+    Parent class for all of instructlab-eval exceptions
+    """
+
+
+class ModelNotFoundError(EvalError):
+    """
+    Exception raised when model is not able to be found
+    """
+
+    def __init__(self, model) -> None:
+        super().__init__()
+        self.model = model
+        self.message = f"Model {self.model} could not be found"

src/instructlab/eval/mmlu.py

@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# Local
+from .evaluator import Evaluator
+
+
+class MMLU_Evaluator(Evaluator):
+    """
+    Child class of an Evaluator for Massive Multitask Language Understanding (MMLU)
+    """
+
+    def __init__(self, model, tasks: list[str], fewshots: int, batchsize: int) -> None:
+        super().__init__(model)
+        self.tasks = tasks
+        self.fewshots = fewshots
+        self.batchsize = batchsize
+
+
+class PR_MMLU_Evaluator(Evaluator):
+    """
+    Child class of an Evaluator for PR Massive Multitask Language Understanding (PR MMLU)
+    """
+
+    def __init__(
+        self, model, task: str, sdg_path: str, fewshots: int, batchsize: int
+    ) -> None:
+        super().__init__(model)
+        self.task = task
+        self.sdg_path = sdg_path
+        self.fewshots = fewshots
+        self.batchsize = batchsize

src/instructlab/eval/mtbench.py

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# Local
+from .evaluator import Evaluator
+
+
+class MT_Bench_Evaluator(Evaluator):
+    """
+    Child class of an Evaluator for Multi-turn Benchmark (MT-Bench)
+    """
+
+    def __init__(self, model, server: str) -> None:
+        super().__init__(model)
+        self.server = server
+
+
+class PR_Bench_Evaluator(Evaluator):
+    """
+    Child class of an Evaluator for PR-Bench Benchmark (PR-Bench)
+    """
+
+    def __init__(self, model, server: str, questions: str) -> None:
+        super().__init__(model)
+        self.server = server
+        self.questions = questions