Initial skeleton for Evaluator classes and exceptions

src/instructlab/eval/evaluator.py

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: Apache-2.0
+
+
+class Evaluator:
+    """
+    Parent class for Evaluators
+
+    Atttributes:
+        model   The model to be evaluated
+    """
+
+    def __init__(self, model: str) -> None:
+        self.model = model
+
+    def run(self) -> dict:
+        return {}

src/instructlab/eval/exceptions.py

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+
+
+class EvalError(Exception):
+    """
+    Parent class for all of instructlab-eval exceptions
+    """
+
+
+class ModelNotFoundError(EvalError):
+    """
+    Exception raised when model is not able to be found
+
+    Attributes
+        model   model that is being operated on
+    """
+
+    def __init__(self, model) -> None:
+        super().__init__()
+        self.model = model
+        self.message = f"Model {self.model} could not be found"

src/instructlab/eval/mmlu.py

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# Local
+from .evaluator import Evaluator
+
+
+class MMLU_Evaluator(Evaluator):
+    """
+    Child class of an Evaluator for Massive Multitask Language Understanding (MMLU)
+
+    Attributes:
+        tasks       list of tasks for MMLU to test the model with
+        fewshots    number of examples
+        batchsize   number of GPUs
+    """
+
+    def __init__(
+        self, model, tasks: list[str], fewshots: int = 2, batchsize: int = 5
+    ) -> None:
+        super().__init__(model)
+        self.tasks = tasks
+        self.fewshots = fewshots
+        self.batchsize = batchsize
+
+
+class PR_MMLU_Evaluator(Evaluator):
+    """
+    Child class of an Evaluator for PR Massive Multitask Language Understanding (PR MMLU)
+
+    Attributes:
+        sdg_path    path where all the PR MMLU tasks are stored
+        task        group name that is shared by all the PR MMLU tasks
+        fewshots    number of examples
+        batchsize   number of GPUs
+    """
+
+    def __init__(
+        self,
+        model,
+        sdg_path: str,
+        task: str = "mmlu_pr",
+        fewshots: int = 2,
+        batchsize: int = 5,
+    ) -> None:
+        super().__init__(model)
+        self.sdg_path = sdg_path
+        self.task = task
+        self.fewshots = fewshots
+        self.batchsize = batchsize

src/instructlab/eval/mtbench.py

@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# Local
+from .evaluator import Evaluator
+
+
+class MT_Bench_Evaluator(Evaluator):
+    """
+    Child class of an Evaluator for Multi-turn Benchmark (MT-Bench)
+
+    Attributes
+        server  vLLM server endpoint
+    """
+
+    def __init__(self, model, server: str) -> None:
+        super().__init__(model)
+        self.server = server
+
+
+class PR_Bench_Evaluator(Evaluator):
+    """
+    Child class of an Evaluator for PR-Bench Benchmark (PR-Bench)
+
+    Attributes
+        server      vLLM server endpoint
+        questions   questions to be asked
+    """
+
+    def __init__(self, model, server: str, questions: str) -> None:
+        super().__init__(model)
+        self.server = server
+        self.questions = questions