Add facilities for unit and functional tests

There was one unit test already existing and this commit adds a couple of starter unit tests for mt_bench and mmlu.

Signed-off-by: Dan McPherson <[email protected]>

.github/workflows/test.yml

@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      - '**.py'
+      - 'pyproject.toml'
+      - 'requirements**.txt'
+      - 'tox.ini'
+      - 'scripts/*.sh' # Used by this workflow
+      - '.github/workflows/test.yml' # This workflow
+  pull_request:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      - '**.py'
+      - 'pyproject.toml'
+      - 'requirements**.txt'
+      - 'tox.ini'
+      - 'scripts/*.sh' # Used by this workflow
+      - '.github/workflows/test.yml' # This workflow
+
+env:
+  LC_ALL: en_US.UTF-8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: "test: ${{ matrix.python }} on ${{ matrix.platform }}"
+    runs-on: "${{ matrix.platform }}"
+    strategy:
+      matrix:
+        python:
+          - "3.10"
+          - "3.11"
+        platform:
+          - "ubuntu-latest"
+        include:
+          - python: "3.11"
+            platform: "macos-latest"
+    steps:
+      - name: "Harden Runner"
+        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          # https://github.com/actions/checkout/issues/249
+          fetch-depth: 0
+
+      - name: Free disk space
+        if: matrix.platform != 'macos-latest'
+        uses: ./.github/actions/free-disk-space
+
+      - name: Install the expect package
+        if: startsWith(matrix.platform, 'ubuntu')
+        run: |
+          sudo apt-get install -y expect
+
+      - name: Install tools on MacOS
+        if: startsWith(matrix.platform, 'macos')
+        run: |
+          brew install expect coreutils bash
+
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+          cache-dependency-path: |
+            **/pyproject.toml
+            **/requirements*.txt
+
+      - name: Remove llama-cpp-python from cache
+        run: |
+          pip cache remove llama_cpp_python
+
+      - name: Cache huggingface
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+        with:
+          path: ~/.cache/huggingface
+          # config contains DEFAULT_MODEL
+          key: huggingface-${{ hashFiles('src/instructlab/configuration.py') }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh>=1.2
+
+      - name: Run unit and functional tests with tox
+        run: |
+          tox
+
+      - name: Remove llama-cpp-python from cache
+        if: always()
+        run: |
+          pip cache remove llama_cpp_python
+
+  test-workflow-complete:
+    needs: ["test"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Test Workflow Complete
+        run: echo "Test Workflow Complete"

Makefile

@@ -54,3 +54,9 @@ spellcheck-sort: .spellcheck-en-custom.txt ## Sort spellcheck directory
 .PHONY: verify
 verify: check-tox ## Run linting, typing, and formatting checks via tox
 	tox p -e fastlint,mypy,ruff
+
+##@ Development
+
+.PHONY: tests
+tests: check-tox ## Run unit and type checks
+	tox -e py3-unit,mypy

README.md

@@ -107,8 +107,8 @@ In another shell window
 ```shell
 export INSTRUCTLAB_EVAL_FIRST_N_QUESTIONS=10 # Optional if you want to shorten run times
 # Commands relative to eval directory
-python3 tests/test_gen_answers.py
-python3 tests/test_branch_gen_answers.py
+python3 scripts/test_gen_answers.py
+python3 scripts/test_branch_gen_answers.py
 ```
 
 Example output tree
@@ -139,8 +139,8 @@ eval_output/
 ```
 
 ```shell
-python3 tests/test_judge_answers.py
-python3 tests/test_branch_judge_answers.py
+python3 scripts/test_judge_answers.py
+python3 scripts/test_branch_judge_answers.py
 ```
 
 Example output tree