From 87ab89784eaeaf7ae822833bf12791fe61848829 Mon Sep 17 00:00:00 2001
From: Nathan Weinberg <nweinber@redhat.com>
Date: Tue, 25 Jun 2024 08:04:29 -0400
Subject: [PATCH 1/2] Add e2e job to CI

Signed-off-by: Nathan Weinberg <nweinber@redhat.com>
---
 .github/actionlint.yaml   |  3 ++
 .github/workflows/e2e.yml | 92 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)
 create mode 100644 .github/actionlint.yaml
 create mode 100644 .github/workflows/e2e.yml

diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml
new file mode 100644
index 0000000..53889f9
--- /dev/null
+++ b/.github/actionlint.yaml
@@ -0,0 +1,3 @@
+self-hosted-runner:
+  labels:
+  - ubuntu-gpu
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
new file mode 100644
index 0000000..be8c420
--- /dev/null
+++ b/.github/workflows/e2e.yml
@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: E2E test
+
+on:
+  push:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      - '**.py'
+      - 'pyproject.toml'
+      - 'requirements*.txt'
+      - '.github/workflows/e2e.yml'
+  pull_request:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      - '**.py'
+      - 'pyproject.toml'
+      - 'requirements*.txt'
+      - '.github/workflows/e2e.yml'
+
+jobs:
+  e2e:
+    runs-on: ubuntu-gpu
+
+    permissions:
+      pull-requests: write
+
+    steps:
+      # No step-security/harden-runner since this is a self-hosted runner
+      - name: Checkout
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          # https://github.com/actions/checkout/issues/249
+          fetch-depth: 0
+
+      - name: Install Packages
+        run: |
+          sudo apt-get install -y cuda-toolkit git cmake build-essential virtualenv
+          nvidia-smi
+          ls -l /dev/nvidia*
+
+      - name: Setup Python 3.11
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: 3.11
+          cache: pip
+          cache-dependency-path: |
+            **/pyproject.toml
+            **/requirements*.txt
+
+      - name: Remove llama-cpp-python from cache
+        run: |
+          pip cache remove llama_cpp_python
+
+      - name: Cache huggingface
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: ~/.cache/huggingface
+          # config contains DEFAULT_MODEL
+          key: huggingface-${{ hashFiles('src/instructlab/configuration.py') }}
+
+      - name: Install instructlab and instructlab-eval
+        run: |
+          export PATH="/home/runner/.local/bin:/usr/local/cuda/bin:$PATH"
+          python3 -m venv venv
+          . venv/bin/activate
+          git clone https://github.com/instructlab/instructlab.git
+          cd instructlab
+          sed 's/\[.*\]//' requirements.txt > constraints.txt
+          python3 -m pip cache remove llama_cpp_python
+          CMAKE_ARGS="-DLLAMA_CUBLAS=on" python3 -m pip install --no-binary llama_cpp_python -c constraints.txt llama_cpp_python
+          # needed for --4-bit-quant option to ilab train
+          python3 -m pip install bitsandbytes
+          # install instructlab
+          python3 -m pip install .
+          cd ..
+          # Install instructlab-eval
+          python3 -m pip install .
+
+      - name: Run e2e test
+        run: |
+          . venv/bin/activate
+          ./instructlab/scripts/basic-workflow-tests.sh -cm
+
+      - name: Remove llama-cpp-python from cache
+        if: always()
+        run: |
+          pip cache remove llama_cpp_python

From 8c0ff1d9d864678dc43d6702d6557e0c70b0eefd Mon Sep 17 00:00:00 2001
From: Nathan Weinberg <nweinber@redhat.com>
Date: Tue, 25 Jun 2024 13:00:51 -0400
Subject: [PATCH 2/2] Updated flow based on BJ's feedback

Co-authored-by: BJ Hargrave <hargrave@us.ibm.com>
Signed-off-by: Nathan Weinberg <nweinber@redhat.com>
---
 .github/workflows/e2e.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index be8c420..6478b26 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -31,12 +31,19 @@ jobs:
 
     steps:
       # No step-security/harden-runner since this is a self-hosted runner
-      - name: Checkout
+      - name: Checkout instructlab/eval
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           # https://github.com/actions/checkout/issues/249
           fetch-depth: 0
 
+      - name: Checkout instructlab/instructlab
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          repository: "instructlab/instructlab"
+          path: "instructlab"
+          fetch-depth: 0
+
       - name: Install Packages
         run: |
           sudo apt-get install -y cuda-toolkit git cmake build-essential virtualenv
@@ -68,7 +75,6 @@ jobs:
           export PATH="/home/runner/.local/bin:/usr/local/cuda/bin:$PATH"
           python3 -m venv venv
           . venv/bin/activate
-          git clone https://github.com/instructlab/instructlab.git
           cd instructlab
           sed 's/\[.*\]//' requirements.txt > constraints.txt
           python3 -m pip cache remove llama_cpp_python