Add CI option to run full standard test on Debug

Also: - Minor edits to comments
KhiopsML · Jan 24, 2024 · 2e12ffb · 2e12ffb
1 parent 2894fdf
commit 2e12ffb
Showing 1 changed file with 114 additions and 64 deletions.
diff --git a/.github/workflows/run-standard-tests.yml b/.github/workflows/run-standard-tests.yml
@@ -1,13 +1,33 @@
 ---
 name: Run Standard Tests
 # Run minimal end-to-end tests for Khiops, Khiops Coclustering and KNITransfer
-# Tests are lauched on Linux, Windows and macOS ; in serial and in parallel (3 procs)
-# On release, it runs Standard test for binary. Plus MultiTables/SpliceJunction for KNITransfer
-# On debug, it runs only Standard/Iris for Coclustering and KNITransfer and Standard/IrisLight for Khiops
-# In case of errors, test results are available as artifacts for 7 days
-# It is automatically triggered on PR when sources files are modified (files in src directory)
+# The tests are launched on:
+# - OS: Linux, Windows and macOS
+# - Running mode: Serial and parallel (3 processes)
+#   - Only Khiops is tested in parallel
+# - Build: Release and Debug
+#
+# Tests for Release build or when debug-test-all is enabled:
+# - TestKhiops/Standard
+# - TestCoclustering/Standard
+# - TestKNITransfer/Standard
+# - TestKNITransfer/MultiTables/SpliceJunction
+#
+# Test for Debug build
+# - TestKhiops/Standard/IrisLight
+# - TestCoclustering/Standard/Iris
+# - TestKNITransfer/Standard/Iris
+#
+# In case of errors, test results are available as artifacts for 7 days.  It's automatically
+# triggered on PR when sources files, the LearningTest directory or the workflow itself are
+# modified.
 on:
   workflow_dispatch:
+    inputs:
+      debug-test-all:
+        type: boolean
+        default: false
+        description: Run all Standard tests on Debug
   pull_request:
     paths:
       - '**CMakeLists.txt'
@@ -19,12 +39,18 @@ on:
       - src/**.lex
       - src/**.yac
       - test/LearningTest/**
+      - .github/workflows/run-standard-tests.yml
+# Cancel a run in progress if the workflow is automatically re-triggered in the same PR
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref
+    }}
+  cancel-in-progress: true
 env:
   KhiopsBatchMode: true
 jobs:
-  # Build the project on each platform for all targets to ensure that all sources and cmake files are correct
-  # The useful binaries are cached for the current run_id. It will be restored only in the same run of the
-  # build. The cached binaries are delete in the last job (cleanup-cache)
+  # Build the project on each platform for all targets to ensure that all sources and cmake files
+  # are correct. The useful binaries are cached for the current run_id. It will be restored only in
+  # the same run of the build. The cached binaries are delete in the last job (cleanup-cache).
   build-full-project:
     strategy:
       matrix:
@@ -38,16 +64,17 @@ jobs:
       PRESET_NAME: ${{ matrix.build-setup.cmake-preset }}-${{ matrix.config }}
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Build all binaries
         uses: ./.github/actions/build-khiops
         with:
           preset-name: ${{ env.PRESET_NAME }}
       - name: Cache binaries
         id: cache-binaries-unix
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
-          # We add binaries path one by one to avoid *.pdb and other msvc stuffs that generate a cache of 200Mo for windows
+          # We add binaries path one by one to avoid *.pdb and other msvc stuffs that generate a
+          # cache of 200Mo for windows
           path: |
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL_Coclustering
@@ -77,10 +104,10 @@ jobs:
       PRESET_NAME: ${{ matrix.build-setup.cmake-preset }}-${{ matrix.config }}
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Restore cached binaries
         id: restore-binaries
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: |
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL
@@ -96,70 +123,97 @@ jobs:
             }}
           fail-on-cache-miss: true
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
+      - name: Setup MPI
+        uses: mpi4py/setup-mpi@v1
       - name: Set environment variables
         shell: bash
         run: |
+          # Set the paths of the test scripts
           echo "TEST_PY=test/LearningTest/cmd/python/test_khiops.py" >> "$GITHUB_ENV"
           echo "APPLY_PY=test/LearningTest/cmd/python/apply_command.py" >> "$GITHUB_ENV"
-          echo "BIN_PATH=${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin" >> "$GITHUB_ENV"
-      - name: Setup MPI
-        uses: mpi4py/setup-mpi@v1
-      - name: Run Tests
-      # Run test on linux and macOS as describe in the top comment:
-      # - In parallel, run only MODL tests
-      # - In debug run only Standard/Iris for KNITRansfer and MODL_Coclustering, Standard/IrisLight for MODL
-      # - In release run all Standard tests
+
+          # Set the paths of the Khiops executables to test
+          BIN_DIR="${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin"
+          if [[ "${{ matrix.build-setup.os }}"  == 'windows-2022' ]]
+          then
+            EXT=".exe"
+          else
+            EXT=""
+          fi
+          echo "MODL_BIN=${BIN_DIR}/MODL${EXT}" >> "$GITHUB_ENV"
+          echo "MODL_CC_BIN=${BIN_DIR}/MODL_Coclustering${EXT}" >> "$GITHUB_ENV"
+          echo "KNI_BIN=${BIN_DIR}/KNITransfer${EXT}" >> "$GITHUB_ENV"
+
+          # Set the MPI settings if necessary
+          if [[ "${{ matrix.running-mode }}" == "parallel" ]] ; then
+            echo "KhiopsMPIProcessNumber=3" >> "$GITHUB_ENV"
+          fi
+      - name: Run Tests (Release or Debug Full)
         shell: bash
+        if: matrix.config == 'release' || inputs.debug-test-all
         run: |
-          if [[ "${{ matrix.build-setup.os }}"  == 'windows-2022' ]]; then
-            export EXT=".exe"
+          set -x
+          # See the top comment to see which tests are launched
+          python "${TEST_PY}" Khiops $MODL_BIN Standard
+          if [[ "${{ matrix.running-mode }}" == "serial" ]]
+          then
+            python "${TEST_PY}" Coclustering $MODL_CC_BIN Standard
+            python "${TEST_PY}" KNI $KNI_BIN Standard
+            python "${TEST_PY}" KNI $KNI_BIN MultiTables SpliceJunction
           fi
-          if [[ "${{ matrix.running-mode }}" == "parallel" ]] ; then
-            export KhiopsMPIProcessNumber=3
+      - name: Run Tests (Debug)
+        shell: bash
+        if: matrix.config == 'debug' && ! inputs.debug-test-all
+        run: |
+          set -x
+          # See the top comment to see which tests are launched
+          python "${TEST_PY}" Khiops $MODL_BIN Standard IrisLight
+          if [[ "${{ matrix.running-mode }}" == "serial" ]]
+          then
+            python "${TEST_PY}" Coclustering $MODL_CC_BIN Standard Iris
+            python "${TEST_PY}" KNI $KNI_BIN Standard Iris
           fi
-          if [[ "${{ matrix.config }}" == "release" ]] ; then
-            python $TEST_PY Khiops ${BIN_PATH}/MODL${EXT} Standard
-            if [[ "${{ matrix.running-mode }}" != "parallel" ]] ; then
-              python $TEST_PY Coclustering ${BIN_PATH}/MODL_Coclustering${EXT} Standard
-              python $TEST_PY KNI ${BIN_PATH}/KNITransfer${EXT} Standard
-              python $TEST_PY KNI ${BIN_PATH}/KNITransfer${EXT} MultiTables SpliceJunction
-            fi
-          else
-            python $TEST_PY Khiops ${BIN_PATH}/MODL${EXT} Standard IrisLight
-            if [[ "${{ matrix.running-mode }}" != "parallel" ]] ; then
-              python $TEST_PY Coclustering ${BIN_PATH}/MODL_Coclustering${EXT} Standard Iris
-              python $TEST_PY KNI ${BIN_PATH}/KNITransfer${EXT} Standard Iris
-            fi
+      - name: Collect Test Results (Release or Debug Full)
+        shell: bash
+        if: (success() || failure()) && (matrix.config == 'release' || inputs.debug-test-all)
+        run: |
+          python $APPLY_PY errors test/LearningTest/TestKhiops/Standard \
+          | tee test/LearningTest/TestKhiops/Standard/errors.txt
+          if [[ "${{ matrix.running-mode }}" == "serial" ]]
+          then
+            python $APPLY_PY errors test/LearningTest/TestCoclustering/Standard \
+              | tee test/LearningTest/TestCoclustering/Standard/errors.txt
+            python $APPLY_PY errors test/LearningTest/TestKNITransfer/Standard \
+              | tee test/LearningTest/TestKNITransfer/Standard/errors.txt
+            python $APPLY_PY errors test/LearningTest/TestKNITransfer/MultiTables SpliceJunction \
+              | tee test/LearningTest/TestKNITransfer/MultiTables/errors.txt
           fi
-      - name: Collect results
+      - name: Collect Test Results (Debug)
         shell: bash
-        if: success() || failure()
+        if: (success() || failure()) && (matrix.config == 'debug' && ! inputs.debug-test-all)
         run: |
-          if [[ "${{ matrix.config }}" == "release" ]] ; then
-            python3 $APPLY_PY errors test/LearningTest/TestKhiops/Standard | tee test/LearningTest/TestKhiops/Standard/errors.txt
-            if [[ "${{ matrix.running-mode }}" != "parallel" ]] ; then
-              python $APPLY_PY errors test/LearningTest/TestCoclustering/Standard | tee test/LearningTest/TestCoclustering/Standard/errors.txt
-              python $APPLY_PY errors test/LearningTest/TestKNITransfer/Standard | tee test/LearningTest/TestKNITransfer/Standard/errors.txt
-              python $APPLY_PY errors test/LearningTest/TestKNITransfer/MultiTables SpliceJunction | tee test/LearningTest/TestKNITransfer/MultiTables/errors.txt
-            fi
-          else
-            python3 $APPLY_PY errors test/LearningTest/TestKhiops/Standard IrisLight | tee test/LearningTest/TestKhiops/Standard/errors.txt
-            if [[ "${{ matrix.running-mode }}" != "parallel" ]] ; then
-              python $APPLY_PY errors test/LearningTest/TestCoclustering/Standard Iris | tee test/LearningTest/TestCoclustering/Standard/errors.txt
-              python $APPLY_PY errors test/LearningTest/TestKNITransfer/Standard Iris | tee test/LearningTest/TestKNITransfer/Standard/errors.txt
-            fi
+          python $APPLY_PY errors test/LearningTest/TestKhiops/Standard IrisLight \
+            | tee test/LearningTest/TestKhiops/Standard/errors.txt
+          if [[ "${{ matrix.running-mode }}" == "serial" ]]
+          then
+            python $APPLY_PY errors test/LearningTest/TestCoclustering/Standard Iris \
+              | tee test/LearningTest/TestCoclustering/Standard/errors.txt
+            python $APPLY_PY errors test/LearningTest/TestKNITransfer/Standard Iris \
+              | tee test/LearningTest/TestKNITransfer/Standard/errors.txt
           fi
       - name: Check results
         shell: bash
         run: |
-          if grep -qr "error" --include="errors.txt" test/LearningTest/ ; then
+          if grep -qr "error" --include="errors.txt" test/LearningTest/
+          then
             echo "::error::Errors in test"
             false
           fi
-          if grep -qr "The test has not been launched" --include="errors.txt" test/LearningTest/ ; then
+          if grep -qr "not been launched" --include="errors.txt" test/LearningTest/
+          then
             echo "::error::Test not launched"
             false
           fi
@@ -170,22 +224,18 @@ jobs:
           name: test-results-${{ env.PRESET_NAME }}-${{ matrix.running-mode}}
           retention-days: 7
           path: |-
-            test/LearningTest/**/results
-            test/LearningTest/**/results.ref
-            test/LearningTest/**/errors.txt
             test/LearningTest/**/comparisonResults.log
+            test/LearningTest/**/errors.txt
+            test/LearningTest/**/results
+            test/LearningTest/**/results.ref*
   cleanup-cache:
     # Clean up all caches belonging to the current run
-    if: ${{ always() }}
+    if: always()
     needs: run-standard-tests
     runs-on: ubuntu-latest
     permissions:
       actions: write
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-        with:
-          sparse-checkout: .git
       - name: Cleanup cache
         shell: bash
         env: