diff --git a/.github/workflows/run-standard-tests.yml b/.github/workflows/run-standard-tests.yml index 472e6f165..f5799d1b0 100644 --- a/.github/workflows/run-standard-tests.yml +++ b/.github/workflows/run-standard-tests.yml @@ -1,13 +1,33 @@ --- name: Run Standard Tests # Run minimal end-to-end tests for Khiops, Khiops Coclustering and KNITransfer -# Tests are lauched on Linux, Windows and macOS ; in serial and in parallel (3 procs) -# On release, it runs Standard test for binary. Plus MultiTables/SpliceJunction for KNITransfer -# On debug, it runs only Standard/Iris for Coclustering and KNITransfer and Standard/IrisLight for Khiops -# In case of errors, test results are available as artifacts for 7 days -# It is automatically triggered on PR when sources files are modified (files in src directory) +# The tests are launched on: +# - OS: Linux, Windows and macOS +# - Running mode: Serial and parallel (3 processes) +# - Only Khiops is tested in parallel +# - Build: Release and Debug +# +# Tests for Release build or when debug-test-all is enabled: +# - TestKhiops/Standard +# - TestCoclustering/Standard +# - TestKNITransfer/Standard +# - TestKNITransfer/MultiTables/SpliceJunction +# +# Test for Debug build +# - TestKhiops/Standard/IrisLight +# - TestCoclustering/Standard/Iris +# - TestKNITransfer/Standard/Iris +# +# In case of errors, test results are available as artifacts for 7 days. It's automatically +# triggered on PR when sources files, the LearningTest directory or the workflow itself are +# modified. on: workflow_dispatch: + inputs: + debug-test-all: + type: boolean + default: false + description: Run all Standard tests on Debug pull_request: paths: - '**CMakeLists.txt' @@ -19,12 +39,18 @@ on: - src/**.lex - src/**.yac - test/LearningTest/** + - .github/workflows/run-standard-tests.yml +# Cancel a run in progress if the workflow is automatically re-triggered in the same PR +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref + }} + cancel-in-progress: true env: KhiopsBatchMode: true jobs: - # Build the project on each platform for all targets to ensure that all sources and cmake files are correct - # The useful binaries are cached for the current run_id. It will be restored only in the same run of the - # build. The cached binaries are delete in the last job (cleanup-cache) + # Build the project on each platform for all targets to ensure that all sources and cmake files + # are correct. The useful binaries are cached for the current run_id. It will be restored only in + # the same run of the build. The cached binaries are delete in the last job (cleanup-cache). build-full-project: strategy: matrix: @@ -38,16 +64,17 @@ jobs: PRESET_NAME: ${{ matrix.build-setup.cmake-preset }}-${{ matrix.config }} steps: - name: Checkout sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build all binaries uses: ./.github/actions/build-khiops with: preset-name: ${{ env.PRESET_NAME }} - name: Cache binaries id: cache-binaries-unix - uses: actions/cache@v3 + uses: actions/cache@v4 with: - # We add binaries path one by one to avoid *.pdb and other msvc stuffs that generate a cache of 200Mo for windows + # We add binaries path one by one to avoid *.pdb and other msvc stuffs that generate a + # cache of 200Mo for windows path: | ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL_Coclustering @@ -77,10 +104,10 @@ jobs: PRESET_NAME: ${{ matrix.build-setup.cmake-preset }}-${{ matrix.config }} steps: - name: Checkout sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore cached binaries id: restore-binaries - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: | ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL @@ -96,70 +123,97 @@ jobs: }} fail-on-cache-miss: true - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' + - name: Setup MPI + uses: mpi4py/setup-mpi@v1 - name: Set environment variables shell: bash run: | + # Set the paths of the test scripts echo "TEST_PY=test/LearningTest/cmd/python/test_khiops.py" >> "$GITHUB_ENV" echo "APPLY_PY=test/LearningTest/cmd/python/apply_command.py" >> "$GITHUB_ENV" - echo "BIN_PATH=${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin" >> "$GITHUB_ENV" - - name: Setup MPI - uses: mpi4py/setup-mpi@v1 - - name: Run Tests - # Run test on linux and macOS as describe in the top comment: - # - In parallel, run only MODL tests - # - In debug run only Standard/Iris for KNITRansfer and MODL_Coclustering, Standard/IrisLight for MODL - # - In release run all Standard tests + + # Set the paths of the Khiops executables to test + BIN_DIR="${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin" + if [[ "${{ matrix.build-setup.os }}" == 'windows-2022' ]] + then + EXT=".exe" + else + EXT="" + fi + echo "MODL_BIN=${BIN_DIR}/MODL${EXT}" >> "$GITHUB_ENV" + echo "MODL_CC_BIN=${BIN_DIR}/MODL_Coclustering${EXT}" >> "$GITHUB_ENV" + echo "KNI_BIN=${BIN_DIR}/KNITransfer${EXT}" >> "$GITHUB_ENV" + + # Set the MPI settings if necessary + if [[ "${{ matrix.running-mode }}" == "parallel" ]] ; then + echo "KhiopsMPIProcessNumber=3" >> "$GITHUB_ENV" + fi + - name: Run Tests (Release or Debug Full) shell: bash + if: matrix.config == 'release' || inputs.debug-test-all run: | - if [[ "${{ matrix.build-setup.os }}" == 'windows-2022' ]]; then - export EXT=".exe" + set -x + # See the top comment to see which tests are launched + python "${TEST_PY}" Khiops $MODL_BIN Standard + if [[ "${{ matrix.running-mode }}" == "serial" ]] + then + python "${TEST_PY}" Coclustering $MODL_CC_BIN Standard + python "${TEST_PY}" KNI $KNI_BIN Standard + python "${TEST_PY}" KNI $KNI_BIN MultiTables SpliceJunction fi - if [[ "${{ matrix.running-mode }}" == "parallel" ]] ; then - export KhiopsMPIProcessNumber=3 + - name: Run Tests (Debug) + shell: bash + if: matrix.config == 'debug' && ! inputs.debug-test-all + run: | + set -x + # See the top comment to see which tests are launched + python "${TEST_PY}" Khiops $MODL_BIN Standard IrisLight + if [[ "${{ matrix.running-mode }}" == "serial" ]] + then + python "${TEST_PY}" Coclustering $MODL_CC_BIN Standard Iris + python "${TEST_PY}" KNI $KNI_BIN Standard Iris fi - if [[ "${{ matrix.config }}" == "release" ]] ; then - python $TEST_PY Khiops ${BIN_PATH}/MODL${EXT} Standard - if [[ "${{ matrix.running-mode }}" != "parallel" ]] ; then - python $TEST_PY Coclustering ${BIN_PATH}/MODL_Coclustering${EXT} Standard - python $TEST_PY KNI ${BIN_PATH}/KNITransfer${EXT} Standard - python $TEST_PY KNI ${BIN_PATH}/KNITransfer${EXT} MultiTables SpliceJunction - fi - else - python $TEST_PY Khiops ${BIN_PATH}/MODL${EXT} Standard IrisLight - if [[ "${{ matrix.running-mode }}" != "parallel" ]] ; then - python $TEST_PY Coclustering ${BIN_PATH}/MODL_Coclustering${EXT} Standard Iris - python $TEST_PY KNI ${BIN_PATH}/KNITransfer${EXT} Standard Iris - fi + - name: Collect Test Results (Release or Debug Full) + shell: bash + if: (success() || failure()) && (matrix.config == 'release' || inputs.debug-test-all) + run: | + python $APPLY_PY errors test/LearningTest/TestKhiops/Standard \ + | tee test/LearningTest/TestKhiops/Standard/errors.txt + if [[ "${{ matrix.running-mode }}" == "serial" ]] + then + python $APPLY_PY errors test/LearningTest/TestCoclustering/Standard \ + | tee test/LearningTest/TestCoclustering/Standard/errors.txt + python $APPLY_PY errors test/LearningTest/TestKNITransfer/Standard \ + | tee test/LearningTest/TestKNITransfer/Standard/errors.txt + python $APPLY_PY errors test/LearningTest/TestKNITransfer/MultiTables SpliceJunction \ + | tee test/LearningTest/TestKNITransfer/MultiTables/errors.txt fi - - name: Collect results + - name: Collect Test Results (Debug) shell: bash - if: success() || failure() + if: (success() || failure()) && (matrix.config == 'debug' && ! inputs.debug-test-all) run: | - if [[ "${{ matrix.config }}" == "release" ]] ; then - python3 $APPLY_PY errors test/LearningTest/TestKhiops/Standard | tee test/LearningTest/TestKhiops/Standard/errors.txt - if [[ "${{ matrix.running-mode }}" != "parallel" ]] ; then - python $APPLY_PY errors test/LearningTest/TestCoclustering/Standard | tee test/LearningTest/TestCoclustering/Standard/errors.txt - python $APPLY_PY errors test/LearningTest/TestKNITransfer/Standard | tee test/LearningTest/TestKNITransfer/Standard/errors.txt - python $APPLY_PY errors test/LearningTest/TestKNITransfer/MultiTables SpliceJunction | tee test/LearningTest/TestKNITransfer/MultiTables/errors.txt - fi - else - python3 $APPLY_PY errors test/LearningTest/TestKhiops/Standard IrisLight | tee test/LearningTest/TestKhiops/Standard/errors.txt - if [[ "${{ matrix.running-mode }}" != "parallel" ]] ; then - python $APPLY_PY errors test/LearningTest/TestCoclustering/Standard Iris | tee test/LearningTest/TestCoclustering/Standard/errors.txt - python $APPLY_PY errors test/LearningTest/TestKNITransfer/Standard Iris | tee test/LearningTest/TestKNITransfer/Standard/errors.txt - fi + python $APPLY_PY errors test/LearningTest/TestKhiops/Standard IrisLight \ + | tee test/LearningTest/TestKhiops/Standard/errors.txt + if [[ "${{ matrix.running-mode }}" == "serial" ]] + then + python $APPLY_PY errors test/LearningTest/TestCoclustering/Standard Iris \ + | tee test/LearningTest/TestCoclustering/Standard/errors.txt + python $APPLY_PY errors test/LearningTest/TestKNITransfer/Standard Iris \ + | tee test/LearningTest/TestKNITransfer/Standard/errors.txt fi - name: Check results shell: bash run: | - if grep -qr "error" --include="errors.txt" test/LearningTest/ ; then + if grep -qr "error" --include="errors.txt" test/LearningTest/ + then echo "::error::Errors in test" false fi - if grep -qr "The test has not been launched" --include="errors.txt" test/LearningTest/ ; then + if grep -qr "not been launched" --include="errors.txt" test/LearningTest/ + then echo "::error::Test not launched" false fi @@ -170,22 +224,18 @@ jobs: name: test-results-${{ env.PRESET_NAME }}-${{ matrix.running-mode}} retention-days: 7 path: |- - test/LearningTest/**/results - test/LearningTest/**/results.ref - test/LearningTest/**/errors.txt test/LearningTest/**/comparisonResults.log + test/LearningTest/**/errors.txt + test/LearningTest/**/results + test/LearningTest/**/results.ref* cleanup-cache: # Clean up all caches belonging to the current run - if: ${{ always() }} + if: always() needs: run-standard-tests runs-on: ubuntu-latest permissions: actions: write steps: - - name: Checkout sources - uses: actions/checkout@v3 - with: - sparse-checkout: .git - name: Cleanup cache shell: bash env: