Reactivate the integration tests for remote file access (GCS, S3) #61
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Tests | |
| env: | |
| DEFAULT_SAMPLES_REVISION: 10.2.4 | |
| DEFAULT_KHIOPS_DESKTOP_REVISION: 10.2.4 | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| samples-revision: | |
| default: 10.2.4 | |
| description: Git Tag/Branch/Commit for the khiops-samples Repo | |
| image-tag: | |
| default: latest | |
| description: Development Docker Image Tag | |
| khiops-desktop-revision: | |
| default: 10.2.4 | |
| description: Khiops Windows Desktop Application Version | |
| run-long-tests: | |
| type: boolean | |
| required: false | |
| default: false | |
| description: Execute long tests | |
| pull_request: | |
| paths: | |
| - khiops/**.py | |
| - pykhiops/**.py # TODO: Remove for Khiops 11 | |
| - tests/**.py | |
| - tests/resources/** | |
| - '!tests/resources/**.md' | |
| - .github/workflows/unit-tests.yml | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| run: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] | |
| container: | |
| # 'latest' default image tag cannot be set as an environment variable, | |
| # because the `env` context is only accessible at the step level; | |
| # hence, it is hard-coded | |
| image: |- | |
| ghcr.io/khiopsml/khiops-python/khiopspydev-ubuntu22.04:${{ inputs.image-tag || '10.2.4.0.s3-gcs-remote-files' }} | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| checks: write | |
| packages: read | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| with: | |
| # Get Git tags so that versioneer can function correctly | |
| # See issue https://github.com/actions/checkout/issues/701 | |
| fetch-depth: 0 | |
| # We move SAMPLES_REVISION to the environment so that we can use | |
| # them in both push and workflow_dispatch events | |
| - name: Set SAMPLES_REVISION on 'pull_request' events | |
| if: github.event_name == 'pull_request' | |
| run: echo "SAMPLES_REVISION=${DEFAULT_SAMPLES_REVISION}" >> "$GITHUB_ENV" | |
| - name: Set SAMPLES_REVISION on 'workflow_dispatch' events | |
| if: github.event_name == 'workflow_dispatch' | |
| run: echo "SAMPLES_REVISION=${{ inputs.samples-revision }}" >> "$GITHUB_ENV" | |
| - name: Checkout Khiops samples | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: khiopsml/khiops-samples | |
| ref: ${{ env.SAMPLES_REVISION }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| path: khiops-samples | |
| - name: Setup and Install Test Requirements | |
| if: success() || failure() | |
| run: | | |
| CONDA="/root/miniforge3/bin/conda" | |
| # Native Khiops-based Conda environment, and | |
| # `khiops-core`-based Conda environment | |
| CONDA_ENVS="py${{ matrix.python-version }} py${{ matrix.python-version }}_conda" | |
| for CONDA_ENV in $CONDA_ENVS | |
| do | |
| mkdir -p -m u+rwx reports/"$CONDA_ENV" | |
| # install within the conda environments without activating them | |
| $CONDA install -y -n "$CONDA_ENV" unittest-xml-reporting | |
| $CONDA install -y -n "$CONDA_ENV" --file test-requirements.txt | |
| done | |
| - name: Install khiops-python dependencies | |
| if: success() || failure() | |
| run: | | |
| # The following git command is required, | |
| # as the Git repository is in a directory the current user does not own, | |
| # Python versioneer fails to compute the current version correctly otherwise | |
| git config --global --add safe.directory $(realpath .) | |
| CONDA="/root/miniforge3/bin/conda" | |
| # Native Khiops-based Conda environment, and | |
| # `khiops-core`-based Conda environment | |
| CONDA_ENVS="py${{ matrix.python-version }} py${{ matrix.python-version }}_conda" | |
| for CONDA_ENV in $CONDA_ENVS | |
| do | |
| # Since Python 3.13, setuptools is not installed automatically anymore | |
| $CONDA install -y -n "$CONDA_ENV" setuptools | |
| $CONDA run --no-capture-output -n "$CONDA_ENV" python setup.py egg_info | |
| $CONDA install -y -n "$CONDA_ENV" `grep -v "^\[" khiops.egg-info/requires.txt` | |
| rm -rf khiops.egg-info | |
| done | |
| - name: Prepare Unit Tests Environment | |
| if: github.ref != 'dev' && github.ref != 'main' && ! inputs.run-long-tests | |
| run: echo "UNITTEST_ONLY_SHORT_TESTS=true" >> "$GITHUB_ENV" | |
| - name: Prepare Integration Tests on remote files | |
| env: | |
| AWS_ENDPOINT_URL: http://localhost:4569 | |
| shell: bash | |
| run: | | |
| # Prepare AWS-S3 credentials and configuration | |
| mkdir -p ${GITHUB_WORKSPACE}/.aws/ | |
| cat << EOF > ${GITHUB_WORKSPACE}/.aws/credentials | |
| [default] | |
| aws_access_key_id=KEY | |
| aws_secret_access_key=SECRET | |
| EOF | |
| cat << EOF > ${GITHUB_WORKSPACE}/.aws/configuration | |
| [default] | |
| endpoint_url=${AWS_ENDPOINT_URL} | |
| region=eu-north-1 | |
| EOF | |
| echo "Generated AWS credentials..." | |
| cat ${GITHUB_WORKSPACE}/.aws/credentials | |
| echo "Generated AWS configuration..." | |
| cat ${GITHUB_WORKSPACE}/.aws/configuration | |
| # Prepare GCS credentials | |
| touch ${GITHUB_WORKSPACE}/google-credentials.json | |
| /scripts/run_fake_remote_file_servers.sh . # launch the servers in the background | |
| - name: Run Unit & Integration Tests | |
| env: | |
| KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples | |
| KHIOPS_DOCKER_RUNNER_URL: https://localhost:11000 | |
| KHIOPS_DOCKER_RUNNER_SHARED_DIR: /tmp/sandbox | |
| KHIOPS_RUNNER_SERVICE_PATH: /scripts/run_service.sh | |
| # Force > 2 CPU cores to launch mpiexec | |
| KHIOPS_PROC_NUMBER: 4 | |
| # Oversubscribe for MPI 4.x | |
| rmaps_base_oversubscribe: true | |
| # Oversubscribe for MPI > 4.x | |
| OMPI_MCA_rmaps_base_oversubscribe: true | |
| # for the tests with GCS | |
| GCS_BUCKET_NAME: gcs-bucket | |
| # we take advantage of the built-in `STORAGE_EMULATOR_HOST` env variable | |
| # that every GCS client can read and lets us use a local fake file server | |
| STORAGE_EMULATOR_HOST: http://localhost:4443 | |
| # the C++ SDK recognizes the `CLOUD_STORAGE_EMULATOR_ENDPOINT` env variable | |
| # instead of `STORAGE_EMULATOR_HOST` | |
| CLOUD_STORAGE_EMULATOR_ENDPOINT: http://localhost:4443 | |
| # even in GCS emulation mode, the credentials file must exist | |
| GOOGLE_APPLICATION_CREDENTIALS: ${{ github.workspace }}/google-credentials.json | |
| GCS_DRIVER_LOGLEVEL: info # set to debug for diagnosis | |
| S3_DRIVER_LOGLEVEL: info # set to debug for diagnosis | |
| # for the tests with S3 | |
| S3_BUCKET_NAME: s3-bucket | |
| AWS_SHARED_CREDENTIALS_FILE: ${{ github.workspace }}/.aws/credentials | |
| AWS_CONFIG_FILE: ${{ github.workspace }}/.aws/configuration | |
| # common var for tests with GCS & S3 | |
| no_proxy: localhost | |
| run: | | |
| # This is needed so that the Git tag is parsed and the khiops-python | |
| # version is retrieved | |
| git config --global --add safe.directory $(realpath .) | |
| CONDA="/root/miniforge3/bin/conda" | |
| # Native Khiops-based Conda environment, and | |
| # `khiops-core`-based Conda environment | |
| CONDA_ENVS="py${{ matrix.python-version }} py${{ matrix.python-version }}_conda" | |
| for CONDA_ENV in $CONDA_ENVS | |
| do | |
| $CONDA run --no-capture-output -n "$CONDA_ENV" coverage run -m xmlrunner -o "reports/$CONDA_ENV" -v | |
| $CONDA run --no-capture-output -n "$CONDA_ENV" coverage report -m | |
| $CONDA run --no-capture-output -n "$CONDA_ENV" coverage xml -o "reports/$CONDA_ENV/py-coverage.xml" | |
| done | |
| - name: Display Test Reports | |
| uses: dorny/test-reporter@v1 | |
| with: | |
| name: Run Tests ${{ matrix.python-version }} | |
| path: >- | |
| reports/py${{ matrix.python-version }}/TEST-tests.*.*.xml, | |
| reports/py${{ matrix.python-version }}_conda/TEST-tests.*.*.xml | |
| reporter: java-junit | |
| path-replace-backslashes: 'true' # Necessary for windows paths | |
| - name: Upload Test Reports as Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-reports-${{ matrix.python-version }} | |
| path: |- | |
| reports/py${{ matrix.python-version }}/TEST-tests.*.*.xml | |
| reports/py${{ matrix.python-version }}/py-coverage.xml | |
| reports/py${{ matrix.python-version }}_conda/TEST-tests.*.*.xml | |
| reports/py${{ matrix.python-version }}_conda/py-coverage.xml | |
| tests/resources/scenario_generation/*/ref/*._kh | |
| tests/resources/scenario_generation/*/output/*._kh | |
| tests/resources/*/output_reports/*.txt | |
| tests/resources/*/ref_reports/*.txt | |
| tests/resources/dictionary/ref_kdic/*.kdic | |
| tests/resources/dictionary/output_kdic/*.kdic | |
| tests/resources/dictionary/copy_output_kdic/*.kdic | |
| tests/resources/general_options/general_options/*/*._kh | |
| retention-days: 7 | |
| check-khiops-integration-on-windows: | |
| runs-on: windows-2019 | |
| steps: | |
| - name: Download the Khiops Desktop NSIS Installer | |
| shell: pwsh | |
| run: | | |
| $KHIOPS_DESKTOP_REVISION = '${{ inputs.khiops-desktop-revision || env.DEFAULT_KHIOPS_DESKTOP_REVISION }}' | |
| $KHIOPS_DOWNLOAD_URL = "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_DESKTOP_REVISION}/khiops-${KHIOPS_DESKTOP_REVISION}-setup.exe" | |
| Invoke-WebRequest "${KHIOPS_DOWNLOAD_URL}" ` | |
| -OutFile .\khiops-setup.exe ` | |
| -UseBasicParsing | |
| Unblock-File .\khiops-setup.exe | |
| - name: Install the Khiops Desktop Application | |
| shell: pwsh | |
| run: | | |
| # Execute the installer | |
| $ErrorActionPreference = 'Stop' | |
| $ProgressPreference = 'SilentlyContinue' | |
| Start-Process ` | |
| -FilePath .\khiops-setup.exe ` | |
| -ArgumentList '/S' ` | |
| -Wait | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| with: | |
| # Get Git tags so that versioneer can function correctly | |
| # See issue https://github.com/actions/checkout/issues/701 | |
| fetch-depth: 0 | |
| - name: Checkout Khiops samples | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: khiopsml/khiops-samples | |
| ref: ${{ env.SAMPLES_REVISION }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| path: khiops-samples | |
| - name: Setup Python | |
| id: setup-python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install khiops-python dev dependencies | |
| shell: pwsh | |
| run: | | |
| # The following git command is required, | |
| # as the Git repository is in a directory the current user does not own, | |
| # Python versioneer fails to compute the current version correctly otherwise | |
| git config --global --add safe.directory $(Resolve-Path '.' | % {$_.toString()}) | |
| python setup.py egg_info | |
| # Install the Python requirements | |
| Get-Content .\khiops.egg-info\requires.txt ` | |
| | Select-String -Pattern '^\[' -NotMatch ` | |
| | Select-String -Pattern '^$' -NotMatch ` | |
| | % {$_.Line} ` | |
| | ForEach-Object {python -m pip install $_.toString()} | |
| Remove-Item -r -force khiops.egg-info | |
| - name: Setup and Install Test Requirements | |
| run: python -m pip install -r test-requirements.txt | |
| - name: Test Khiops Integration | |
| env: | |
| # Force > 2 CPU cores to launch MPI | |
| KHIOPS_PROC_NUMBER: 4 | |
| KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples | |
| shell: pwsh | |
| run: | | |
| $ErrorActionPreference = 'Stop' | |
| Set-StrictMode -Version Latest | |
| $PSNativeCommandUseErrorActionPreference = $true | |
| # Refresh environment variables by using the Chocolatey tool | |
| # Otherwise, the env vars set in the registry by the Khiops installer do not get updated | |
| # See also https://github.com/actions/runner-images/discussions/6065#discussioncomment-3517318 | |
| Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 | |
| refreshenv | |
| # Invoke the Python version set-up by the `setup-python` step / action | |
| # Otherwise invoking `python` directly invokes the system-cached Python on the runner | |
| # This is a side-effect of refreshing the environment | |
| $Python = "${{ steps.setup-python.outputs.python-path }}" | |
| # Print status | |
| Invoke-Expression -Command "$Python -c 'import khiops.core as kh; kh.get_runner().print_status()'" | |
| # Run integration tests on Windows | |
| Invoke-Expression -Command "$Python -m unittest -v tests.test_khiops_integrations" | |
| # Execute Khiops sample (train and deploy model) | |
| Invoke-Expression -Command "$Python -m khiops.samples.samples -i deploy_model -e" | |
| # Execute Khiops Coclustering sample (train and deploy model) | |
| Invoke-Expression -Command "$Python -m khiops.samples.samples -i deploy_coclustering -e" | |
| check-khiops-integration-on-linux: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| container: [ubuntu22.04, rocky8, rocky9] | |
| runs-on: ubuntu-latest | |
| container: | |
| # 'latest' default image tag cannot be set as an environment variable, | |
| # because the `env` context is only accessible at the step level; | |
| # hence, it is hard-coded | |
| image: |- | |
| ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:${{ inputs.image-tag || 'latest' }} | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| checks: write | |
| packages: read | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| with: | |
| # Get Git tags so that versioneer can function correctly | |
| # See issue https://github.com/actions/checkout/issues/701 | |
| fetch-depth: 0 | |
| - name: Checkout Khiops samples | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: khiopsml/khiops-samples | |
| ref: ${{ env.SAMPLES_REVISION }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| path: khiops-samples | |
| - name: Install khiops-python dev dependencies | |
| run: | | |
| # The following git command is required, | |
| # as the Git repository is in a directory the current user does not own, | |
| # Python versioneer fails to compute the current version correctly otherwise | |
| git config --global --add safe.directory $(realpath .) | |
| python setup.py egg_info | |
| pip install `grep -v "^\[" khiops.egg-info/requires.txt` | |
| rm -rf khiops.egg-info | |
| - name: Setup and Install Test Requirements | |
| run: | | |
| pip install -r test-requirements.txt | |
| - name: Test Khiops Integration | |
| env: | |
| # Force > 2 CPU cores to launch mpiexec | |
| KHIOPS_PROC_NUMBER: 4 | |
| KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples | |
| # Oversubscribe for MPI 4.x | |
| rmaps_base_oversubscribe: true | |
| # Oversubscribe for MPI > 4.x | |
| OMPI_MCA_rmaps_base_oversubscribe: true | |
| run: |- | |
| # Make sure MPI support is not loaded through env modules | |
| # Note: As Docker container's shell is non-interactive, environment | |
| # modules are currently not initializing the shell anyway | |
| if [ -n "$MODULESHOME" ]; then module unload mpi; fi | |
| # Print status | |
| python -c "import khiops.core as kh; kh.get_runner().print_status()" | |
| # Run integration tests on Ubuntu and Rocky | |
| python -m unittest -v tests.test_khiops_integrations | |
| # Execute Khiops sample (train and deploy model), which uses MPI | |
| python -m khiops.samples.samples -i deploy_model -e |