diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml deleted file mode 100644 index 45e9a7de..00000000 --- a/.github/workflows/integration-tests.yml +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: Integration Tests -env: - DEFAULT_SAMPLES_REVISION: main -on: - workflow_dispatch: - inputs: - samples-revision: - default: main - description: Git tag, branch or commit for the khiops-samples repository - run-long-tests: - type: boolean - required: false - default: false - description: Execute long tests - pull_request: -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true -jobs: - test-access-to-remote-files: - runs-on: ubuntu-latest - container: - image: ghcr.io/khiopsml/khiops-python/khiopspydev-ubuntu22.04:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged # as docker will run inside the container we need this option otherwise docker would not start - permissions: - id-token: write - contents: read - checks: write - packages: read - steps: - - name: Checkout sources - uses: actions/checkout@v4 - with: - # Get Git tags so that versioneer can function correctly - # See issue https://github.com/actions/checkout/issues/701 - fetch-depth: 0 - - name: Install khiops-python dev dependencies - run: | - # The following git command is required, - # as the Git repository is in a directory the current user does not own, - # Python versioneer fails to compute the current version correctly otherwise - git config --global --add safe.directory $(realpath .) - python setup.py egg_info - pip install `grep -v "^\[" khiops.egg-info/requires.txt` - rm -rf khiops.egg-info - - name: Setup and Install Test Requirements - run: | - pip install -r test-requirements.txt - pip install unittest-xml-reporting - - name: Launch the tests - env: - # for the tests with GCS - GCS_BUCKET_NAME: gcs-bucket - STORAGE_EMULATOR_HOST: http://localhost:4443 - # for the tests with S3 - S3_BUCKET_NAME: s3-bucket - AWS_SECRET_ACCESS_KEY: XXX - AWS_ENDPOINT_URL: http://localhost:4569 - AWS_ACCESS_KEY_ID: XXX - # common vars - # Force > 2 CPU cores to launch mpiexec - KHIOPS_PROC_NUMBER: 4 - run: |- - /scripts/run_fake_remote_file_servers.sh # launch the servers in the background - python -m xmlrunner tests.test_remote_access.KhiopsGCSRemoteFileTests -o reports/ - python -m xmlrunner tests.test_remote_access.KhiopsS3RemoteFileTests -o reports/ - - name: Display Integration Test Reports - uses: dorny/test-reporter@v1 - with: - name: Integration Tests - path: >- - reports/TEST-tests.test_remote_access.*.xml, - reporter: java-junit - path-replace-backslashes: 'true' # Necessary for windows paths - - name: Upload Test Reports as Artifacts - uses: actions/upload-artifact@v4 - with: - name: integration-test-reports - path: |- - reports/TEST-tests.test_remote_access.*.xml - test/resources/tmp/khiops-cicd/*/* - tests/resources/scenario_generation/*/ref/*._kh - tests/resources/scenario_generation/*/output/*._kh - tests/resources/*/output_reports/*.txt - tests/resources/*/ref_reports/*.txt - tests/resources/dictionary/ref_kdic/*.kdic - tests/resources/dictionary/output_kdic/*.kdic - tests/resources/dictionary/copy_output_kdic/*.kdic - tests/resources/general_options/general_options/*/*._kh - retention-days: 7 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-and-integration-tests.yml similarity index 74% rename from .github/workflows/unit-tests.yml rename to .github/workflows/unit-and-integration-tests.yml index cc9a2382..d9d6bf2c 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-and-integration-tests.yml @@ -1,5 +1,5 @@ --- -name: Unit Tests +name: Unit & Integration Tests env: DEFAULT_SAMPLES_REVISION: main on: @@ -188,3 +188,77 @@ jobs: # modules are currently not initializing the shell anyway if [ -n "$MODULESHOME" ]; then module unload mpi; fi python -m unittest -v tests.test_khiops_integrations.KhiopsRunnerEnvironmentTests.test_runner_has_mpiexec_on_linux + test-access-to-remote-files: + runs-on: ubuntu-latest + container: + image: ghcr.io/khiopsml/khiops-python/khiopspydev-ubuntu22.04:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + permissions: + id-token: write + contents: read + checks: write + packages: read + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + # Get Git tags so that versioneer can function correctly + # See issue https://github.com/actions/checkout/issues/701 + fetch-depth: 0 + - name: Install khiops-python dev dependencies + run: | + # The following git command is required, + # as the Git repository is in a directory the current user does not own, + # Python versioneer fails to compute the current version correctly otherwise + git config --global --add safe.directory $(realpath .) + python setup.py egg_info + pip install `grep -v "^\[" khiops.egg-info/requires.txt` + rm -rf khiops.egg-info + - name: Setup and Install Test Requirements + run: | + pip install -r test-requirements.txt + pip install unittest-xml-reporting + - name: Launch the tests + env: + # for the tests with GCS + GCS_BUCKET_NAME: gcs-bucket + STORAGE_EMULATOR_HOST: http://localhost:4443 + # for the tests with S3 + S3_BUCKET_NAME: s3-bucket + AWS_SECRET_ACCESS_KEY: XXX + AWS_ENDPOINT_URL: http://localhost:4569 + AWS_ACCESS_KEY_ID: XXX + # common vars + no_proxy: localhost + # Force > 2 CPU cores to launch mpiexec + KHIOPS_PROC_NUMBER: 4 + run: |- + /scripts/run_fake_remote_file_servers.sh . # launch the servers in the background + python -m xmlrunner tests.test_remote_access.KhiopsGCSRemoteFileTests -o reports/ + python -m xmlrunner tests.test_remote_access.KhiopsS3RemoteFileTests -o reports/ + - name: Display Integration Test Reports on remote files + uses: dorny/test-reporter@v1 + with: + name: Integration Tests on remote files + path: >- + reports/TEST-tests.test_remote_access.*.xml, + reporter: java-junit + path-replace-backslashes: 'true' # Necessary for windows paths + - name: Upload Test Reports as Artifacts + uses: actions/upload-artifact@v4 + with: + name: integration-test-reports-on-remote-files + path: |- + reports/TEST-tests.test_remote_access.*.xml + test/resources/tmp/khiops-cicd/*/* + tests/resources/scenario_generation/*/ref/*._kh + tests/resources/scenario_generation/*/output/*._kh + tests/resources/*/output_reports/*.txt + tests/resources/*/ref_reports/*.txt + tests/resources/dictionary/ref_kdic/*.kdic + tests/resources/dictionary/output_kdic/*.kdic + tests/resources/dictionary/copy_output_kdic/*.kdic + tests/resources/general_options/general_options/*/*._kh + retention-days: 7 diff --git a/packaging/docker/khiopspydev/Dockerfile.ubuntu b/packaging/docker/khiopspydev/Dockerfile.ubuntu index 71b0ab7c..f8719a03 100644 --- a/packaging/docker/khiopspydev/Dockerfile.ubuntu +++ b/packaging/docker/khiopspydev/Dockerfile.ubuntu @@ -10,14 +10,14 @@ ARG KHIOPS_REVISION RUN true \ # Install git (for khiops-python version calculation) and pip \ && apt-get -y update \ - && apt-get -y --no-install-recommends install git python3-pip zip pandoc wget \ + && apt-get -y --no-install-recommends install git python3-pip zip pandoc wget ruby-dev \ # Get Linux distribution codename \ && if [ -f /etc/os-release ]; then . /etc/os-release; fi \ # Obtain the Khiops native package \ && KHIOPS_PKG_FILE=$KHIOPS_REVISION/khiops-core-openmpi_$KHIOPS_REVISION-1-$VERSION_CODENAME.amd64.deb \ && wget -O KHIOPS_CORE.deb "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_PKG_FILE}" \ # Install the Khiops native package : do not break immediately if dpkg fails because apt will install the missing dependencies \ - && (dpkg -i --force-all KHIOPS_CORE.deb || apt-get -f -y install) \ + && (dpkg -i --force-all KHIOPS_CORE.deb || true) && apt-get -f -y install \ && rm -f KHIOPS_CORE.deb \ # Set python to python3 \ && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ @@ -33,18 +33,6 @@ RUN true \ && rm -fr /var/lib/apt/lists/* \ && apt-get clean \ && rm -rf ./khiops \ - # Install docker within the image to be able to launch the fake remote file servers for S3 & GCS. - # Please note that the container based on the current image must be ran with the --privileged flag - # otherwise docker would not start - && apt-get update \ - && apt-get install -y ca-certificates curl \ - && mkdir -p /etc/apt/keyrings && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \ - && chmod a+r /etc/apt/keyrings/docker.asc \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ - && apt-get update \ - && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ - && sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker \ && true # set up all the supported Python environments under conda (for the unit tests) @@ -64,5 +52,16 @@ RUN chmod +x /scripts/*.sh && \ useradd -rm -d /home/ubuntu -s /bin/bash -g root -u 1000 ubuntu FROM ghcr.io/khiopsml/khiops-server:${SERVER_REVISION} AS server +FROM fsouza/fake-gcs-server:1.49 AS gcs-server + FROM khiopsdev AS base COPY --from=server /service /usr/bin/service + +# GCS fake file server (only in the ubuntu container) +COPY --from=gcs-server /bin/fake-gcs-server /bin/fake-gcs-server +EXPOSE 4443 + +# S3 fake file server (only in the ubuntu container) +RUN gem install fakes3:0.2.5 sorted_set +EXPOSE 4569 + diff --git a/packaging/docker/khiopspydev/run_fake_remote_file_servers.sh b/packaging/docker/khiopspydev/run_fake_remote_file_servers.sh index 09ecde6c..417d1071 100644 --- a/packaging/docker/khiopspydev/run_fake_remote_file_servers.sh +++ b/packaging/docker/khiopspydev/run_fake_remote_file_servers.sh @@ -1,27 +1,25 @@ #!/bin/bash -# first start the docker daemon -service docker start -# File server for GCS +# We cannot start the fake remote files servers until the source code is fetched +# because we expose pre-provisioned files to be read + +ROOT_FOLDER=${1:-.} # defaults to current folder + +# File server for GCS (runs in background) # WARNING : there are 3 major features actived by the options ... -# -v : the volume mounted exposes pre-provisioned files to be read remotely : the direct child folders will be the bucket names +# -data : exposes pre-provisioned files to be read remotely : the direct child folders will be the bucket names # -filesystem-root : let upload and read new files remotely at the same location as the source # -public-host : must expose localhost (https://github.com/fsouza/fake-gcs-server/issues/201) -docker run --rm \ - -d \ - -p 4443:4443 \ - --name fake-gcs-server \ - -v ./tests/resources/remote-access:/data fsouza/fake-gcs-server \ +nohup /bin/fake-gcs-server \ + -data "${ROOT_FOLDER}"/tests/resources/remote-access \ + -filesystem-root "${ROOT_FOLDER}"/tests/resources/remote-access \ -scheme http \ - -filesystem-root /data \ - -public-host localhost + -public-host localhost & -# File server for S3 +# File server for S3 (runs in background) # WARNING : -# -v : the volume mounted exposes pre-provisioned files : the direct child folders will be the bucket names +# -r : exposes pre-provisioned files : the direct child folders will be the bucket names # these files were uploaded once because fake-s3 creates metadata -docker run --rm \ - -d \ - -p 4569:4569 \ - --name my_s3 \ - -v ./tests/resources/remote-access:/fakes3_root lphoward/fake-s3 +nohup /usr/local/bin/fakes3 \ + -r "${ROOT_FOLDER}"/tests/resources/remote-access \ + -p 4569 &