Simplification : avoid using a docker host in a container

.github/workflows/unit-tests.yml → .github/workflows/unit-and-integration-tests.yml

@@ -1,5 +1,5 @@
 ---
-name: Unit Tests
+name: Unit & Integration Tests
 env:
   DEFAULT_SAMPLES_REVISION: main
 on:
@@ -188,3 +188,77 @@ jobs:
           # modules are currently not initializing the shell anyway
           if [ -n "$MODULESHOME" ]; then module unload mpi; fi
           python -m unittest -v tests.test_khiops_integrations.KhiopsRunnerEnvironmentTests.test_runner_has_mpiexec_on_linux
+  test-access-to-remote-files:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/khiopsml/khiops-python/khiopspydev-ubuntu22.04:latest
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    permissions:
+      id-token: write
+      contents: read
+      checks: write
+      packages: read
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          # Get Git tags so that versioneer can function correctly
+          # See issue https://github.com/actions/checkout/issues/701
+          fetch-depth: 0
+      - name: Install khiops-python dev dependencies
+        run: |
+          # The following git command is required, 
+          # as the Git repository is in a directory the current user does not own,
+          # Python versioneer fails to compute the current version correctly otherwise
+          git config --global --add safe.directory $(realpath .)
+          python setup.py egg_info
+          pip install `grep -v "^\[" khiops.egg-info/requires.txt`
+          rm -rf khiops.egg-info
+      - name: Setup and Install Test Requirements
+        run: |
+          pip install -r test-requirements.txt
+          pip install unittest-xml-reporting
+      - name: Launch the tests
+        env:
+          # for the tests with GCS
+          GCS_BUCKET_NAME: gcs-bucket
+          STORAGE_EMULATOR_HOST: http://localhost:4443
+          # for the tests with S3
+          S3_BUCKET_NAME: s3-bucket
+          AWS_SECRET_ACCESS_KEY: XXX
+          AWS_ENDPOINT_URL: http://localhost:4569
+          AWS_ACCESS_KEY_ID: XXX
+          # common vars
+          no_proxy: localhost
+          # Force > 2 CPU cores to launch mpiexec
+          KHIOPS_PROC_NUMBER: 4
+        run: |-
+          /scripts/run_fake_remote_file_servers.sh .  # launch the servers in the background
+          python -m xmlrunner tests.test_remote_access.KhiopsGCSRemoteFileTests -o reports/
+          python -m xmlrunner tests.test_remote_access.KhiopsS3RemoteFileTests -o reports/
+      - name: Display Integration Test Reports on remote files
+        uses: dorny/test-reporter@v1
+        with:
+          name: Integration Tests on remote files
+          path: >-
+            reports/TEST-tests.test_remote_access.*.xml,
+          reporter: java-junit
+          path-replace-backslashes: 'true'  # Necessary for windows paths
+      - name: Upload Test Reports as Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-reports-on-remote-files
+          path: |-
+            reports/TEST-tests.test_remote_access.*.xml
+            test/resources/tmp/khiops-cicd/*/*
+            tests/resources/scenario_generation/*/ref/*._kh
+            tests/resources/scenario_generation/*/output/*._kh
+            tests/resources/*/output_reports/*.txt
+            tests/resources/*/ref_reports/*.txt
+            tests/resources/dictionary/ref_kdic/*.kdic
+            tests/resources/dictionary/output_kdic/*.kdic
+            tests/resources/dictionary/copy_output_kdic/*.kdic
+            tests/resources/general_options/general_options/*/*._kh
+          retention-days: 7

packaging/docker/khiopspydev/Dockerfile.ubuntu

@@ -10,14 +10,14 @@ ARG KHIOPS_REVISION
 RUN true \
   # Install git (for khiops-python version calculation) and pip \
   && apt-get -y update \
-  && apt-get -y --no-install-recommends install git python3-pip zip pandoc wget \
+  && apt-get -y --no-install-recommends install git python3-pip zip pandoc wget ruby-dev \
   # Get Linux distribution codename \
   && if [ -f /etc/os-release ]; then . /etc/os-release; fi \
   # Obtain the Khiops native package \
   && KHIOPS_PKG_FILE=$KHIOPS_REVISION/khiops-core-openmpi_$KHIOPS_REVISION-1-$VERSION_CODENAME.amd64.deb \
   && wget -O KHIOPS_CORE.deb "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_PKG_FILE}" \
   # Install the Khiops native package : do not break immediately if dpkg fails because apt will install the missing dependencies \
-  && (dpkg -i --force-all KHIOPS_CORE.deb || apt-get -f -y install) \
+  && (dpkg -i --force-all KHIOPS_CORE.deb || true) && apt-get -f -y install \
   && rm -f KHIOPS_CORE.deb \
   # Set python to python3 \
   && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
@@ -33,18 +33,6 @@ RUN true \
   && rm -fr /var/lib/apt/lists/* \
   && apt-get clean \
   && rm -rf ./khiops \
-  # Install docker within the image to be able to launch the fake remote file servers for S3 & GCS.
-  # Please note that the container based on the current image must be ran with the --privileged flag
-  # otherwise docker would not start
-  && apt-get update \
-  && apt-get install -y ca-certificates curl \
-  && mkdir -p /etc/apt/keyrings && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \
-  && chmod a+r /etc/apt/keyrings/docker.asc \
-  && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
-            $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \
-  && apt-get update \
-  && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \
-  && sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker \
   && true
 
 # set up all the supported Python environments under conda (for the unit tests)
@@ -64,5 +52,16 @@ RUN chmod +x /scripts/*.sh && \
   useradd -rm -d /home/ubuntu -s /bin/bash -g root -u 1000 ubuntu
 
 FROM ghcr.io/khiopsml/khiops-server:${SERVER_REVISION} AS server
+FROM fsouza/fake-gcs-server:1.49 AS gcs-server
+
 FROM khiopsdev AS base
 COPY --from=server /service /usr/bin/service
+
+# GCS fake file server (only in the ubuntu container)
+COPY --from=gcs-server /bin/fake-gcs-server /bin/fake-gcs-server
+EXPOSE 4443
+
+# S3 fake file server (only in the ubuntu container)
+RUN gem install fakes3:0.2.5 sorted_set
+EXPOSE 4569
+

packaging/docker/khiopspydev/run_fake_remote_file_servers.sh

@@ -1,27 +1,25 @@
 #!/bin/bash
-# first start the docker daemon
-service docker start
 
-# File server for GCS
+# We cannot start the fake remote files servers until the source code is fetched
+# because we expose pre-provisioned files to be read
+
+ROOT_FOLDER=${1:-.} # defaults to current folder
+
+# File server for GCS (runs in background)
 # WARNING : there are 3 major features actived by the options ...
-# -v : the volume mounted exposes pre-provisioned files to be read remotely : the direct child folders will be the bucket names
+# -data : exposes pre-provisioned files to be read remotely : the direct child folders will be the bucket names
 # -filesystem-root : let upload and read new files remotely at the same location as the source
 # -public-host : must expose localhost (https://github.com/fsouza/fake-gcs-server/issues/201)
-docker run --rm \
-  -d \
-  -p 4443:4443 \
-  --name fake-gcs-server \
-  -v ./tests/resources/remote-access:/data fsouza/fake-gcs-server \
+nohup /bin/fake-gcs-server \
+  -data "${ROOT_FOLDER}"/tests/resources/remote-access \
+  -filesystem-root "${ROOT_FOLDER}"/tests/resources/remote-access \
   -scheme http \
-  -filesystem-root /data \
-  -public-host localhost
+  -public-host localhost &
 
-# File server for S3
+# File server for S3 (runs in background)
 # WARNING :
-# -v : the volume mounted exposes pre-provisioned files : the direct child folders will be the bucket names
+# -r : exposes pre-provisioned files : the direct child folders will be the bucket names
 #      these files were uploaded once because fake-s3 creates metadata
-docker run --rm \
-  -d \
-  -p 4569:4569 \
-  --name my_s3 \
-  -v ./tests/resources/remote-access:/fakes3_root lphoward/fake-s3
+nohup /usr/local/bin/fakes3 \
+  -r "${ROOT_FOLDER}"/tests/resources/remote-access \
+  -p 4569 &