switch to OpenMPI

Build new docker images with MPICH and OpenMPI. They are named with "-new" e.g. khiopsdev-new-ubuntu22.04 Generates DEB and RPM packages for multiple mpi implementations (mpich and openMPI) - Add mpi implemntation discovering in cmake - Add a suffix to package names corresponding to the mpi implementation (as it advised in fedora packaging guidline) - Add mpi implementation in packages and binaries name - To distinguish between different mpi implementation, we add the mpi suffix both to packages name and binaries name - Add mpich vs openMPI option in debian and rpm workflows (openMPI is the default) - Use ctest to run unit tests Cherry pick LearningTestTool and test hierarchy from dev Switch to openMPI in kht_test and standard tests notes: OpenMpi is used without problem on the linux host. There are some changes: - the flags used with mpiexec are different from mpich, we have to add: - --oversubscribe to launch more process than physical cores - --allow-run-as-root
KhiopsML · Apr 10, 2024 · d1a28a5 · d1a28a5
1 parent f63a655
commit d1a28a5
Show file tree

Hide file tree

Showing 147 changed files with 6,205 additions and 4,162 deletions.
diff --git a/.clang-format b/.clang-format
@@ -2,6 +2,7 @@
 BasedOnStyle: LLVM
 Language: Cpp
 ColumnLimit: 120
+ReflowComments: false
 IndentWidth: 8
 AccessModifierOffset: -8
 BreakBeforeBraces: Allman

diff --git a/.github/actions/build-khiops/action.yml b/.github/actions/build-khiops/action.yml
@@ -13,8 +13,22 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Setup MPI
+    - name: Setup MPI (windows)
+      if: ${{ runner.os == 'Windows' }}
       uses: mpi4py/setup-mpi@v1
+    - name: Setup MPI (macOS)
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: |
+        brew install mpich
+    - name: Setup MPI (Linux)
+      if: ${{ runner.os == 'Linux' }}
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install openmpi-bin libopenmpi-dev
+        sudo update-alternatives --set mpi /usr/bin/mpicc.openmpi
+        sudo update-alternatives --set mpirun /usr/bin/mpirun.openmpi
     - name: Setup Ninja
       uses: ashutoshvarma/[email protected]
     - name: Load Visual C++ Environment Variables (Windows only)

diff --git a/.github/actions/test-khiops-install/action.yml b/.github/actions/test-khiops-install/action.yml
@@ -61,11 +61,11 @@ runs:
         touch test/LearningTest/TestCoclustering/Standard/Iris/results/time.log
         touch test/LearningTest/TestKhiops/Standard/Iris/results/time.log
         echo "Check test results"
-        cd test/LearningTest/cmd/python/
-        $PYTHON test_khiops.py Khiops nul Standard
-        $PYTHON test_khiops.py Coclustering nul Standard
-        $PYTHON apply_command.py errors ../../TestKhiops/Standard/ Iris | tee /tmp/khiops-log.txt
-        $PYTHON apply_command.py errors ../../TestCoclustering/Standard/ Iris | tee /tmp/coclustering-log.txt
+        cd test/LearningTestTool/py/
+        $PYTHON kht_test.py ../../LearningTest/TestKhiops/Standard/Iris check
+        $PYTHON kht_test.py ../../LearningTest/TestCoclustering/Standard/Iris check
+        $PYTHON kht_apply.py ../../LearningTest/TestKhiops/Standard/Iris errors | tee /tmp/khiops-log.txt
+        $PYTHON kht_apply.py ../../LearningTest/TestCoclustering/Standard/Iris errors | tee /tmp/khiops-log.txt
         if (grep -q error /tmp/khiops-log.txt || grep -q error /tmp/coclustering-log.txt); then
           echo "::error::Errors in Khiops run"
           false

diff --git a/.github/workflows/build-linux-pack-containers.yml b/.github/workflows/build-linux-pack-containers.yml
@@ -38,7 +38,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           file: ./packaging/dockerfiles/Dockerfile.${{ matrix.os }}
-          tags: ghcr.io/${{ env.GH_REPOSITORY }}/khiopsdev-${{ matrix.os }}:latest
+          tags: ghcr.io/${{ env.GH_REPOSITORY }}/khiopsdev-new-${{ matrix.os }}:latest
           push: true
       - name: Display the image digest
         run: echo ${{ steps.docker-build.outputs.digest }}
diff --git a/.github/workflows/pack-debian.yml b/.github/workflows/pack-debian.yml
@@ -2,6 +2,14 @@
 name: DEB Packages
 on:
   workflow_dispatch:
+    inputs:
+      mpi_implementation:
+        description: mpi implementation
+        type: choice
+        default: openmpi
+        options: 
+          - mpich
+          - openmpi
   pull_request:
     paths:
       - '**CMakeLists.txt'
@@ -24,7 +32,7 @@ jobs:
       matrix:
         os: [debian10, debian11, debian12, ubuntu22.04, ubuntu20.04]
     container:
-      image: ghcr.io/khiopsml/khiops/khiopsdev-${{ matrix.os }}:latest
+      image: ghcr.io/khiopsml/khiops/khiopsdev-new-${{ matrix.os }}:latest
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -33,8 +41,17 @@ jobs:
           source /etc/os-release
           echo "ID=$ID" >> "$GITHUB_ENV"
           echo "VERSION_CODENAME=$VERSION_CODENAME" >> "$GITHUB_ENV"
+          if [ -z "${{ github.event.inputs.mpi_implementation }}" ] ; 
+          then
+            echo "MPI_IMPLEMENTATION=openmpi" >> "$GITHUB_ENV"
+          else
+            echo "MPI_IMPLEMENTATION=${{ github.event.inputs.mpi_implementation }}" >> "$GITHUB_ENV"
+          fi
+
       - name: Configure CMake
         run: |
+          update-alternatives --set mpi /usr/bin/mpicc.${{ env.MPI_IMPLEMENTATION }}
+          update-alternatives --set mpirun /usr/bin/mpirun.${{ env.MPI_IMPLEMENTATION }}
           cmake --preset linux-gcc-release -DBUILD_JARS=ON
       - name: Build Khiops binaries
         run: |
@@ -82,29 +99,38 @@ jobs:
           source /etc/os-release
           echo "ID=$ID" >> "$GITHUB_ENV"
           echo "VERSION_CODENAME=$VERSION_CODENAME" >> "$GITHUB_ENV"
+          if [ -z "${{ github.event.inputs.mpi_implementation }}" ] ; 
+          then
+            echo "MPI_IMPLEMENTATION=openmpi" >> "$GITHUB_ENV"
+          else
+            echo "MPI_IMPLEMENTATION=${{ github.event.inputs.mpi_implementation }}" >> "$GITHUB_ENV"
+          fi
+
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
           name: deb-${{ env.ID }}-${{ env.VERSION_CODENAME }}
           path: artifacts
       - name: Install Khiops core
         run: |
+          export DEBIAN_FRONTEND="noninteractive"
           apt-get update
           dpkg -i ./artifacts/khiops-core* || true
           apt-get -f install -y
       - name: Check Khiops core installation
-        continue-on-error: ${{ matrix.os == 'debian:11' }}
+        #continue-on-error: ${{ matrix.os == 'debian:11' }}
         run: |
           khiops-env --env
           khiops -v
           khiops_coclustering -v
       - name: Install Khiops Desktop (with java)
         run: |
-          dpkg -i ./khiops_* || true
+          dpkg -i ./artifacts//khiops_* || true
           apt-get -f install -y
       - name: Check Khiops installation
-        continue-on-error: ${{ matrix.os == 'debian:11' }}
+        #continue-on-error: ${{ matrix.os == 'debian:11' }}
         run: |
+          khiops-env --env
           khiops -v
           khiops_coclustering -v
       - name: Test Khiops installation
@@ -122,7 +148,7 @@ jobs:
       matrix:
         os: [debian10, debian11, debian12, ubuntu22.04, ubuntu20.04]
     container:
-      image: ghcr.io/khiopsml/khiops/khiopsdev-${{ matrix.os }}:latest
+      image: ghcr.io/khiopsml/khiops/khiopsdev-new-${{ matrix.os }}:latest
     steps:
       - name: Put OS info on the environment
         run: |

diff --git a/.github/workflows/pack-rpm.yml b/.github/workflows/pack-rpm.yml
@@ -2,6 +2,12 @@
 name: RPM Packages
 on:
   workflow_dispatch:
+    inputs:
+      mpi_implementation:
+        description: mpi implementation
+        type: choice
+        default: openmpi
+        options: [mpich, openmpi]
   pull_request:
     paths: ['**CMakeLists.txt', '**.cmake', .github/workflows/pack-rpm.yml]
   push:
@@ -22,7 +28,7 @@ jobs:
       matrix:
         os: [rocky8, rocky9]
     container:
-      image: ghcr.io/khiopsml/khiops/khiopsdev-${{ matrix.os }}:latest
+      image: ghcr.io/khiopsml/khiops/khiopsdev-new-${{ matrix.os }}:latest
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -32,17 +38,22 @@ jobs:
           echo "ID=$ID" >> "$GITHUB_ENV"
           VERSION_CODENAME=$(echo $PLATFORM_ID | cut -d":" -f2)
           echo "VERSION_CODENAME=$VERSION_CODENAME" >> "$GITHUB_ENV"
+          if [ -z "${{ github.event.inputs.mpi_implementation }}" ] ; 
+          then
+            echo "MPI_IMPLEMENTATION=openmpi" >> "$GITHUB_ENV"
+          else
+            echo "MPI_IMPLEMENTATION=${{ github.event.inputs.mpi_implementation }}" >> "$GITHUB_ENV"
+          fi
       - name: Configure CMake
         run: |
           source /etc/profile.d/modules.sh
-          module load mpi/mpich-x86_64
-          echo "MPI_SUFFIX=$MPI_SUFFIX" >> "$GITHUB_ENV"
+          module unload mpi
+          module load mpi/${{ env.MPI_IMPLEMENTATION}}-x86_64
           cmake --preset linux-gcc-release -DBUILD_JARS=ON
       - name: Build Khiops binaries
         run: |
           cmake --build --preset linux-gcc-release --parallel \
-            --target MODL${{ env.MPI_SUFFIX }} MODL_Coclustering${{ env.MPI_SUFFIX }} \
-            KhiopsNativeInterface norm_jar khiops_jar
+            --target MODL MODL_Coclustering KhiopsNativeInterface norm_jar khiops_jar
       - name: Build package with CPack
         run: cd build/linux-gcc-release && cpack -G RPM
       - name: Rename packages with codename
@@ -88,6 +99,12 @@ jobs:
           echo "ID=$ID" >> "$GITHUB_ENV"
           VERSION=$(echo $PLATFORM_ID | cut -d":" -f2)
           echo "VERSION=$VERSION" >> "$GITHUB_ENV"
+          if [ -z "${{ github.event.inputs.mpi_implementation }}" ] ; 
+          then
+            echo "MPI_IMPLEMENTATION=openmpi" >> "$GITHUB_ENV"
+          else
+            echo "MPI_IMPLEMENTATION=${{ github.event.inputs.mpi_implementation }}" >> "$GITHUB_ENV"
+          fi
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
@@ -97,9 +114,19 @@ jobs:
       - name: Install Khiops core
         run: |
           yum install -y ./artifacts/khiops-core*
+      - name: Check Khiops core installation
+        run: |
+          khiops-env --env
+          khiops -v
+          khiops_coclustering -v
       - name: Install Khiops Desktop (with java)
         run: |
           yum install -y ./artifacts/khiops-*
+      - name: Check Khiops  installation
+        run: |
+          khiops-env --env
+          khiops -v
+          khiops_coclustering -v
       - name: Test Khiops installation
         uses: ./.github/actions/test-khiops-install
   test-kni:
@@ -114,7 +141,7 @@ jobs:
       matrix:
         os: [rocky8, rocky9]
     container:
-      image: ghcr.io/khiopsml/khiops/khiopsdev-${{ matrix.os }}:latest
+      image: ghcr.io/khiopsml/khiops/khiopsdev-new-${{ matrix.os }}:latest
     steps:
       - name: Put OS info on the environment
         run: |