switch to OpenMPI

Build new docker images with MPICH and OpenMPI. Generates DEB and RPM packages for multiple mpi implementations (mpich and openMPI) - Add mpi implemntation discovering in cmake (not working in conda) - Add a suffix to package names corresponding to the mpi implementation (as it advised in fedora packaging guidline) - Add mpi implementation name in packages names - packages khiops-core-openmpi with a version >> 10.2.1 replace khiops-core-10.2.1 without trouble with apt and dnf - Add mpich vs openMPI option in debian and rpm workflows (openMPI is the default) - Use ctest to run unit tests (issue #23) Cherry pick LearningTestTool and test hierarchy from dev Handle binary renaming in LearningTestTool - Switch to openMPI in kht_test and standard tests - The kht_test script handles the new naming of the parallel binaries: MODL_openmpi or MODL_mpich or MODL (the last one has priority) Conda: - Fix mpich build - Remove the use of the obsolete cmake option MPI_SUFFIX in conda build Fix DEB and RPM version renaming in pack-debian and pack-rpm cherry pick from dev-v10
KhiopsML · May 7, 2024 · 21fb91a · 21fb91a
1 parent c45cf9a
commit 21fb91a
Show file tree

Hide file tree

Showing 31 changed files with 337 additions and 215 deletions.
diff --git a/.github/actions/build-khiops/action.yml b/.github/actions/build-khiops/action.yml
@@ -13,12 +13,26 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Setup MPI
+    - name: Setup MPI (windows)
+      if: runner.os == 'Windows'
       uses: mpi4py/setup-mpi@v1
+    - name: Setup MPI (macOS)
+      if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        brew install mpich
+    - name: Setup MPI (Linux)
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install openmpi-bin libopenmpi-dev
+        sudo update-alternatives --set mpi /usr/bin/mpicc.openmpi
+        sudo update-alternatives --set mpirun /usr/bin/mpirun.openmpi
     - name: Setup Ninja
       uses: ashutoshvarma/[email protected]
     - name: Load Visual C++ Environment Variables (Windows only)
-      if: ${{ runner.os == 'Windows' }}
+      if: runner.os == 'Windows'
       shell: cmd
       run: |
         call "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Auxiliary\\Build\\vcvars64.bat"

diff --git a/.github/actions/test-khiops-install/action.yml b/.github/actions/test-khiops-install/action.yml
@@ -41,8 +41,10 @@ runs:
     - name: Check Khiops installation
       shell: bash
       run: |
-        "$KHIOPS_SCRIPT" -v
-        "$KHIOPS_CC_SCRIPT" -v
+        export KHIOPS_PROC_NUMBER=4 
+        export OMPI_MCA_rmaps_base_oversubscribe=true
+        "$KHIOPS_SCRIPT" -s
+        "$KHIOPS_CC_SCRIPT" -s
     - name: Run Khiops tests
       shell: bash
       run: |
@@ -65,7 +67,7 @@ runs:
         $PYTHON kht_test.py ../../LearningTest/TestKhiops/Standard/Iris check
         $PYTHON kht_test.py ../../LearningTest/TestCoclustering/Standard/Iris check
         $PYTHON kht_apply.py ../../LearningTest/TestKhiops/Standard/Iris errors | tee /tmp/khiops-log.txt
-        $PYTHON kht_apply.py ../../LearningTest/TestCoclustering/Standard/Iris errors | tee /tmp/khiops-log.txt
+        $PYTHON kht_apply.py ../../LearningTest/TestCoclustering/Standard/Iris errors | tee /tmp/coclustering-log.txt
         if (grep -q error /tmp/khiops-log.txt || grep -q error /tmp/coclustering-log.txt); then
           echo "::error::Errors in Khiops run"
           false

diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
@@ -27,17 +27,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          # We use the oldest ubuntu available to have forward compatibility of the libc
-          - os: ubuntu-20.04
-            os-family: linux-64
-          - os: windows-2022
-            os-family: win-64
-          - os: macos-13
-            os-family: osx-64
-          - os: macos-14
-            os-family: osx-arm64
-    runs-on: ${{ matrix.os }}
+        setup:
+          # Linux: We use the Khiops dev debian:10 container to build with glibc 2.28
+          #        It ensures compatibility with glibc >= 2.28 (in particular Rocky 8)
+          - {os: ubuntu-22.04, os-family: linux-64, json-image: '{"image": "ghcr.io/khiopsml/khiops/khiopsdev-debian10:latest"}'}
+          - {os: windows-2022, os-family: win-64, json-image: '{"image": null}'}
+          - {os: macos-13, os-family: osx-64, json-image: '{"image": null}'}
+          - {os: macos-14, os-family: osx-arm64, json-image: '{"image": null}'}
+    container: ${{ fromJSON(matrix.setup.json-image) }}
+    runs-on: ${{ matrix.setup.os }}
     steps:
       - name: Checkout Sources
         uses: actions/checkout@v4
@@ -78,7 +76,7 @@ jobs:
       - name: Upload conda packages artifact
         uses: actions/upload-artifact@v4
         with:
-          name: khiops-conda-${{ matrix.os-family }}
+          name: khiops-conda-${{ matrix.setup.os-family }}
           path: ./build/conda
           retention-days: 7
   # Test Conda package on brand new environments

diff --git a/.github/workflows/pack-debian.yml b/.github/workflows/pack-debian.yml
@@ -2,6 +2,14 @@
 name: DEB Packages
 on:
   workflow_dispatch:
+    inputs:
+      mpi_implementation:
+        description: mpi implementation
+        type: choice
+        default: openmpi
+        options: 
+          - mpich
+          - openmpi
   pull_request:
     paths:
       - '**CMakeLists.txt'
@@ -33,9 +41,13 @@ jobs:
           source /etc/os-release
           echo "ID=$ID" >> "$GITHUB_ENV"
           echo "VERSION_CODENAME=$VERSION_CODENAME" >> "$GITHUB_ENV"
+          echo "MPI_IMPLEMENTATION=${{ inputs.mpi_implementation || 'openmpi' }}" >> "$GITHUB_ENV"
+          
       - name: Configure CMake
         run: |
-          cmake --preset linux-gcc-release -DBUILD_JARS=ON -DTESTING=OFF
+          update-alternatives --set mpi /usr/bin/mpicc.$MPI_IMPLEMENTATION
+          update-alternatives --set mpirun /usr/bin/mpirun.$MPI_IMPLEMENTATION
+          cmake --preset linux-gcc-release -DBUILD_JARS=ON
       - name: Build Khiops binaries
         run: |
           cmake --build --preset linux-gcc-release --parallel \
@@ -82,31 +94,33 @@ jobs:
           source /etc/os-release
           echo "ID=$ID" >> "$GITHUB_ENV"
           echo "VERSION_CODENAME=$VERSION_CODENAME" >> "$GITHUB_ENV"
+          echo "MPI_IMPLEMENTATION=${{ inputs.mpi_implementation || 'openmpi' }}" >> "$GITHUB_ENV"
+
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
           name: deb-${{ env.ID }}-${{ env.VERSION_CODENAME }}
           path: artifacts
       - name: Install Khiops core
         run: |
+          export DEBIAN_FRONTEND="noninteractive"
           apt-get update
           dpkg -i ./artifacts/khiops-core* || true
           apt-get -f install -y
       - name: Check Khiops core installation
-        continue-on-error: ${{ matrix.os == 'debian:11' }}
         run: |
           khiops-env --env
-          khiops -v
-          khiops_coclustering -v
+          khiops -s
+          khiops_coclustering -s
       - name: Install Khiops Desktop (with java)
         run: |
-          dpkg -i ./khiops_* || true
+          dpkg -i ./artifacts//khiops_* || true
           apt-get -f install -y
       - name: Check Khiops installation
-        continue-on-error: ${{ matrix.os == 'debian:11' }}
         run: |
-          khiops -v
-          khiops_coclustering -v
+          khiops-env --env
+          khiops -s
+          khiops_coclustering -s
       - name: Test Khiops installation
         continue-on-error: ${{ matrix.os == 'debian:11' }}
         uses: ./.github/actions/test-khiops-install
@@ -164,12 +178,18 @@ jobs:
           merge-multiple: true
       - name: Rename packages with tag version
         run: |
-          SOURCE_VERSION=$(./scripts/khiops-version)
-          for PKG_FILE in *.deb
-          do
-            NEW_PKG_FILE=$(echo $PKG_FILE | sed "s/_${SOURCE_VERSION}-/_${{ github.ref_name }}-/")
-            mv $PKG_FILE $NEW_PKG_FILE
-          done
+         # Renames the packge only if the source version differs from the tag
+         SOURCE_VERSION=$(./scripts/khiops-version | sed 's/-/_/')
+         TAG_VERSION=$(echo "${{ github.ref_name }}" | sed 's/-/_/')
+         for PKG_FILE in *.deb
+         do
+           NEW_PKG_FILE=$(echo $PKG_FILE | sed "s/_${SOURCE_VERSION}-/_${TAG_VERSION}-/")
+           if [[ "$PKG_FILE" != "$NEW_PKG_FILE" ]]
+           then
+             echo "Renaming '$PKG_FILE' to '$NEW_PKG_FILE'"
+             mv $PKG_FILE $NEW_PKG_FILE
+           fi
+         done
       - name: Upload packages to the release
         uses: ncipollo/[email protected]
         with:

diff --git a/.github/workflows/pack-nsis.yml b/.github/workflows/pack-nsis.yml
@@ -158,10 +158,16 @@ jobs:
           name: KNI
       - name: Rename packages with tag version
         run: |
+          # Build the package file names
           SOURCE_VERSION="${{ needs.build.outputs.khiops-version }}"
           PKG_FILE=$(ls *-setup.exe)
           NEW_PKG_FILE=$(echo $PKG_FILE | sed "s/-${SOURCE_VERSION}-/-${{ github.ref_name }}-/")
-          mv $PKG_FILE $NEW_PKG_FILE
+
+          # Rename to the tag version only if it is not coherent with the source version
+          if [[ "$PKG_FILE" != "$NEW_PKG_FILE" ]]
+          then
+            mv $PKG_FILE $NEW_PKG_FILE
+          fi
       - name: Upload NSIS installer and KNI to the release
         uses: ncipollo/[email protected]
         with:

diff --git a/.github/workflows/pack-rpm.yml b/.github/workflows/pack-rpm.yml
@@ -2,6 +2,12 @@
 name: RPM Packages
 on:
   workflow_dispatch:
+    inputs:
+      mpi_implementation:
+        description: mpi implementation
+        type: choice
+        default: openmpi
+        options: [mpich, openmpi]
   pull_request:
     paths: ['**CMakeLists.txt', '**.cmake', .github/workflows/pack-rpm.yml]
   push:
@@ -32,17 +38,17 @@ jobs:
           echo "ID=$ID" >> "$GITHUB_ENV"
           VERSION_CODENAME=$(echo $PLATFORM_ID | cut -d":" -f2)
           echo "VERSION_CODENAME=$VERSION_CODENAME" >> "$GITHUB_ENV"
+          echo "MPI_IMPLEMENTATION=${{ inputs.mpi_implementation || 'openmpi' }}" >> "$GITHUB_ENV"
       - name: Configure CMake
         run: |
           source /etc/profile.d/modules.sh
-          module load mpi/mpich-x86_64
-          echo "MPI_SUFFIX=$MPI_SUFFIX" >> "$GITHUB_ENV"
-          cmake --preset linux-gcc-release -DBUILD_JARS=ON -DTESTING=OFF
+          module unload mpi
+          module load mpi/${MPI_IMPLEMENTATION}-x86_64
+          cmake --preset linux-gcc-release -DBUILD_JARS=ON
       - name: Build Khiops binaries
         run: |
           cmake --build --preset linux-gcc-release --parallel \
-            --target MODL${{ env.MPI_SUFFIX }} MODL_Coclustering${{ env.MPI_SUFFIX }} \
-            KhiopsNativeInterface norm_jar khiops_jar
+            --target MODL MODL_Coclustering KhiopsNativeInterface norm_jar khiops_jar
       - name: Build package with CPack
         run: cd build/linux-gcc-release && cpack -G RPM
       - name: Rename packages with codename
@@ -88,6 +94,7 @@ jobs:
           echo "ID=$ID" >> "$GITHUB_ENV"
           VERSION=$(echo $PLATFORM_ID | cut -d":" -f2)
           echo "VERSION=$VERSION" >> "$GITHUB_ENV"
+          echo "MPI_IMPLEMENTATION=${{ inputs.mpi_implementation || 'openmpi' }}" >> "$GITHUB_ENV"
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
@@ -97,9 +104,19 @@ jobs:
       - name: Install Khiops core
         run: |
           yum install -y ./artifacts/khiops-core*
+      - name: Check Khiops core installation
+        run: |
+          khiops-env --env
+          khiops -s
+          khiops_coclustering -s
       - name: Install Khiops Desktop (with java)
         run: |
           yum install -y ./artifacts/khiops-*
+      - name: Check Khiops  installation
+        run: |
+          khiops-env --env
+          khiops -s
+          khiops_coclustering -s
       - name: Test Khiops installation
         uses: ./.github/actions/test-khiops-install
   test-kni:
@@ -156,11 +173,17 @@ jobs:
           merge-multiple: true
       - name: Rename packages with tag version
         run: |
-          SOURCE_VERSION=$(./scripts/khiops-version)
+          # Renames the packge only if the source version differs from the tag
+          SOURCE_VERSION=$(./scripts/khiops-version | sed 's/-/_/')
+          TAG_VERSION=$(echo "${{ github.ref_name }}" | sed 's/-/_/')
           for PKG_FILE in *.rpm
           do
-            NEW_PKG_FILE=$(echo $PKG_FILE | sed "s/-${SOURCE_VERSION}-/-${{ github.ref_name }}-/")
-            mv $PKG_FILE $NEW_PKG_FILE
+            NEW_PKG_FILE=$(echo $PKG_FILE | sed "s/-${SOURCE_VERSION}-/-${TAG_VERSION}-/")
+            if [[ "$PKG_FILE" != "$NEW_PKG_FILE" ]]
+            then
+              echo "Renaming '$PKG_FILE' to '$NEW_PKG_FILE'"
+              mv $PKG_FILE $NEW_PKG_FILE
+            fi
           done
       - name: Upload packages to the release
         uses: ncipollo/[email protected]