Switch to openMPI in kht_test and standard tests

KhiopsML · Apr 8, 2024 · bd452bd · bd452bd
1 parent 112e843
commit bd452bd
Show file tree

Hide file tree

Showing 11 changed files with 168 additions and 96 deletions.
diff --git a/.github/workflows/run-standard-tests.yml b/.github/workflows/run-standard-tests.yml
@@ -48,24 +48,32 @@ jobs:
         uses: ./.github/actions/build-khiops
         with:
           preset-name: ${{ env.PRESET_NAME }}
-      - name: Cache binaries
-        id: cache-binaries-unix
+      - name: Cache binaries (windows)
+        id: cache-binaries-windows
+        if: ${{ runner.os == 'Windows' }}
         uses: actions/cache@v3
         with:
           # We add binaries path one by one to avoid *.pdb and other msvc stuffs that generate a cache of 200Mo for windows
           path: |
-            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL
-            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL_Coclustering
-            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/KNITransfer
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL.exe
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL_Coclustering.exe
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/KNITransfer.exe
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/KhiopsNativeInterface.dll
-            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/lib/libKhiopsNativeInterface.*
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/lib/KhiopsNativeInterface.lib
           key: ${{ github.run_id }}-${{ matrix.build-setup.os }}-${{ env.PRESET_NAME
             }}
           fail-on-cache-miss: false
+      - name: Cache binaries (macOS, Linux)
+        id: cache-binaries-unix
+        if: ${{ runner.os != 'Windows' }}
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/*          
+            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/lib/libKhiopsNativeInterface.*
+          key: ${{ github.run_id }}-${{ matrix.build-setup.os }}-${{ env.PRESET_NAME
+            }}
+          fail-on-cache-miss: false
   run-standard-tests:
     needs: build-full-project
     strategy:
@@ -83,23 +91,36 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
-      - name: Restore cached binaries
-        id: restore-binaries
+      - name: Restore cached binaries (windows)
+        if: ${{ runner.os == 'Windows' }}
+        id: restore-binaries-windows
         uses: actions/cache/restore@v3
         with:
           path: |
-            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL
-            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL_Coclustering
-            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/KNITransfer
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL.exe
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL_Coclustering.exe
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/KNITransfer.exe
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/KhiopsNativeInterface.dll
-            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/lib/libKhiopsNativeInterface.*
             ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/lib/KhiopsNativeInterface.lib
           key: ${{ github.run_id }}-${{ matrix.build-setup.os }}-${{ env.PRESET_NAME
             }}
           fail-on-cache-miss: true
+      - name: Restore cached binaries (macOS, Linux)
+        if: ${{ runner.os != 'Windows' }}
+        id: restore-binaries
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/*           
+            ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/lib/libKhiopsNativeInterface.*
+          key: ${{ github.run_id }}-${{ matrix.build-setup.os }}-${{ env.PRESET_NAME
+            }}
+          fail-on-cache-miss: true
+      - name: Rename mpi binaries
+        if: ${{ runner.os == 'Linux' }}
+        shell: bash
+        run: |
+          mv ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL_openmpi ${{ github.workspace }}/build/${{ env.PRESET_NAME }}/bin/MODL
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
@@ -134,8 +155,6 @@ jobs:
       # - In release run all Standard tests
         shell: bash
         run: |
-          export OMPI_MCA_orte_execute_quiet=true
-          export OMPI_MCA_rmaps_base_oversubscribe=true
           if [[ "${{ matrix.build-setup.os }}"  == 'windows-2022' ]]; then
             export EXT=".exe"
           fi
@@ -159,6 +178,7 @@ jobs:
         shell: bash
         if: success() || failure()
         run: |
+          mkdir results
           if [[ "${{ matrix.config }}" == "release" ]] ; then
             python $APPLY_PY test/LearningTest/TestKhiops/Standard errors | tee -a results/errors.txt
             if [[ "${{ matrix.running-mode }}" != "parallel" ]] ; then

diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml
@@ -50,33 +50,39 @@ jobs:
           build/${{ matrix.build-setup.cmake-preset }}/bin/norm_test \
             --gtest_output="xml:reports/report-norm.xml"
       - name: Run Unit Tests for the Parallel Module
-        if: success() || failure()
+        if: always()
         run: |
           build/${{ matrix.build-setup.cmake-preset }}/bin/parallel_test \
             --gtest_output="xml:reports/report-parallel.xml"
       - name: Run Unit Tests for the Parallel MPI Module in serial
-        if: success() || failure()
+        if: always()
         run: |
           build/${{ matrix.build-setup.cmake-preset }}/bin/parallel_mpi_test \
             --gtest_output="xml:reports/report-parallel-mpi-serial.xml"
-      - name: Run Unit Tests for the Parallel MPI Module in parallel
-        if: success() || failure()
+      - name: Run Unit Tests for the Parallel MPI Module in parallel (Linux)
+        if: ${{ always() && runner.os == 'Linux' }}
         run: |
-          mpiexec -n 4 build/${{ matrix.build-setup.cmake-preset }}/bin/parallel_mpi_test \
+          mpirun --oversubscribe --use-hwthread-cpus -n 4 build/${{ matrix.build-setup.cmake-preset }}/bin/parallel_mpi_test \
             --gtest_output="xml:reports/report-parallel-mpi-parallel.xml"
+      - name: Run Unit Tests for the Parallel MPI Module in parallel (Windows and
+          macOS)
+        if: ${{ always() && runner.os != 'Linux' }}
+        run: |
+          mpiexec -n 4 build/${{ matrix.build-setup.cmake-preset }}/bin/parallel_mpi_test \
+              --gtest_output="xml:reports/report-parallel-mpi-parallel.xml"
       - name: Run Unit Tests for the Learning Module
-        if: success() || failure()
+        if: always()
         run: |
           build/${{ matrix.build-setup.cmake-preset }}/bin/learning_test \
             --gtest_output="xml:reports/report-learning.xml"
       - name: Run Unit Tests for the KNI Module
-        if: success() || failure()
+        if: always()
         run: |
           build/${{ matrix.build-setup.cmake-preset }}/bin/KNITest \
             --gtest_output="xml:reports/report-kni.xml"
       - name: Create Unit Test Reports Dashboards
         uses: phoenix-actions/test-reporting@v12
-        if: success() || failure()
+        if: always()
         with:
           name: Unit Tests Reports (${{ matrix.build-setup.os }})
           reporter: jest-junit

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -376,12 +376,12 @@ if(TESTING)
   enable_testing()
 
   # Add testing targets
-  add_subdirectory(test/Norm)
-  add_subdirectory(test/Parallel)
-  add_subdirectory(test/Parallel-mpi)
-  add_subdirectory(test/Learning)
-  add_subdirectory(test/KNITest)
-  add_subdirectory(test/Utils)
+  add_subdirectory(test/UnitTests/Norm)
+  add_subdirectory(test/UnitTests/Parallel)
+  add_subdirectory(test/UnitTests/Parallel-mpi)
+  add_subdirectory(test/UnitTests/Learning)
+  add_subdirectory(test/UnitTests/KNITest)
+  add_subdirectory(test/UnitTests/Utils)
 endif(TESTING)
 
 # Exclude googletest from the installation

diff --git a/test/LearningTestTool/py/kht_test.py b/test/LearningTestTool/py/kht_test.py
@@ -17,7 +17,7 @@
     mpi_exe_name = "mpiexec.exe"
 # mpiexec sous Linux
 else:
-    mpi_exe_name = "mpiexec"
+    mpi_exe_name = "mpirun"
 
 
 def build_tool_exe_path(tool_binaries_dir, tool_name):
@@ -274,10 +274,19 @@ def evaluate_tool_on_test_dir(
             khiops_params.append(mpi_exe_name)
             # Option -l, specifique a mpich, valide au moins pour Windows:
             #    "Label standard out and standard error (stdout and stderr) with the rank of the process"
-            khiops_params.append("-l")
+            if platform.system() == "Windows":
+                khiops_params.append("-l")
             if platform.system() == "Darwin":
                 khiops_params.append("-host")
                 khiops_params.append("localhost")
+            # Options specifiques a Open MPI
+            if platform.system() == "Linux":
+                # permet de lancer plus de processus qu'il n'y a de coeurs
+                khiops_params.append("--oversubscribe")
+                # permet de lancer en tant que root
+                khiops_params.append("--allow-run-as-root ")
+                # Ajoute le rang du processus dans les traces
+                khiops_params.append("--tag-output")
             khiops_params.append("-n")
             khiops_params.append(str(tool_process_number))
         khiops_params.append(tool_exe_path)
@@ -389,7 +398,8 @@ def evaluate_tool_on_test_dir(
             lines = utils.filter_copyright_lines(
                 lines
             )  # Supression eventuelle des lignes de copyright
-            lines = utils.filter_empty_lines(lines)  # Suopression des lignes vides
+            # Suopression des lignes vides
+            lines = utils.filter_empty_lines(lines)
 
             # Pour les test KNI, le stdout contient une ligne avec le nombre de records
             if is_kni:

diff --git a/test/UnitTests/KNITest/KNITest.cpp b/test/UnitTests/KNITest/KNITest.cpp
@@ -9,7 +9,7 @@
 
 #include "KNITest.h"
 #include "TestServices.h"
-#include "../../src/Learning/KNITransfer/KNIRecodeFile.cpp"
+#include "../../../src/Learning/KNITransfer/KNIRecodeFile.cpp"
 
 #define MAXITER 1000
 #define MAXBUFFERSIZE 1000
@@ -199,7 +199,7 @@ void TestIris()
 
 	sTestPath = FileService::GetPathName(__FILE__);
 	sDictionaryPath = FileService::BuildFilePathName(sTestPath, "ModelingIris.kdic");
-	sDataPath = FileService::BuildFilePathName(sTestPath, "../LearningTest/datasets/Iris/Iris.txt");
+	sDataPath = FileService::BuildFilePathName(sTestPath, "../../LearningTest/datasets/Iris/Iris.txt");
 	sOutputPath = sTestPath + "results" + FileService::GetFileSeparator() + "R_Iris.txt";
 	sRefFilePath = sTestPath + "results.ref" + FileService::GetFileSeparator() + "R_Iris.txt";
 
@@ -232,8 +232,8 @@ void TestAdult()
 	boolean bOk;
 
 	sTestPath = FileService::GetPathName(__FILE__);
-	sDictionaryPath = FileService::BuildFilePathName(sTestPath, "../LearningTest/datasets/Adult/Adult.kdic");
-	sDataPath = FileService::BuildFilePathName(sTestPath, "../LearningTest/datasets/Adult/Adult.txt");
+	sDictionaryPath = FileService::BuildFilePathName(sTestPath, "../../LearningTest/datasets/Adult/Adult.kdic");
+	sDataPath = FileService::BuildFilePathName(sTestPath, "../../LearningTest/datasets/Adult/Adult.txt");
 	sOutputPath = sTestPath + "results" + FileService::GetFileSeparator() + "R_Adult.txt";
 
 	// Test de deploiement
@@ -246,20 +246,20 @@ void TestAdult()
 
 	// Saut du header
 	FileService::OpenInputBinaryFile(sDataPath, fRef);
-	ch = getc(fRef);
+	ch = (char)getc(fRef);
 	while (ch != '\n' and ch != EOF)
 	{
-		ch = getc(fRef);
+		ch = (char)getc(fRef);
 	}
 
 	// Copie du fichier tel quel
 	sRefFilePath = sTestPath + "results" + FileService::GetFileSeparator() + "ref_Adult.txt";
 	FileService::OpenOutputBinaryFile(sRefFilePath, fWithoutHeader);
-	ch = getc(fRef);
+	ch = (char)getc(fRef);
 	while (ch != EOF)
 	{
 		putc(ch, fWithoutHeader);
-		ch = getc(fRef);
+		ch = (char)getc(fRef);
 	}
 	FileService::CloseInputBinaryFile(sDataPath, fRef);
 	FileService::CloseOutputBinaryFile(sRefFilePath, fWithoutHeader);

diff --git a/test/UnitTests/KNITest/KNITest.h b/test/UnitTests/KNITest/KNITest.h
@@ -10,4 +10,4 @@
 #include <string.h>
 
 #include "KhiopsNativeInterface.h"
-#include "../../src/Learning/KNITransfer/KNIRecodeFile.h"
+#include "../../../src/Learning/KNITransfer/KNIRecodeFile.h"
diff --git a/test/UnitTests/KNITest/results.ref/KNI_Adult.txt b/test/UnitTests/KNITest/results.ref/KNI_Adult.txt
@@ -1,5 +1,5 @@
 
-Recode records of @ROOT_DIR@/test/KNITest/../LearningTest/datasets/Adult/Adult.txt to @ROOT_DIR@/test/KNITest/results/R_Adult.txt
+Recode records of @ROOT_DIR@/UnitTests/KNITest/../../LearningTest/datasets/Adult/Adult.txt to @ROOT_DIR@/UnitTests/KNITest/results/R_Adult.txt
 Recoded record number: 48842
 
-End test Adult
+End test Adult
diff --git a/test/UnitTests/KNITest/results.ref/KNI_Iris.txt b/test/UnitTests/KNITest/results.ref/KNI_Iris.txt
@@ -1,7 +1,7 @@
 Begin test KNI
 SYS KNI version: 110
 SYS KNI full version: 10.1.5
-Open stream @ROOT_DIR@/test/KNITest/ModelingIris.kdic SNB_Iris: 1
+Open stream @ROOT_DIR@/UnitTests/KNITest/ModelingIris.kdic SNB_Iris: 1
 Recode stream record Iris-setosa	Iris-setosa	0.9960418693	0.001979090503	0.001979040221: 0
 Recode stream record Iris-virginica	Iris-virginica	0.00198971666	0.03512915094	0.9628811324: 0
 Recode invalid stream record : -14
@@ -14,12 +14,12 @@ Recode stream record with negative output size : -15
 Wrong parameter tests
 Open stream with NULL dictionary file NULL SNB_Iris: -2
 Open stream with missing dictionary file WrongFile SNB_Iris: -3
-Open stream with invalid dictionary file @ROOT_DIR@/test/KNITest/../LearningTest/datasets/Iris/Iris.txt SNB_Iris: -4
-Open stream with NULL dictionary @ROOT_DIR@/test/KNITest/ModelingIris.kdic NULL: -5
-Open stream with missing dictionary @ROOT_DIR@/test/KNITest/ModelingIris.kdic WrongDic: -6
-Open stream with NULL header line @ROOT_DIR@/test/KNITest/ModelingIris.kdic SNB_Iris: -8
-Open stream with bad header line @ROOT_DIR@/test/KNITest/ModelingIris.kdic SNB_Iris: -23
-Open stream with eol separator @ROOT_DIR@/test/KNITest/ModelingIris.kdic SNB_Iris: -9
+Open stream with invalid dictionary file @ROOT_DIR@/UnitTests/KNITest/../../LearningTest/datasets/Iris/Iris.txt SNB_Iris: -4
+Open stream with NULL dictionary @ROOT_DIR@/UnitTests/KNITest/ModelingIris.kdic NULL: -5
+Open stream with missing dictionary @ROOT_DIR@/UnitTests/KNITest/ModelingIris.kdic WrongDic: -6
+Open stream with NULL header line @ROOT_DIR@/UnitTests/KNITest/ModelingIris.kdic SNB_Iris: -8
+Open stream with bad header line @ROOT_DIR@/UnitTests/KNITest/ModelingIris.kdic SNB_Iris: -23
+Open stream with eol separator @ROOT_DIR@/UnitTests/KNITest/ModelingIris.kdic SNB_Iris: -9
 Recode with wrong stream : -10
 Close wrong stream (-1): -10
 Close wrong stream (0): -10
@@ -30,7 +30,7 @@ SYS	=> stream index 317
 	=> Error -22
 Muliple close stream
 
-Recode records of @ROOT_DIR@/test/KNITest/../LearningTest/datasets/Iris/Iris.txt to @ROOT_DIR@/test/KNITest/results/R_Iris.txt
+Recode records of @ROOT_DIR@/UnitTests/KNITest/../../LearningTest/datasets/Iris/Iris.txt to @ROOT_DIR@/UnitTests/KNITest/results/R_Iris.txt
 Recoded record number: 150
 
-End test KNI
+End test KNI
diff --git a/test/UnitTests/Utils/ParallelTest.h b/test/UnitTests/Utils/ParallelTest.h
@@ -4,8 +4,8 @@
 
 #include "gtest/gtest.h"
 
-// Fixture a utiliser pour realiser les test en sequentiel, en parallele et en simule.
-// Les tests qui utilisent cette fixture vont etre lancer en sequentiel puis en simule (sauf si il y a plus d'un
+// Fixture a utiliser pour realiser les tests en sequentiel, en parallele et en simule.
+// Les tests qui utilisent cette fixture vont etre lance en sequentiel puis en simule (sauf si il y a plus d'un
 // processus, auquel cas le test est skippe) Pour lancer les test il faut definir les tests avec parametres
 // (Value-parameterized tests) et les instancier INSTANTIATE_TEST_SUITE_P(nom_du_test, nom_de_la_classe ,
 // testing::Values(RunType::MPI, RunType::SIMULATED));