From f12563767a6f8f3d5af1dd4ab4e3d8fbdb67b943 Mon Sep 17 00:00:00 2001 From: bruno Date: Tue, 9 Apr 2024 09:36:48 +0200 Subject: [PATCH] Use ctest to run unit tests --- .github/workflows/pack-debian.yml | 12 ++++- .github/workflows/pack-rpm.yml | 9 +++- .github/workflows/run-unit-tests.yml | 46 ++----------------- packaging/install.cmake | 21 +++++++-- .../common/{khiops-env => khiops-env.in} | 19 +++----- test/LearningTestTool/py/_kht_families.py | 8 ++-- test/UnitTests/Parallel-mpi/CMakeLists.txt | 9 ++++ 7 files changed, 58 insertions(+), 66 deletions(-) rename packaging/linux/common/{khiops-env => khiops-env.in} (84%) diff --git a/.github/workflows/pack-debian.yml b/.github/workflows/pack-debian.yml index b3a47e26a..ccef716bf 100644 --- a/.github/workflows/pack-debian.yml +++ b/.github/workflows/pack-debian.yml @@ -6,6 +6,7 @@ on: mpi_implementation: description: mpi implementation type: choice + default: openmpi options: - mpich - openmpi @@ -40,10 +41,17 @@ jobs: source /etc/os-release echo "ID=$ID" >> "$GITHUB_ENV" echo "VERSION_CODENAME=$VERSION_CODENAME" >> "$GITHUB_ENV" + if [ -z "${{ github.event.inputs.mpi_implementation }}" ] ; + then + echo "MPI_IMPLEMENTATION=openmpi" >> "$GITHUB_ENV" + else + echo "MPI_IMPLEMENTATION=${{ github.event.inputs.mpi_implementation }}" >> "$GITHUB_ENV" + fi + - name: Configure CMake run: | - update-alternatives --set mpi /usr/bin/mpicc.${{ github.event.inputs.mpi_implementation }} - update-alternatives --set mpirun /usr/bin/mpirun.${{ github.event.inputs.mpi_implementation }} + update-alternatives --set mpi /usr/bin/mpicc.${{ env.MPI_IMPLEMENTATION }} + update-alternatives --set mpirun /usr/bin/mpirun.${{ env.MPI_IMPLEMENTATION }} cmake --preset linux-gcc-release -DBUILD_JARS=ON - name: Build Khiops binaries run: | diff --git a/.github/workflows/pack-rpm.yml b/.github/workflows/pack-rpm.yml index 6224cabda..45c1ff46f 100644 --- a/.github/workflows/pack-rpm.yml +++ b/.github/workflows/pack-rpm.yml @@ -6,6 +6,7 @@ on: mpi_implementation: description: mpi implementation type: choice + default: openmpi options: [mpich, openmpi] pull_request: paths: ['**CMakeLists.txt', '**.cmake', .github/workflows/pack-rpm.yml] @@ -37,11 +38,17 @@ jobs: echo "ID=$ID" >> "$GITHUB_ENV" VERSION_CODENAME=$(echo $PLATFORM_ID | cut -d":" -f2) echo "VERSION_CODENAME=$VERSION_CODENAME" >> "$GITHUB_ENV" + if [ -z "${{ github.event.inputs.mpi_implementation }}" ] ; + then + echo "MPI_IMPLEMENTATION=openmpi" >> "$GITHUB_ENV" + else + echo "MPI_IMPLEMENTATION=${{ github.event.inputs.mpi_implementation }}" >> "$GITHUB_ENV" + fi - name: Configure CMake run: | source /etc/profile.d/modules.sh module unload mpi - module load mpi/${{ github.event.inputs.mpi_implementation }}-x86_64 + module load mpi/${{ env.MPI_IMPLEMENTATION}}-x86_64 cmake --preset linux-gcc-release -DBUILD_JARS=ON - name: Build Khiops binaries run: | diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 0e5981df1..b728c33fe 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -45,46 +45,6 @@ jobs: preset-name: ${{ matrix.build-setup.cmake-preset }} targets: norm_test parallel_test parallel_mpi_test learning_test KNITest override-flags: -DTESTING=ON - - name: Run Unit Tests for the Norm Module - run: | - build/${{ matrix.build-setup.cmake-preset }}/bin/norm_test \ - --gtest_output="xml:reports/report-norm.xml" - - name: Run Unit Tests for the Parallel Module - if: always() - run: | - build/${{ matrix.build-setup.cmake-preset }}/bin/parallel_test \ - --gtest_output="xml:reports/report-parallel.xml" - - name: Run Unit Tests for the Parallel MPI Module in serial - if: always() - run: | - build/${{ matrix.build-setup.cmake-preset }}/bin/parallel_mpi_test \ - --gtest_output="xml:reports/report-parallel-mpi-serial.xml" - - name: Run Unit Tests for the Parallel MPI Module in parallel (Linux) - if: ${{ always() && runner.os == 'Linux' }} - run: | - mpirun --oversubscribe --use-hwthread-cpus -n 4 build/${{ matrix.build-setup.cmake-preset }}/bin/parallel_mpi_test \ - --gtest_output="xml:reports/report-parallel-mpi-parallel.xml" - - name: Run Unit Tests for the Parallel MPI Module in parallel (Windows and - macOS) - if: ${{ always() && runner.os != 'Linux' }} - run: | - mpiexec -n 4 build/${{ matrix.build-setup.cmake-preset }}/bin/parallel_mpi_test \ - --gtest_output="xml:reports/report-parallel-mpi-parallel.xml" - - name: Run Unit Tests for the Learning Module - if: always() - run: | - build/${{ matrix.build-setup.cmake-preset }}/bin/learning_test \ - --gtest_output="xml:reports/report-learning.xml" - - name: Run Unit Tests for the KNI Module - if: always() - run: | - build/${{ matrix.build-setup.cmake-preset }}/bin/KNITest \ - --gtest_output="xml:reports/report-kni.xml" - - name: Create Unit Test Reports Dashboards - uses: phoenix-actions/test-reporting@v12 - if: always() - with: - name: Unit Tests Reports (${{ matrix.build-setup.os }}) - reporter: jest-junit - path: reports/report-*.xml - path-replace-backslashes: 'true' # Necessary for windows paths + - name: Run Unit Tests + run: |- + ctest --preset ${{ matrix.build-setup.cmake-preset }} diff --git a/packaging/install.cmake b/packaging/install.cmake index b18787090..71ab8d3da 100644 --- a/packaging/install.cmake +++ b/packaging/install.cmake @@ -59,14 +59,26 @@ configure_file(${PROJECT_SOURCE_DIR}/packaging/common/KNI/template-README.md ${T # ######################################## Khiops and Khiops Coclustering installation +# replace MPIEXEC MPIEXEC_NUMPROC_FLAG and MPI_IMPL +configure_file(${PROJECT_SOURCE_DIR}/packaging/linux/common/khiops-env.in ${TMP_DIR}/khiops-env @ONLY + NEWLINE_STYLE UNIX) + if(NOT IS_FEDORA_LIKE) install(TARGETS MODL MODL_Coclustering RUNTIME DESTINATION usr/bin COMPONENT KHIOPS_CORE) + + # We install the binary with mpi suffix and create a symlink without the suffix + get_target_property(MODL_NAME MODL OUTPUT_NAME) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink /usr/bin/${MODL_NAME} ${CMAKE_BINARY_DIR}/MODL) + install( + FILES ${CMAKE_BINARY_DIR}/MODL + DESTINATION usr/bin + COMPONENT KHIOPS_CORE) else() # On fedora binaries built with mpi must follow these rules : # # - the binaries MUST be suffixed with $MPI_SUFFIX - # - MPI implementation specific files MUST be installed in the directories used by the used MPI compiler e.g. $MPI_BIN + # - MPI implementation specific files MUST be installed in the directories used by the MPI compiler e.g. $MPI_BIN # # see https://docs.fedoraproject.org/en-US/packaging-guidelines/MPI/ # @@ -74,8 +86,8 @@ else() install(TARGETS MODL_Coclustering RUNTIME DESTINATION /usr/bin COMPONENT KHIOPS_CORE) # We install the binary under $MPI_BIN and create a symlink to it - execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${MPI_BIN}/khiops/MODL${MPI_SUFFIX} - ${CMAKE_BINARY_DIR}/MODL) + get_target_property(MODL_NAME MODL OUTPUT_NAME) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${MPI_BIN}/khiops/${MODL_NAME} ${CMAKE_BINARY_DIR}/MODL) install( FILES ${CMAKE_BINARY_DIR}/MODL DESTINATION usr/bin @@ -85,8 +97,7 @@ endif() install( PROGRAMS ${PROJECT_SOURCE_DIR}/packaging/linux/common/khiops - ${PROJECT_SOURCE_DIR}/packaging/linux/common/khiops_coclustering - ${PROJECT_SOURCE_DIR}/packaging/linux/common/khiops-env + ${PROJECT_SOURCE_DIR}/packaging/linux/common/khiops_coclustering ${TMP_DIR}/khiops-env DESTINATION usr/bin COMPONENT KHIOPS_CORE) diff --git a/packaging/linux/common/khiops-env b/packaging/linux/common/khiops-env.in similarity index 84% rename from packaging/linux/common/khiops-env rename to packaging/linux/common/khiops-env.in index 214e39412..5d52efe67 100644 --- a/packaging/linux/common/khiops-env +++ b/packaging/linux/common/khiops-env.in @@ -53,15 +53,8 @@ if ! command -v mpiexec &> /dev/null then if source /etc/profile.d/modules.sh &> /dev/null then - # find the good module - # mpich_module="$(module avail |& sed -E -e 's/[[:blank:]]+/\n/g' | grep mpich-[0-9] | grep $(arch) | sort | tail -n 1)" - # if [ -z "$mpich_module" ] ; then - # mpich_module=$(module avail |& sed -E -e 's/[[:blank:]]+/\n/g' | grep mpich-$(arch)) - # fi - - # and load it module unload mpi - module load mpi/openmpi-x86_64 + module load mpi/@MPI_IMPL@-x86_64 fi fi @@ -79,12 +72,14 @@ if [[ -z $KHIOPS_PROC_NUMBER ]]; then KHIOPS_PROC_NUMBER=$(( $(lscpu -b -p=Core,Socket | grep -v '^#' | sort -u | wc -l) + 1 )) fi -if command -v mpiexec &> /dev/null +${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} + +if command -v @MPIEXEC@ &> /dev/null then - MPIEXEC_PATH=$(command -v mpiexec) - KHIOPS_MPI_COMMAND="${MPIEXEC_PATH} --allow-run-as-root --quiet --oversubscribe -n $KHIOPS_PROC_NUMBER" + MPIEXEC_PATH=$(command -v @MPIEXEC@) + KHIOPS_MPI_COMMAND="${MPIEXEC_PATH} -bind-to hwthread -map-by core @MPIEXEC_NUMPROC_FLAG@ $KHIOPS_PROC_NUMBER" else - echo "We didn't find mpiexec in the regular path. Parallel computation is unavailable: Khiops is launched in serial" + echo "We didn't find @MPIEXEC@ in the regular path. Parallel computation is unavailable: Khiops is launched in serial" KHIOPS_MPI_COMMAND="" fi diff --git a/test/LearningTestTool/py/_kht_families.py b/test/LearningTestTool/py/_kht_families.py index ff4b5a74e..cc88c5db4 100644 --- a/test/LearningTestTool/py/_kht_families.py +++ b/test/LearningTestTool/py/_kht_families.py @@ -22,9 +22,11 @@ def check_family(family): # Liste des familles, de la plus simple a la plus complete BASIC = "basic" # Famille elementaire, tres rapide a tester -FULL = "full" # Famille correspondant a tous les tests de non regression (environ une heure a tester) +# Famille correspondant a tous les tests de non regression (environ une heure a tester) +FULL = "full" FULL_NO_KNI = "full-no-kni" # Idem, sans KNI -COMPLETE = "complete" # Famille complete, tres lourde a tester (environ une journee) +# Famille complete, tres lourde a tester (environ une journee) +COMPLETE = "complete" ALL = "all" # Tous les repertoire de suite exhaustivement: ne pas utiliser pour les test, mais pour leur gestion TEST_FAMILIES = [BASIC, FULL, FULL_NO_KNI, COMPLETE, ALL] assert len(set(TEST_FAMILIES)) == len(TEST_FAMILIES), ( @@ -36,7 +38,7 @@ def check_family(family): assert DEFAULT_TEST_FAMILY in TEST_FAMILIES # Cas de la version V11, qui inclus des suites de test additionnelles -KHIOPS_V11 = True +KHIOPS_V11 = False # Ensuite des suites de test par famille et par outils # Pour le cas particlier le la famille ALL, les suite ne sont pas specifiee: diff --git a/test/UnitTests/Parallel-mpi/CMakeLists.txt b/test/UnitTests/Parallel-mpi/CMakeLists.txt index 87a26991b..11533a1a3 100644 --- a/test/UnitTests/Parallel-mpi/CMakeLists.txt +++ b/test/UnitTests/Parallel-mpi/CMakeLists.txt @@ -12,3 +12,12 @@ target_include_directories(parallel_mpi_test PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_compile_options(parallel_mpi_test PUBLIC ${GTEST_CFLAGS}) include(GoogleTest) gtest_discover_tests(parallel_mpi_test) + +# Add the same test in CTest framework. It allows to run the test directly with the right mpi command. The command +# "ctest --preset linux-gcc-release -L mpi" will run (depending on the mpi implementation and the core number): mpiexec +# -n 8 build/linux-gcc-release/parallel_mpi_test +add_test(NAME ctest_parallel_mpi_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} + ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) + +# Add "mpi" label to the test +set_tests_properties(ctest_parallel_mpi_test PROPERTIES LABELS "mpi")