Update khiops_env: remove -hostfile and -n flags (#540)

* Changes made to improve the usability on K8s:

- remove the KHIOPS_HOST_FILE: the host flag is not mandatory, it can be replaced by mpi environment variables
- by default, the -n flag is removed for openmpi, it will be easier to use on K8s

* Refactoring

we use _MPIEXEC to set the path of mpiexec (different on conda and system wide)
And after all tests, we set KHIOPS_MPI_COMMAND. In tha way, on conda or not we're sure to have the same command line

packaging/install.cmake

@@ -91,9 +91,9 @@ if(UNIX)
 
   endif(IS_CONDA)
 
-  # replace MPIEXEC MPIEXEC_NUMPROC_FLAG and MPI_IMPL KHIOPS_MPI_EXTRA_FLAG ADDITIONAL_ENV_VAR
+  # replace MPIEXEC MPIEXEC_NUMPROC_FLAG and MPI_IMPL MPI_EXTRA_FLAG ADDITIONAL_ENV_VAR
   if("${MPI_IMPL}" STREQUAL "openmpi")
-    set(KHIOPS_MPI_EXTRA_FLAG "--allow-run-as-root")
+    set(MPI_EXTRA_FLAG "--allow-run-as-root")
     set(KHIOPS_MPI_QUIET "--quiet")
     set(ADDITIONAL_ENV_VAR "export OMPI_MCA_btl_vader_single_copy_mechanism=none # issue on docker")
     set(ADDITIONAL_ENV_VAR_DISPLAY
@@ -102,13 +102,10 @@ if(UNIX)
       set(ADDITIONAL_ENV_VAR "${ADDITIONAL_ENV_VAR}\nexport PSM3_DEVICES=self # issue on rocky linux")
       set(ADDITIONAL_ENV_VAR_DISPLAY "${ADDITIONAL_ENV_VAR_DISPLAY}\n    echo PSM3_DEVICES \"$PSM3_DEVICES\"")
     endif()
-    set(MPIEXEC_HOSTFILE_FLAG "--hostfile")
-    file(READ ${PROJECT_SOURCE_DIR}/packaging/linux/common/khiops_env/export_env_variables.sh EXPORT_ENV_VARIABLES)
   elseif("${MPI_IMPL}" STREQUAL "mpich")
-    set(MPIEXEC_HOSTFILE_FLAG "-f")
     # Set localhost on MacOS (see issue # https://github.com/pmodels/mpich/issues/4710)
     if(APPLE)
-      set(KHIOPS_MPI_EXTRA_FLAG "-host localhost")
+      set(MPI_EXTRA_FLAG "-host localhost")
     endif(APPLE)
   endif()
 
@@ -126,14 +123,26 @@ if(UNIX)
     set(MODL_NAME "MODL")
   endif()
 
+  # For all mpi implementation except openmpi, we compute the proc number (with openmpi, the -n flag is not mandatory)
+  if(NOT "${MPI_IMPL}" STREQUAL "openmpi")
+    # Replace the path of _khiopsgetprocnumber in set_proc_number.in (with the variable GET_PROC_NUMBER_PATH)
+    configure_file(${PROJECT_SOURCE_DIR}/packaging/linux/common/khiops_env/set_proc_number.in
+                   ${TMP_DIR}/set_proc_number.sh @ONLY NEWLINE_STYLE UNIX)
+    # The variable SET_PROC_NUMBER is filled with the content of set_proc_number.sh This variable will be replaced in
+    # khiops_env.in with configure_file
+    file(READ ${TMP_DIR}/set_proc_number.sh SET_PROC_NUMBER)
+
+    # Add _khiopsgetprocnumber to the khiops_core package except for openmpi
+    install(TARGETS _khiopsgetprocnumber RUNTIME DESTINATION ./${GET_PROC_NUMBER_PATH} COMPONENT KHIOPS_CORE)
+  endif()
+
   configure_file(${PROJECT_SOURCE_DIR}/packaging/linux/common/khiops_env/khiops_env.in ${TMP_DIR}/khiops_env @ONLY
                  NEWLINE_STYLE UNIX)
   configure_file(${PROJECT_SOURCE_DIR}/packaging/linux/debian/khiops-core/postinst.in ${TMP_DIR}/postinst @ONLY
                  NEWLINE_STYLE UNIX)
 
   install(TARGETS MODL RUNTIME DESTINATION ./${KHIOPS_PATH} COMPONENT KHIOPS_CORE)
   install(TARGETS MODL_Coclustering RUNTIME DESTINATION ./${KHIOPS_COCLUSTERING_PATH} COMPONENT KHIOPS_CORE)
-  install(TARGETS _khiopsgetprocnumber RUNTIME DESTINATION ./${GET_PROC_NUMBER_PATH} COMPONENT KHIOPS_CORE)
 
   install(
     PROGRAMS ${PROJECT_SOURCE_DIR}/packaging/linux/common/khiops

packaging/linux/common/khiops_env/khiops_env.in

@@ -42,9 +42,9 @@ help() {
     echo "  . default behavior if not set: depending on the file drivers available for Khiops"
     echo "  . set to 'true' to allow file name selection with uri schemas"
     echo "  . set to 'false' to allow local file name selection only with a file selection dialog"
-    echo "KHIOPS_MPI_HOST_FILE: provide hostfile to mpi."
-    echo
     echo "KHIOPS_MPI_VERBOSE: true (default) or false, print messages from mpi (OpenMPI only)."
+    echo "KHIOPS_MPI_EXTRA_FLAGS: extra flags added to the mpi command line"
+    echo
     echo "In case of configuration problems, the variables KHIOPS_JAVA_ERROR and KHIOPS_MPI_ERROR contain error messages."
 
 }
@@ -71,44 +71,41 @@ fi
 
 if ! [ "$KHIOPS_MPI_VERBOSE" == "true" ]; then
     # Mute mpi by adding flags (OpenMPI only)
-    MPI_EXTRA_FLAGS="@KHIOPS_MPI_QUIET@"
-fi
-
-# Setting up mpi for multiple machines
-if [[ -n $KHIOPS_MPI_HOST_FILE ]]; then
-    MPI_EXTRA_FLAGS="$MPI_EXTRA_FLAGS @MPIEXEC_HOSTFILE_FLAG@ $KHIOPS_MPI_HOST_FILE"
-    @EXPORT_ENV_VARIABLES@
+    _MPI_EXTRA_FLAGS="@KHIOPS_MPI_QUIET@"
 fi
 
 KHIOPS_PATH=@KHIOPS_PATH@@MODL_NAME@
 KHIOPS_COCLUSTERING_PATH=@KHIOPS_COCLUSTERING_PATH@MODL_Coclustering
 
-# Number of processes in use (must be set according to the physical cores number)
-if [[ -z $KHIOPS_PROC_NUMBER ]]; then
-    KHIOPS_PROC_NUMBER=$("@GET_PROC_NUMBER_PATH@"_khiopsgetprocnumber | head -n 1)
-fi
-
 if command -v mpiexec &>/dev/null; then
     KHIOPS_MPI_ERROR=""
-    KHIOPS_MPI_COMMAND="$(type -P mpiexec) $MPI_EXTRA_FLAGS @KHIOPS_MPI_EXTRA_FLAG@ @MPIEXEC_NUMPROC_FLAG@ $KHIOPS_PROC_NUMBER"
+    _MPIEXEC=$(type -P mpiexec)
 else
     # Fallback for Conda-based environments where `mpiexec` is not in PATH,
     # because $CONDA_PREFIX/bin is not in PATH
     _MPIEXEC=@KHIOPS_PATH@mpiexec
     if command -v $_MPIEXEC &>/dev/null; then
         KHIOPS_MPI_ERROR=""
-        KHIOPS_MPI_COMMAND="$_MPIEXEC $MPI_EXTRA_FLAGS @KHIOPS_MPI_EXTRA_FLAG@ @MPIEXEC_NUMPROC_FLAG@ $KHIOPS_PROC_NUMBER"
     else
         KHIOPS_MPI_ERROR="We didn't find mpiexec in the expected paths. Parallel computation is unavailable: Khiops is launched in serial"
-        KHIOPS_MPI_COMMAND=""
     fi
-    unset _MPIEXEC
+
 fi
 
-# without more than 2 procs, we use the serial khiops
-if [[ $KHIOPS_PROC_NUMBER -le 2 ]]; then
-    KHIOPS_MPI_COMMAND=""
+if [[ -z $KHIOPS_MPI_ERROR ]]; then
+    KHIOPS_MPI_COMMAND="$_MPIEXEC $_MPI_EXTRA_FLAGS @MPI_EXTRA_FLAG@ $KHIOPS_MPI_EXTRA_FLAGS"
+    @SET_PROC_NUMBER@
+    if [[ -n $KHIOPS_PROC_NUMBER ]]; then
+        KHIOPS_MPI_COMMAND="$KHIOPS_MPI_COMMAND @MPIEXEC_NUMPROC_FLAG@ $KHIOPS_PROC_NUMBER"
+    fi
+
+    # without more than 2 procs, we use the serial khiops
+    if [[ -n $KHIOPS_PROC_NUMBER && $KHIOPS_PROC_NUMBER -le 2 ]]; then
+        KHIOPS_MPI_COMMAND=""
+    fi
 fi
+unset _MPIEXEC
+unset _MPI_EXTRA_FLAGS
 
 if [ "$1" = "--env" ]; then
     echo KHIOPS_PATH "$KHIOPS_PATH"
@@ -126,5 +123,6 @@ if [ "$1" = "--env" ]; then
     echo KHIOPS_JAVA_ERROR "$KHIOPS_JAVA_ERROR"
     echo KHIOPS_MPI_ERROR "$KHIOPS_MPI_ERROR"
     echo KHIOPS_MPI_VERBOSE "$KHIOPS_MPI_VERBOSE"
+    echo KHIOPS_MPI_EXTRA_FLAGS "$KHIOPS_MPI_EXTRA_FLAGS"
     @ADDITIONAL_ENV_VAR_DISPLAY@
 fi

packaging/linux/common/khiops_env/set_proc_number.in

@@ -0,0 +1,5 @@
+
+    # Number of processes in use (must be set according to the physical cores number)
+    if [[ -z $KHIOPS_PROC_NUMBER ]]; then
+        KHIOPS_PROC_NUMBER=$("@GET_PROC_NUMBER_PATH@"_khiopsgetprocnumber | head -n 1)
+    fi