From f4c554bca023696ab78d3b06f42ed5966813d51f Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 9 Oct 2024 09:12:57 +0100 Subject: [PATCH] Option to use Julia-provided MPI in machine setup --- machines/generic-pc/compile_dependencies.sh | 98 +++--- .../shared/add_dependencies_to_project.jl | 295 +++++++++--------- machines/shared/machine_setup.jl | 10 + 3 files changed, 214 insertions(+), 189 deletions(-) diff --git a/machines/generic-pc/compile_dependencies.sh b/machines/generic-pc/compile_dependencies.sh index 15c3c5860..ae70bd6b9 100755 --- a/machines/generic-pc/compile_dependencies.sh +++ b/machines/generic-pc/compile_dependencies.sh @@ -12,62 +12,68 @@ ARTIFACT_DIR=$PWD # HDF5 ###### -# Get default response for whether to download/build HDF5 -DEFAULT_BUILDHDF5=$(../../bin/julia --project ../shared/get_mk_preference.jl build_hdf5 "y") +SYSTEM_MPI=$(../../bin/julia --project ../shared/get_mk_preference.jl use_system_mpi) -if [[ $DEFAULT_BUILDHDF5 == "y" ]]; then - echo "Do you want to download, and compile a local version of HDF5 (if you do" - echo "not do this, you will be given the option to choose an HDF5 library to" - echo "link later)? [y]/n" - read -p "> " input - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - echo - echo "$input is not a valid response: y/[n]" +if [[ $SYSTEM_MPI == "y" ]]; then + # Get default response for whether to download/build HDF5 + DEFAULT_BUILDHDF5=$(../../bin/julia --project ../shared/get_mk_preference.jl build_hdf5 "y") + + if [[ $DEFAULT_BUILDHDF5 == "y" ]]; then + echo "Do you want to download, and compile a local version of HDF5 (if you do" + echo "not do this, you will be given the option to choose an HDF5 library to" + echo "link later)? [y]/n" read -p "> " input - done - if [[ -z $input || $input == "y" ]]; then - BUILDHDF5="y" + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + done + if [[ -z $input || $input == "y" ]]; then + BUILDHDF5="y" + else + BUILDHDF5="n" + fi else - BUILDHDF5="n" - fi -else - echo "Do you want to download, and compile a local version of HDF5 (if you do" - echo "not do this, you will be given the option to choose an HDF5 library to" - echo "link later)? y/[n]" - read -p "> " input - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - echo - echo "$input is not a valid response: y/[n]" + echo "Do you want to download, and compile a local version of HDF5 (if you do" + echo "not do this, you will be given the option to choose an HDF5 library to" + echo "link later)? y/[n]" read -p "> " input - done - if [[ -z $input || $input == "n" ]]; then - BUILDHDF5="n" - else - BUILDHDF5="y" + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + done + if [[ -z $input || $input == "n" ]]; then + BUILDHDF5="n" + else + BUILDHDF5="y" + fi fi -fi -# Save current response for whether to download/build HDF5 as default -../../bin/julia --project ../shared/set_mk_preference.jl build_hdf5 $BUILDHDF5 + # Save current response for whether to download/build HDF5 as default + ../../bin/julia --project ../shared/set_mk_preference.jl build_hdf5 $BUILDHDF5 -if [[ $BUILDHDF5 == "y" && -d hdf5-build ]]; then - echo "HDF5 appears to have been downloaded, compiled and installed already." - echo "Do you want to download, compile and install again, overwriting the existing " - echo "version? y/[n]" - read -p "> " input - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - echo - echo "$input is not a valid response: y/[n]" + if [[ $BUILDHDF5 == "y" && -d hdf5-build ]]; then + echo "HDF5 appears to have been downloaded, compiled and installed already." + echo "Do you want to download, compile and install again, overwriting the existing " + echo "version? y/[n]" read -p "> " input - done - if [[ -z $input || $input == "n" ]]; then - BUILDHDF5="n" - else - # Remove the install directory if it exists already - if [[ -d hdf5-build ]]; then - rm -r hdf5-build + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + done + if [[ -z $input || $input == "n" ]]; then + BUILDHDF5="n" + else + # Remove the install directory if it exists already + if [[ -d hdf5-build ]]; then + rm -r hdf5-build + fi fi fi +else + BUILD_HDF5="n" fi if [[ $BUILDHDF5 == "y" ]]; then diff --git a/machines/shared/add_dependencies_to_project.jl b/machines/shared/add_dependencies_to_project.jl index 008045e93..71298ab2e 100644 --- a/machines/shared/add_dependencies_to_project.jl +++ b/machines/shared/add_dependencies_to_project.jl @@ -84,179 +84,188 @@ Pkg.resolve() # MPI setup ########### -println("\n** Setting up to use system MPI\n") -using MPIPreferences - -if "mpi_library_names" ∈ keys(machine_settings) || "mpiexec" ∈ keys(machine_settings) - MPIPreferences.use_system_binary(library_names=machine_settings["mpi_library_names"], - mpiexec=machine_settings["mpiexec"]) -elseif Sys.isapple() - # On macOS, MPIPreferences.use_system_binary() does not automatically find the MPI - # library when MPI was installed with homebrew, so prompt the user for the library - # path instead. - # ?? Could we attempt to auto-detect the MPI library before prompting the user?? - if prompt_for_lib_paths - try - # See if MPIPreferences can auto-detect the system MPI library path - MPIPreferences.use_system_binary() - catch - println("Failed to auto-detect path of MPI library...") - - local mpi_library_path - - default_mpi_library_path = get(mk_preferences, "mpi_library_path", "") - mpi_library_path = get_input_with_path_completion( - "\nEnter the full path to your MPI library (e.g. something like " - * "'libmpi.dylib'): [$default_mpi_library_path]") - if mpi_library_path == "" - mpi_library_path = default_mpi_library_path - end +if mk_preferences["use_system_mpi"] == "y" + println("\n** Setting up to use system MPI\n") + using MPIPreferences + + if "mpi_library_names" ∈ keys(machine_settings) || "mpiexec" ∈ keys(machine_settings) + MPIPreferences.use_system_binary(library_names=machine_settings["mpi_library_names"], + mpiexec=machine_settings["mpiexec"]) + elseif Sys.isapple() + # On macOS, MPIPreferences.use_system_binary() does not automatically find the MPI + # library when MPI was installed with homebrew, so prompt the user for the library + # path instead. + # ?? Could we attempt to auto-detect the MPI library before prompting the user?? + if prompt_for_lib_paths + try + # See if MPIPreferences can auto-detect the system MPI library path + MPIPreferences.use_system_binary() + catch + println("Failed to auto-detect path of MPI library...") + + local mpi_library_path + + default_mpi_library_path = get(mk_preferences, "mpi_library_path", "") + mpi_library_path = get_input_with_path_completion( + "\nEnter the full path to your MPI library (e.g. something like " + * "'libmpi.dylib'): [$default_mpi_library_path]") + if mpi_library_path == "" + mpi_library_path = default_mpi_library_path + end - MPIPreferences.use_system_binary(library_names=mpi_library_path) + MPIPreferences.use_system_binary(library_names=mpi_library_path) - global mk_preferences, local_preferences + global mk_preferences, local_preferences - # Just got the value for the setting, now write it to LocalPreferences.toml, - # but first reload the preferences from the LocalPreferences.toml file so that - # we don't overwrite the values that MPIPreferences has set. - local_preferences = TOML.parsefile(local_preferences_filename) - mk_preferences = local_preferences["moment_kinetics"] - mk_preferences["mpi_library_path"] = mpi_library_path - open(local_preferences_filename, "w") do io - TOML.print(io, local_preferences, sorted=true) + # Just got the value for the setting, now write it to LocalPreferences.toml, + # but first reload the preferences from the LocalPreferences.toml file so that + # we don't overwrite the values that MPIPreferences has set. + local_preferences = TOML.parsefile(local_preferences_filename) + mk_preferences = local_preferences["moment_kinetics"] + mk_preferences["mpi_library_path"] = mpi_library_path + open(local_preferences_filename, "w") do io + TOML.print(io, local_preferences, sorted=true) + end + # Re-read local_preferences file, so we can modify it again below, keeping the + # changes here + local_preferences = TOML.parsefile(local_preferences_filename) + mk_preferences = local_preferences["moment_kinetics"] end - # Re-read local_preferences file, so we can modify it again below, keeping the - # changes here - local_preferences = TOML.parsefile(local_preferences_filename) - mk_preferences = local_preferences["moment_kinetics"] - end - else - if "mpi_library_path" ∈ keys(mk_preferences) - mpi_library_path = mk_preferences["mpi_library_path"] - MPIPreferences.use_system_binary(library_names=mpi_library_path) else - # Must have auto-detected MPI library before, so do the same here - MPIPreferences.use_system_binary() + if "mpi_library_path" ∈ keys(mk_preferences) + mpi_library_path = mk_preferences["mpi_library_path"] + MPIPreferences.use_system_binary(library_names=mpi_library_path) + else + # Must have auto-detected MPI library before, so do the same here + MPIPreferences.use_system_binary() + end end + else + # If settings for MPI library are not given explicitly, then auto-detection by + # MPIPreferences.use_system_binary() should work. + MPIPreferences.use_system_binary() end else - # If settings for MPI library are not given explicitly, then auto-detection by - # MPIPreferences.use_system_binary() should work. - MPIPreferences.use_system_binary() + using MPI + MPI.install_mpiexecjl(; destdir=project_dir, force=true) end # HDF5 setup ############ -println("\n** Setting up to use system HDF5\n") - -if machine_settings["hdf5_library_setting"] == "system" - hdf5_dir = joinpath(ENV["HDF5_DIR"], "lib") # system hdf5 - using HDF5 - HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), - joinpath(hdf5_dir, "libhdf5_hl.so")) -elseif machine_settings["hdf5_library_setting"] == "download" - artifact_dir = joinpath(repo_dir, "machines", "artifacts") - hdf5_dir = joinpath(artifact_dir, "hdf5-build", "lib") - using HDF5 - HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), - joinpath(hdf5_dir, "libhdf5_hl.so")) -elseif machine_settings["hdf5_library_setting"] == "prompt" - # Prompt user to select what HDF5 to use - if mk_preferences["build_hdf5"] == "y" - local_hdf5_install_dir = joinpath("machines", "artifacts", "hdf5-build", "lib") - local_hdf5_install_dir = realpath(local_hdf5_install_dir) - # We have downloaded and compiled HDF5, so link that - hdf5_dir = local_hdf5_install_dir - hdf5_lib = joinpath(local_hdf5_install_dir, "libhdf5.so") - hdf5_lib_hl = joinpath(local_hdf5_install_dir, "libhdf5_hl.so") - elseif !prompt_for_lib_paths - hdf5_dir = mk_preferences["hdf5_dir"] - if hdf5_dir != "default" - hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") - hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") - end - else - println("\n** Setting up to use system HDF5\n") +if mk_preferences["use_system_mpi"] == "y" + # Only need to do this if using 'system MPI'. If we are using the Julia-provided MPI, + # then the Julia-provided HDF5 is already MPI-enabled + println("\n** Setting up to use system HDF5\n") + + if machine_settings["hdf5_library_setting"] == "system" + hdf5_dir = joinpath(ENV["HDF5_DIR"], "lib") # system hdf5 + using HDF5 + HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), + joinpath(hdf5_dir, "libhdf5_hl.so")) + elseif machine_settings["hdf5_library_setting"] == "download" + artifact_dir = joinpath(repo_dir, "machines", "artifacts") + hdf5_dir = joinpath(artifact_dir, "hdf5-build", "lib") + using HDF5 + HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), + joinpath(hdf5_dir, "libhdf5_hl.so")) + elseif machine_settings["hdf5_library_setting"] == "prompt" + # Prompt user to select what HDF5 to use + if mk_preferences["build_hdf5"] == "y" + local_hdf5_install_dir = joinpath("machines", "artifacts", "hdf5-build", "lib") + local_hdf5_install_dir = realpath(local_hdf5_install_dir) + # We have downloaded and compiled HDF5, so link that + hdf5_dir = local_hdf5_install_dir + hdf5_lib = joinpath(local_hdf5_install_dir, "libhdf5.so") + hdf5_lib_hl = joinpath(local_hdf5_install_dir, "libhdf5_hl.so") + elseif !prompt_for_lib_paths + hdf5_dir = mk_preferences["hdf5_dir"] + if hdf5_dir != "default" + hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") + hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") + end + else + println("\n** Setting up to use system HDF5\n") - default_hdf5_dir = get(ENV, "HDF5_DIR", "") # try to find a path to a system hdf5, may not work on all systems + default_hdf5_dir = get(ENV, "HDF5_DIR", "") # try to find a path to a system hdf5, may not work on all systems - default_hdf5_dir = get(mk_preferences, "hdf5_dir", default_hdf5_dir) + default_hdf5_dir = get(mk_preferences, "hdf5_dir", default_hdf5_dir) - hdf5_dir = "" - hdf5_lib = "" - hdf5_lib_hl = "" - while true - global hdf5_dir, hdf5_lib, hdf5_lib_hl - hdf5_dir = get_input_with_path_completion( - "\nAn HDF5 installation compiled with your system MPI is required to use\n" - * "parallel I/O. Enter the directory where the libhdf5.so and libhdf5_hl.so are\n" - * "located (enter 'default' to use the Julia-provided HDF5, which does not\n" - * "support parallel I/O): [$default_hdf5_dir]") + hdf5_dir = "" + hdf5_lib = "" + hdf5_lib_hl = "" + while true + global hdf5_dir, hdf5_lib, hdf5_lib_hl + hdf5_dir = get_input_with_path_completion( + "\nAn HDF5 installation compiled with your system MPI is required to use\n" + * "parallel I/O. Enter the directory where the libhdf5.so and libhdf5_hl.so are\n" + * "located (enter 'default' to use the Julia-provided HDF5, which does not\n" + * "support parallel I/O): [$default_hdf5_dir]") - if hdf5_dir == "" - hdf5_dir = default_hdf5_dir - end + if hdf5_dir == "" + hdf5_dir = default_hdf5_dir + end - if hdf5_dir == "default" - break - end + if hdf5_dir == "default" + break + end - if isdir(hdf5_dir) - hdf5_dir = realpath(hdf5_dir) - end - hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") - hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") - if isfile(hdf5_lib) && isfile(hdf5_lib_hl) - break - else - # Remove trailing slash if it exists so that we can print a single trailing slash - # consistently - hdf5_dir = rstrip(hdf5_dir, '/') - print("HDF5 libraries not found in '$hdf5_dir/'.") - if !isfile(hdf5_lib) - print(" $hdf5_lib does not exist.") + if isdir(hdf5_dir) + hdf5_dir = realpath(hdf5_dir) end - if !isfile(hdf5_lib_hl) - print(" $hdf5_lib_hl does not exist.") + hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") + hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") + if isfile(hdf5_lib) && isfile(hdf5_lib_hl) + break + else + # Remove trailing slash if it exists so that we can print a single trailing slash + # consistently + hdf5_dir = rstrip(hdf5_dir, '/') + print("HDF5 libraries not found in '$hdf5_dir/'.") + if !isfile(hdf5_lib) + print(" $hdf5_lib does not exist.") + end + if !isfile(hdf5_lib_hl) + print(" $hdf5_lib_hl does not exist.") + end end end end - end - # Reload local_preferences and mk_preferences as they may have been modified by MPI - # setup - local_preferences_filename = joinpath(project_dir, "LocalPreferences.toml") - local_preferences = TOML.parsefile(local_preferences_filename) - if abspath(PROGRAM_FILE) == @__FILE__ - # Only need to do this for the top-level project. When adding dependencies to - # makie_post_processing or plots_post_processing, do not need to set "hdf5_dir" in - # `mk_preferences` to go in the `moment_kinetics` section - this only needs to be - # done for the top-level project. - mk_preferences = local_preferences["moment_kinetics"] - - mk_preferences["hdf5_dir"] = hdf5_dir - end + # Reload local_preferences and mk_preferences as they may have been modified by MPI + # setup + local_preferences_filename = joinpath(project_dir, "LocalPreferences.toml") + local_preferences = TOML.parsefile(local_preferences_filename) + if abspath(PROGRAM_FILE) == @__FILE__ + # Only need to do this for the top-level project. When adding dependencies to + # makie_post_processing or plots_post_processing, do not need to set "hdf5_dir" in + # `mk_preferences` to go in the `moment_kinetics` section - this only needs to be + # done for the top-level project. + mk_preferences = local_preferences["moment_kinetics"] - # Delete any existing preferences for HDF5 and HDF5.jll because they may prevent - # `using HDF5` if the libraries do not exist. - pop!(local_preferences, "HDF5", nothing) - pop!(local_preferences, "HDF5_jll", nothing) + mk_preferences["hdf5_dir"] = hdf5_dir + end - open(local_preferences_filename, "w") do io - TOML.print(io, local_preferences, sorted=true) - end + # Delete any existing preferences for HDF5 and HDF5.jll because they may prevent + # `using HDF5` if the libraries do not exist. + pop!(local_preferences, "HDF5", nothing) + pop!(local_preferences, "HDF5_jll", nothing) + + open(local_preferences_filename, "w") do io + TOML.print(io, local_preferences, sorted=true) + end - using HDF5 - if hdf5_dir == "default" - HDF5.API.set_libraries!() + using HDF5 + if hdf5_dir == "default" + HDF5.API.set_libraries!() + else + HDF5.API.set_libraries!(hdf5_lib, hdf5_lib_hl) + end else - HDF5.API.set_libraries!(hdf5_lib, hdf5_lib_hl) + error("Unrecognized setting " + * "hdf5_library_setting=$(machine_settings["hdf5_library_setting"])") end -else - error("Unrecognized setting " - * "hdf5_library_setting=$(machine_settings["hdf5_library_setting"])") end diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index d8c228969..e85a477cd 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -22,6 +22,7 @@ default_settings["base"] = Dict("account"=>"", "use_makie"=>"n", "use_plots"=>"n", "separate_postproc_projects"=>"n", + "use_system_mpi"=>"y", "use_netcdf"=>"n", "enable_mms"=>"n", "use_revise"=>"n") @@ -232,6 +233,15 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals * "post-processing for example)?", machine, mk_preferences, ["y", "n"]) end + get_setting("use_system_mpi", + "Normally you probably want to use the system-provided MPI library. However\n" + * "occasionally it can be useful to use the Julia-provided MPI instead. If you\n" + * "choose the Julia-provided MPI, a link to `mpiexecjl` will be installed to the\n" + * "project directory, which you should use instead of `mpirun`/`mpiexec` when\n" + * "you want to run an MPI job - i.e. use `./mpiexecjl -np 4 julia ...` instead\n" + * "of `mpirun -np 4 julia ...`.\n" + * "Do you want to use the system-provided MPI?", + machine, mk_preferences, ["y", "n"]) get_setting("use_netcdf", "Would you like to enable optional NetCDF I/O (warning: using NetCDF sometimes\n" * "causes errors when using a local or system install of HDF5)?",