From b4fa7617b6daf5d844c09ea14b70a71d295b4c62 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Wed, 5 Oct 2022 12:04:44 -0700 Subject: [PATCH 1/5] add functions to check (and error) if the MPI implementation has changed --- docs/src/reference/mpipreferences.md | 1 + lib/MPIPreferences/src/MPIPreferences.jl | 56 +++++++++++++++++++----- src/MPI.jl | 5 ++- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/docs/src/reference/mpipreferences.md b/docs/src/reference/mpipreferences.md index 09234dbfb..d17849e71 100644 --- a/docs/src/reference/mpipreferences.md +++ b/docs/src/reference/mpipreferences.md @@ -20,6 +20,7 @@ MPIPreferences.use_jll_binary ## Utils ```@docs +MPIPreferences.check_unchanged MPIPreferences.identify_abi ``` diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl index 66d76b45b..6d1993e6b 100644 --- a/lib/MPIPreferences/src/MPIPreferences.jl +++ b/lib/MPIPreferences/src/MPIPreferences.jl @@ -1,11 +1,16 @@ module MPIPreferences +export use_jll_binary, use_system_binary + using Preferences, Libdl if !(VersionNumber(@load_preference("_format", "1.0")) <= v"1.0") error("The preferences attached to MPIPreferences are incompatible with this version of the package.") end +const PREFS_CHANGED = Ref(false) +const DEPS_LOADED = Ref(false) + """ MPIPreferences.binary :: String @@ -50,17 +55,19 @@ end end """ - MPIPreferences.use_jll_binary([binary]; export_prefs=false, force=true) + use_jll_binary([binary]; export_prefs=false, force=true) + +Switches the underlying MPI implementation to one provided by JLL packages. A +restart of Julia is required for the changes to take effect. -Switches the underlying MPI implementation to one provided by JLL packages. Available options are: - `"MicrosoftMPI_jll"` (Only option and default on Winddows) - `"MPICH_jll"` (Default on all other platform) - `"OpenMPI_jll"` - `"MPItrampoline_jll"` -The `export_prefs` option determines whether the preferences being set should be stored -within `LocalPreferences.toml` or `Project.toml`. +The `export_prefs` option determines whether the preferences being set should be +stored within `LocalPreferences.toml` or `Project.toml`. """ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_jll"; export_prefs=false, force=true) binary in ["MicrosoftMPI_jll", "MPICH_jll", "OpenMPI_jll", "MPItrampoline_jll"] || @@ -75,7 +82,8 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j force=force ) - @warn "The underlying MPI implementation has changed. You will need to restart Julia for this change to take effect" binary + PREFS_CHANGED[] = true + @info "MPIPreferences changed" binary if VERSION <= v"1.6.5" || VERSION == v"1.7.0" @warn """ @@ -85,19 +93,24 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j """ end + if DEPS_LOADED[] + error("You will need to restart Julia for the changes to take effect") + end + return nothing end """ - MPIPreferences.use_system_binary(; + use_system_binary(; library_names = ["libmpi", "libmpi_ibm", "msmpi", "libmpich", "libmpi_cray", "libmpitrampoline"], mpiexec = "mpiexec", abi = nothing, export_prefs = false, force = true) -Switches the underlying MPI implementation to a system provided one. +Switches the underlying MPI implementation to a system provided one. A restart +of Julia is required for the changes to take effect. Options: @@ -113,9 +126,11 @@ Options: to include specific command line options. - `abi`: the ABI of the MPI library. By default this is determined automatically - using [`identify_abi`](@ref). See [`abi`](@ref) for currently supported values. + using [`identify_abi`](@ref). See [`abi`](@ref) for currently supported + values. -- `export_prefs`: if `true`, the preferences into the `Project.toml` instead of `LocalPreferences.toml`. +- `export_prefs`: if `true`, the preferences into the `Project.toml` instead of + `LocalPreferences.toml`. - `force`: if `true`, the preferences are set even if they are already set. """ @@ -154,7 +169,8 @@ function use_system_binary(; force=force ) - @warn "The underlying MPI implementation has changed. You will need to restart Julia for this change to take effect" binary libmpi abi mpiexec + PREFS_CHANGED[] = true + @info "MPIPreferences changed" binary libmpi abi mpiexec if VERSION <= v"1.6.5" || VERSION == v"1.7.0" @warn """ @@ -164,9 +180,29 @@ function use_system_binary(; """ end + if DEPS_LOADED[] + error("You will need to restart Julia for the changes to take effect") + end + return nothing end +""" + MPIPreferences.check_unchanged() + +This will throw an error if the MPIPreferences have been modified in the current +Julia session, or if they are modified after this function is called. + +This is typically called from the `__init__()` function of any package which +relies on the values of MPIPreferences. +""" +function check_unchanged() + if PREFS_CHANGED[] + error("MPIPreferences have changed, you will need to restart Julia for the changes to take effect") + end + DEPS_LOADED[] = true +end + function identify_implementation_version_abi(version_string::AbstractString) impl = "unknown" version = v"0" diff --git a/src/MPI.jl b/src/MPI.jl index 4c70e7fca..a43f40f02 100644 --- a/src/MPI.jl +++ b/src/MPI.jl @@ -3,6 +3,7 @@ module MPI using Libdl, Serialization using Requires using DocStringExtensions +import MPIPreferences export mpiexec, UBuffer, VBuffer @@ -58,7 +59,6 @@ function run_load_time_hooks() nothing end -using MPIPreferences include("implementations.jl") include("error.jl") include("info.jl") @@ -81,12 +81,13 @@ include("misc.jl") include("deprecated.jl") function __init__() + MPIPreferences.check_unchanged() # an empty string was used to indicate "default" # https://github.com/JuliaParallel/MPI.jl/blob/v0.19.2/deps/build.jl#L142 mpi_env_binary = get(ENV, "JULIA_MPI_BINARY", "") if mpi_env_binary != "" && mpi_env_binary != MPIPreferences.binary - @info """ + @warn """ The JULIA_MPI_BINARY environment variable is no longer used to configure the MPI binary. Please use the MPIPreferences.jl package instead: From cc738c49d6207fd8186972756fed4d725f196e0b Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Wed, 5 Oct 2022 12:20:51 -0700 Subject: [PATCH 2/5] fix invalidations check --- .github/workflows/Invalidations.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Invalidations.yml b/.github/workflows/Invalidations.yml index 6a9169c1f..6cb6db31f 100644 --- a/.github/workflows/Invalidations.yml +++ b/.github/workflows/Invalidations.yml @@ -22,18 +22,32 @@ jobs: - uses: julia-actions/setup-julia@v1 with: version: '1' + # current branch - uses: actions/checkout@v3 + - name: add MPIPreferences + shell: julia --color=yes --project=. {0} + run: | + using Pkg + Pkg.develop(path="lib/MPIPreferences") + - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-invalidations@v1 id: invs_pr + # default branch - uses: actions/checkout@v3 with: ref: ${{ github.event.repository.default_branch }} + - name: add MPIPreferences + shell: julia --color=yes --project=. {0} + run: | + using Pkg + Pkg.develop(path="lib/MPIPreferences") - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-invalidations@v1 id: invs_default - + + # report - name: Report invalidation counts run: | echo "Invalidations on default branch: ${{ steps.invs_default.outputs.total }} (${{ steps.invs_default.outputs.deps }} via deps)" >> $GITHUB_STEP_SUMMARY From 2a3dbc78b392f3861f41e3aa7870a1636ba3a5d2 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Wed, 5 Oct 2022 13:16:43 -0700 Subject: [PATCH 3/5] check if preferences actually changed --- lib/MPIPreferences/src/MPIPreferences.jl | 31 +++++++++++++++--------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl index 6d1993e6b..24ab08e02 100644 --- a/lib/MPIPreferences/src/MPIPreferences.jl +++ b/lib/MPIPreferences/src/MPIPreferences.jl @@ -82,9 +82,6 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j force=force ) - PREFS_CHANGED[] = true - @info "MPIPreferences changed" binary - if VERSION <= v"1.6.5" || VERSION == v"1.7.0" @warn """ Due to a bug in Julia (until 1.6.5 and 1.7.1), setting preferences in transitive dependencies @@ -93,10 +90,16 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j """ end - if DEPS_LOADED[] - error("You will need to restart Julia for the changes to take effect") - end + if binary == MPIPreferences.binary + @info "MPIPreferences unchanged" binary + else + PREFS_CHANGED[] = true + @info "MPIPreferences changed" binary + if DEPS_LOADED[] + error("You will need to restart Julia for the changes to take effect") + end + end return nothing end @@ -169,8 +172,6 @@ function use_system_binary(; force=force ) - PREFS_CHANGED[] = true - @info "MPIPreferences changed" binary libmpi abi mpiexec if VERSION <= v"1.6.5" || VERSION == v"1.7.0" @warn """ @@ -180,10 +181,16 @@ function use_system_binary(; """ end - if DEPS_LOADED[] - error("You will need to restart Julia for the changes to take effect") - end + if binary == MPIPreferences.binary && abi == MPIPreferences.abi && libmpi == System.libmpi && mpiexec == System.mpiexec_path + @info "MPIPreferences unchanged" binary libmpi abi mpiexec + else + PREFS_CHANGED[] = true + @info "MPIPreferences changed" binary libmpi abi mpiexec + if DEPS_LOADED[] + error("You will need to restart Julia for the changes to take effect") + end + end return nothing end @@ -351,7 +358,7 @@ function identify_abi(libmpi) # 2) try to identify the MPI implementation impl, version, abi = identify_implementation_version_abi(version_string) - @info "MPI implementation" libmpi version_string impl version abi + @info "MPI implementation identified" libmpi version_string impl version abi return abi end From 7185108f4d02b581bb28e11f183b22d2aec2b055 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Thu, 6 Oct 2022 13:12:22 -0700 Subject: [PATCH 4/5] fix docstring --- lib/MPIPreferences/src/MPIPreferences.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl index 24ab08e02..fcef689bd 100644 --- a/lib/MPIPreferences/src/MPIPreferences.jl +++ b/lib/MPIPreferences/src/MPIPreferences.jl @@ -197,10 +197,10 @@ end """ MPIPreferences.check_unchanged() -This will throw an error if the MPIPreferences have been modified in the current -Julia session, or if they are modified after this function is called. +Throws an error if the preferences have been modified in the current Julia +session, or if they are modified after this function is called. -This is typically called from the `__init__()` function of any package which +This is should be called from the `__init__()` function of any package which relies on the values of MPIPreferences. """ function check_unchanged() @@ -208,6 +208,7 @@ function check_unchanged() error("MPIPreferences have changed, you will need to restart Julia for the changes to take effect") end DEPS_LOADED[] = true + return nothing end function identify_implementation_version_abi(version_string::AbstractString) From 01c7de80f5838087e14210a1d331b84b8d79fe72 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Thu, 6 Oct 2022 13:16:06 -0700 Subject: [PATCH 5/5] bump versions --- Project.toml | 4 ++-- lib/MPIPreferences/Project.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index a5d4fa241..2e5fa5f8f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MPI" uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195" authors = [] -version = "0.20.1" +version = "0.20.2" [deps] Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" @@ -18,6 +18,6 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" [compat] DocStringExtensions = "0.8, 0.9" -MPIPreferences = "0.1.3" +MPIPreferences = "0.1.6" Requires = "~0.5, 1.0" julia = "1.6" diff --git a/lib/MPIPreferences/Project.toml b/lib/MPIPreferences/Project.toml index 3a8ade53a..6fc3ee6c6 100644 --- a/lib/MPIPreferences/Project.toml +++ b/lib/MPIPreferences/Project.toml @@ -1,7 +1,7 @@ name = "MPIPreferences" uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" authors = [] -version = "0.1.5" +version = "0.1.6" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"