diff --git a/.github/workflows/Invalidations.yml b/.github/workflows/Invalidations.yml
index 6a9169c1f..6cb6db31f 100644
--- a/.github/workflows/Invalidations.yml
+++ b/.github/workflows/Invalidations.yml
@@ -22,18 +22,32 @@ jobs:
     - uses: julia-actions/setup-julia@v1
       with:
         version: '1'
+    # current branch
     - uses: actions/checkout@v3
+    - name: add MPIPreferences
+      shell: julia --color=yes --project=. {0}
+      run: |
+        using Pkg
+        Pkg.develop(path="lib/MPIPreferences")
+
     - uses: julia-actions/julia-buildpkg@v1
     - uses: julia-actions/julia-invalidations@v1
       id: invs_pr
 
+    # default branch
     - uses: actions/checkout@v3
       with:
         ref: ${{ github.event.repository.default_branch }}
+    - name: add MPIPreferences
+      shell: julia --color=yes --project=. {0}
+      run: |
+        using Pkg
+        Pkg.develop(path="lib/MPIPreferences")
     - uses: julia-actions/julia-buildpkg@v1
     - uses: julia-actions/julia-invalidations@v1
       id: invs_default
-    
+
+    # report
     - name: Report invalidation counts
       run: |
         echo "Invalidations on default branch: ${{ steps.invs_default.outputs.total }} (${{ steps.invs_default.outputs.deps }} via deps)" >> $GITHUB_STEP_SUMMARY
diff --git a/Project.toml b/Project.toml
index a5d4fa241..2e5fa5f8f 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,7 +1,7 @@
 name = "MPI"
 uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195"
 authors = []
-version = "0.20.1"
+version = "0.20.2"
 
 [deps]
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
@@ -18,6 +18,6 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 
 [compat]
 DocStringExtensions = "0.8, 0.9"
-MPIPreferences = "0.1.3"
+MPIPreferences = "0.1.6"
 Requires = "~0.5, 1.0"
 julia = "1.6"
diff --git a/docs/src/reference/mpipreferences.md b/docs/src/reference/mpipreferences.md
index 09234dbfb..d17849e71 100644
--- a/docs/src/reference/mpipreferences.md
+++ b/docs/src/reference/mpipreferences.md
@@ -20,6 +20,7 @@ MPIPreferences.use_jll_binary
 ## Utils
 
 ```@docs
+MPIPreferences.check_unchanged
 MPIPreferences.identify_abi
 ```
 
diff --git a/lib/MPIPreferences/Project.toml b/lib/MPIPreferences/Project.toml
index 3a8ade53a..6fc3ee6c6 100644
--- a/lib/MPIPreferences/Project.toml
+++ b/lib/MPIPreferences/Project.toml
@@ -1,7 +1,7 @@
 name = "MPIPreferences"
 uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 authors = []
-version = "0.1.5"
+version = "0.1.6"
 
 [deps]
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index 66d76b45b..fcef689bd 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -1,11 +1,16 @@
 module MPIPreferences
 
+export use_jll_binary, use_system_binary
+
 using Preferences, Libdl
 
 if !(VersionNumber(@load_preference("_format", "1.0")) <= v"1.0")
     error("The preferences attached to MPIPreferences are incompatible with this version of the package.")
 end
 
+const PREFS_CHANGED = Ref(false)
+const DEPS_LOADED = Ref(false)
+
 """
     MPIPreferences.binary :: String
 
@@ -50,17 +55,19 @@ end
 end
 
 """
-    MPIPreferences.use_jll_binary([binary]; export_prefs=false, force=true)
+    use_jll_binary([binary]; export_prefs=false, force=true)
+
+Switches the underlying MPI implementation to one provided by JLL packages. A
+restart of Julia is required for the changes to take effect.
 
-Switches the underlying MPI implementation to one provided by JLL packages.
 Available options are:
 - `"MicrosoftMPI_jll"` (Only option and default on Winddows)
 - `"MPICH_jll"` (Default on all other platform)
 - `"OpenMPI_jll"`
 - `"MPItrampoline_jll"`
 
-The `export_prefs` option determines whether the preferences being set should be stored
-within `LocalPreferences.toml` or `Project.toml`.
+The `export_prefs` option determines whether the preferences being set should be
+stored within `LocalPreferences.toml` or `Project.toml`.
 """
 function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_jll"; export_prefs=false, force=true)
     binary in ["MicrosoftMPI_jll", "MPICH_jll", "OpenMPI_jll", "MPItrampoline_jll"] ||
@@ -75,8 +82,6 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j
         force=force
     )
 
-    @warn "The underlying MPI implementation has changed. You will need to restart Julia for this change to take effect" binary
-
     if VERSION <= v"1.6.5" || VERSION == v"1.7.0"
         @warn """
         Due to a bug in Julia (until 1.6.5 and 1.7.1), setting preferences in transitive dependencies
@@ -85,19 +90,30 @@ function use_jll_binary(binary = Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_j
         """
     end
 
+    if binary == MPIPreferences.binary
+        @info "MPIPreferences unchanged" binary
+    else
+        PREFS_CHANGED[] = true
+        @info "MPIPreferences changed" binary
+
+        if DEPS_LOADED[]
+            error("You will need to restart Julia for the changes to take effect")
+        end
+    end
     return nothing
 end
 
 
 """
-    MPIPreferences.use_system_binary(;
+    use_system_binary(;
         library_names = ["libmpi", "libmpi_ibm", "msmpi", "libmpich", "libmpi_cray", "libmpitrampoline"],
         mpiexec = "mpiexec",
         abi = nothing,
         export_prefs = false,
         force = true)
 
-Switches the underlying MPI implementation to a system provided one.
+Switches the underlying MPI implementation to a system provided one. A restart
+of Julia is required for the changes to take effect.
 
 Options:
 
@@ -113,9 +129,11 @@ Options:
   to include specific command line options.
 
 - `abi`: the ABI of the MPI library. By default this is determined automatically
-  using [`identify_abi`](@ref). See [`abi`](@ref) for currently supported values.
+  using [`identify_abi`](@ref). See [`abi`](@ref) for currently supported
+  values.
 
-- `export_prefs`: if `true`, the preferences into the `Project.toml` instead of `LocalPreferences.toml`.
+- `export_prefs`: if `true`, the preferences into the `Project.toml` instead of
+  `LocalPreferences.toml`.
 
 - `force`: if `true`, the preferences are set even if they are already set.
 """
@@ -154,7 +172,6 @@ function use_system_binary(;
         force=force
     )
 
-    @warn "The underlying MPI implementation has changed. You will need to restart Julia for this change to take effect" binary libmpi abi mpiexec
 
     if VERSION <= v"1.6.5" || VERSION == v"1.7.0"
         @warn """
@@ -164,6 +181,33 @@ function use_system_binary(;
         """
     end
 
+    if binary == MPIPreferences.binary && abi == MPIPreferences.abi && libmpi == System.libmpi && mpiexec == System.mpiexec_path
+        @info "MPIPreferences unchanged" binary libmpi abi mpiexec
+    else
+        PREFS_CHANGED[] = true
+        @info "MPIPreferences changed" binary libmpi abi mpiexec
+
+        if DEPS_LOADED[]
+            error("You will need to restart Julia for the changes to take effect")
+        end
+    end
+    return nothing
+end
+
+"""
+    MPIPreferences.check_unchanged()
+
+Throws an error if the preferences have been modified in the current Julia
+session, or if they are modified after this function is called.
+
+This is should be called from the `__init__()` function of any package which
+relies on the values of MPIPreferences.
+"""
+function check_unchanged()
+    if PREFS_CHANGED[]
+        error("MPIPreferences have changed, you will need to restart Julia for the changes to take effect")
+    end
+    DEPS_LOADED[] = true
     return nothing
 end
 
@@ -315,7 +359,7 @@ function identify_abi(libmpi)
     # 2) try to identify the MPI implementation
     impl, version, abi = identify_implementation_version_abi(version_string)
 
-    @info "MPI implementation" libmpi version_string impl version abi
+    @info "MPI implementation identified" libmpi version_string impl version abi
 
     return abi
 end
diff --git a/src/MPI.jl b/src/MPI.jl
index 4c70e7fca..a43f40f02 100644
--- a/src/MPI.jl
+++ b/src/MPI.jl
@@ -3,6 +3,7 @@ module MPI
 using Libdl, Serialization
 using Requires
 using DocStringExtensions
+import MPIPreferences
 
 export mpiexec, UBuffer, VBuffer
 
@@ -58,7 +59,6 @@ function run_load_time_hooks()
     nothing
 end
 
-using MPIPreferences
 include("implementations.jl")
 include("error.jl")
 include("info.jl")
@@ -81,12 +81,13 @@ include("misc.jl")
 include("deprecated.jl")
 
 function __init__()
+    MPIPreferences.check_unchanged()
 
     # an empty string was used to indicate "default"
     # https://github.com/JuliaParallel/MPI.jl/blob/v0.19.2/deps/build.jl#L142
     mpi_env_binary = get(ENV, "JULIA_MPI_BINARY", "")
     if mpi_env_binary != "" && mpi_env_binary != MPIPreferences.binary
-        @info """
+        @warn """
         The JULIA_MPI_BINARY environment variable is no longer used to configure the MPI binary.
         Please use the MPIPreferences.jl package instead: