Use Preferences for using custom hdf5 library

.gitignore

@@ -4,3 +4,4 @@
 *.jl.mem
 /docs/build/
 Manifest.toml
+LocalPreferences.toml

Project.toml

@@ -2,21 +2,22 @@ name = "HDF5"
 uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 version = "0.16.14"
 
+[compat]
+Compat = "3.1.0, 4"
+HDF5_jll = "~1.10.5, ~1.12.0"
+Requires = "1.0"
+julia = "1.3"
+
 [deps]
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
-[compat]
-Compat = "3.1.0, 4"
-HDF5_jll = "~1.10.5, ~1.12.0"
-Requires = "1.0"
-julia = "1.3"
-
 [extras]
 CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
@@ -27,6 +28,7 @@ H5Zlz4 = "eb20ec05-5464-47b5-ba41-098e3c1068a3"
 H5Zzstd = "f6f2d980-1ec6-471c-a70d-0270e22f1103"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
+MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

docs/src/index.md

@@ -27,24 +27,35 @@ Starting from Julia 1.3, the HDF5 binaries are by default downloaded using the `
 
 ### Using custom or system provided HDF5 binaries
 
-To use system-provided HDF5 binaries instead, set the environment variable `JULIA_HDF5_PATH` to the top-level installation directory HDF5, i.e. the library should be located in `${JULIA_HDF5_PATH}/lib` or `${JULIA_HDF5_PATH}/lib64`, or alternatively simply in `${JULIA_HDF5_PATH}`. Then run `import Pkg; Pkg.build("HDF5")`. In particular, this is required if you need parallel HDF5 support, which is not provided by the `HDF5_jll` binaries.
+To use system-provided HDF5 binaries instead, set the preference `libhdf5path`, see also [Preferences.jl](https://github.com/JuliaPackaging/Preferences.jl). This has to be the top-level installation directory of HDF5, i.e. the library should be located in `${JULIA_HDF5_PATH}/lib` or `${JULIA_HDF5_PATH}/lib64`, or alternatively simply in `${JULIA_HDF5_PATH}`. In particular, this is required if you need parallel HDF5 support, which is not provided by the `HDF5_jll` binaries.
 
-If the library is in your library search path, then `JULIA_HDF5_PATH` can be set to an empty string.
+If the library is in your library search path, then `libhdf5path` can be set to an empty string.
 
-For example, to use HDF5 (`libhdf5-mpich-dev`) with MPI using system libraries on Ubuntu 20.04, you would run:
+For example, to use HDF5 (`libhdf5-mpich-dev`) with MPI using system libraries on Ubuntu 20.04, you would run
 
 ```sh
 $ sudo apt install mpich libhdf5-mpich-dev
-$ JULIA_HDF5_PATH=/usr/lib/x86_64-linux-gnu/hdf5/mpich/
-$ JULIA_MPI_BINARY=system
 ```
 
 Then in Julia, run:
 
 ```julia
-pkg> build
+using MPIPreferences
+MPIPreferences.use_system_binary()
 ```
 
+to set the MPI preferences, see the [documentation of MPI.jl](https://juliaparallel.org/MPI.jl/stable/configuration/) and then:
+
+```julia
+using Preferences, UUIDs
+
+set_preferences!(
+    UUID("f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"), # UUID of HDF5.jl
+    "libhdf5path" => "/usr/lib/x86_64-linux-gnu/hdf5/mpich/", force = true)
+```
+
+to set the preference for the custom HDF5 library.
+
 ## Opening and closing files
 
 "Plain" (i.e., with no extra formatting conventions) HDF5 files are created and/or opened with the `h5open` command:

docs/src/mpi.md

@@ -34,9 +34,16 @@ for details.
 ### 2. Using parallel HDF5 libraries
 
 As detailed in [Using custom or system provided HDF5 binaries](@ref), set the
-`JULIA_HDF5_PATH` environment variable to the path where the parallel HDF5
-binaries are located.
-Then run `]build HDF5` from Julia.
+preference `libhdf5path` to the path where the parallel HDF5 binaries are located.
+This can be done by:
+
+```julia
+julia> using Preferences, UUIDs
+
+julia> set_preferences!(
+           UUID("f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"), # UUID of HDF5.jl
+           "libhdf5path" => "/path/to/your/hdf5/binaries/", force = true)
+```
 
 ### 3. Loading MPI-enabled HDF5
 

src/api/api.jl

@@ -1,16 +1,46 @@
 module API
 
-import Libdl
+using Libdl
 using Base: StringVector
+using Preferences: @load_preference
 
-const depsfile = joinpath(@__DIR__, "..", "..", "deps", "deps.jl")
-if isfile(depsfile)
-    include(depsfile)
+# We avoid calling Libdl.find_library to avoid possible segfault when calling
+# dlclose (#929).
+# The only difference with Libdl.find_library is that we allow custom dlopen
+# flags via the `flags` argument.
+function find_library_alt(libnames, extrapaths=String[]; flags=RTLD_LAZY)
+    for lib in libnames
+        for path in extrapaths
+            l = joinpath(path, lib)
+            p = dlopen(l, flags; throw_error=false)
+            if p !== nothing
+                dlclose(p)
+                return l
+            end
+        end
+        p = dlopen(lib, flags; throw_error=false)
+        if p !== nothing
+            dlclose(p)
+            return lib
+        end
+    end
+    return ""
+end
+
+const libpath = @load_preference("libhdf5path", nothing)
+if libpath === nothing
+    using HDF5_JLL
 else
-    @error(
-        "HDF5 is not properly installed. Please run Pkg.build(\"HDF5\") ",
-        "and restart Julia."
-    )
+    libpaths = [libpath, joinpath(libpath, "lib"), joinpath(libpath, "lib64")]
+    flags = RTLD_LAZY | RTLD_NODELETE  # RTLD_NODELETE may be needed to avoid segfault (#929)
+
+    libhdf5 = find_library_alt(["libhdf5", "libhdf5_openmpi", "libhdf5_mpich"], libpaths; flags=flags)
+    libhdf5_hl = find_library_alt(["libhdf5_hl", "libhdf5_openmpi_hl", "libhdf5_mpich_hl"], libpaths; flags=flags)
+
+    isempty(libhdf5) && error("libhdf5 could not be found")
+    isempty(libhdf5_hl) && error("libhdf5_hl could not be found")
+
+    libhdf5_size = filesize(dlpath(libhdf5))
 end
 
 include("lock.jl")
@@ -22,11 +52,8 @@ include("helpers.jl")
 function __init__()
     # HDF5.API.__init__() is run before HDF5.__init__()
 
-    # From deps.jl
-    check_deps()
-
     # Ensure this is reinitialized on using
-    libhdf5handle[] = Libdl.dlopen(libhdf5)
+    libhdf5handle[] = dlopen(libhdf5)
 
     # Disable file locking as that can cause problems with mmap'ing.
     # File locking is disabled in HDF5.init!(::FileAccessPropertyList)