rewrite

JuliaComputing · Dec 13, 2024 · 0c13b27 · 0c13b27
1 parent 9ca352c
commit 0c13b27
Show file tree

Hide file tree

Showing 7 changed files with 142 additions and 152 deletions.
diff --git a/Project.toml b/Project.toml
@@ -1,15 +1,19 @@
 name = "DepotDelivery"
 uuid = "5c353f05-31b8-41b6-b09f-65e5456b8b1e"
 authors = ["Josh Day <[email protected]> and contributors"]
-version = "0.1.7"
+version = "0.2.0"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
+Dates = "1.11.0"
+InteractiveUtils = "1.11.0"
+Pkg = "1.11.0"
+UUIDs = "1.11.0"
 julia = "1"
 
 [extras]

diff --git a/src/DepotDelivery.jl b/src/DepotDelivery.jl
@@ -1,135 +1,50 @@
 module DepotDelivery
 
-using Dates, InteractiveUtils, Pkg, TOML
+public build
 
-#-----------------------------------------------------------------------------# State
-# Things we change before Pkg.instantiate-ing and need to restore.
-@kwdef struct State
-    depot_path = copy(DEPOT_PATH)
-    precomp = get(ENV, "JULIA_PKG_PRECOMPILE_AUTO", nothing)
-    project = Base.current_project()
-end
-function restore!!(s::State)
-    isnothing(s.project) ? Pkg.activate() : Pkg.activate(s.project)
-    isnothing(s.precomp) || (ENV["JULIA_PKG_PRECOMPILE_AUTO"] = s.precomp)
-    append!(empty!(DEPOT_PATH), s.depot_path)
-end
-
-function sandbox(f::Function)
-    state = State()
-    try
-        f()
-    catch ex
-        @warn "DepotDelivery.sandbox failed"
-        rethrow(ex)
-    finally
-        restore!!(state)
-    end
-end
-
-#-----------------------------------------------------------------------------#
+#-----------------------------------------------------------------------------# build
 """
-Builds the depot for a specified project.
+    build(src = pwd(), dest; triplet, platform, verbose, precompiled)
 
 Arguments:
-- `path::String`: The path to the project directory containing `Project.toml` or `JuliaProject.toml`.
-- `platform::AbstractPlatform`: The target platform for building (default is the host platform).
-- `verbose::Bool`: Whether to display verbose output during the build process (default is `true`).
-- `depot::String`: The path to the depot directory (default is a temporary directory).
-- `precompiled::Bool`: Whether to enable precompilation of packages (default is `false`).
+- `src = pwd()`: A `String`/`Vector{String}` of the project path/paths containing `Project.toml` or `JuliaProject.toml` files.
+- `dest::String = <tempdir>`: The depot directory to populate.
+- `platform::AbstractPlatform = <host platform>`: The target `Base.BinaryPlatforms.Platform`.
+- `triplet = nothing`: The target triplet of the platform to build for.  If not `nothing`, it overrides `platform`.
+- `verbose = true`: Whether to display verbose output during the build process (default is `true`).
+- `precompiled = false`: Whether to enable precompilation of packages.
 
 Returns:
-- The path to the built depot.
+- `dest::String`
 
 Example:
-```julia
-depot_path = build("/path/to/your/project")
+    depot_path = build("/path/to/your/project")
 """
-function build(path::String; platform = Base.BinaryPlatforms.HostPlatform(), verbose=true, depot = mktempdir(), precompiled=false)
-    path = abspath(path)
-    mkpath(depot)
-    sandbox() do
-        proj_file = joinpath(path, "Project.toml")
-        proj_file = isfile(proj_file) ? proj_file : joinpath(path, "JuliaProject.toml")
-        isfile(proj_file) || error("No Project.toml or JuliaProject.toml found in `$path`.")
-        proj = TOML.parsefile(proj_file)
-        # Defines a project name for Project.toml that don't come from a package
-        name = haskey(proj, "name") ? proj["name"] : Base.basename(Base.dirname(proj_file))
-        build_spec = Dict(
-            :datetime => Dates.now(),
-            :versioninfo => sprint(InteractiveUtils.versioninfo),
-            :project_file => proj_file,
-            :project => proj,
-            :platform => string(platform)
+function build(
+        src::String = pwd(),  # paths separated with ':'
+        dest::String = joinpath(mktempdir(), "depot");
+        triplet = nothing,
+        platform = Base.BinaryPlatforms.HostPlatform(),
+        verbose = true,
+        precompiled = false,
+        offline = false
         )
-        mkpath(joinpath(depot, "config"))
-        mkpath(joinpath(depot, "dev", name))
-        push!(empty!(DEPOT_PATH), depot)
-
-        # Disabling precompile for non-host platforms
-        precompiled ? delete!(ENV, "JULIA_PKG_PRECOMPILE_AUTO") : ENV["JULIA_PKG_PRECOMPILE_AUTO"] = "0" 
-
-        cp(path, joinpath(depot, "dev", name), force=true)  # Copy project into dev/
-
-        Pkg.activate(joinpath(depot, "dev", name))
-        Pkg.instantiate(; platform, verbose)
-
-
-        # Add local/dev-ed packages to Project
-        manifest_file = joinpath(dirname(proj_file), replace(basename(proj_file), "Project" => "Manifest"))
-        for (uuid, entry) in Pkg.Types.read_manifest(manifest_file)
-            !isnothing(entry.path) && Pkg.dev(entry.path)
-        end
-
-
-        open(io -> TOML.print(io, build_spec), joinpath(depot, "config", "depot_build.toml"), "w")
-        open(io -> print(io, startup_script(name)), joinpath(depot, "config", "depot_startup.jl"), "w")
-    end
-
-    return depot
+    julia = Base.julia_cmd()
+    script_jl = joinpath(@__DIR__, "build_script.jl")
+    triplet = isnothing(triplet) ? Base.BinaryPlatforms.triplet(platform) : triplet
+    cmd = `$julia $script_jl $verbose $triplet $src $dest $precompiled $offline`
+    read(cmd, String)
 end
 
-"""
-Builds depots for multiple projects specified by their paths.
-
-Arguments:
-- `paths::Vector{String}`: An array of directories of project paths.
-- `platform::AbstractPlatform`: The target platform for building (default is the host platform).
-- `verbose::Bool`: Whether to display verbose output during the build process (default is `true`).
-- `depot::String`: The path to the depot directory (default is a temporary directory).
-- `precompiled::Bool`: Whether to enable precompilation (default is `false`).
+build(sources::Vector{String}, dest::String=joinpath(mktempdir(), "depot"); kw...) = build(join(sources, ':'), dest; kw...)
 
-Example:
-```julia
-project_paths = ["/path/to/project1", "/path/to/project2"]
-build(project_paths)
-"""
-function build(paths::Vector{String}; platform = Base.BinaryPlatforms.HostPlatform(), verbose=true, depot=mktempdir(), precompiled=false)
-    for path in paths
-        build(path; platform=platform, verbose=verbose, depot=depot, precompiled=precompiled)
-    end
-    return depot
- end
-
 
-#-----------------------------------------------------------------------------# startup_script
-startup_script(name) = """
-    import Pkg
-    let
-        depot = abspath(joinpath(@__DIR__, ".."))
-        Pkg.activate(joinpath(depot, "dev", "$name"))
-        ENV["JULIA_DEPOT_PATH"] = depot   # For Distributed.jl workers
-        push!(empty!(DEPOT_PATH), depot)  # For current process
-        @info "Initializing Depot `\$depot` with project `$name`."
-    end
-    using $name
-    """
 
 #-----------------------------------------------------------------------------# test
-function test(depot_path::String)
+function test(depot::String)
     script = """
-    @info "DepotDelivery.test: Loading the depot_startup.jl script"
-    include(raw"$(joinpath(depot_path, "config", "depot_startup.jl"))")
+    @info "DepotDelivery.test: Loading the startup.jl script"
+    include(raw"$(joinpath(depot, "config", "startup.jl"))")
     """
     process = run(`$(Base.julia_cmd()) --startup-file=no -e $script`)
     process.exitcode == 0
@@ -144,6 +59,7 @@ function _check_artifacts(depot_path::String, not=[".dylib"])
             ext = lowercase(splitext(file)[2])
             if ext ∈ not
                 @warn "Found unexpected artifact: $file"
+                return false
             end
         end
     end

diff --git a/src/build_script.jl b/src/build_script.jl
@@ -0,0 +1,87 @@
+using Pkg, Dates, UUIDs, InteractiveUtils
+
+verbose, triplet, sources, dest, precomp, offline = ARGS
+
+function get_project(path)
+    file = joinpath(path, "Project.toml")
+    file = isfile(file) ? file : joinpath(path, "JuliaProject.toml")
+    isfile(file) || error("No Project.toml or JuliaProject.toml file found in `$path`.")
+    return Pkg.Types.read_project(file)
+end
+
+get_project_name(path) = (p = get_project(path); isnothing(p.name) ? splitpath(path)[end] : p.name)
+
+#-----------------------------------------------------------------------------# ARGS
+verbose = verbose == "true"
+offline = offline == "true"
+source_dict = Dict(x => get_project_name(x) for x in abspath.(split(sources, ':')))
+platform = Base.parse(Base.BinaryPlatforms.Platform, triplet)
+
+for src in keys(source_dict)
+    isdir(src) || error("Source directory `$src` does not exist.")
+end
+isdir(dest) || mkpath(dest)
+
+verbose && @info """
+    Building depot
+
+    - Sources:
+        - $(join(["$v: $k" for (k,v) in source_dict], "\n    - "))
+    - Destination: $dest
+    - Triplet: $triplet
+    - Platform: $platform
+    """
+
+spec = Dict(
+    :datetime => Dates.now(),
+    :versioninfo => sprint(InteractiveUtils.versioninfo),
+    :sources => source_dict,
+    :destination => dest,
+    :triplet => triplet,
+    :platform => string(platform)
+)
+
+#-----------------------------------------------------------------------------# populate dest
+ENV["JULIA_PKG_PRECOMPILE_AUTO"] = precomp == "true" ? 1 : 0
+ENV["JULIA_DEPOT_PATH"] = dest
+DEPOT_PATH[1] = dest
+
+# PIRACY: This is more reliable than setting the `platform` keyword argument in `Pkg` functions.
+Base.BinaryPlatforms.HostPlatform() = platform
+
+Pkg.activate()
+mkpath(joinpath(dest, "dev"))
+for (path, name) in source_dict
+    path = cp(path, joinpath(dest, "dev", name), force=true)
+    proj = get_project(path)
+    if any(isnothing, (proj.name, proj.uuid))
+        @info "$path is not a valid Julia project.  Assigning necessary name/uuid..."
+        proj.name = isnothing(proj.name) ? get_project_name(path) : proj.name
+        proj.uuid = isnothing(proj.uuid) ? UUIDs.uuid4() : proj.uuid
+        file = isfile(joinpath(path, "Project.toml")) ? "Project.toml" : "JuliaProject.toml"
+        Pkg.Types.write_project(proj, joinpath(path, file))
+    end
+    Pkg.develop(path=path)
+end
+Pkg.instantiate(; verbose)
+
+#-----------------------------------------------------------------------------# config
+mkpath(joinpath(dest, "config"))
+
+startup = """
+# This file was automatically generated by DepotDelivery.jl
+# Created at: $(Dates.now())
+$(offline ? "import Pkg; Pkg.offline(true)" : "")
+let
+    depot = abspath(joinpath(@__DIR__, ".."))
+    ENV["JULIA_DEPOT_PATH"] = depot     # For Distributed workers
+    DEPOT_PATH[1] = depot               # For current process
+end
+@info "DepotDelivery startup: `using $(join(values(source_dict), ", "))`"
+using $(join(values(source_dict), ", "))
+"""
+
+write(joinpath(dest, "config", "startup.jl"), startup)
+write(joinpath(dest, "config", "DepotDeliveryBuild.toml"), sprint(Pkg.TOML.print, spec))
+
+print(dest)
diff --git a/test/MultipleWorkflows/BitFlags/src/BitFlags.jl b/test/MultipleWorkflows/BitFlags/src/BitFlags.jl
@@ -0,0 +1,2 @@
+module BitFlags
+end
diff --git a/test/MultipleWorkflows/URIs/src/URIs.jl b/test/MultipleWorkflows/URIs/src/URIs.jl
@@ -0,0 +1,2 @@
+module URIs
+end
diff --git a/test/MultipleWorkflows/Unzip/src/Unzip.jl b/test/MultipleWorkflows/Unzip/src/Unzip.jl
@@ -0,0 +1,2 @@
+module Unzip
+end
diff --git a/test/runtests.jl b/test/runtests.jl
@@ -1,47 +1,24 @@
 using DepotDelivery
-using Pkg
 using Test
 
 
-depot = DepotDelivery.build(joinpath(@__DIR__, "TestProject"))
-
+#-----------------------------------------------------------------------------# Single Project
+depot = DepotDelivery.build(joinpath(@__DIR__, "TestProject"), precompiled=false)
+@test !isdir(joinpath(depot, "compiled"))
 @test DepotDelivery.test(depot)
+@test DepotDelivery._check_artifacts(depot, [".dll"])
 
-DepotDelivery.sandbox() do
-    include(joinpath(depot, "config", "depot_startup.jl"))
-    @test !any(x -> occursin(".julia", x), DEPOT_PATH) # Ensure DEPOT_PATH changed
-    @test !occursin(".julia", pathof(TestProject))
-    @test !occursin(".julia", pathof(TestProject.HDF5))
-    @test !occursin(".julia", pathof(TestProject.HDF5.API.HDF5_jll))
-end
-
-
-depot2 = DepotDelivery.build(joinpath(@__DIR__, "TestProject"), platform = Pkg.BinaryPlatforms.Windows(:x86_64))
-
-path = joinpath(depot2, "packages", "HDF5_jll")
-
+depot = DepotDelivery.build(joinpath(@__DIR__, "TestProject"), precompiled=true)
+@test isdir(joinpath(depot, "compiled"))
+@test DepotDelivery.test(depot)
 
-#-----------------------------------------------------------------------------# Testing multiple workflows
-packages_list = readdir(joinpath(@__DIR__, "MultipleWorkflows/"));
-proj_paths = joinpath.(@__DIR__, "MultipleWorkflows/", packages_list);
-depot = DepotDelivery.build(proj_paths, precompiled=true)
 
-DepotDelivery.sandbox() do 
-    push!(empty!(DEPOT_PATH), depot)
-
-    # Test that for every project instantiated, their dependencies exist 
-    # and the depot path does not point to the default value
-    @testset for (proj, package) in zip(proj_paths, packages_list)
-        Pkg.activate(proj)
-        Pkg.instantiate()
-        package_symbol = Symbol(package)
-        @eval using $package_symbol
-        package_value = eval(package_symbol)
-        @test !occursin(".julia", pathof(package_value))
-    end
+depot = DepotDelivery.build(joinpath(@__DIR__, "TestProject"), triplet="x86_64-w64-mingw32")
+@test !isdir(joinpath(depot, "compiled"))
+@test DepotDelivery._check_artifacts(depot, [".dylib"])
 
-    # Ensure compiled folders are populated
-    @testset for package in packages_list
-        @test length(readdir(joinpath(depot, "compiled", "v$(VERSION.major).$(VERSION.minor)", package))) > 0
-    end
+#-----------------------------------------------------------------------------# Multiple Projects
+cd(joinpath(@__DIR__, "MultipleWorkflows")) do
+    depot = DepotDelivery.build(readdir())
+    @test DepotDelivery.test(depot)
 end