fix: make sure that created Project.toml gets included in the appbu…

…ndle (#44)

* test: rename, group packagebundler test

* fix: always include the Project.toml in the appbundle

create_pkg_context generates the Project.toml, but is currently called
after the files have been copied over to the temporary directory that
gets tarred up. This patch moves the copying of the appbundle contents
after the Project.toml has been created.

* fix: support relative paths in appbundle paths

* docs: clarify the appbundle() docstring

* Add CHANGELOG, set version to 0.1.6

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Version v0.1.6 - 2023-11-27
+
+### Fixed
+
+* `JuliaHub.appbundle`, when it has to generate a `Projec.toml` file, now correctly includes it in the appbundle tarball. (#44)
+* `JuliaHub.appbundle` now works with relative paths such as `"."`. (#44)
+
 ## Version v0.1.5 - 2023-09-27
 
 ### Added

Project.toml

@@ -1,7 +1,7 @@
 name = "JuliaHub"
 uuid = "bc7fa6ce-b75e-4d60-89ad-56c957190b6e"
 authors = ["JuliaHub Inc."]
-version = "0.1.5"
+version = "0.1.6"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/PackageBundler/PackageBundler.jl

@@ -62,6 +62,9 @@ The return value is used when requesting a sysimage build, in which case we have
 manifest's hash with the submit request.
 """
 function bundle(dir; output="", force=false, allownoenv=false, verbose=true)::String
+    # We'll normalize the path passed by the user, in case they pass a relative path
+    # like `.`
+    dir = abspath(dir)
     if !isdir(dir)
         error("'$(dir)' is not a directory")
     end
@@ -74,12 +77,11 @@ function bundle(dir; output="", force=false, allownoenv=false, verbose=true)::St
             error("file '$output_tar' already exists")
         end
     end
-    tmp_dir = mktempdir()
-    output_dir = joinpath(tmp_dir, name)
-    cp(dir, output_dir; follow_symlinks=true)
 
     packages_tracked_pkg_server = find_packages_tracked_pkg_server()
 
+    # This step may modify the user's bundle directory (`dir`) by creating a Project.toml
+    # and Manifest.toml, if either or both are missing.
     ctx = create_pkg_context(dir, allownoenv)
     if isempty(ctx.env.manifest)
         @warn "No Manifest available. Resolving environment."
@@ -90,6 +92,11 @@ function bundle(dir; output="", force=false, allownoenv=false, verbose=true)::St
         ctx = create_pkg_context(dir, allownoenv)
     end
 
+    # We'll copy the files we want to bundle to a temporary directory. We then
+    # add the depot and such, and finally tar all that up.
+    tmp_dir = mktempdir()
+    output_dir = joinpath(tmp_dir, name)
+    cp(dir, output_dir; follow_symlinks=true)
     bundle_dir = joinpath(output_dir, ".bundle")
     mkpath(bundle_dir)
     # Bundle artifacts

src/jobsubmission.jl

@@ -723,7 +723,9 @@ The following should be kept in mind about how appbundles are handled:
 
 * The bundler looks for a Julia environment (i.e. `Project.toml` and/or `Manifest.toml` files)
   at the root of the directory. If the environment does not exist (i.e. the files are missing),
-  one is created.
+  the missing files are created. If the manifest is missing, then the environment is re-instantiated
+  from scratch based on the contents of `Project.toml`. The generated files will also be left
+  in the user-provided directory `directory`.
 
 * Development dependencies of the environment (i.e. packages added with `pkg> develop` or
   `Pkg.develop()`) are also bundled up into the archive that gets submitted to JuliaHub

test/fixtures/ignorefiles/Pkg1/bundle.noenv/run.jl

@@ -0,0 +1,10 @@
+using Pkg1
+
+@assert Pkg1.whoami() == "pkg1"
+@assert Pkg1.Pkg2.whoami() == "pkg2"
+@assert Pkg1.Pkg3.whoami() == "pkg3"
+
+for i in 1:100
+    println(i)
+    sleep(1)
+end

test/fixtures/ignorefiles/Pkg1/bundle.nomanifest/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+Example = "7876af07-990d-54b4-ab0e-23690620f79a"

test/fixtures/ignorefiles/Pkg1/bundle.nomanifest/run.jl

@@ -0,0 +1,10 @@
+using Pkg1
+
+@assert Pkg1.whoami() == "pkg1"
+@assert Pkg1.Pkg2.whoami() == "pkg2"
+@assert Pkg1.Pkg3.whoami() == "pkg3"
+
+for i in 1:100
+    println(i)
+    sleep(1)
+end

test/fixtures/ignorefiles/Pkg1/bundle.noproject-throw/run.jl

@@ -0,0 +1,10 @@
+using Pkg1
+
+@assert Pkg1.whoami() == "pkg1"
+@assert Pkg1.Pkg2.whoami() == "pkg2"
+@assert Pkg1.Pkg3.whoami() == "pkg3"
+
+for i in 1:100
+    println(i)
+    sleep(1)
+end

test/fixtures/ignorefiles/Pkg1/bundle.noproject/Manifest.toml

@@ -0,0 +1,22 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.7.1"
+manifest_format = "2.0"
+project_hash = "8aa8902f5725df756a200ded26d78d775619a452"
+
+[[deps.Pkg1]]
+deps = ["Pkg2", "Pkg3"]
+path = ".."
+uuid = "0d733eb5-97ac-4a4a-9738-0f33664ba196"
+version = "0.1.0"
+
+[[deps.Pkg2]]
+deps = ["Pkg3"]
+path = "../../Pkg2"
+uuid = "df0ae394-da6f-45d5-857b-fe46f2dd6ae6"
+version = "0.1.0"
+
+[[deps.Pkg3]]
+path = "../../Pkg3"
+uuid = "8e8c87e8-d389-41a9-b801-d72778e29757"
+version = "0.1.0"

test/fixtures/ignorefiles/Pkg1/bundle.noproject/run.jl

@@ -0,0 +1,10 @@
+using Pkg1
+
+@assert Pkg1.whoami() == "pkg1"
+@assert Pkg1.Pkg2.whoami() == "pkg2"
+@assert Pkg1.Pkg3.whoami() == "pkg3"
+
+for i in 1:100
+    println(i)
+    sleep(1)
+end

test/fixtures/ignorefiles/Pkg1/bundle.standard/Manifest.toml

@@ -0,0 +1,22 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.7.1"
+manifest_format = "2.0"
+project_hash = "8aa8902f5725df756a200ded26d78d775619a452"
+
+[[deps.Pkg1]]
+deps = ["Pkg2", "Pkg3"]
+path = ".."
+uuid = "0d733eb5-97ac-4a4a-9738-0f33664ba196"
+version = "0.1.0"
+
+[[deps.Pkg2]]
+deps = ["Pkg3"]
+path = "../../Pkg2"
+uuid = "df0ae394-da6f-45d5-857b-fe46f2dd6ae6"
+version = "0.1.0"
+
+[[deps.Pkg3]]
+path = "../../Pkg3"
+uuid = "8e8c87e8-d389-41a9-b801-d72778e29757"
+version = "0.1.0"

test/fixtures/ignorefiles/Pkg1/bundle.standard/run.jl

@@ -0,0 +1,10 @@
+using Pkg1
+
+@assert Pkg1.whoami() == "pkg1"
+@assert Pkg1.Pkg2.whoami() == "pkg2"
+@assert Pkg1.Pkg3.whoami() == "pkg3"
+
+for i in 1:100
+    println(i)
+    sleep(1)
+end

test/jobs.jl

@@ -109,6 +109,13 @@ end
         @test_throws JuliaHub.AppBundleSizeError JuliaHub.appbundle(path; code="")
         rm(bigfile_path, force=true)
     end
+
+    # Testing relative paths to the appbundle directory
+    cd(jobfile()) do
+        bundle = JuliaHub.appbundle(".", "script.jl")
+        @test isfile(bundle.environment.tarball_path)
+        @test bundle.code == read(jobfile("script.jl"), String)
+    end
 end
 
 # We'll re-use this further down in job submission tests.