feat: add progress kwarg for upload_dataset (#48)

* feat: add progress kwarg for upload_dataset progress=false disables rclone's progress printing * Update Project.toml * Update CHANGELOG.md * Update src/datasets.jl Co-authored-by: Morten Piibeleht <[email protected]> * update docstring --------- Co-authored-by: Morten Piibeleht <[email protected]>
JuliaComputing · Feb 20, 2024 · 6799ba1 · 6799ba1
1 parent 129d1c5
commit 6799ba1
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 7 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Version v0.1.8 - 2024-02-21
+
+### Added
+
+* The progress output printing in `JuliaHub.upload_dataset` can now be disabled by setting `progress=false`. (#48)
+
 ## Version v0.1.7 - 2024-01-22
 
 ### Fixed

diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "JuliaHub"
 uuid = "bc7fa6ce-b75e-4d60-89ad-56c957190b6e"
 authors = ["JuliaHub Inc."]
-version = "0.1.7"
+version = "0.1.8"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

diff --git a/src/datasets.jl b/src/datasets.jl
@@ -495,6 +495,8 @@ If a dataset already exists, then these fields are updated as if [`update_datase
 
 The function will throw an `ArgumentError` for invalid argument combinations.
 
+Use the `progress` keyword argument to suppress upload progress from being printed.
+
 !!! note
     Presently, it is only possible to upload datasets for the currently authenticated user.
 """
@@ -503,6 +505,7 @@ function upload_dataset end
 @_authuser function upload_dataset(
     dsref::_DatasetRefTuple,
     local_path::AbstractString;
+    progress::Bool=true,
     # Operation type
     create::Bool=true,
     update::Bool=false,
@@ -604,7 +607,7 @@ function upload_dataset end
     end
     # Upload the actual data
     try
-        _upload_dataset(upload_config, local_path)
+        _upload_dataset(upload_config, local_path; progress)
     catch e
         throw(JuliaHubError("Data upload failed", e, catch_backtrace()))
     end
@@ -679,7 +682,7 @@ function _open_dataset_version(name; auth::Authentication=__auth__())::_RESTResp
     _restcall(auth, :POST, "user", "datasets", name, "versions")
 end
 
-function _upload_dataset(upload_config, local_path)
+function _upload_dataset(upload_config, local_path; progress::Bool)
     type = upload_config["upload_type"]
     vendor = upload_config["vendor"]
     if type != "S3" || vendor != "aws"
@@ -704,13 +707,20 @@ function _upload_dataset(upload_config, local_path)
             #   --s3-upload-cutoff 1M --s3-chunk-size 5M
 
             # FIXME: remove `--s3-no-head` once policies are figured out (again)
+            args = [
+                "--s3-no-check-bucket",
+                "--s3-no-head",
+                "--no-check-dest",
+            ]
+
+            if progress
+                pushfirst!(args, "--progress")
+            end
+
             run(```
                 $rclone_exe copyto $local_path "juliahub_remote:$remote_path"
                 --config $rclone_conf_path
-                --progress
-                --s3-no-check-bucket
-                --s3-no-head
-                --no-check-dest
+                $args
                 ```)
         end
     end