[DAR-3513][DAR-3514][DAR-3515][DAR-3516][DAR-3517][External] Multi-fi…

…le `push` (#923)

* Added  to CLI & SDK  + created MultiFileItem parsing & layout building

* Unit test coverage

* Cleaned up tests

* Added natural sorting to the order of files in multi-file items

* Fix for failing test on Windows

* Working upload for multi-slot, multi-channel, & DICOM series items

* Multi-slot, multi-channel, & DICOM series

* Test improvements

* Improved checking of args that are incompatible with `item_merge_mode`

* Fix for Windows filepaths

* Always use LayoutV3 + misc improvements

* Better API response typing + other misc changes

darwin/cli.py

@@ -126,6 +126,7 @@ def _run(args: Namespace, parser: ArgumentParser) -> None:
                 args.extract_views,
                 args.preserve_folders,
                 args.verbose,
+                args.item_merge_mode,
             )
         # Remove a project (remotely)
         elif args.action == "remove":

darwin/cli_functions.py

@@ -70,6 +70,7 @@
     prompt,
     secure_continue_request,
     validate_file_against_schema,
+    BLOCKED_UPLOAD_ERROR_ALREADY_EXISTS,
 )
 
 
@@ -656,6 +657,7 @@ def upload_data(
     extract_views: bool = False,
     preserve_folders: bool = False,
     verbose: bool = False,
+    item_merge_mode: Optional[str] = None,
 ) -> None:
     """
     Uploads the provided files to the remote dataset.
@@ -684,6 +686,14 @@ def upload_data(
         Specify whether or not to preserve folder paths when uploading.
     verbose : bool
         Specify whether to have full traces print when uploading files or not.
+    item_merge_mode : Optional[str]
+        If set, each file path passed to `files_to_upload` behaves as follows:
+        - Paths pointing directly to individual files are ignored
+        - Paths pointing to folders of files will be uploaded according to the following mode rules.
+          Note that folders will not be recursively searched, so only files in the first level of the folder will be uploaded:
+            - "slots": Each file in the folder will be uploaded to a different slot of the same item.
+            - "series": All `.dcm` files in the folder will be concatenated into a single slot. All other files are ignored.
+            - "channels": Each file in the folder will be uploaded to a different channel of the same item.
     """
     client: Client = _load_client()
     try:
@@ -773,6 +783,7 @@ def file_upload_callback(
                 preserve_folders=preserve_folders,
                 progress_callback=progress_callback,
                 file_upload_callback=file_upload_callback,
+                item_merge_mode=item_merge_mode,
             )
         console = Console(theme=_console_theme())
 
@@ -788,10 +799,13 @@ def file_upload_callback(
         already_existing_items = []
         other_skipped_items = []
         for item in upload_manager.blocked_items:
-            if (item.reason is not None) and (item.reason.upper() == "ALREADY_EXISTS"):
-                already_existing_items.append(item)
-            else:
-                other_skipped_items.append(item)
+            for slot in item.slots:
+                if (slot.reason is not None) and (
+                    slot.reason.upper() == BLOCKED_UPLOAD_ERROR_ALREADY_EXISTS
+                ):
+                    already_existing_items.append(item)
+                else:
+                    other_skipped_items.append(item)
 
         if already_existing_items:
             console.print(
@@ -819,17 +833,18 @@ def file_upload_callback(
             show_header=True,
             header_style="bold cyan",
         )
-
         for item in upload_manager.blocked_items:
-            if item.reason != "ALREADY_EXISTS":
-                error_table.add_row(
-                    str(item.dataset_item_id),
-                    item.filename,
-                    item.path,
-                    "UPLOAD_REQUEST",
-                    item.reason,
-                )
-
+            for slot in item.slots:
+                if (slot.reason is not None) and (
+                    slot.reason.upper() != BLOCKED_UPLOAD_ERROR_ALREADY_EXISTS
+                ):
+                    error_table.add_row(
+                        str(item.dataset_item_id),
+                        item.filename,
+                        item.path,
+                        "UPLOAD_REQUEST",
+                        slot.reason,
+                    )
         for error in upload_manager.errors:
             for local_file in upload_manager.local_files:
                 if local_file.local_path != error.file_path:
@@ -855,8 +870,8 @@ def file_upload_callback(
         _error(f"No dataset with name '{e.name}'")
     except UnsupportedFileType as e:
         _error(f"Unsupported file type {e.path.suffix} ({e.path.name})")
-    except ValueError:
-        _error("No files found")
+    except ValueError as e:
+        _error(f"{e}")
 
 
 def dataset_import(

darwin/dataset/download_manager.py

@@ -672,7 +672,7 @@ def _get_planned_image_paths(
         for slot in annotation.slots:
             slot_name = Path(slot.name)
             for source_file in slot.source_files:
-                file_name = source_file["file_name"]
+                file_name = source_file.file_name
                 if use_folders and annotation.remote_path != "/":
                     file_paths.append(
                         images_path

darwin/dataset/remote_dataset.py

@@ -138,6 +138,7 @@ def push(
         preserve_folders: bool = False,
         progress_callback: Optional[ProgressCallback] = None,
         file_upload_callback: Optional[FileUploadCallback] = None,
+        item_merge_mode: Optional[str] = None,
     ) -> UploadHandler:
         pass