diff --git a/e2e_tests/cli/test_import.py b/e2e_tests/cli/test_import.py index 79ba83220..2441b7ebe 100644 --- a/e2e_tests/cli/test_import.py +++ b/e2e_tests/cli/test_import.py @@ -1,17 +1,19 @@ from pathlib import Path + from e2e_tests.helpers import ( assert_cli, run_cli_command, export_and_download_annotations, delete_annotation_uuids, + list_items, ) from e2e_tests.objects import E2EDataset, ConfigValues from darwin.utils.utils import parse_darwin_json import tempfile import zipfile import darwin.datatypes as dt -from typing import List, Dict +from typing import List, Dict, Optional def get_actual_annotation_filename( @@ -57,7 +59,7 @@ def find_matching_actual_annotation( def assert_same_annotation_data( expected_annotation: dt.Annotation, actual_annotation: dt.Annotation -): +) -> None: """ Ensures that `expected_annotation.data` is equivalent to `actual_annotation.data` """ @@ -66,7 +68,7 @@ def assert_same_annotation_data( def assert_same_annotation_properties( expected_annotation: dt.Annotation, actual_annotation: dt.Annotation -): +) -> None: """ Ensures that `expected_annotation.properties` is equivalent to `actual_annotation.properties` """ @@ -78,15 +80,57 @@ def assert_same_annotation_properties( assert expected_property in actual_properties # type : ignore +def get_base_slot_name_of_item( + config_values: ConfigValues, dataset_id: int, item_name: str +) -> str: + """ + Returns the base slot name for the item with the given name in a specific `E2EDataset`. + The base slot is always the first listed slot + """ + items = list_items( + config_values.api_key, + dataset_id, + config_values.team_slug, + config_values.server, + ) + for item in items: + if item["name"] == item_name: + return item["slots"][0]["slot_name"] + + +def assert_same_annotation_slot_name( + expected_annotation: dt.Annotation, + actual_annotation: dt.Annotation, + item_type: str, + base_slot: Optional[str], +) -> None: + """ + Ensures that the slot tied to an `actual_annotation` is aligned depending on the + value of `item_type`: + - `single_slotted`: Perform no checks + - `multi_slotted`: Ensures `actual_annotation.slot_names` is equivalent to + `expected_annotation.slot_names` + - `multi_channel`: Ensures the `actual_annotation` is tied to the base slot + """ + if item_type == "multi_slotted": + if expected_annotation.slot_names: + assert expected_annotation.slot_names == actual_annotation.slot_names + else: + assert actual_annotation.slot_names == [base_slot] + elif item_type == "multi_channel": + assert actual_annotation.slot_names == [base_slot] + + def compare_annotations_export( actual_annotations_dir: Path, expected_annotations_dir: Path, + item_type: str, + base_slot: Optional[str] = "0", ): """ Compares a set of downloaded annotation files with the imported files that resulted in those annotations. Ensures equality """ - with zipfile.ZipFile(actual_annotations_dir / "dataset.zip") as z: z.extractall(actual_annotations_dir) @@ -121,193 +165,214 @@ def compare_annotations_export( actual_annotation = find_matching_actual_annotation( expected_annotation, actual_annotations ) - assert_same_annotation_data(actual_annotation, expected_annotation) + assert_same_annotation_data(expected_annotation, actual_annotation) assert_same_annotation_properties(expected_annotation, actual_annotation) + assert_same_annotation_slot_name( + expected_annotation, actual_annotation, item_type, base_slot + ) -def test_import_annotations_without_subtypes_to_images( - local_dataset: E2EDataset, config_values: ConfigValues +def run_import_test( + local_dataset: E2EDataset, + config_values: ConfigValues, + item_type: str, + annotations_subdir: str, + item_name: Optional[str] = None, + additional_flags: str = "", + exit_code: int = 0, + expect_warning: Optional[str] = None, + expect_error: Optional[str] = None, ) -> None: """ - Test importing a set of basic annotations (no sub-types or properties) to a set of - pre-registered files in a dataset. + Helper function to run import tests for different item types and annotation configurations. """ - local_dataset.register_read_only_items(config_values) + local_dataset.register_read_only_items(config_values, item_type) expected_annotations_dir = ( - Path(__file__).parents[1] - / "data" - / "import" - / "image_annotations_without_subtypes" + Path(__file__).parents[1] / "data" / "import" / annotations_subdir ) result = run_cli_command( - f"darwin dataset import {local_dataset.name} darwin {expected_annotations_dir}" + f"darwin dataset import {local_dataset.name} darwin {expected_annotations_dir} {additional_flags}" + ) + assert_cli(result, exit_code) + + if expect_warning: + assert expect_warning in result.stdout + + if expect_error: + assert expect_error in result.stdout + return + + base_slot = ( + get_base_slot_name_of_item(config_values, local_dataset.id, item_name) + if item_name + else None ) - assert_cli(result, 0) with tempfile.TemporaryDirectory() as tmp_dir_str: actual_annotations_dir = Path(tmp_dir_str) export_and_download_annotations( actual_annotations_dir, local_dataset, config_values ) - compare_annotations_export(actual_annotations_dir, expected_annotations_dir) + compare_annotations_export( + actual_annotations_dir, expected_annotations_dir, item_type, base_slot + ) -def test_import_annotations_with_subtypes_to_images( +def test_import_annotations_without_subtypes_to_images( local_dataset: E2EDataset, config_values: ConfigValues ) -> None: - """ - Test importing a set of annotations that includes subtypes & properties to a set of - pre-registered files in a dataset. - """ - local_dataset.register_read_only_items(config_values) - expected_annotations_dir = ( - Path(__file__).parents[1] - / "data" - / "import" - / "image_annotations_with_subtypes" + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_annotations_without_subtypes", ) - result = run_cli_command( - f"darwin dataset import {local_dataset.name} darwin {expected_annotations_dir}" + + +def test_import_annotations_with_subtypes_to_images( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_annotations_with_subtypes", ) - assert_cli(result, 0) - with tempfile.TemporaryDirectory() as tmp_dir_str: - actual_annotations_dir = Path(tmp_dir_str) - export_and_download_annotations( - actual_annotations_dir, local_dataset, config_values - ) - compare_annotations_export(actual_annotations_dir, expected_annotations_dir) def test_annotation_classes_are_created_on_import( local_dataset: E2EDataset, config_values: ConfigValues ) -> None: - """ - Test that importing non-existent annotation classes creates those classes in the - target Darwin team - """ - local_dataset.register_read_only_items(config_values) - expected_annotations_dir = ( - Path(__file__).parents[1] - / "data" - / "import" - / "image_annotations_without_subtypes" - ) - result = run_cli_command( - f"darwin dataset import {local_dataset.name} darwin {expected_annotations_dir} --yes" + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_annotations_without_subtypes", + additional_flags="--yes", ) - assert_cli(result, 0) - with tempfile.TemporaryDirectory() as tmp_dir_str: - actual_annotations_dir = Path(tmp_dir_str) - export_and_download_annotations( - actual_annotations_dir, local_dataset, config_values - ) - compare_annotations_export(actual_annotations_dir, expected_annotations_dir) def test_annotation_classes_are_created_with_properties_on_import( local_dataset: E2EDataset, config_values: ConfigValues ) -> None: - """ - Test that importing non-existent annotation classes with properties creates those - classes and properties in the target Darwin team - """ - local_dataset.register_read_only_items(config_values) - expected_annotations_dir = ( - Path(__file__).parents[1] - / "data" - / "import" - / "image_new_annotations_with_properties" + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_new_annotations_with_properties", + additional_flags="--yes", ) - result = run_cli_command( - f"darwin dataset import {local_dataset.name} darwin {expected_annotations_dir} --yes" - ) - assert_cli(result, 0) - with tempfile.TemporaryDirectory() as tmp_dir_str: - actual_annotations_dir = Path(tmp_dir_str) - export_and_download_annotations( - actual_annotations_dir, local_dataset, config_values - ) - compare_annotations_export(actual_annotations_dir, expected_annotations_dir) def test_appending_annotations( local_dataset: E2EDataset, config_values: ConfigValues ) -> None: - """ - Test that appending annotations to an item with already existing annotations does - not overwrite the original annotations - """ - local_dataset.register_read_only_items(config_values) - expected_annotations_dir = ( - Path(__file__).parents[1] - / "data" - / "import" - / "image_annotations_split_in_two_files" + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_annotations_split_in_two_files", + additional_flags="--append", ) - result = run_cli_command( - f"darwin dataset import {local_dataset.name} darwin {expected_annotations_dir} --append" - ) - assert_cli(result, 0) - with tempfile.TemporaryDirectory() as tmp_dir_str: - actual_annotations_dir = Path(tmp_dir_str) - export_and_download_annotations( - actual_annotations_dir, local_dataset, config_values - ) - compare_annotations_export(actual_annotations_dir, expected_annotations_dir) def test_overwriting_annotations( local_dataset: E2EDataset, config_values: ConfigValues ) -> None: - """ - Test that the `--overwrite` flag allows bypassing of the overwrite warning when - importing to items with already existing annotations - """ - local_dataset.register_read_only_items(config_values) - expected_annotations_dir = ( - Path(__file__).parents[1] - / "data" - / "import" - / "image_annotations_without_subtypes" - ) - # 1st import to create annotations - result = run_cli_command( - f"darwin dataset import {local_dataset.name} darwin {expected_annotations_dir}" + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_annotations_without_subtypes", ) - assert_cli(result, 0) - # 2nd import to overwrite annotations - result = run_cli_command( - f" darwin dataset import {local_dataset.name} darwin {expected_annotations_dir} --overwrite" + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_annotations_without_subtypes", + additional_flags="--overwrite", ) - assert_cli(result, 0) - with tempfile.TemporaryDirectory() as tmp_dir_str: - actual_annotations_dir = Path(tmp_dir_str) - export_and_download_annotations( - actual_annotations_dir, local_dataset, config_values - ) - compare_annotations_export(actual_annotations_dir, expected_annotations_dir) def test_annotation_overwrite_warning( local_dataset: E2EDataset, config_values: ConfigValues ) -> None: - """ - Test that importing annotations to an item with already existing annotations throws - a warning if not using the `--append` or `--overwrite` flags - """ + # 1st import to create annotations local_dataset.register_read_only_items(config_values) + annotations_subdir = "image_annotations_without_subtypes" expected_annotations_dir = ( - Path(__file__).parents[1] - / "data" - / "import" - / "image_annotations_without_subtypes" + Path(__file__).parents[1] / "data" / "import" / annotations_subdir ) - # 1st import to create annotations - result = run_cli_command( + run_cli_command( f"darwin dataset import {local_dataset.name} darwin {expected_annotations_dir}" ) - assert_cli(result, 0) - # 2nd import to trigger overwrite warning - result = run_cli_command( - f"darwin dataset import {local_dataset.name} darwin {expected_annotations_dir}" + + # Run the 2nd import to trigger the overwrite warning + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_annotations_without_subtypes", + expect_warning="will be overwritten", + exit_code=255, + ) + + +def test_import_annotations_to_multi_slotted_item_without_slots_defined( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + run_import_test( + local_dataset, + config_values, + item_type="multi_slotted", + item_name="multi_slotted_item", + annotations_subdir="multi_slotted_annotations_without_slots_defined", + ) + + +def test_import_annotations_to_multi_slotted_item_with_slots_defined( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + run_import_test( + local_dataset, + config_values, + item_type="multi_slotted", + item_name="multi_slotted_item", + annotations_subdir="multi_slotted_annotations_with_slots_defined", + ) + + +def test_import_annotations_to_multi_channel_item_without_slots_defined( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + run_import_test( + local_dataset, + config_values, + item_type="multi_channel", + item_name="multi_channel_item", + annotations_subdir="multi_channel_annotations_without_slots_defined", + ) + + +def test_import_annotations_to_multi_channel_item_with_slots_defined( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + run_import_test( + local_dataset, + config_values, + item_type="multi_channel", + item_name="multi_channel_item", + annotations_subdir="multi_channel_annotations_with_slots_defined", + ) + + +def test_import_annotations_to_multi_channel_item_non_base_slot( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + run_import_test( + local_dataset, + config_values, + item_type="multi_channel", + item_name="multi_channel_item", + annotations_subdir="multi_channel_annotations_aligned_with_non_base_slot", + expect_error="WARNING: 1 file(s) have the following blocking issues and will not be imported", ) - assert "will be overwritten" in result.stdout diff --git a/e2e_tests/cli/test_push.py b/e2e_tests/cli/test_push.py index b53b569c2..fd7a38272 100644 --- a/e2e_tests/cli/test_push.py +++ b/e2e_tests/cli/test_push.py @@ -13,34 +13,41 @@ import zipfile -def test_push_mixed_filetypes( - local_dataset: E2EDataset, config_values: ConfigValues -) -> None: - """ - Test pushing a directory of files containing various fileytypes: - - .jpg - - .png - - .mp4 - - .dcm - - .pdf - """ - push_dir = Path(__file__).parents[1] / "data" / "push" / "mixed_filetypes.zip" +def extract_and_push( + push_dir, local_dataset, config_values, expected_push_dir, extra_args="" +): with tempfile.TemporaryDirectory() as tmp_dir_str: tmp_dir = Path(tmp_dir_str) with zipfile.ZipFile(push_dir) as z: z.extractall(tmp_dir) result = run_cli_command( - f"darwin dataset push {local_dataset.name} {tmp_dir}/mixed_filetypes" + f"darwin dataset push {local_dataset.name} {tmp_dir}/{expected_push_dir} {extra_args}" ) assert_cli(result, 0) wait_until_items_processed(config_values, local_dataset.id) - items = list_items( + return list_items( config_values.api_key, local_dataset.id, config_values.team_slug, config_values.server, ) - assert len(items) == 5 + + +def test_push_mixed_filetypes( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + """ + Test pushing a directory of files containing various fileytypes: + - .jpg + - .png + - .mp4 + - .dcm + - .pdf + """ + expected_push_dir = "mixed_filetypes" + push_dir = Path(__file__).parents[1] / "data" / "push" / f"{expected_push_dir}.zip" + items = extract_and_push(push_dir, local_dataset, config_values, expected_push_dir) + assert len(items) == 5 def test_push_nested_directory_of_images( @@ -49,6 +56,7 @@ def test_push_nested_directory_of_images( """ Test pushing a nested directory structure of some images with the `preserve_folders` flag. """ + expected_push_dir = "nested_directory_of_images" expected_paths = { "image_1.jpg": "/dir1", "image_2.jpg": "/dir1", @@ -57,27 +65,13 @@ def test_push_nested_directory_of_images( "image_5.jpg": "/dir1/dir3", "image_6.jpg": "/dir1/dir3", } - push_dir = ( - Path(__file__).parents[1] / "data" / "push" / "nested_directory_of_images.zip" + push_dir = Path(__file__).parents[1] / "data" / "push" / f"{expected_push_dir}.zip" + items = extract_and_push( + push_dir, local_dataset, config_values, expected_push_dir, "--preserve-folders" ) - with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = Path(tmp_dir_str) - with zipfile.ZipFile(push_dir) as z: - z.extractall(tmp_dir) - result = run_cli_command( - f"darwin dataset push {local_dataset.name} {tmp_dir}/nested_directory_of_images --preserve-folders" - ) - assert_cli(result, 0) - wait_until_items_processed(config_values, local_dataset.id) - items = list_items( - config_values.api_key, - local_dataset.id, - config_values.team_slug, - config_values.server, - ) - assert len(items) == 6 - for item in items: - assert expected_paths[item["name"]] == item["path"] + assert len(items) == 6 + for item in items: + assert expected_paths[item["name"]] == item["path"] def test_push_videos_with_non_native_fps( @@ -86,27 +80,144 @@ def test_push_videos_with_non_native_fps( """ Test that if FPS is set, that the value is respected in the resulting dataset items """ - push_dir = Path(__file__).parents[1] / "data" / "push" / "25_frame_video.zip" + expected_push_dir = "25_frame_video" + push_dir = Path(__file__).parents[1] / "data" / "push" / f"{expected_push_dir}.zip" fps = 5 - with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = Path(tmp_dir_str) - with zipfile.ZipFile(push_dir) as z: - z.extractall(tmp_dir) - result = run_cli_command( - f"darwin dataset push {local_dataset.name} {tmp_dir}/25_frame_video --fps {fps}" - ) - assert_cli(result, 0) - wait_until_items_processed(config_values, local_dataset.id) - items = list_items( - config_values.api_key, - local_dataset.id, - config_values.team_slug, - config_values.server, - ) - video_metadata = items[0]["slots"][0]["metadata"] - assert len(items) == 1 - assert 1 == 1 - assert items[0]["slots"][0]["fps"] == fps - assert video_metadata["native_fps"] == 10 - assert video_metadata["frames_manifests"][0]["total_frames"] == 25 - assert video_metadata["frames_manifests"][0]["visible_frames"] == 13 + items = extract_and_push( + push_dir, local_dataset, config_values, expected_push_dir, f"--fps {fps}" + ) + video_metadata = items[0]["slots"][0]["metadata"] + assert len(items) == 1 + assert items[0]["slots"][0]["fps"] == fps + assert video_metadata["native_fps"] == 10 + assert video_metadata["frames_manifests"][0]["total_frames"] == 25 + assert video_metadata["frames_manifests"][0]["visible_frames"] == 13 + + +def test_push_multi_slotted_item( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + """ + Test pushing a multi-slotted item with the CLI. Check the resulting item is + structured as expected + """ + expected_push_dir = "flat_directory_of_6_images" + expected_name = "flat_directory_of_6_images" + expected_slot_types = ["image", "image", "image", "image", "image", "image"] + expected_layout = { + "slots_grid": [[["0"], ["3"]], [["1"], ["4"]], [["2"], ["5"]]], + "version": 3, + } + expected_file_names = [ + "image_1.jpg", + "image_2.jpg", + "image_3.jpg", + "image_4.jpg", + "image_5.jpg", + "image_6.jpg", + ] + push_dir = Path(__file__).parents[1] / "data" / "push" / f"{expected_push_dir}.zip" + items = extract_and_push( + push_dir, + local_dataset, + config_values, + expected_push_dir, + "--item-merge-mode slots", + ) + assert len(items) == 1 + multi_slotted_item = items[0] + assert multi_slotted_item["name"] == expected_name + assert multi_slotted_item["slot_types"] == expected_slot_types + assert multi_slotted_item["layout"] == expected_layout + for num, slot in enumerate(multi_slotted_item["slots"]): + assert slot["slot_name"] == str(num) + assert slot["file_name"] == expected_file_names[num] + + +def test_push_multi_channel_item( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + """ + Test pushing a multi-channel item with the CLI. Check the resulting item is + structured as expected + """ + expected_push_dir = "flat_directory_of_6_images" + expected_name = "flat_directory_of_6_images" + expected_slot_types = ["image", "image", "image", "image", "image", "image"] + expected_layout = { + "slots_grid": [ + [ + [ + "image_1.jpg", + "image_2.jpg", + "image_3.jpg", + "image_4.jpg", + "image_5.jpg", + "image_6.jpg", + ] + ] + ], + "version": 3, + } + expected_file_names = [ + "image_1.jpg", + "image_2.jpg", + "image_3.jpg", + "image_4.jpg", + "image_5.jpg", + "image_6.jpg", + ] + push_dir = Path(__file__).parents[1] / "data" / "push" / f"{expected_push_dir}.zip" + items = extract_and_push( + push_dir, + local_dataset, + config_values, + expected_push_dir, + "--item-merge-mode channels", + ) + assert len(items) == 1 + multi_channel_item = items[0] + assert multi_channel_item["name"] == expected_name + assert multi_channel_item["slot_types"] == expected_slot_types + assert multi_channel_item["layout"] == expected_layout + for num, slot in enumerate(multi_channel_item["slots"]): + assert slot["slot_name"] == expected_file_names[num] + assert slot["file_name"] == expected_file_names[num] + + +def test_push_dicom_series( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + """ + Test pushing a multi-file DICOM item with the CLI. Check the resulting item is + structured as expected + """ + expected_push_dir = "flat_directory_of_2_dicom_files" + expected_name = "flat_directory_of_2_dicom_files" + expected_slot_types = ["dicom", "dicom"] + expected_layout = { + "slots_grid": [ + [ + [ + "flat_directory_of_2_dicom_files", + ] + ] + ], + "version": 3, + } + expected_file_names = ["flat_directory_of_2_dicom_files"] + push_dir = Path(__file__).parents[1] / "data" / "push" / f"{expected_push_dir}.zip" + items = extract_and_push( + push_dir, + local_dataset, + config_values, + expected_push_dir, + "--item-merge-mode series", + ) + assert len(items) == 1 + dicom_series_item = items[0] + assert dicom_series_item["name"] == expected_name + assert dicom_series_item["slot_types"] == expected_slot_types + assert dicom_series_item["layout"] == expected_layout + for num, slot in enumerate(dicom_series_item["slots"]): + assert slot["slot_name"] == expected_file_names[num] diff --git a/e2e_tests/data/import/multi_channel_annotations_aligned_with_non_base_slot/multi_channel_item.json b/e2e_tests/data/import/multi_channel_annotations_aligned_with_non_base_slot/multi_channel_item.json new file mode 100644 index 000000000..aa7df5ae9 --- /dev/null +++ b/e2e_tests/data/import/multi_channel_annotations_aligned_with_non_base_slot/multi_channel_item.json @@ -0,0 +1,138 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "multi_channel_item", + "path": "/", + "source_info": { + "item_id": "01923066-0bbc-6dd2-1b00-cae3d0a9d1fa", + "dataset": { + "name": "test_dataset_04c8d85d-dbf4-459c-b1c6-68b3d5a4382a", + "slug": "test_dataset_04c8d85d-dbf4-459c-b1c6-68b3d5a4382a", + "dataset_management_url": "https://staging.v7labs.com/datasets/360910/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=360910&item=01923066-0bbc-6dd2-1b00-cae3d0a9d1fa" + }, + "slots": [ + { + "type": "image", + "slot_name": "image_1.jpg", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/1adfde54-61cc-41bd-b591-83ca4026c3db/thumbnail", + "source_files": [ + { + "file_name": "image_1.jpg", + "storage_key": "darwin-py/images/image_1.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/1a9c71f7-8801-4783-ab35-bcc06b4f658d" + } + ] + }, + { + "type": "image", + "slot_name": "image_2.jpg", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/c83d4e06-b43d-4a5b-9e88-93446cf9a554/thumbnail", + "source_files": [ + { + "file_name": "image_2.jpg", + "storage_key": "darwin-py/images/image_2.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/f6a302aa-f641-48a6-9427-6bbe073205b0" + } + ] + }, + { + "type": "image", + "slot_name": "image_3.jpg", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/d6861a96-8cf2-4bfb-a7ec-2cd9192afe7e/thumbnail", + "source_files": [ + { + "file_name": "image_3.jpg", + "storage_key": "darwin-py/images/image_3.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/97d3a43e-b229-4156-ab13-18694cae6834" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 1.0018, + "w": 8.3482, + "x": 7.2124, + "y": 13.8891 + }, + "id": "6ab44579-31d9-4823-81fa-d318a6025978", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "this_slot_does_not_exist" + ] + }, + { + "bounding_box": { + "h": 6.6785, + "w": 13.691, + "x": 38.9354, + "y": 29.2498 + }, + "id": "6a41e1fc-10b1-4791-aa43-bca1fadd2614", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "this_slot_does_not_exist" + ] + }, + { + "bounding_box": { + "h": 11.0196, + "w": 12.3553, + "x": 14.5588, + "y": 53.9604 + }, + "id": "4a8cd4dc-a87e-4736-ae01-45ffbdaabdfa", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "this_slot_does_not_exist" + ] + }, + { + "bounding_box": { + "h": 6.3446, + "w": 19.0338, + "x": -5.1429, + "y": 41.6051 + }, + "id": "b195cbc3-bdf9-498e-afc3-c557bac9e3ba", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "this_slot_does_not_exist" + ] + }, + { + "bounding_box": { + "h": 10.3517, + "w": 11.3535, + "x": 39.9372, + "y": -14.4947 + }, + "id": "3ad79017-88cc-49db-a758-dd6dc5051720", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "this_slot_does_not_exist" + ] + } + ], + "properties": [] +} \ No newline at end of file diff --git a/e2e_tests/data/import/multi_channel_annotations_with_slots_defined/multi_channel_item.json b/e2e_tests/data/import/multi_channel_annotations_with_slots_defined/multi_channel_item.json new file mode 100644 index 000000000..561503a39 --- /dev/null +++ b/e2e_tests/data/import/multi_channel_annotations_with_slots_defined/multi_channel_item.json @@ -0,0 +1,138 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "multi_channel_item", + "path": "/", + "source_info": { + "item_id": "01923066-0bbc-6dd2-1b00-cae3d0a9d1fa", + "dataset": { + "name": "test_dataset_04c8d85d-dbf4-459c-b1c6-68b3d5a4382a", + "slug": "test_dataset_04c8d85d-dbf4-459c-b1c6-68b3d5a4382a", + "dataset_management_url": "https://staging.v7labs.com/datasets/360910/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=360910&item=01923066-0bbc-6dd2-1b00-cae3d0a9d1fa" + }, + "slots": [ + { + "type": "image", + "slot_name": "image_1.jpg", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/1adfde54-61cc-41bd-b591-83ca4026c3db/thumbnail", + "source_files": [ + { + "file_name": "image_1.jpg", + "storage_key": "darwin-py/images/image_1.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/1a9c71f7-8801-4783-ab35-bcc06b4f658d" + } + ] + }, + { + "type": "image", + "slot_name": "image_2.jpg", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/c83d4e06-b43d-4a5b-9e88-93446cf9a554/thumbnail", + "source_files": [ + { + "file_name": "image_2.jpg", + "storage_key": "darwin-py/images/image_2.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/f6a302aa-f641-48a6-9427-6bbe073205b0" + } + ] + }, + { + "type": "image", + "slot_name": "image_3.jpg", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/d6861a96-8cf2-4bfb-a7ec-2cd9192afe7e/thumbnail", + "source_files": [ + { + "file_name": "image_3.jpg", + "storage_key": "darwin-py/images/image_3.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/97d3a43e-b229-4156-ab13-18694cae6834" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 1.0018, + "w": 8.3482, + "x": 7.2124, + "y": 13.8891 + }, + "id": "6ab44579-31d9-4823-81fa-d318a6025978", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "image_1.jpg" + ] + }, + { + "bounding_box": { + "h": 6.6785, + "w": 13.691, + "x": 38.9354, + "y": 29.2498 + }, + "id": "6a41e1fc-10b1-4791-aa43-bca1fadd2614", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "image_1.jpg" + ] + }, + { + "bounding_box": { + "h": 11.0196, + "w": 12.3553, + "x": 14.5588, + "y": 53.9604 + }, + "id": "4a8cd4dc-a87e-4736-ae01-45ffbdaabdfa", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "image_1.jpg" + ] + }, + { + "bounding_box": { + "h": 6.3446, + "w": 19.0338, + "x": -5.1429, + "y": 41.6051 + }, + "id": "b195cbc3-bdf9-498e-afc3-c557bac9e3ba", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "image_1.jpg" + ] + }, + { + "bounding_box": { + "h": 10.3517, + "w": 11.3535, + "x": 39.9372, + "y": -14.4947 + }, + "id": "3ad79017-88cc-49db-a758-dd6dc5051720", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "image_1.jpg" + ] + } + ], + "properties": [] +} \ No newline at end of file diff --git a/e2e_tests/data/import/multi_channel_annotations_without_slots_defined/multi_channel_item.json b/e2e_tests/data/import/multi_channel_annotations_without_slots_defined/multi_channel_item.json new file mode 100644 index 000000000..76f0b42ab --- /dev/null +++ b/e2e_tests/data/import/multi_channel_annotations_without_slots_defined/multi_channel_item.json @@ -0,0 +1,123 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "multi_channel_item", + "path": "/", + "source_info": { + "item_id": "01923066-0bbc-6dd2-1b00-cae3d0a9d1fa", + "dataset": { + "name": "test_dataset_04c8d85d-dbf4-459c-b1c6-68b3d5a4382a", + "slug": "test_dataset_04c8d85d-dbf4-459c-b1c6-68b3d5a4382a", + "dataset_management_url": "https://staging.v7labs.com/datasets/360910/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=360910&item=01923066-0bbc-6dd2-1b00-cae3d0a9d1fa" + }, + "slots": [ + { + "type": "image", + "slot_name": "image_1.jpg", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/1adfde54-61cc-41bd-b591-83ca4026c3db/thumbnail", + "source_files": [ + { + "file_name": "image_1.jpg", + "storage_key": "darwin-py/images/image_1.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/1a9c71f7-8801-4783-ab35-bcc06b4f658d" + } + ] + }, + { + "type": "image", + "slot_name": "image_2.jpg", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/c83d4e06-b43d-4a5b-9e88-93446cf9a554/thumbnail", + "source_files": [ + { + "file_name": "image_2.jpg", + "storage_key": "darwin-py/images/image_2.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/f6a302aa-f641-48a6-9427-6bbe073205b0" + } + ] + }, + { + "type": "image", + "slot_name": "image_3.jpg", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/d6861a96-8cf2-4bfb-a7ec-2cd9192afe7e/thumbnail", + "source_files": [ + { + "file_name": "image_3.jpg", + "storage_key": "darwin-py/images/image_3.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/97d3a43e-b229-4156-ab13-18694cae6834" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 1.0018, + "w": 8.3482, + "x": 7.2124, + "y": 13.8891 + }, + "id": "6ab44579-31d9-4823-81fa-d318a6025978", + "name": "test_bounding_box_basic", + "properties": [] + }, + { + "bounding_box": { + "h": 6.6785, + "w": 13.691, + "x": 38.9354, + "y": 29.2498 + }, + "id": "6a41e1fc-10b1-4791-aa43-bca1fadd2614", + "name": "test_bounding_box_basic", + "properties": [] + }, + { + "bounding_box": { + "h": 11.0196, + "w": 12.3553, + "x": 14.5588, + "y": 53.9604 + }, + "id": "4a8cd4dc-a87e-4736-ae01-45ffbdaabdfa", + "name": "test_bounding_box_basic", + "properties": [] + }, + { + "bounding_box": { + "h": 6.3446, + "w": 19.0338, + "x": -5.1429, + "y": 41.6051 + }, + "id": "b195cbc3-bdf9-498e-afc3-c557bac9e3ba", + "name": "test_bounding_box_basic", + "properties": [] + }, + { + "bounding_box": { + "h": 10.3517, + "w": 11.3535, + "x": 39.9372, + "y": -14.4947 + }, + "id": "3ad79017-88cc-49db-a758-dd6dc5051720", + "name": "test_bounding_box_basic", + "properties": [] + } + ], + "properties": [] +} \ No newline at end of file diff --git a/e2e_tests/data/import/multi_slotted_annotations_with_slots_defined/multi_slotted_item.json b/e2e_tests/data/import/multi_slotted_annotations_with_slots_defined/multi_slotted_item.json new file mode 100644 index 000000000..faa0376ee --- /dev/null +++ b/e2e_tests/data/import/multi_slotted_annotations_with_slots_defined/multi_slotted_item.json @@ -0,0 +1,166 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "multi_slotted_item", + "path": "/", + "source_info": { + "item_id": "01923065-fe73-590a-119b-59cf1cd9b0ea", + "dataset": { + "name": "test_dataset_04c8d85d-dbf4-459c-b1c6-68b3d5a4382a", + "slug": "test_dataset_04c8d85d-dbf4-459c-b1c6-68b3d5a4382a", + "dataset_management_url": "https://staging.v7labs.com/datasets/360910/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=360910&item=01923065-fe73-590a-119b-59cf1cd9b0ea" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/14a7245b-40b8-4c90-aa7e-b1ce6f121b3c/thumbnail", + "source_files": [ + { + "file_name": "image_1.jpg", + "storage_key": "darwin-py/images/image_1.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/9fe7ed5e-8e61-425d-9125-562fa5fd7fb6" + } + ] + }, + { + "type": "image", + "slot_name": "1", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/8a06403e-b9fb-4a4c-9ba2-09d2fcdb94ca/thumbnail", + "source_files": [ + { + "file_name": "image_2.jpg", + "storage_key": "darwin-py/images/image_2.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/78939254-966b-496f-b66f-e1c2ab2236bc" + } + ] + }, + { + "type": "image", + "slot_name": "2", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/dff647fb-676c-4799-866a-604a4bb0798b/thumbnail", + "source_files": [ + { + "file_name": "image_3.jpg", + "storage_key": "darwin-py/images/image_3.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/7a9608b4-0509-45f9-8fbb-7c73130cd56c" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 108.5714, + "w": 251.4286, + "x": 737.1429, + "y": 508.5714 + }, + "id": "3f9c8b2e-4493-412d-8a3b-216ac91e4b92", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "bounding_box": { + "h": 102.8571, + "w": 177.1429, + "x": 742.8571, + "y": 920.0 + }, + "id": "8bdd21c4-e0a2-494d-9491-e4ef0a82d492", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "bounding_box": { + "h": 251.4286, + "w": 182.8571, + "x": 474.2857, + "y": 520.0 + }, + "id": "88c95406-e728-423d-8dd5-67046956e476", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "bounding_box": { + "h": 7.9365, + "w": 11.7725, + "x": 8.7963, + "y": 8.2011 + }, + "id": "2a4fc53e-52e2-4c29-a558-90307f7beb7a", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "1" + ] + }, + { + "bounding_box": { + "h": 3.4392, + "w": 3.1746, + "x": 9.9868, + "y": 13.4921 + }, + "id": "ed640687-b83f-4b23-a08b-dc88f4fe290e", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "2" + ] + }, + { + "bounding_box": { + "h": 3.5714, + "w": 8.3333, + "x": 9.0608, + "y": 3.836 + }, + "id": "8bd9616d-b0fc-4d04-a40a-1882f63dcc3e", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "2" + ] + }, + { + "bounding_box": { + "h": 5.291, + "w": 2.5132, + "x": 2.5794, + "y": 4.7619 + }, + "id": "3a2b9b85-c423-4509-a7fe-b7a8e5761817", + "name": "test_bounding_box_basic", + "properties": [], + "slot_names": [ + "2" + ] + } + ], + "properties": [] +} \ No newline at end of file diff --git a/e2e_tests/data/import/multi_slotted_annotations_without_slots_defined/multi_slotted_item.json b/e2e_tests/data/import/multi_slotted_annotations_without_slots_defined/multi_slotted_item.json new file mode 100644 index 000000000..2c1b40d30 --- /dev/null +++ b/e2e_tests/data/import/multi_slotted_annotations_without_slots_defined/multi_slotted_item.json @@ -0,0 +1,145 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "multi_slotted_item", + "path": "/", + "source_info": { + "item_id": "01923065-fe73-590a-119b-59cf1cd9b0ea", + "dataset": { + "name": "test_dataset_04c8d85d-dbf4-459c-b1c6-68b3d5a4382a", + "slug": "test_dataset_04c8d85d-dbf4-459c-b1c6-68b3d5a4382a", + "dataset_management_url": "https://staging.v7labs.com/datasets/360910/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=360910&item=01923065-fe73-590a-119b-59cf1cd9b0ea" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/14a7245b-40b8-4c90-aa7e-b1ce6f121b3c/thumbnail", + "source_files": [ + { + "file_name": "image_1.jpg", + "storage_key": "darwin-py/images/image_1.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/9fe7ed5e-8e61-425d-9125-562fa5fd7fb6" + } + ] + }, + { + "type": "image", + "slot_name": "1", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/8a06403e-b9fb-4a4c-9ba2-09d2fcdb94ca/thumbnail", + "source_files": [ + { + "file_name": "image_2.jpg", + "storage_key": "darwin-py/images/image_2.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/78939254-966b-496f-b66f-e1c2ab2236bc" + } + ] + }, + { + "type": "image", + "slot_name": "2", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/dff647fb-676c-4799-866a-604a4bb0798b/thumbnail", + "source_files": [ + { + "file_name": "image_3.jpg", + "storage_key": "darwin-py/images/image_3.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/7a9608b4-0509-45f9-8fbb-7c73130cd56c" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 108.5714, + "w": 251.4286, + "x": 737.1429, + "y": 508.5714 + }, + "id": "3f9c8b2e-4493-412d-8a3b-216ac91e4b92", + "name": "test_bounding_box_basic", + "properties": [] + }, + { + "bounding_box": { + "h": 102.8571, + "w": 177.1429, + "x": 742.8571, + "y": 920.0 + }, + "id": "8bdd21c4-e0a2-494d-9491-e4ef0a82d492", + "name": "test_bounding_box_basic", + "properties": [] + }, + { + "bounding_box": { + "h": 251.4286, + "w": 182.8571, + "x": 474.2857, + "y": 520.0 + }, + "id": "88c95406-e728-423d-8dd5-67046956e476", + "name": "test_bounding_box_basic", + "properties": [] + }, + { + "bounding_box": { + "h": 7.9365, + "w": 11.7725, + "x": 8.7963, + "y": 8.2011 + }, + "id": "2a4fc53e-52e2-4c29-a558-90307f7beb7a", + "name": "test_bounding_box_basic", + "properties": [] + }, + { + "bounding_box": { + "h": 3.4392, + "w": 3.1746, + "x": 9.9868, + "y": 13.4921 + }, + "id": "ed640687-b83f-4b23-a08b-dc88f4fe290e", + "name": "test_bounding_box_basic", + "properties": [] + }, + { + "bounding_box": { + "h": 3.5714, + "w": 8.3333, + "x": 9.0608, + "y": 3.836 + }, + "id": "8bd9616d-b0fc-4d04-a40a-1882f63dcc3e", + "name": "test_bounding_box_basic", + "properties": [] + }, + { + "bounding_box": { + "h": 5.291, + "w": 2.5132, + "x": 2.5794, + "y": 4.7619 + }, + "id": "3a2b9b85-c423-4509-a7fe-b7a8e5761817", + "name": "test_bounding_box_basic", + "properties": [] + } + ], + "properties": [] +} \ No newline at end of file diff --git a/e2e_tests/data/push/flat_directory_of_2_dicom_files.zip b/e2e_tests/data/push/flat_directory_of_2_dicom_files.zip new file mode 100644 index 000000000..19f0cdb9a Binary files /dev/null and b/e2e_tests/data/push/flat_directory_of_2_dicom_files.zip differ diff --git a/e2e_tests/data/push/flat_directory_of_6_images.zip b/e2e_tests/data/push/flat_directory_of_6_images.zip new file mode 100644 index 000000000..e523b3274 Binary files /dev/null and b/e2e_tests/data/push/flat_directory_of_6_images.zip differ diff --git a/e2e_tests/helpers.py b/e2e_tests/helpers.py index 6d0896e86..13ca62f1c 100644 --- a/e2e_tests/helpers.py +++ b/e2e_tests/helpers.py @@ -181,7 +181,7 @@ def wait_until_items_processed( items = list_items(api_key, dataset_id, team_slug, base_url) if not items: return - if all(item["processing_status"] != "processing" for item in items): + if all(item["processing_status"] == "complete" for item in items): break print(f"Waiting {sleep_duration} seconds for items to finish processing...") time.sleep(sleep_duration) diff --git a/e2e_tests/objects.py b/e2e_tests/objects.py index a10c978c6..cece8b321 100644 --- a/e2e_tests/objects.py +++ b/e2e_tests/objects.py @@ -55,91 +55,16 @@ def __init__( def add_item(self, item: E2EItem) -> None: self.items.append(item) - def register_read_only_items(self, config_values: ConfigValues) -> None: + def register_read_only_items( + self, config_values: ConfigValues, item_type: str = "single_slotted" + ) -> None: """ Registers a set of images from an external bucket in the dataset in a read-only fashion: Useful for creating dataset to test `pull` or `import` operations on without having to wait for items to finish processing """ + payload = get_read_only_registration_payload(item_type, dataset_slug=self.slug) api_key = config_values.api_key - payload = { - "items": [ - { - "path": "/", - "type": "image", - "storage_key": "darwin-py/images/image_1.jpg", - "storage_thumbnail_key": "darwin-py/images/image_1_thumbnail.jpg", - "height": 1080, - "width": 1920, - "name": "image_1", - }, - { - "path": "/", - "type": "image", - "storage_key": "darwin-py/images/image_2.jpg", - "storage_thumbnail_key": "darwin-py/images/image_2_thumbnail.jpg", - "height": 1080, - "width": 1920, - "name": "image_2", - }, - { - "path": "dir1", - "type": "image", - "storage_key": "darwin-py/images/image_3.jpg", - "storage_thumbnail_key": "darwin-py/images/image_3_thumbnail.jpg", - "height": 1080, - "width": 1920, - "name": "image_3", - }, - { - "path": "dir1", - "type": "image", - "storage_key": "darwin-py/images/image_4.jpg", - "storage_thumbnail_key": "darwin-py/images/image_4_thumbnail.jpg", - "height": 1080, - "width": 1920, - "name": "image_4", - }, - { - "path": "dir2", - "type": "image", - "storage_key": "darwin-py/images/image_5.jpg", - "storage_thumbnail_key": "darwin-py/images/image_5_thumbnail.jpg", - "height": 1080, - "width": 1920, - "name": "image_5", - }, - { - "path": "dir2", - "type": "image", - "storage_key": "darwin-py/images/image_6.jpg", - "storage_thumbnail_key": "darwin-py/images/image_6_thumbnail.jpg", - "height": 1080, - "width": 1920, - "name": "image_6", - }, - { - "path": "dir1/dir3", - "type": "image", - "storage_key": "darwin-py/images/image_7.jpg", - "storage_thumbnail_key": "darwin-py/images/image_7_thumbnail.jpg", - "height": 1080, - "width": 1920, - "name": "image_7", - }, - { - "path": "dir1/dir3", - "type": "image", - "storage_key": "darwin-py/images/image_8.jpg", - "storage_thumbnail_key": "darwin-py/images/image_8_thumbnail.jpg", - "height": 1080, - "width": 1920, - "name": "image_8", - }, - ], - "dataset_slug": self.slug, - "storage_slug": "darwin-e2e-data", - } headers = { "Content-Type": "application/json", "Accept": "application/json", @@ -203,6 +128,185 @@ def get_annotation_data( return item_annotations, annotation_classes, properties +def get_read_only_registration_payload( + item_type: str, dataset_slug: str +) -> Dict[str, str]: + """ + Returns a payload for registering items from external storage in a read-only + fashion. `item_type` determines what type of items will be uploaded: + - `single_slotted`: A series of single-slotted items + - `multi_slotted`: A single item with 3 image slots + - `multi_channel`: A single item with 3 image channels + """ + items = { + "single_slotted": [ + { + "path": "/", + "type": "image", + "storage_key": "darwin-py/images/image_1.jpg", + "storage_thumbnail_key": "darwin-py/images/image_1_thumbnail.jpg", + "height": 1080, + "width": 1920, + "name": "image_1", + }, + { + "path": "/", + "type": "image", + "storage_key": "darwin-py/images/image_2.jpg", + "storage_thumbnail_key": "darwin-py/images/image_2_thumbnail.jpg", + "height": 1080, + "width": 1920, + "name": "image_2", + }, + { + "path": "dir1", + "type": "image", + "storage_key": "darwin-py/images/image_3.jpg", + "storage_thumbnail_key": "darwin-py/images/image_3_thumbnail.jpg", + "height": 1080, + "width": 1920, + "name": "image_3", + }, + { + "path": "dir1", + "type": "image", + "storage_key": "darwin-py/images/image_4.jpg", + "storage_thumbnail_key": "darwin-py/images/image_4_thumbnail.jpg", + "height": 1080, + "width": 1920, + "name": "image_4", + }, + { + "path": "dir2", + "type": "image", + "storage_key": "darwin-py/images/image_5.jpg", + "storage_thumbnail_key": "darwin-py/images/image_5_thumbnail.jpg", + "height": 1080, + "width": 1920, + "name": "image_5", + }, + { + "path": "dir2", + "type": "image", + "storage_key": "darwin-py/images/image_6.jpg", + "storage_thumbnail_key": "darwin-py/images/image_6_thumbnail.jpg", + "height": 1080, + "width": 1920, + "name": "image_6", + }, + { + "path": "dir1/dir3", + "type": "image", + "storage_key": "darwin-py/images/image_7.jpg", + "storage_thumbnail_key": "darwin-py/images/image_7_thumbnail.jpg", + "height": 1080, + "width": 1920, + "name": "image_7", + }, + { + "path": "dir1/dir3", + "type": "image", + "storage_key": "darwin-py/images/image_8.jpg", + "storage_thumbnail_key": "darwin-py/images/image_8_thumbnail.jpg", + "height": 1080, + "width": 1920, + "name": "image_8", + }, + ], + "multi_slotted": [ + { + "path": "/", + "layout": { + "slots_grid": [[["0"], ["1"], ["2"]]], + "version": 3, + }, + "slots": [ + { + "slot_name": "0", + "type": "image", + "storage_key": "darwin-py/images/image_1.jpg", + "storage_thumbnail_key": "darwin-py/images/image_1_thumbnail.jpg", + "height": 1080, + "width": 1920, + "file_name": "image_1.jpg", + }, + { + "slot_name": "1", + "type": "image", + "storage_key": "darwin-py/images/image_2.jpg", + "storage_thumbnail_key": "darwin-py/images/image_2_thumbnail.jpg", + "height": 1080, + "width": 1920, + "file_name": "image_2.jpg", + }, + { + "slot_name": "2", + "type": "image", + "storage_key": "darwin-py/images/image_3.jpg", + "storage_thumbnail_key": "darwin-py/images/image_3_thumbnail.jpg", + "height": 1080, + "width": 1920, + "file_name": "image_3.jpg", + }, + ], + "name": "multi_slotted_item", + }, + ], + "multi_channel": [ + { + "path": "/", + "layout": { + "slots_grid": [ + [ + [ + "image_1.jpg", + "image_2.jpg", + "image_3.jpg", + ] + ] + ], + "version": 3, + }, + "slots": [ + { + "slot_name": "image_1.jpg", + "type": "image", + "storage_key": "darwin-py/images/image_1.jpg", + "storage_thumbnail_key": "darwin-py/images/image_1_thumbnail.jpg", + "height": 1080, + "width": 1920, + "file_name": "image_1.jpg", + }, + { + "slot_name": "image_2.jpg", + "type": "image", + "storage_key": "darwin-py/images/image_2.jpg", + "storage_thumbnail_key": "darwin-py/images/image_2_thumbnail.jpg", + "height": 1080, + "width": 1920, + "file_name": "image_2.jpg", + }, + { + "slot_name": "image_3.jpg", + "type": "image", + "storage_key": "darwin-py/images/image_3.jpg", + "storage_thumbnail_key": "darwin-py/images/image_3_thumbnail.jpg", + "height": 1080, + "width": 1920, + "file_name": "image_3.jpg", + }, + ], + "name": "multi_channel_item", + }, + ], + } + return { + "items": items[item_type], # type: ignore + "dataset_slug": dataset_slug, + "storage_slug": "darwin-e2e-data", + } + + @dataclass class E2ETestRunInfo: prefix: str