diff --git a/darwin/future/core/items/uploads.py b/darwin/future/core/items/uploads.py new file mode 100644 index 000000000..459759811 --- /dev/null +++ b/darwin/future/core/items/uploads.py @@ -0,0 +1,495 @@ +import asyncio +from logging import getLogger +from pathlib import Path +from typing import Dict, List, Tuple, Union + +from darwin.future.core.client import ClientCore +from darwin.future.data_objects.item import UploadItem +from darwin.future.data_objects.typing import UnknownType +from darwin.future.exceptions import DarwinException + +logger = getLogger(__name__) + + +async def _build_slots(item: UploadItem) -> List[Dict]: + """ + (internal) Builds the slots for an item + + Parameters + ---------- + item: UploadItem + The item to build slots for + + Returns + ------- + List[Dict] + The built slots + """ + + if not item.slots: + return [] + + slots_to_return: List[Dict] = [] + + for slot in item.slots: + slot_dict: Dict[str, UnknownType] = { + "slot_name": slot.slot_name, + "file_name": slot.file_name, + "fps": slot.fps, + } + + if slot.storage_key is not None: + slot_dict["storage_key"] = slot.storage_key + + if slot.as_frames is not None: + slot_dict["as_frames"] = slot.as_frames + + if slot.extract_views is not None: + slot_dict["extract_views"] = slot.extract_views + + if slot.metadata is not None: + slot_dict["metadata"] = slot.metadata + + if slot.tags is not None: + slot_dict["tags"] = slot.tags + + if slot.type is not None: + slot_dict["type"] = slot.type + + slots_to_return.append(slot_dict) + + return slots_to_return + + +async def _build_layout(item: UploadItem) -> Dict: + if not item.layout: + return {} + + if item.layout.version == 1: + return { + "slots": item.layout.slots, + "type": item.layout.type, + "version": item.layout.version, + } + + if item.layout.version == 2: + return { + "slots": item.layout.slots, + "type": item.layout.type, + "version": item.layout.version, + "layout_shape": item.layout.layout_shape, + } + + raise DarwinException(f"Invalid layout version {item.layout.version}") + + +async def _build_payload_items( + items_and_paths: List[Tuple[UploadItem, Path]] +) -> List[Dict]: + """ + Builds the payload for the items to be registered for upload + + Parameters + ---------- + items_and_paths: List[Tuple[UploadItem, Path]] + + Returns + ------- + List[Dict] + The payload for the items to be registered for upload + """ + + return_list = [] + for item, path in items_and_paths: + base_item = { + "name": getattr(item, "name"), + "slots": await _build_slots(item), + "path": str(path), + } + + if getattr(item, "tags", None): + base_item["tags"] = item.tags + + if getattr(item, "layout", None): + base_item["layout"] = await _build_layout(item) + + return_list.append(base_item) + + return return_list + + +async def async_register_upload( + api_client: ClientCore, + team_slug: str, + dataset_slug: str, + items_and_paths: Union[Tuple[UploadItem, Path], List[Tuple[UploadItem, Path]]], + force_tiling: bool = False, + handle_as_slices: bool = False, + ignore_dicom_layout: bool = False, +) -> Dict: + """ + Registers an upload for a dataset that can then be used to upload files to Darwin + + Parameters + ---------- + api_client: ClientCore + The client to use for the request + team_slug: str + The slug of the team to register the upload for + dataset_slug: str + The slug of the dataset to register the upload for + items_and_paths: Union[Tuple[UploadItem, Path], List[Tuple[UploadItem, Path]]] + A list of tuples of Items and Paths to register for upload + force_tiling: bool + Whether to force tiling for the upload + handle_as_slices: bool + Whether to handle the upload as slices + ignore_dicom_layout: bool + Whether to ignore the dicom layout + """ + + if isinstance(items_and_paths, tuple): + items_and_paths = [items_and_paths] + assert all( + (isinstance(item, UploadItem) and isinstance(path, Path)) + for item, path in items_and_paths + ), "items must be a list of Items" + + payload_items = await _build_payload_items(items_and_paths) + + options = { + "force_tiling": force_tiling, + "handle_as_slices": handle_as_slices, + "ignore_dicom_layout": ignore_dicom_layout, + } + + payload = { + "dataset_slug": dataset_slug, + "items": payload_items, + "options": options, + } + + try: + response = api_client.post( + f"/v2/teams/{team_slug}/items/register_upload", payload + ) + except Exception as exc: + logger.error(f"Failed to register upload in {__name__}", exc_info=exc) + raise DarwinException(f"Failed to register upload in {__name__}") from exc + + assert isinstance(response, dict), "Unexpected return type from register upload" + + return response + + +async def async_create_signed_upload_url( + api_client: ClientCore, + team_slug: str, + upload_id: str, +) -> str: + """ + Asynchronously create a signed upload URL for an upload or uploads + + Parameters + ---------- + api_client: ClientCore + The client to use for the request + team_slug: str + The slug of the team to register the upload for + upload_id: str + The ID of the upload to confirm + + Returns + ------- + JSONType + The response from the API + """ + try: + response = api_client.get( + f"/v2/teams/{team_slug}/items/uploads/{upload_id}/sign" + ) + except Exception as exc: + logger.error(f"Failed to create signed upload url in {__name__}", exc_info=exc) + raise DarwinException( + f"Failed to create signed upload url in {__name__}" + ) from exc + + assert isinstance( + response, dict + ), "Unexpected return type from create signed upload url" + + if not response: + logger.error( + f"Failed to create signed upload url in {__name__}, got no response" + ) + raise DarwinException( + f"Failed to create signed upload url in {__name__}, got no response" + ) + + if "errors" in response: + logger.error( + f"Failed to create signed upload url in {__name__}, got errors: {response['errors']}" + ) + raise DarwinException(f"Failed to create signed upload url in {__name__}") + + if "upload_url" not in response: + logger.error( + f"Failed to create signed upload url in {__name__}, got no upload_url" + ) + raise DarwinException( + f"Failed to create signed upload url in {__name__}, got no upload_url" + ) + + return response["upload_url"] + + +async def async_register_and_create_signed_upload_url( + api_client: ClientCore, + team_slug: str, + dataset_slug: str, + items_and_paths: Union[Tuple[UploadItem, Path], List[Tuple[UploadItem, Path]]], + force_tiling: bool = False, + handle_as_slices: bool = False, + ignore_dicom_layout: bool = False, +) -> List[Tuple[str, str]]: + """ + Asynchronously register and create a signed upload URL for an upload or uploads + + Parameters + ---------- + api_client: ClientCore + The client to use for the request + team_slug: str + The slug of the team to register the upload for + dataset_slug: str + The slug of the dataset to register the upload for + items_and_paths: Union[Tuple[Item, Path], List[Tuple[Item, Path]]] + A list of tuples, or a single tuple of Items and Paths to register for upload + force_tiling: bool + Whether to force tiling for the upload + handle_as_slices: bool + Whether to handle the upload as slices + ignore_dicom_layout: bool + Whether to ignore the dicom layout + + Returns + ------- + List[Tuple[str, str]] + List of tuples of signed upload urls and upload ids + """ + + register = await async_register_upload( + api_client, + team_slug, + dataset_slug, + items_and_paths, + force_tiling, + handle_as_slices, + ignore_dicom_layout, + ) + + if "errors" in register: + raise DarwinException(f"Failed to register upload in {__name__}") + + if ( + "blocked_items" in register + and isinstance(register["blocked_items"], list) + and len(register["blocked_items"]) > 0 + ): + raise DarwinException( + f"Failed to register upload in {__name__}, got blocked items: {register['blocked_items']}" + ) + + assert "items" in register, "Unexpected return type from register upload" + assert "blocked_items" in register, "Unexpected return type from register upload" + + uploaded_items = register["items"] + + upload_ids = [] + for item in uploaded_items: + if "slots" in item: + for slot in item["slots"]: + if "upload_id" in slot: + upload_ids.append(slot["upload_id"]) + + return [ + (await async_create_signed_upload_url(api_client, team_slug, id), id) + for id in upload_ids + ] + + +async def async_confirm_upload( + api_client: ClientCore, team_slug: str, upload_id: str +) -> None: + """ + Asynchronously confirm an upload/uploads was successful by ID + + Parameters + ---------- + api_client: ClientCore + The client to use for the request + team_slug: str + The slug of the team to confirm the upload for + upload_id: str + The ID of the upload to confirm + + Returns + ------- + JSONType + The response from the API + """ + + try: + response = api_client.post( + f"/v2/teams/{team_slug}/items/uploads/{upload_id}/confirm", data={} + ) + except Exception as exc: + logger.error(f"Failed to confirm upload in {__name__}", exc_info=exc) + raise DarwinException(f"Failed to confirm upload in {__name__}") from exc + + assert isinstance(response, dict), "Unexpected return type from confirm upload" + + if "errors" in response: + logger.error( + f"Failed to confirm upload in {__name__}, got errors: {response['errors']}" + ) + raise DarwinException( + f"Failed to confirm upload in {__name__}: {str(response['errors'])}" + ) + + +def register_upload( + api_client: ClientCore, + team_slug: str, + dataset_slug: str, + items_and_paths: Union[Tuple[UploadItem, Path], List[Tuple[UploadItem, Path]]], + force_tiling: bool = False, + handle_as_slices: bool = False, + ignore_dicom_layout: bool = False, +) -> Dict: + """ + Asynchronously register an upload/uploads for a dataset that can then be used to upload files to Darwin + + Parameters + ---------- + api_client: ClientCore + The client to use for the request + team_slug: str + The slug of the team to register the upload for + dataset_slug: str + The slug of the dataset to register the upload for + items_and_paths: Union[Tuple[Item, Path], List[Tuple[Item, Path]]] + A list of tuples, or a single tuple of Items and Paths to register for upload + force_tiling: bool + Whether to force tiling for the upload + handle_as_slices: bool + Whether to handle the upload as slices + ignore_dicom_layout: bool + Whether to ignore the dicom layout + """ + + response = asyncio.run( + async_register_upload( + api_client, + team_slug, + dataset_slug, + items_and_paths, + force_tiling, + handle_as_slices, + ignore_dicom_layout, + ) + ) + return response + + +def create_signed_upload_url( + api_client: ClientCore, + upload_id: str, + team_slug: str, +) -> str: + """ + Create a signed upload URL for an upload or uploads + + Parameters + ---------- + api_client: ClientCore + The client to use for the request + team_slug: str + The slug of the team to register the upload for + upload_id: str + The ID of the upload to confirm + + Returns + ------- + JSONType + The response from the API + """ + + return asyncio.run(async_create_signed_upload_url(api_client, upload_id, team_slug)) + + +def register_and_create_signed_upload_url( + api_client: ClientCore, + team_slug: str, + dataset_slug: str, + items_and_paths: Union[List[Tuple[UploadItem, Path]], Tuple[UploadItem, Path]], + force_tiling: bool = False, + handle_as_slices: bool = False, + ignore_dicom_layout: bool = False, +) -> List[Tuple[str, str]]: + """ + Register and create a signed upload URL for an upload or uploads + + Parameters + ---------- + api_client: ClientCore + The client to use for the request + team_slug: str + The slug of the team to register the upload for + dataset_slug: str + The slug of the dataset to register the upload for + + Returns + ------- + JSONType + The response from the API + """ + + return asyncio.run( + async_register_and_create_signed_upload_url( + api_client, + team_slug, + dataset_slug, + items_and_paths, + force_tiling, + handle_as_slices, + ignore_dicom_layout, + ) + ) + + +def confirm_upload(api_client: ClientCore, team_slug: str, upload_id: str) -> None: + """ + Confirm an upload/uploads was successful by ID + + Parameters + ---------- + api_client: ClientCore + The client to use for the request + team_slug: str + The slug of the team to confirm the upload for + upload_id: str + The ID of the upload to confirm + + Returns + ------- + None + + Raises + ------ + DarwinException + If the upload could not be confirmed + """ + + response = asyncio.run(async_confirm_upload(api_client, team_slug, upload_id)) + return response diff --git a/darwin/future/data_objects/item.py b/darwin/future/data_objects/item.py index a52f542b7..e3639be7a 100644 --- a/darwin/future/data_objects/item.py +++ b/darwin/future/data_objects/item.py @@ -2,7 +2,7 @@ from typing import Dict, List, Literal, Optional, Union from uuid import UUID -from pydantic import Field, validator +from pydantic import root_validator, validator from darwin.datatypes import NumberLike from darwin.future.data_objects.pydantic_base import DefaultDarwin @@ -14,26 +14,46 @@ def validate_no_slashes(v: UnknownType) -> str: assert isinstance(v, str), "Must be a string" assert len(v) > 0, "cannot be empty" - assert r"^[^/].*$".find(v) == -1, "cannot start with a slash" + assert "/" not in v, "cannot contain slashes" + assert " " not in v, "cannot contain spaces" return v +class ItemLayout(DefaultDarwin): + # GraphotateWeb.Schemas.DatasetsV2.Common.ItemLayoutV1 + + # Required fields + slots: List[str] + type: Literal["grid", "horizontal", "vertical", "simple"] + version: Literal[1, 2] + + # Required only in version 2 + layout_shape: Optional[List[int]] = None + + @validator("layout_shape", always=True) + def layout_validator(cls, value: UnknownType, values: Dict) -> Dict: + if not value and values.get("version") == 2: + raise ValueError("layout_shape must be specified for version 2 layouts") + + return value + + class ItemSlot(DefaultDarwin): # GraphotateWeb.Schemas.DatasetsV2.ItemRegistration.ExistingSlot # Required fields slot_name: str file_name: str + fps: Optional[ItemFrameRate] = None # Optional fields - storage_key: Optional[str] - as_frames: Optional[bool] - extract_views: Optional[bool] - fps: Optional[ItemFrameRate] = Field(None, alias="fps") - metadata: Optional[Dict[str, UnknownType]] = Field({}, alias="metadata") - tags: Optional[Union[List[str], Dict[str, str]]] = Field(None, alias="tags") - type: Literal["image", "video", "pdf", "dicom"] = Field(..., alias="type") + storage_key: Optional[str] = None + as_frames: Optional[bool] = None + extract_views: Optional[bool] = None + metadata: Optional[Dict[str, UnknownType]] = None + tags: Optional[Union[List[str], Dict[str, str]]] = None + type: Optional[Literal["image", "video", "pdf", "dicom"]] = None @validator("slot_name") def validate_slot_name(cls, v: UnknownType) -> str: @@ -41,30 +61,88 @@ def validate_slot_name(cls, v: UnknownType) -> str: assert len(v) > 0, "slot_name cannot be empty" return v - @validator("storage_key") - def validate_storage_key(cls, v: UnknownType) -> str: - return validate_no_slashes(v) + @classmethod + def validate_fps(cls, values: dict): + value = values.get("fps") + + if value is None: + values["fps"] = 0 + return values + + assert isinstance(value, (int, float, str)), "fps must be a number or 'native'" + if isinstance(value, str): + assert value == "native", "fps must be 'native' or a number greater than 0" + elif isinstance(value, (int, float)): + type = values.get("type") + if type == "image": + assert value == 0, "fps must be 0 for images" + else: + assert value >= 0, "fps must be greater than or equal to 0 for videos" + + return values + + @classmethod + def infer_type(cls, values: Dict[str, UnknownType]) -> Dict[str, UnknownType]: + file_name = values.get("file_name") + + if file_name is not None: + # TODO - Review types + if file_name.endswith((".jpg", ".jpeg", ".png", ".bmp", ".gif")): + values["type"] = "image" + elif file_name.endswith(".pdf"): + values["type"] = "pdf" + elif file_name.endswith((".dcm", ".nii", ".nii.gz")): + values["type"] = "dicom" + elif file_name.endswith((".mp4", ".avi", ".mov", ".wmv", ".mkv")): + values["type"] = "video" + + return values + + class Config: + smart_union = True + + @root_validator + def root(cls, values: Dict) -> Dict: + values = cls.infer_type(values) + values = cls.validate_fps(values) + + return values - @validator("fps") - def validate_fps(cls, v: UnknownType) -> ItemFrameRate: - assert isinstance(v, (int, float, str)), "fps must be a number or 'native'" - if isinstance(v, (int, float)): - assert v >= 0.0, "fps must be a positive number" - if isinstance(v, str): - assert v == "native", "fps must be 'native' or a number greater than 0" - return v + +class UploadItem(DefaultDarwin): + # GraphotateWeb.Schemas.DatasetsV2.ItemRegistration.NewItem + + # Required fields + name: str + slots: List[ItemSlot] = [] + + # Optional fields + description: Optional[str] = None + path: str = "/" + tags: Optional[Union[List[str], Dict[str, str]]] = [] + layout: Optional[ItemLayout] = None + + @validator("name") + def validate_name(cls, v: UnknownType) -> str: + return validate_no_slashes(v) class Item(DefaultDarwin): + # GraphotateWeb.Schemas.DatasetsV2.ItemRegistration.NewItem + + # Required fields name: str - path: str - archived: bool - dataset_id: int id: UUID - layout: Dict[str, UnknownType] - slots: List[ItemSlot] + slots: List[ItemSlot] = [] + path: str = "/" + dataset_id: int processing_status: str - priority: int + + # Optional fields + archived: Optional[bool] = False + priority: Optional[int] = None + tags: Optional[Union[List[str], Dict[str, str]]] = [] + layout: Optional[ItemLayout] = None @validator("name") def validate_name(cls, v: UnknownType) -> str: diff --git a/darwin/future/tests/core/items/fixtures.py b/darwin/future/tests/core/items/fixtures.py index 806997042..b0fcb9402 100644 --- a/darwin/future/tests/core/items/fixtures.py +++ b/darwin/future/tests/core/items/fixtures.py @@ -13,9 +13,7 @@ def base_items() -> List[Item]: name=f"test_{i}", path="test_path", dataset_id=1, - id=uuid4(), - archived=False, - layout={}, + id=UUID("00000000-0000-0000-0000-000000000000"), slots=[], processing_status="complete", priority=0, diff --git a/darwin/future/tests/core/items/test_item_data_object.py b/darwin/future/tests/core/items/test_item_data_object.py new file mode 100644 index 000000000..5bbb47772 --- /dev/null +++ b/darwin/future/tests/core/items/test_item_data_object.py @@ -0,0 +1,92 @@ +from typing import List, Literal, Optional, Tuple + +import pytest + +from darwin.future.data_objects.item import ItemSlot, validate_no_slashes +from darwin.future.data_objects.typing import UnknownType + + +def generate_extension_expectations( + extension: str, expectation: Optional[Literal["image", "video", "pdf", "dicom"]] +) -> List[Tuple[str, Optional[Literal["image", "video", "pdf", "dicom"]]]]: + """ + Generate a list of tuples of the form (file_name, expectation) where + """ + return [ + (f"file.{extension}", expectation), + (f"file.with.dots.{extension}", expectation), + (f"/file/with/slashes.{extension}", expectation), + (f"file/with/slashes.{extension}", expectation), + ] + + +expectations_list = [ + # Supported images + *generate_extension_expectations("jpg", "image"), + *generate_extension_expectations("jpeg", "image"), + *generate_extension_expectations("png", "image"), + *generate_extension_expectations("gif", "image"), + *generate_extension_expectations("bmp", "image"), + # Supported documents + *generate_extension_expectations("pdf", "pdf"), + # Supported medical imaging + *generate_extension_expectations("dcm", "dicom"), + *generate_extension_expectations("nii", "dicom"), + *generate_extension_expectations("nii.gz", "dicom"), + # Supported videos + *generate_extension_expectations("mp4", "video"), + *generate_extension_expectations("avi", "video"), + *generate_extension_expectations("mov", "video"), + *generate_extension_expectations("wmv", "video"), + *generate_extension_expectations("mkv", "video"), + # Unsupported + *generate_extension_expectations("unsupported", None), +] + + +class TestValidateNoSlashes: + @pytest.mark.parametrize( + "string", [("validname"), ("valid-name"), ("valid_name_still")] + ) + def test_happy_paths(self, string: str) -> None: + assert validate_no_slashes(string) == string + + @pytest.mark.parametrize("string", [(""), (123), ("/invalid_string")]) + def test_sad_paths(self, string: UnknownType) -> None: + with pytest.raises(AssertionError): + validate_no_slashes(string) + + +class TestSlotNameValidator: + @pytest.mark.parametrize( + "string", [("validname"), ("valid/name"), ("valid/name/still")] + ) + def test_happy_paths(self, string: str) -> None: + assert ItemSlot.validate_slot_name(string) == string + + @pytest.mark.parametrize("string", [(""), (123)]) + def test_sad_paths(self, string: UnknownType) -> None: + with pytest.raises(AssertionError): + ItemSlot.validate_slot_name(string) + + +class TestFpsValidator: + def test_sets_value_if_absent(self) -> None: + assert ItemSlot.validate_fps({}) == {"fps": 0} + + @pytest.mark.parametrize("fps", [(0), (1), (1.0), ("native")]) + def test_happy_paths(self, fps: UnknownType) -> None: + assert ItemSlot.validate_fps({"fps": fps}) == {"fps": fps} + + @pytest.mark.parametrize("fps", [(-1), ("invalid")]) + def test_sad_paths(self, fps: UnknownType) -> None: + with pytest.raises(AssertionError): + ItemSlot.validate_fps({"fps": fps}) + + +class TestRootValidator: + @pytest.mark.parametrize("file_name, expectation", expectations_list) + def test_happy_paths(self, file_name: str, expectation: str) -> None: + assert ( + ItemSlot.infer_type({"file_name": file_name}).get("type") == expectation + ), f"Failed for {file_name}, got {expectation}" diff --git a/darwin/future/tests/core/items/test_upload_items.py b/darwin/future/tests/core/items/test_upload_items.py new file mode 100644 index 000000000..03794154d --- /dev/null +++ b/darwin/future/tests/core/items/test_upload_items.py @@ -0,0 +1,818 @@ +import asyncio +from pathlib import Path +from typing import Dict, Generator, List, Tuple +from unittest.mock import MagicMock, Mock, patch + +import orjson as json +import pytest +import responses + +import darwin.future.core.items.uploads as uploads +from darwin.future.core.client import ClientCore, DarwinConfig +from darwin.future.data_objects.item import ItemLayout, ItemSlot, UploadItem +from darwin.future.exceptions import DarwinException +from darwin.future.tests.core.fixtures import * # noqa: F401,F403 + +from .fixtures import * # noqa: F401,F403 + + +class TestBuildSlots: + BUILD_SLOT_RETURN_TYPE = List[Dict] + + items_and_expectations: List[Tuple[UploadItem, BUILD_SLOT_RETURN_TYPE]] = [] + + # Test empty slots + items_and_expectations.append( + ( + UploadItem(name="name_with_no_slots", slots=[]), + [], + ) + ) + + # Test Simple slot with no non-required fields + items_and_expectations.append( + ( + UploadItem( + name="name_with_simple_slot", + slots=[ + ItemSlot( + slot_name="slot_name_simple", + file_name="file_name", + storage_key="storage_key", + ) + ], + ), + [ + { + "slot_name": "slot_name_simple", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + } + ], + ) + ) + + # Test with multiple slots + items_and_expectations.append( + ( + UploadItem( + name="name_with_multiple_slots", + slots=[ + ItemSlot( + slot_name="slot_name1", + file_name="file_name1", + storage_key="storage_key1", + ), + ItemSlot( + slot_name="slot_name2", + file_name="file_name2", + storage_key="storage_key2", + ), + ], + ), + [ + { + "slot_name": "slot_name1", + "file_name": "file_name1", + "storage_key": "storage_key1", + "fps": 0, + }, + { + "slot_name": "slot_name2", + "file_name": "file_name2", + "storage_key": "storage_key2", + "fps": 0, + }, + ], + ) + ) + + # Test with `as_frames` optional field + items_and_expectations.append( + ( + UploadItem( + name="name_testing_as_frames", + slots=[ + ItemSlot( + slot_name="slot_name1", + file_name="file_name", + storage_key="storage_key", + as_frames=True, + ), + ItemSlot( + slot_name="slot_name2", + file_name="file_name", + storage_key="storage_key", + as_frames=False, + ), + ItemSlot( + slot_name="slot_name3", + file_name="file_name", + storage_key="storage_key", + ), + ], + ), + [ + { + "slot_name": "slot_name1", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + "as_frames": True, + }, + { + "slot_name": "slot_name2", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + "as_frames": False, + }, + { + "slot_name": "slot_name3", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + }, + ], + ) + ) + + # Test with `extract_views` optional field + items_and_expectations.append( + ( + UploadItem( + name="name_testing_extract_views", + slots=[ + ItemSlot( + slot_name="slot_name1", + file_name="file_name", + storage_key="storage_key", + extract_views=True, + ), + ItemSlot( + slot_name="slot_name2", + file_name="file_name", + storage_key="storage_key", + extract_views=False, + ), + ItemSlot( + slot_name="slot_name3", + file_name="file_name", + storage_key="storage_key", + ), + ], + ), + [ + { + "slot_name": "slot_name1", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + "extract_views": True, + }, + { + "slot_name": "slot_name2", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + "extract_views": False, + }, + { + "slot_name": "slot_name3", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + }, + ], + ) + ) + + # Test with `fps` semi-optional field - field defaults to 0 if not provided + items_and_expectations.append( + ( + UploadItem( + name="name_with_simple_slot", + slots=[ + ItemSlot( + slot_name="slot_name25", + file_name="file_name", + storage_key="storage_key", + fps=25, # Testing int + ), + ItemSlot( + slot_name="slot_name29.997", + file_name="file_name", + storage_key="storage_key", + fps=29.997, # Testing float + ), + ItemSlot( + slot_name="slot_namenative", + file_name="file_name", + storage_key="storage_key", + fps="native", # Testing literal + ), + ItemSlot( + slot_name="slot_name", + file_name="file_name", + storage_key="storage_key", + ), + ], + ), + [ + { + "slot_name": "slot_name25", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 25, + }, + { + "slot_name": "slot_name29.997", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 29.997, + }, + { + "slot_name": "slot_namenative", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": "native", + }, + { + "slot_name": "slot_name", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + }, + ], + ) + ) + + # Test with `metadata` optional field + items_and_expectations.append( + ( + UploadItem( + name="name_with_simple_slot", + slots=[ + ItemSlot( + slot_name="slot_name", + file_name="file_name", + storage_key="storage_key", + metadata={"key": "value"}, + ), + ItemSlot( + slot_name="slot_name", + file_name="file_name", + storage_key="storage_key", + metadata=None, + ), + ItemSlot( + slot_name="slot_name", + file_name="file_name", + storage_key="storage_key", + ), + ], + ), + [ + { + "slot_name": "slot_name", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + "metadata": {"key": "value"}, + }, + { + "slot_name": "slot_name", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + }, + { + "slot_name": "slot_name", + "file_name": "file_name", + "storage_key": "storage_key", + "fps": 0, + }, + ], + ) + ) + + # Test with `tags` optional field + items_and_expectations.append( + ( + UploadItem( + name="name_testing_tags", + slots=[ + ItemSlot( + slot_name="slot_name_with_string_list", + file_name="file_name", + storage_key="storage_key", + tags=["tag1", "tag2"], + ), + ItemSlot( + slot_name="slot_name_with_kv_pairs", + file_name="file_name", + storage_key="storage_key", + tags={"key": "value"}, + ), + ], + ), + [ + { + "slot_name": "slot_name_with_string_list", + "file_name": "file_name", + "storage_key": "storage_key", + "tags": ["tag1", "tag2"], + "fps": 0, + }, + { + "slot_name": "slot_name_with_kv_pairs", + "file_name": "file_name", + "storage_key": "storage_key", + "tags": {"key": "value"}, + "fps": 0, + }, + ], + ) + ) + + @pytest.mark.parametrize("item,expected", items_and_expectations) + def test_build_slots(self, item: UploadItem, expected: List[Dict]) -> None: + result = asyncio.run(uploads._build_slots(item)) + assert result == expected + + +class TestBuildLayout: + @pytest.mark.parametrize( + "item, expected", + [ + ( + UploadItem( + name="test_item", + layout=ItemLayout(version=1, type="grid", slots=["slot1", "slot2"]), + ), + { + "slots": ["slot1", "slot2"], + "type": "grid", + "version": 1, + }, + ), + ( + UploadItem( + name="test_item", + layout=ItemLayout( + version=2, + type="grid", + slots=["slot1", "slot2"], + layout_shape=[3, 4], + ), + ), + { + "slots": ["slot1", "slot2"], + "type": "grid", + "version": 2, + "layout_shape": [3, 4], + }, + ), + ], + ) + def test_build_layout(self, item: UploadItem, expected: Dict) -> None: + assert asyncio.run(uploads._build_layout(item)) == expected + + +class TestBuildPayloadItems: + @pytest.mark.parametrize( + "items_and_paths, expected", + [ + ( + [ + ( + UploadItem( + name="test_item", + slots=[ + ItemSlot( + slot_name="slot_name_with_string_list", + file_name="file_name", + storage_key="storage_key", + tags=["tag1", "tag2"], + ), + ItemSlot( + slot_name="slot_name_with_kv_pairs", + file_name="file_name", + storage_key="storage_key", + tags={"key": "value"}, + ), + ], + ), + Path("test_path"), + ) + ], + [ + { + "name": "test_item", + "path": "test_path", + "slots": [ + { + "slot_name": "slot_name_with_string_list", + "file_name": "file_name", + "storage_key": "storage_key", + "tags": ["tag1", "tag2"], + "fps": 0, + }, + { + "slot_name": "slot_name_with_kv_pairs", + "file_name": "file_name", + "storage_key": "storage_key", + "tags": {"key": "value"}, + "fps": 0, + }, + ], + } + ], + ) + ], + ) + def test_build_payload_items( + self, items_and_paths: List[Tuple[UploadItem, Path]], expected: List[Dict] + ) -> None: + result = asyncio.run(uploads._build_payload_items(items_and_paths)) + + assert result == expected + + +class SetupTests: + @pytest.fixture + def default_url(self, base_client: ClientCore) -> str: + return f"{base_client.config.base_url}api/v2/teams/my-team/items" + + +class TestRegisterUpload(SetupTests): + team_slug = "my-team" + dataset_slug = "my-dataset" + + @pytest.fixture + def items_and_paths(self) -> List[Tuple[UploadItem, Path]]: + return [ + ( + UploadItem( + name="test_item", + slots=[ + ItemSlot( + slot_name="slot_name_with_string_list", + file_name="file_name", + storage_key="storage_key", + tags=["tag1", "tag2"], + ), + ItemSlot( + slot_name="slot_name_with_kv_pairs", + file_name="file_name", + storage_key="storage_key", + tags={"key": "value"}, + ), + ], + ), + Path("test_path"), + ) + ] + + @responses.activate + @patch.object(uploads, "_build_payload_items") + def test_async_register_upload_happy_path( + self, + mock_build_payload_items: MagicMock, + base_client: ClientCore, + default_url: str, + items_and_paths: List[Tuple[UploadItem, Path]], + ) -> None: + items = [ + { + "name": "test_item", + "path": "test_path", + "tags": [], + "slots": [ + { + "slot_name": "slot_name_with_string_list", + "file_name": "file_name", + "storage_key": "storage_key", + "tags": ["tag1", "tag2"], + "fps": 0, + "type": "image", + }, + { + "slot_name": "slot_name_with_kv_pairs", + "file_name": "file_name", + "storage_key": "storage_key", + "tags": {"key": "value"}, + "fps": 0, + "type": "image", + }, + ], + } + ] + mock_build_payload_items.return_value = items + + responses.add( + responses.POST, f"{default_url}/register_upload", json={"status": "success"} + ) + asyncio.run( + uploads.async_register_upload( + api_client=base_client, + team_slug=self.team_slug, + dataset_slug=self.dataset_slug, + items_and_paths=items_and_paths, + force_tiling=True, + handle_as_slices=True, + ignore_dicom_layout=True, + ) + ) + # noqa: E501 + assert responses.calls[0].request.url == f"{default_url}/register_upload" # type: ignore + # noqa: E501 + received_call = json.loads(responses.calls[0].request.body.decode("utf-8")) # type: ignore + assert received_call == { + "dataset_slug": "my-dataset", + "items": items, + "options": { + "force_tiling": True, + "handle_as_slices": True, + "ignore_dicom_layout": True, + }, + }, "The request body should be empty" + + @patch.object(uploads, "_build_payload_items") + def test_async_register_upload_raises( + self, mock_build_payload_items: MagicMock, base_client: ClientCore + ) -> None: + with pytest.raises(DarwinException) as exc: + mock_build_payload_items.side_effect = DarwinException("Error1") + + asyncio.run( + uploads.async_register_upload( + api_client=base_client, + team_slug=self.team_slug, + dataset_slug=self.dataset_slug, + items_and_paths=[], + force_tiling=True, + handle_as_slices=True, + ignore_dicom_layout=True, + ) + ) + + assert str(exc) == "Error1", "Failed to raise on failed payload build" + + with pytest.raises(DarwinException) as exc: + base_client.post = MagicMock() # type: ignore + base_client.post.side_effect = DarwinException("Error2") + + asyncio.run( + uploads.async_register_upload( + api_client=base_client, + team_slug=self.team_slug, + dataset_slug=self.dataset_slug, + items_and_paths=[], + force_tiling=True, + handle_as_slices=True, + ignore_dicom_layout=True, + ) + ) + + assert str(exc) == "Error2", "Failed to raise on failed API call" + + +class TestCreateSignedUploadUrl(SetupTests): + def test_async_create_signed_upload_url( + self, default_url: str, base_config: DarwinConfig + ) -> None: + with responses.RequestsMock() as rsps: + # Mock the API response + expected_response = {"upload_url": "https://signed.url"} + rsps.add( + rsps.GET, + f"{default_url}/uploads/1/sign", + json=expected_response, + ) + + # Call the function with mocked arguments + api_client = ClientCore(base_config) + actual_response = asyncio.run( + uploads.async_create_signed_upload_url(api_client, "my-team", "1") + ) + + # Check that the response matches the expected response + if not actual_response: + pytest.fail("Response was None") + + assert actual_response == expected_response["upload_url"] + + def test_async_create_signed_upload_url_raises( + self, base_client: ClientCore + ) -> None: + base_client.post = MagicMock() # type: ignore + base_client.post.side_effect = DarwinException("Error") + + with pytest.raises(DarwinException): + asyncio.run( + uploads.async_create_signed_upload_url(base_client, "1", "my-team") + ) + + +class TestRegisterAndCreateSignedUploadUrl: + @pytest.fixture + def mock_async_register_upload(self) -> Generator: + with patch.object(uploads, "async_register_upload") as mock: + yield mock + + @pytest.fixture + def mock_async_create_signed_upload_url(self) -> Generator: + with patch.object(uploads, "async_create_signed_upload_url") as mock: + yield mock + + def test_async_register_and_create_signed_upload_url( + self, + mock_async_register_upload: MagicMock, + mock_async_create_signed_upload_url: MagicMock, + ) -> None: + # Set up mock responses + mock_async_register_upload.return_value = { + "blocked_items": [], + "items": [{"id": "123", "slots": [{"upload_id": "321"}]}], + } + mock_signed_url_response = "https://signed.url" + mock_async_create_signed_upload_url.return_value = mock_signed_url_response + + # Set up mock API client + mock_api_client = MagicMock() + + # Call the function with mocked arguments + actual_response = asyncio.run( + uploads.async_register_and_create_signed_upload_url( + mock_api_client, + "my-team", + "my-dataset", + [(Mock(), Mock())], + False, + False, + False, + ) + ) + + # Check that the function called the correct sub-functions with the correct arguments + mock_async_register_upload.assert_called_once() + mock_async_create_signed_upload_url.assert_called_once_with( + mock_api_client, + "my-team", + "321", + ) + + # Check that the response matches the expected response + assert actual_response == [("https://signed.url", "321")] + + def test_async_register_and_create_signed_upload_url_raises( + self, + mock_async_register_upload: MagicMock, + mock_async_create_signed_upload_url: MagicMock, + ) -> None: + # Set up mock responses + mock_async_register_upload.return_value = {"id": "123", "errors": ["error"]} + mock_signed_url_response = {"upload_url": "https://signed.url"} + mock_async_create_signed_upload_url.return_value = mock_signed_url_response + + # Set up mock API client + mock_api_client = MagicMock() + + # Check that the response matches the expected response + with pytest.raises(DarwinException): + asyncio.run( + uploads.async_register_and_create_signed_upload_url( + mock_api_client, + "my-team", + "my-dataset", + [(Mock(), Mock())], + False, + False, + False, + ) + ) + + +class TestConfirmUpload(SetupTests): + @responses.activate + def test_async_confirm_upload( + self, base_client: ClientCore, default_url: str + ) -> None: + # Call the function with mocked arguments + responses.add( + "POST", + f"{default_url}/uploads/123/confirm", + status=200, + json={}, + ) + + actual_response = asyncio.run( + uploads.async_confirm_upload( + base_client, + "my-team", + "123", + ) + ) + + assert actual_response is None # Function doesn't return anything on success + + @responses.activate + def test_async_confirm_upload_raises_on_returned_errors( + self, base_client: ClientCore, default_url: str + ) -> None: + # Call the function with mocked arguments + responses.add( + "POST", + f"{default_url}/uploads/123/confirm", + status=200, # API will not normally return this on success, but we're not test API + json={"errors": ["error1", "error2"]}, + ) + + with pytest.raises(DarwinException) as exc: + asyncio.run( + uploads.async_confirm_upload( + base_client, + "my-team", + "123", + ) + ) + + assert ( + "Failed to confirm upload in darwin.future.core.items.uploads: ['error1', 'error2']" + in str(exc) + ) + + def test_async_confirm_upload_raises(self, base_client: ClientCore) -> None: + base_client.post = MagicMock() # type: ignore + base_client.post.side_effect = DarwinException("Error") + + with pytest.raises(DarwinException): + asyncio.run(uploads.async_confirm_upload(base_client, "team", "123")) + + +class TestSynchronousMethods: + @pytest.fixture + def mock_async_register_upload(self) -> Generator: + with patch.object(uploads, "async_register_upload") as mock: + yield mock + + @pytest.fixture + def mock_async_create_signed_upload_url(self) -> Generator: + with patch.object(uploads, "async_create_signed_upload_url") as mock: + yield mock + + @pytest.fixture + def mock_async_register_and_create_signed_upload_url(self) -> Generator: + with patch.object( + uploads, "async_register_and_create_signed_upload_url" + ) as mock: + yield mock + + @pytest.fixture + def mock_async_confirm_upload(self) -> Generator: + with patch.object(uploads, "async_confirm_upload") as mock: + yield mock + + def test_register_upload( + self, + mock_async_register_upload: MagicMock, + base_client: ClientCore, + ) -> None: + uploads.register_upload(base_client, "team", "dataset", [(Mock(), Mock())]) + + mock_async_register_upload.assert_called_once() + + def test_create_signed_upload_url( + self, + mock_async_create_signed_upload_url: MagicMock, + base_client: ClientCore, + ) -> None: + uploads.create_signed_upload_url(base_client, "team", "123") + + mock_async_create_signed_upload_url.assert_called_once() + + def test_register_and_create_signed_upload_url( + self, + mock_async_register_and_create_signed_upload_url: MagicMock, + base_client: ClientCore, + ) -> None: + uploads.register_and_create_signed_upload_url( + base_client, "team", "dataset", [(Mock(), Mock())] + ) + + mock_async_register_and_create_signed_upload_url.assert_called_once() + + def test_confirm_upload( + self, + mock_async_confirm_upload: MagicMock, + base_client: ClientCore, + ) -> None: + uploads.confirm_upload(base_client, "team", "123") + + mock_async_confirm_upload.assert_called_once() + + +if __name__ == "__main__": + pytest.main(["-s", "-v", __file__]) diff --git a/deploy/_filter_files.py b/deploy/_filter_files.py index c1777b0df..ae183fa44 100755 --- a/deploy/_filter_files.py +++ b/deploy/_filter_files.py @@ -11,7 +11,11 @@ def main(argv: List[str]) -> None: if file_extension.startswith("."): file_extension = file_extension[1:] - files_out = [file for file in files_in if file.endswith(f".{file_extension}") and 'future' in file] + files_out = [ + file + for file in files_in + if file.endswith(f".{file_extension}") and "future" in file + ] print(" ".join(files_out)) diff --git a/pyproject.toml b/pyproject.toml index e9e2f36e0..41deba54e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ warn_untyped_fields = true [tool.ruff] select = ["E", "F", "C"] -ignore = ["E203", "E402"] +ignore = ["E203", "E402", "E501"] line-length = 88 [tool.ruff.per-file-ignores]