From a8f1a90dba1b0d071e1c7c8144682b03c2841faa Mon Sep 17 00:00:00 2001 From: John Wilkie <124276291+JBWilkie@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:12:28 +0100 Subject: [PATCH] [DAR-2860][External] Import of item-level properties (#936) * Automatic creation of item-level properties & values + dataset assignment from metadata & imported files * Test coverage for automatic property creation & property value creation * Extended overwrite warning to account for item-level properties * Introducing compatibility with annotation-level properties (WIP) * Making item-level properties import fit with annotation-level * Import of item-level property values * E2E tests for import of item-level properties * Light refactoring of * Style improvements --- darwin/backend_v2.py | 15 + darwin/datatypes.py | 6 + darwin/future/data_objects/properties.py | 35 +- darwin/future/tests/core/fixtures.py | 2 +- .../tests/core/properties/test_create.py | 4 +- .../future/tests/core/properties/test_get.py | 6 +- .../tests/core/properties/test_update.py | 2 +- darwin/future/tests/data/.v7/metadata.json | 6 +- .../metadata_no_item_level_properties.json | 102 ++ .../metadata_with_item_level_properties.json | 138 ++ darwin/importer/importer.py | 589 +++++++- darwin/path_utils.py | 8 +- darwin/utils/utils.py | 3 + e2e_tests/cli/test_import.py | 57 +- e2e_tests/conftest.py | 16 +- .../image_1.json | 595 ++++++++ .../image_2.json | 487 +++++++ .../image_3.json | 531 +++++++ .../image_4.json | 583 ++++++++ .../image_5.json | 455 ++++++ .../image_6.json | 467 ++++++ .../image_7.json | 507 +++++++ .../image_8.json | 483 +++++++ .../.v7/metadata.json | 94 ++ .../image_1.json | 595 ++++++++ .../image_2.json | 487 +++++++ .../image_3.json | 531 +++++++ .../image_4.json | 583 ++++++++ .../image_5.json | 455 ++++++ .../image_6.json | 467 ++++++ .../image_7.json | 507 +++++++ .../image_8.json | 483 +++++++ e2e_tests/objects.py | 8 + e2e_tests/setup_tests.py | 144 +- tests/darwin/client_test.py | 16 +- tests/darwin/data/metadata.json | 6 +- .../data/metadata_nested_properties.json | 1 + tests/darwin/importer/importer_test.py | 1253 ++++++++++++++++- 38 files changed, 10565 insertions(+), 162 deletions(-) create mode 100644 darwin/future/tests/data/.v7/metadata_no_item_level_properties.json create mode 100644 darwin/future/tests/data/.v7/metadata_with_item_level_properties.json create mode 100644 e2e_tests/data/import/image_annotations_with_item_level_properties/image_1.json create mode 100644 e2e_tests/data/import/image_annotations_with_item_level_properties/image_2.json create mode 100644 e2e_tests/data/import/image_annotations_with_item_level_properties/image_3.json create mode 100644 e2e_tests/data/import/image_annotations_with_item_level_properties/image_4.json create mode 100644 e2e_tests/data/import/image_annotations_with_item_level_properties/image_5.json create mode 100644 e2e_tests/data/import/image_annotations_with_item_level_properties/image_6.json create mode 100644 e2e_tests/data/import/image_annotations_with_item_level_properties/image_7.json create mode 100644 e2e_tests/data/import/image_annotations_with_item_level_properties/image_8.json create mode 100644 e2e_tests/data/import/image_new_annotations_with_item_level_properties/.v7/metadata.json create mode 100644 e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_1.json create mode 100644 e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_2.json create mode 100644 e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_3.json create mode 100644 e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_4.json create mode 100644 e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_5.json create mode 100644 e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_6.json create mode 100644 e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_7.json create mode 100644 e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_8.json diff --git a/darwin/backend_v2.py b/darwin/backend_v2.py index 615aff00f..69702efcc 100644 --- a/darwin/backend_v2.py +++ b/darwin/backend_v2.py @@ -293,3 +293,18 @@ def _get_remote_annotations( The team slug. """ return self._client._get(f"v2/teams/{team_slug}/items/{item_id}/annotations") + + def _get_properties_state_for_item( + self, item_id: str, team_slug: str + ) -> Dict[str, List[Dict[str, str]]]: + """ + Returns the state of property values for the specified item. + + Parameters + ---------- + item_id: str + The ID of the item to get properties for. + team_slug: str + The slug of the team to get. + """ + return self._client._get(f"/v2/teams/{team_slug}/items/{item_id}/properties") diff --git a/darwin/datatypes.py b/darwin/datatypes.py index 2f75ef8d0..189a6ce94 100644 --- a/darwin/datatypes.py +++ b/darwin/datatypes.py @@ -438,6 +438,9 @@ class Property: # Property options property_values: list[dict[str, Any]] + # Granularity of the property + granularity: PropertyGranularity + # Description of the property description: Optional[str] = None @@ -547,6 +550,9 @@ class AnnotationFile: #: List of ``VideoAnnotation``\s or ``Annotation``\s. annotations: Sequence[Union[Annotation, VideoAnnotation]] + # Item-level properties + item_properties: Optional[list[dict[str, Any]]] = None + # Deprecated #: Whether the annotations in the ``annotations`` attribute are ``VideoAnnotation`` or not. is_video: bool = False diff --git a/darwin/future/data_objects/properties.py b/darwin/future/data_objects/properties.py index f60b917f3..d9e56f390 100644 --- a/darwin/future/data_objects/properties.py +++ b/darwin/future/data_objects/properties.py @@ -2,9 +2,9 @@ import json import os +from enum import Enum from pathlib import Path from typing import List, Literal, Optional, Tuple, Union -from enum import Enum from pydantic import field_validator @@ -81,31 +81,36 @@ class FullProperty(DefaultDarwin): team_id: Optional[int] = None annotation_class_id: Optional[int] = None property_values: Optional[List[PropertyValue]] = None + granularity: PropertyGranularity + dataset_ids: Optional[List[int]] = None options: Optional[List[PropertyValue]] = None granularity: PropertyGranularity = PropertyGranularity("section") def to_create_endpoint( self, ) -> dict: - if self.annotation_class_id is None: - raise ValueError("annotation_class_id must be set") - return self.model_dump( - include={ - "name": True, - "type": True, - "required": True, - "annotation_class_id": True, - "property_values": {"__all__": {"value", "color"}}, - "description": True, - "granularity": True, - } - ) + include_fields = { + "name": True, + "type": True, + "required": True, + "property_values": {"__all__": {"value", "color", "type"}}, + "description": True, + "granularity": True, + } + if self.granularity != PropertyGranularity.item: + if self.annotation_class_id is None: + raise ValueError("annotation_class_id must be set") + include_fields["annotation_class_id"] = True + if self.dataset_ids is not None: + include_fields["dataset_ids"] = True + return self.model_dump(mode="json", include=include_fields) def to_update_endpoint(self) -> Tuple[str, dict]: if self.id is None: raise ValueError("id must be set") + updated_base = self.to_create_endpoint() - del updated_base["annotation_class_id"] # Can't update this field + updated_base.pop("annotation_class_id", None) # Can't update this field del updated_base["granularity"] # Can't update this field return self.id, updated_base diff --git a/darwin/future/tests/core/fixtures.py b/darwin/future/tests/core/fixtures.py index b824232b2..51a46f860 100644 --- a/darwin/future/tests/core/fixtures.py +++ b/darwin/future/tests/core/fixtures.py @@ -41,8 +41,8 @@ def base_property_object(base_property_value: PropertyValue) -> FullProperty: team_id=0, annotation_class_id=0, property_values=[base_property_value], + granularity=PropertyGranularity.section, options=[base_property_value], - granularity=PropertyGranularity("section"), ) diff --git a/darwin/future/tests/core/properties/test_create.py b/darwin/future/tests/core/properties/test_create.py index dcab2d701..20177ae76 100644 --- a/darwin/future/tests/core/properties/test_create.py +++ b/darwin/future/tests/core/properties/test_create.py @@ -14,7 +14,7 @@ def test_create_property( responses.add( responses.POST, f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties", - json=base_property_object.model_dump(), + json=base_property_object.model_dump(mode="json"), status=200, ) # Call the function being tested @@ -38,7 +38,7 @@ def test_create_property_from_json( responses.add( responses.POST, f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties", - json=base_property_object.model_dump(), + json=base_property_object.model_dump(mode="json"), status=200, ) # Call the function being tested diff --git a/darwin/future/tests/core/properties/test_get.py b/darwin/future/tests/core/properties/test_get.py index c59f8e55f..8c34ce04e 100644 --- a/darwin/future/tests/core/properties/test_get.py +++ b/darwin/future/tests/core/properties/test_get.py @@ -19,7 +19,7 @@ def test_get_team_properties( # Mocking the response using responses library base_property_object.options = None base_property_object.property_values = None - response_data = {"properties": [base_property_object.model_dump()]} + response_data = {"properties": [base_property_object.model_dump(mode="json")]} responses.add( responses.GET, f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties", @@ -41,7 +41,7 @@ def test_get_team_full_properties( base_client: ClientCore, base_property_object: FullProperty ) -> None: # Mocking the response using responses library - response_data = {"properties": [base_property_object.model_dump()]} + response_data = {"properties": [base_property_object.model_dump(mode="json")]} responses.add( responses.GET, f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties", @@ -70,7 +70,7 @@ def test_get_property_by_id( responses.add( responses.GET, f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties/{property_id}", - json=base_property_object.model_dump(), + json=base_property_object.model_dump(mode="json"), status=200, ) diff --git a/darwin/future/tests/core/properties/test_update.py b/darwin/future/tests/core/properties/test_update.py index 05cceed6e..68413864b 100644 --- a/darwin/future/tests/core/properties/test_update.py +++ b/darwin/future/tests/core/properties/test_update.py @@ -14,7 +14,7 @@ def test_update_property( responses.add( responses.PUT, f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties/{base_property_object.id}", - json=base_property_object.model_dump(), + json=base_property_object.model_dump(mode="json"), status=200, ) # Call the function being tested diff --git a/darwin/future/tests/data/.v7/metadata.json b/darwin/future/tests/data/.v7/metadata.json index 561711c61..63d98bade 100644 --- a/darwin/future/tests/data/.v7/metadata.json +++ b/darwin/future/tests/data/.v7/metadata.json @@ -26,7 +26,7 @@ { "name": "Property 1", "type": "multi_select", - "options": [ + "property_values": [ { "type": "string", "value": "first value", @@ -38,12 +38,13 @@ "color": "rgba(0,0,0,1.0)" } ], + "granularity": "section", "required": false }, { "name": "Property 2", "type": "single_select", - "options": [ + "property_values": [ { "type": "string", "value": "first value", @@ -55,6 +56,7 @@ "color": "rgba(255,255,255,1.0)" } ], + "granularity": "section", "required": false } ] diff --git a/darwin/future/tests/data/.v7/metadata_no_item_level_properties.json b/darwin/future/tests/data/.v7/metadata_no_item_level_properties.json new file mode 100644 index 000000000..ee015784e --- /dev/null +++ b/darwin/future/tests/data/.v7/metadata_no_item_level_properties.json @@ -0,0 +1,102 @@ +{ + "classes": [ + { + "name": "Test bounding box", + "type": "bounding_box", + "description": null, + "color": "rgba(255,145,82,1)", + "sub_types": [ + "inference" + ], + "properties": [ + { + "name": "Property 1", + "type": "multi_select", + "property_values": [ + { + "type": "string", + "value": "first value", + "color": "rgba(255,92,0,1.0)" + }, + { + "type": "string", + "value": "second value", + "color": "rgba(0,0,0,1.0)" + } + ], + "granularity": "section", + "required": false + }, + { + "name": "Property 2", + "type": "single_select", + "property_values": [ + { + "type": "string", + "value": "first value", + "color": "rgba(0,194,255,1.0)" + }, + { + "type": "string", + "value": "second value", + "color": "rgba(255,255,255,1.0)" + } + ], + "granularity": "section", + "required": false + } + ] + }, + { + "name": "Test Polygon", + "type": "polygon", + "description": null, + "color": "rgba(219,255,0,1.0)", + "sub_types": [ + "directional_vector", + "attributes", + "text", + "instance_id", + "inference" + ], + "properties": [ + { + "name": "Property 1", + "type": "multi_select", + "property_values": [ + { + "type": "string", + "value": "first value", + "color": "rgba(255,92,0,1.0)" + }, + { + "type": "string", + "value": "second value", + "color": "rgba(0,0,0,1.0)" + } + ], + "granularity": "section", + "required": false + }, + { + "name": "Property 2", + "type": "single_select", + "property_values": [ + { + "type": "string", + "value": "first value", + "color": "rgba(0,194,255,1.0)" + }, + { + "type": "string", + "value": "second value", + "color": "rgba(255,255,255,1.0)" + } + ], + "granularity": "section", + "required": false + } + ] + } + ] +} \ No newline at end of file diff --git a/darwin/future/tests/data/.v7/metadata_with_item_level_properties.json b/darwin/future/tests/data/.v7/metadata_with_item_level_properties.json new file mode 100644 index 000000000..bd5945629 --- /dev/null +++ b/darwin/future/tests/data/.v7/metadata_with_item_level_properties.json @@ -0,0 +1,138 @@ +{ + "classes": [ + { + "name": "Test bounding box", + "type": "bounding_box", + "description": null, + "color": "rgba(255,145,82,1)", + "sub_types": [ + "inference" + ], + "properties": [ + { + "name": "Property 1", + "type": "multi_select", + "property_values": [ + { + "type": "string", + "value": "first value", + "color": "rgba(255,92,0,1.0)" + }, + { + "type": "string", + "value": "second value", + "color": "rgba(0,0,0,1.0)" + } + ], + "granularity": "section", + "required": false + }, + { + "name": "Property 2", + "type": "single_select", + "property_values": [ + { + "type": "string", + "value": "first value", + "color": "rgba(0,194,255,1.0)" + }, + { + "type": "string", + "value": "second value", + "color": "rgba(255,255,255,1.0)" + } + ], + "granularity": "section", + "required": false + } + ] + }, + { + "name": "Test Polygon", + "type": "polygon", + "description": null, + "color": "rgba(219,255,0,1.0)", + "sub_types": [ + "directional_vector", + "attributes", + "text", + "instance_id", + "inference" + ], + "properties": [ + { + "name": "Property 1", + "type": "multi_select", + "property_values": [ + { + "type": "string", + "value": "first value", + "color": "rgba(255,92,0,1.0)" + }, + { + "type": "string", + "value": "second value", + "color": "rgba(0,0,0,1.0)" + } + ], + "granularity": "section", + "required": false + }, + { + "name": "Property 2", + "type": "single_select", + "property_values": [ + { + "type": "string", + "value": "first value", + "color": "rgba(0,194,255,1.0)" + }, + { + "type": "string", + "value": "second value", + "color": "rgba(255,255,255,1.0)" + } + ], + "granularity": "section", + "required": false + } + ] + } + ], + "properties": [ + { + "name": "prop1", + "type": "single_select", + "description": "", + "required": true, + "property_values": [ + { + "value": "1", + "color": "rgba(0,0,0,1.0)" + }, + { + "value": "2", + "color": "rgba(255,255,255,1.0)" + } + ], + "granularity": "item" + }, + { + "name": "prop2", + "type": "multi_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(255,92,0,1.0)" + }, + { + "value": "2", + "color": "rgba(0,0,0,1.0)" + } + ], + "granularity": "item" + } + ] +} \ No newline at end of file diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index 1202fe752..21de45e2e 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -20,7 +20,12 @@ Union, ) -from darwin.datatypes import AnnotationFile, Property, parse_property_classes +from darwin.datatypes import ( + AnnotationFile, + Property, + parse_property_classes, + PropertyClass, +) from darwin.future.data_objects.properties import ( FullProperty, PropertyType, @@ -30,6 +35,7 @@ ) from darwin.item import DatasetItem from darwin.path_utils import is_properties_enabled, parse_metadata +from darwin.utils.utils import _parse_annotators Unknown = Any # type: ignore @@ -278,7 +284,21 @@ def _resolve_annotation_classes( return local_classes_not_in_dataset, local_classes_not_in_team -def _get_team_properties_annotation_lookup(client, team_slug): +def _get_team_properties_annotation_lookup( + client: "Client", team_slug: str +) -> Tuple[Dict[Tuple[str, Optional[int]], FullProperty], Dict[str, FullProperty]]: + """ + Returns two lookup dictionaries for team properties: + - team_properties_annotation_lookup: (property-name, annotation_class_id): FullProperty object + - team_item_properties_lookup: property-name: FullProperty object + + Args: + client (Client): Darwin Client object + team_slug (str): Team slug + + Returns: + Tuple[Dict[Tuple[str, Optional[int]], FullProperty], Dict[str, FullProperty]]: Tuple of two dictionaries + """ # get team properties -> List[FullProperty] team_properties = client.get_team_properties(team_slug) @@ -286,10 +306,21 @@ def _get_team_properties_annotation_lookup(client, team_slug): team_properties_annotation_lookup: Dict[Tuple[str, Optional[int]], FullProperty] = ( {} ) + + # property-name: FullProperty object + team_item_properties_lookup: Dict[str, FullProperty] = {} for prop in team_properties: - team_properties_annotation_lookup[(prop.name, prop.annotation_class_id)] = prop + if ( + prop.granularity.value == "section" + or prop.granularity.value == "annotation" + ): + team_properties_annotation_lookup[(prop.name, prop.annotation_class_id)] = ( + prop + ) + elif prop.granularity.value == "item": + team_item_properties_lookup[prop.name] = prop - return team_properties_annotation_lookup + return team_properties_annotation_lookup, team_item_properties_lookup def _update_payload_with_properties( @@ -298,6 +329,10 @@ def _update_payload_with_properties( ) -> None: """ Updates the annotations with the properties that were created/updated during the import. + + Args: + annotations (List[dt.Annotation]): List of annotations + annotation_id_property_map: Dict[str, Dict[str, Dict[str, Set[str]]]]: Dict of annotation.id to frame_index -> property id -> property val ids """ if not annotation_id_property_map: return @@ -318,12 +353,113 @@ def _update_payload_with_properties( annotation["annotation_properties"] = dict(_map) +def _serialize_item_level_properties( + item_property_values: List[Dict[str, str]], + client: "Client", + dataset: "RemoteDataset", + import_annotators: bool, + import_reviewers: bool, +) -> List[Dict[str, Any]]: + """ + Returns serialized item-level properties to be added to the annotation import payload. + + Args: + item_property_values (List[Dict[str, str]]): A list of dictionaries containing item property values. + client (Client): The client instance used to interact with the API. + dataset (RemoteDataset): The remote dataset instance. + import_annotators (bool): Flag indicating whether to import annotators. + import_reviewers (bool): Flag indicating whether to import reviewers. + + Returns: + List[Dict[str, Any]]: A list of serialized item-level properties for the annotation import payload. + """ + if not item_property_values: + return [] + + serialized_item_level_properties: List[Dict[str, Any]] = [] + actors: List[dt.DictFreeForm] = [] + # Get team properties + _, team_item_properties_lookup = _get_team_properties_annotation_lookup( + client, dataset.team + ) + for item_property_value in item_property_values: + item_property = team_item_properties_lookup[item_property_value["name"]] + item_property_id = item_property.id + item_property_value_id = next( + ( + pv.id + for pv in item_property.property_values or [] + if pv.value == item_property_value["value"] + ), + None, + ) + actors: List[dt.DictFreeForm] = [] + actors.extend( + _handle_annotators( + import_annotators, item_property_value=item_property_value + ) + ) + actors.extend( + _handle_reviewers(import_reviewers, item_property_value=item_property_value) + ) + serialized_item_level_properties.append( + { + "actors": actors, + "property_id": item_property_id, + "value": {"id": item_property_value_id}, + } + ) + + return serialized_item_level_properties + + +def _parse_metadata_file( + metadata_path: Union[Path, bool] +) -> Tuple[List[PropertyClass], List[Dict[str, str]]]: + if isinstance(metadata_path, Path): + metadata = parse_metadata(metadata_path) + metadata_property_classes = parse_property_classes(metadata) + metadata_item_props = metadata.get("properties", []) + return metadata_property_classes, metadata_item_props + return [], [] + + +def _build_metadata_lookups( + metadata_property_classes: List[PropertyClass], + metadata_item_props: List[Dict[str, str]], +) -> Tuple[ + Set[Tuple[str, str]], + Dict[Tuple[str, str], Property], + Dict[Tuple[int, str], Property], + Dict[str, Property], +]: + metadata_classes_lookup = set() + metadata_cls_prop_lookup = {} + metadata_cls_id_prop_lookup = {} + metadata_item_prop_lookup = {} + + for _cls in metadata_property_classes: + metadata_classes_lookup.add((_cls.name, _cls.type)) + for _prop in _cls.properties or []: + metadata_cls_prop_lookup[(_cls.name, _prop.name)] = _prop + for _item_prop in metadata_item_props: + metadata_item_prop_lookup[_item_prop["name"]] = _item_prop + + return ( + metadata_classes_lookup, + metadata_cls_prop_lookup, + metadata_cls_id_prop_lookup, + metadata_item_prop_lookup, + ) + + def _import_properties( metadata_path: Union[Path, bool], + item_properties: List[Dict[str, str]], client: "Client", annotations: List[dt.Annotation], annotation_class_ids_map: Dict[Tuple[str, str], str], - team_slug: str, + dataset: "RemoteDataset", ) -> Dict[str, Dict[str, Dict[str, Set[str]]]]: """ Creates/Updates missing/mismatched properties from annotation & metadata.json file to team-properties. @@ -333,9 +469,10 @@ def _import_properties( Args: metadata_path (Union[Path, bool]): Path object to .v7/metadata.json file client (Client): Darwin Client object + item_properties (List[Dict[str, str]]): List of item-level properties present in the annotation file annotations (List[dt.Annotation]): List of annotations annotation_class_ids_map (Dict[Tuple[str, str], str]): Dict of annotation class names/types to annotation class ids - team_slug (str): Team slug + dataset (RemoteDataset): RemoteDataset object Raises: ValueError: raise error if annotation class not present in metadata and in team-properties @@ -348,33 +485,27 @@ def _import_properties( """ annotation_property_map: Dict[str, Dict[str, Dict[str, Set[str]]]] = {} - metadata_property_classes = [] - if isinstance(metadata_path, Path): - # parse metadata.json file -> list[PropertyClass] - metadata = parse_metadata(metadata_path) - metadata_property_classes = parse_property_classes(metadata) + # Parse metadata + metadata_property_classes, metadata_item_props = _parse_metadata_file(metadata_path) - # get team properties - team_properties_annotation_lookup = _get_team_properties_annotation_lookup( - client, team_slug + # Get team properties + team_properties_annotation_lookup, team_item_properties_lookup = ( + _get_team_properties_annotation_lookup(client, dataset.team) ) - # (annotation-cls-name, annotation-cls-name): PropertyClass object - metadata_classes_lookup: Set[Tuple[str, str]] = set() - # (annotation-cls-name, property-name): Property object - metadata_cls_prop_lookup: Dict[Tuple[str, str], Property] = {} - # (annotation-cls-id, property-name): Property object - metadata_cls_id_prop_lookup: Dict[Tuple[int, str], Property] = {} - for _cls in metadata_property_classes: - metadata_classes_lookup.add((_cls.name, _cls.type)) - for _prop in _cls.properties or []: - metadata_cls_prop_lookup[(_cls.name, _prop.name)] = _prop + # Build metadata lookups + ( + metadata_classes_lookup, + metadata_cls_prop_lookup, + metadata_cls_id_prop_lookup, + metadata_item_prop_lookup, + ) = _build_metadata_lookups(metadata_property_classes, metadata_item_props) # (annotation-id): dt.Annotation object annotation_id_map: Dict[str, dt.Annotation] = {} - create_properties: List[FullProperty] = [] - update_properties: List[FullProperty] = [] + annotation_and_section_level_properties_to_create: List[FullProperty] = [] + annotation_and_section_level_properties_to_update: List[FullProperty] = [] for annotation in annotations: annotation_name = annotation.annotation_class.name annotation_type = annotation_type = ( @@ -435,6 +566,7 @@ def _import_properties( ].add(t_prop_val.id) continue + # TODO: Change this so that if a property isn't found in the metadata, we can create it assuming it's an option, multi-select with no description (DAR-2920) raise ValueError( f"Annotation: '{annotation_name}' -> Property '{a_prop.name}' not found in {metadata_path}" ) @@ -462,8 +594,8 @@ def _import_properties( a_prop.name, annotation_class_id, ) not in team_properties_annotation_lookup: - # check if fullproperty exists in create_properties - for full_property in create_properties: + # check if fullproperty exists in annotation_and_section_level_properties_to_create + for full_property in annotation_and_section_level_properties_to_create: if ( full_property.name == a_prop.name and full_property.annotation_class_id == annotation_class_id @@ -509,20 +641,38 @@ def _import_properties( ) break # if it doesn't exist, create it - full_property = FullProperty( - name=a_prop.name, - type=m_prop_type, # type from .v7/metadata.json - required=m_prop.required, # required from .v7/metadata.json - description=m_prop.description - or "property-created-during-annotation-import", - slug=client.default_team, - annotation_class_id=int(annotation_class_id), - property_values=property_values, - granularity=PropertyGranularity(m_prop.granularity.value), - ) + for prop in annotation_and_section_level_properties_to_create: + if ( + prop.name == a_prop.name + and prop.annotation_class_id == annotation_class_id + ): + current_prop_values = [ + value.value for value in prop.property_values + ] + for value in property_values: + if value.value not in current_prop_values: + prop.property_values.append(value) + break + else: + full_property = FullProperty( + name=a_prop.name, + type=m_prop_type, # type from .v7/metadata.json + required=m_prop.required, # required from .v7/metadata.json + description=m_prop.description + or "property-created-during-annotation-import", + slug=client.default_team, + annotation_class_id=int(annotation_class_id), + property_values=property_values, + granularity=PropertyGranularity(m_prop.granularity), + ) # Don't attempt the same propery creation multiple times - if full_property not in create_properties: - create_properties.append(full_property) + if ( + full_property + not in annotation_and_section_level_properties_to_create + ): + annotation_and_section_level_properties_to_create.append( + full_property + ) continue # check if property value is different in m_prop (.v7/metadata.json) options @@ -569,10 +719,16 @@ def _import_properties( color=m_prop_option.get("color"), # type: ignore ) ], + granularity=t_prop.granularity, ) # Don't attempt the same propery update multiple times - if full_property not in update_properties: - update_properties.append(full_property) + if ( + full_property + not in annotation_and_section_level_properties_to_update + ): + annotation_and_section_level_properties_to_update.append( + full_property + ) continue assert t_prop.id is not None @@ -581,37 +737,116 @@ def _import_properties( t_prop.id ].add(t_prop_val.id) + # Create/Update team item properties based on metadata + item_properties_to_create_from_metadata, item_properties_to_update_from_metadata = ( + _create_update_item_properties( + _normalize_item_properties(metadata_item_prop_lookup), + team_item_properties_lookup, + client, + ) + ) + console = Console(theme=_console_theme()) + properties_to_create = ( + annotation_and_section_level_properties_to_create + + item_properties_to_create_from_metadata + ) + properties_to_update = ( + annotation_and_section_level_properties_to_update + + item_properties_to_update_from_metadata + ) + created_properties = [] - if create_properties: - console.print(f"Creating {len(create_properties)} properties", style="info") - for full_property in create_properties: - console.print( - f"Creating property {full_property.name} ({full_property.type})", - style="info", - ) + if properties_to_create: + console.print(f"Creating {len(properties_to_create)} properties:", style="info") + for full_property in properties_to_create: + if full_property.granularity.value == "item": + console.print( + f"- Creating item-level property '{full_property.name}' of type: {full_property.type}" + ) + else: + console.print( + f"- Creating property '{full_property.name}' of type {full_property.type}", + ) prop = client.create_property( team_slug=full_property.slug, params=full_property ) created_properties.append(prop) updated_properties = [] - if update_properties: - console.print(f"Updating {len(update_properties)} properties", style="info") - for full_property in update_properties: + if properties_to_update: + console.print( + f"Performing {len(properties_to_update)} property update(s):", style="info" + ) + for full_property in properties_to_update: + if full_property.granularity.value == "item": + console.print( + f"- Updating item-level property '{full_property.name}' with new value: {full_property.property_values[0].value}", + ) + else: + console.print( + f"- Updating property '{full_property.name}' of type {full_property.type}", + ) + prop = client.update_property( + team_slug=full_property.slug, params=full_property + ) + updated_properties.append(prop) + + # get latest team properties + team_properties_annotation_lookup, team_item_properties_lookup = ( + _get_team_properties_annotation_lookup(client, dataset.team) + ) + + # Create or update item-level properties from annotations + item_property_creations_from_metadata, item_property_updates_from_metadata = ( + _create_update_item_properties( + _normalize_item_properties(item_properties), + team_item_properties_lookup, + client, + ) + ) + + properties_to_create = item_property_creations_from_metadata + properties_to_update = item_property_updates_from_metadata + + if properties_to_create: + console.print(f"Creating {len(properties_to_create)} properties:", style="info") + for full_property in properties_to_create: + if full_property.granularity.value == "item": + console.print( + f"- Creating item-level property '{full_property.name}' of type: {full_property.type}" + ) console.print( - f"Updating property {full_property.name} ({full_property.type})", - style="info", + f"- Creating property '{full_property.name}' of type {full_property.type}", ) + prop = client.create_property( + team_slug=full_property.slug, params=full_property + ) + created_properties.append(prop) + + if item_property_updates_from_metadata: + console.print( + f"Performing {len(item_property_updates_from_metadata)} property update(s):", + style="info", + ) + for full_property in item_property_updates_from_metadata: + if full_property.granularity.value == "item": + console.print( + f"- Updating item-level property '{full_property.name}' with new value: {full_property.property_values[0].value}" + ) + else: + console.print( + f"- Updating property {full_property.name} ({full_property.type})", + ) prop = client.update_property( team_slug=full_property.slug, params=full_property ) updated_properties.append(prop) # get latest team properties - team_properties_annotation_lookup = _get_team_properties_annotation_lookup( - client, team_slug + team_properties_annotation_lookup, team_item_properties_lookup = ( + _get_team_properties_annotation_lookup(client, dataset.team) ) # loop over metadata_cls_id_prop_lookup, and update additional metadata property values @@ -697,7 +932,6 @@ def _import_properties( ] = set() break - # find the property-id and property-value-id in the response for prop_val in prop.property_values or []: if prop_val.value == a_prop.value: annotation_property_map[annotation_id][frame_index][ @@ -705,10 +939,154 @@ def _import_properties( ].add(prop_val.id) break break + _assign_item_properties_to_dataset( + item_properties, team_item_properties_lookup, client, dataset, console + ) return annotation_property_map +def _normalize_item_properties( + item_properties: Union[Dict[str, Dict[str, Any]], List[Dict[str, str]]] +) -> Dict[str, Dict[str, Any]]: + """ + Normalizes item properties to a common dictionary format. + + Args: + item_properties (Union[Dict[str, Dict[str, Any]], List[Dict[str, str]]]): Item properties in different formats. + + Returns: + Dict[str, Dict[str, Any]]: Normalized item properties. + """ + if isinstance(item_properties, dict): + return item_properties + + normalized_properties = defaultdict(lambda: {"property_values": []}) + for item_prop in item_properties: + name = item_prop["name"] + value = item_prop["value"] + normalized_properties[name]["property_values"].append({"value": value}) + + return normalized_properties + + +def _create_update_item_properties( + item_properties: Dict[str, Dict[str, Any]], + team_item_properties_lookup: Dict[str, FullProperty], + client: "Client", +) -> Tuple[List[FullProperty], List[FullProperty]]: + """ + Compares item-level properties present in `item_properties` with the team item properties and plans to create or update them. + + Args: + item_properties (Dict[str, Dict[str, Any]]): Dictionary of item-level properties present in the annotation file + team_item_properties_lookup (Dict[str, FullProperty]): Lookup of team item properties + client (Client): Darwin Client object + + Returns: + Tuple[List[FullProperty], List[FullProperty]]: Tuple of lists of properties to be created and updated + """ + create_properties = [] + update_properties = [] + for item_prop_name, m_prop in item_properties.items(): + m_prop_values = [ + prop_val["value"] for prop_val in m_prop.get("property_values", []) + ] + + # If the property exists in the team, check that all values are present + if item_prop_name in team_item_properties_lookup: + t_prop = team_item_properties_lookup[item_prop_name] + t_prop_values = [ + prop_val.value for prop_val in t_prop.property_values or [] + ] + + # Add one update per missing property value + for m_prop_value in m_prop_values: + if m_prop_value not in t_prop_values: + update_property = FullProperty( + id=t_prop.id, + name=t_prop.name, + type=t_prop.type, + required=t_prop.required, + description=t_prop.description, + slug=client.default_team, + annotation_class_id=t_prop.annotation_class_id, + property_values=[PropertyValue(value=m_prop_value)], + granularity=PropertyGranularity.item, + ) + update_properties.append(update_property) + + # If the property does not exist in the team, create it + else: + + # If we've already planned to create this property, simply extend the property values + for prop in create_properties: + if prop.name == item_prop_name: + current_prop_values = [ + value.value for value in prop.property_values + ] + if prop.property_values is None: + prop.property_values = [] + for val in m_prop_values: + if val.value not in current_prop_values: + prop.property_values.append(PropertyValue(value=val)) + break + else: + full_property = FullProperty( + name=item_prop_name, + type=m_prop.get("type", "multi_select"), + required=bool(m_prop.get("required", False)), + description=m_prop.get( + "description", "property-created-during-annotation-import" + ), + slug=client.default_team, + annotation_class_id=None, + property_values=[PropertyValue(value=val) for val in m_prop_values], + granularity=PropertyGranularity.item, + ) + create_properties.append(full_property) + + return create_properties, update_properties + + +def _assign_item_properties_to_dataset( + item_properties: List[Dict[str, str]], + team_item_properties_lookup: Dict[str, FullProperty], + client: "Client", + dataset: "RemoteDataset", + console: Console, +) -> None: + """ + Ensures that all item-level properties to be imported are assigned to the target dataset + + Args: + item_properties (List[Dict[str, str]]): List of item-level properties present in the annotation file + team_item_properties_lookup (Dict[str, FullProperty]): Server- side state of item-level properties + client (Client): Darwin Client object + dataset (RemoteDataset): RemoteDataset object + console (Console): Rich Console + """ + if item_properties: + item_properties_set = {prop["name"] for prop in item_properties} + for item_property in item_properties_set: + for team_prop in team_item_properties_lookup: + if team_prop == item_property: + prop_datasets = ( + team_item_properties_lookup[team_prop].dataset_ids or [] + ) + if dataset.dataset_id not in prop_datasets: + updated_property = team_item_properties_lookup[team_prop] + updated_property.dataset_ids.append(dataset.dataset_id) + updated_property.property_values = ( + [] + ) # Necessary to clear, otherwise we're trying to add the exsting values to themselves + console.print( + f"Adding item-level property '{updated_property.name}' to the dataset '{dataset.name}' ", + style="info", + ) + client.update_property(dataset.team, updated_property) + + def import_annotations( # noqa: C901 dataset: "RemoteDataset", importer: Callable[[Path], Union[List[dt.AnnotationFile], dt.AnnotationFile, None]], @@ -1004,6 +1382,7 @@ def import_annotation(parsed_file): remote_classes, attributes, parsed_file.annotations, + parsed_file.item_properties, default_slot_name, dataset, append, @@ -1193,24 +1572,38 @@ def _annotators_or_reviewers_to_payload( def _handle_reviewers( - annotation: dt.Annotation, import_reviewers: bool + import_reviewers: bool, + annotation: Optional[dt.Annotation] = None, + item_property_value: Optional[Dict[str, Any]] = None, ) -> List[dt.DictFreeForm]: if import_reviewers: - if annotation.reviewers: + if annotation and annotation.reviewers: return _annotators_or_reviewers_to_payload( annotation.reviewers, dt.AnnotationAuthorRole.REVIEWER ) + elif item_property_value and "reviewers" in item_property_value: + return _annotators_or_reviewers_to_payload( + _parse_annotators(item_property_value["reviewers"]), + dt.AnnotationAuthorRole.REVIEWER, + ) return [] def _handle_annotators( - annotation: dt.Annotation, import_annotators: bool + import_annotators: bool, + annotation: Optional[dt.Annotation] = None, + item_property_value: Optional[Dict[str, Any]] = None, ) -> List[dt.DictFreeForm]: if import_annotators: - if annotation.annotators: + if annotation and annotation.annotators: return _annotators_or_reviewers_to_payload( annotation.annotators, dt.AnnotationAuthorRole.ANNOTATOR ) + elif item_property_value and "annotators" in item_property_value: + return _annotators_or_reviewers_to_payload( + _parse_annotators(item_property_value["annotators"]), + dt.AnnotationAuthorRole.ANNOTATOR, + ) return [] @@ -1362,6 +1755,7 @@ def _import_annotations( remote_classes: dt.DictFreeForm, attributes: dt.DictFreeForm, annotations: List[dt.Annotation], + item_properties: List[Dict[str, str]], default_slot_name: str, dataset: "RemoteDataset", append: bool, @@ -1432,8 +1826,8 @@ def _import_annotations( ) actors: List[dt.DictFreeForm] = [] - actors.extend(_handle_annotators(annotation, import_annotators)) - actors.extend(_handle_reviewers(annotation, import_reviewers)) + actors.extend(_handle_annotators(import_annotators, annotation=annotation)) + actors.extend(_handle_reviewers(import_reviewers, annotation=annotation)) # Insert the default slot name if not available in the import source annotation = _handle_slot_names(annotation, dataset.version, default_slot_name) @@ -1457,14 +1851,21 @@ def _import_annotations( annotation_id_property_map = _import_properties( metadata_path, + item_properties, client, annotations, # type: ignore annotation_class_ids_map, - dataset.team, + dataset, ) + _update_payload_with_properties(serialized_annotations, annotation_id_property_map) + serialized_item_level_properties = _serialize_item_level_properties( + item_properties, client, dataset, import_annotators, import_reviewers + ) payload: dt.DictFreeForm = {"annotations": serialized_annotations} + if serialized_item_level_properties: + payload["properties"] = serialized_item_level_properties payload["overwrite"] = _get_overwrite_value(append) try: @@ -1496,7 +1897,7 @@ def _overwrite_warning( console: Console, ) -> bool: """ - Determines if any dataset items targeted for import already have annotations that will be overwritten. + Determines if any dataset items targeted for import already have annotations or item-level properties that will be overwritten. If they do, a warning is displayed to the user and they are prompted to confirm if they want to proceed with the import. Parameters @@ -1517,22 +1918,56 @@ def _overwrite_warning( bool True if the user wants to proceed with the import, False otherwise. """ - files_to_overwrite = [] + files_with_annotations_to_overwrite = [] + files_with_item_properties_to_overwrite = [] + for local_file in local_files: - item_id = remote_files.get(local_file.full_path)["item_id"] + item_id = remote_files.get(local_file.full_path)["item_id"] # type: ignore + + # Check if the item has annotations that will be overwritten remote_annotations = client.api_v2._get_remote_annotations( item_id, dataset.team, ) - if remote_annotations and local_file.full_path not in files_to_overwrite: - files_to_overwrite.append(local_file.full_path) - if files_to_overwrite: - console.print( - f"The following {len(files_to_overwrite)} dataset items already have annotations that will be overwritten by this import:", - style="warning", - ) - for file in files_to_overwrite: - console.print(f"- {file}", style="warning") + if ( + remote_annotations + and local_file.full_path not in files_with_annotations_to_overwrite + ): + files_with_annotations_to_overwrite.append(local_file.full_path) + + # Check if the item has item-level properties that will be overwritten + if local_file.item_properties: + response: Dict[str, List[Dict[str, str]]] = ( + client.api_v2._get_properties_state_for_item(item_id, dataset.team) + ) + item_property_ids_with_populated_values = [ + property_data["id"] + for property_data in response["properties"] + if property_data["values"] + ] + if item_property_ids_with_populated_values: + files_with_item_properties_to_overwrite.append(local_file.full_path) + + if files_with_annotations_to_overwrite or files_with_item_properties_to_overwrite: + + # Overwriting of annotations + if files_with_annotations_to_overwrite: + console.print( + f"The following {len(files_with_annotations_to_overwrite)} dataset item(s) have annotations that will be overwritten by this import:", + style="warning", + ) + for file in files_with_annotations_to_overwrite: + console.print(f"- {file}", style="warning") + + # Overwriting of item-level-properties + if files_with_item_properties_to_overwrite: + console.print( + f"The following {len(files_with_item_properties_to_overwrite)} dataset item(s) have item-level properties that will be overwritten by this import:", + style="warning", + ) + for file in files_with_item_properties_to_overwrite: + console.print(f"- {file}", style="warning") + proceed = input("Do you want to proceed with the import? [y/N] ") if proceed.lower() != "y": return False diff --git a/darwin/path_utils.py b/darwin/path_utils.py index 904c41fe7..2f573596f 100644 --- a/darwin/path_utils.py +++ b/darwin/path_utils.py @@ -113,10 +113,16 @@ def is_properties_enabled( return False # .v7 directory exists, parse the metadata file and check if any class has properties + # Additionally check if there are any item-level properties metadata_path = v7_path / filename - metadata_classes = parse_metadata(metadata_path).get("classes", []) + metadata = parse_metadata(metadata_path) + metadata_classes = metadata.get("classes", []) + metadata_item_level_properties = metadata.get("properties", []) for _cls in metadata_classes: if _cls.get("properties"): return metadata_path + for _item_level_property in metadata_item_level_properties: + if _item_level_property.get("property_values"): + return metadata_path return False diff --git a/darwin/utils/utils.py b/darwin/utils/utils.py index ba95198c3..b7460e0ca 100644 --- a/darwin/utils/utils.py +++ b/darwin/utils/utils.py @@ -594,6 +594,7 @@ def _parse_darwin_v2(path: Path, data: Dict[str, Any]) -> dt.AnnotationFile: frame_urls=None, remote_path=item["path"], slots=slots, + item_properties=data.get("properties", []), ) else: slot = slots[0] @@ -622,6 +623,7 @@ def _parse_darwin_v2(path: Path, data: Dict[str, Any]) -> dt.AnnotationFile: remote_path=item["path"], slots=slots, frame_count=slot.frame_count, + item_properties=data.get("properties", []), ) return annotation_file @@ -1278,6 +1280,7 @@ def split_video_annotation(annotation: dt.AnnotationFile) -> List[dt.AnnotationF filename, annotation_classes, annotations, + [], False, annotation.image_width, annotation.image_height, diff --git a/e2e_tests/cli/test_import.py b/e2e_tests/cli/test_import.py index 2441b7ebe..2eb4ae350 100644 --- a/e2e_tests/cli/test_import.py +++ b/e2e_tests/cli/test_import.py @@ -121,6 +121,17 @@ def assert_same_annotation_slot_name( assert actual_annotation.slot_names == [base_slot] +def assert_same_item_level_properties( + expected_item_level_properties: List[Dict[str, str]], + actual_item_level_properties: List[Dict[str, str]], +) -> None: + """ + Ensures that all expected item-level properties are present in exported item-level properties + """ + for expected_item_level_property in expected_item_level_properties: + assert expected_item_level_property in actual_item_level_properties + + def compare_annotations_export( actual_annotations_dir: Path, expected_annotations_dir: Path, @@ -151,16 +162,28 @@ def compare_annotations_export( actual_filename = get_actual_annotation_filename( expected_filename, actual_annotation_files ) - expected_annotations: List[dt.Annotation] = parse_darwin_json( - Path(expected_annotation_files[expected_filename]) - ).annotations # type: ignore - actual_annotations: List[dt.Annotation] = parse_darwin_json( - Path(actual_annotation_files[actual_filename]) - ).annotations # type: ignore + expected_annotation_data: List[dt.Annotation] = parse_darwin_json( + Path(expected_annotation_files[expected_filename]) # type: ignore + ) + expected_annotations = expected_annotation_data.annotations # type: ignore + expected_item_level_properties = ( + expected_annotation_data.item_properties # type: ignore + ) + + actual_annotation_data: List[dt.Annotation] = parse_darwin_json( + Path(actual_annotation_files[actual_filename]) # type: ignore + ) + actual_annotations = actual_annotation_data.annotations # type: ignore + actual_item_level_properties = ( + actual_annotation_data.item_properties # type: ignore + ) delete_annotation_uuids(expected_annotations) delete_annotation_uuids(actual_annotations) + assert_same_item_level_properties( + expected_item_level_properties, actual_item_level_properties + ) for expected_annotation in expected_annotations: actual_annotation = find_matching_actual_annotation( expected_annotation, actual_annotations @@ -263,6 +286,28 @@ def test_annotation_classes_are_created_with_properties_on_import( ) +def test_import_existing_item_level_properties( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_annotations_with_item_level_properties", + ) + + +def test_item_level_property_classes_are_created_on_import( + local_dataset: E2EDataset, config_values: ConfigValues +) -> None: + run_import_test( + local_dataset, + config_values, + item_type="single_slotted", + annotations_subdir="image_new_annotations_with_item_level_properties", + ) + + def test_appending_annotations( local_dataset: E2EDataset, config_values: ConfigValues ) -> None: diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 92af9f8f0..c243780fa 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -16,6 +16,8 @@ setup_annotation_classes, setup_datasets, teardown_annotation_classes, + setup_item_level_properties, + teardown_item_level_properties, teardown_tests, ) @@ -54,13 +56,15 @@ def pytest_sessionstart(session: pytest.Session) -> None: config = ConfigValues(server=server, api_key=api_key, team_slug=team_slug) datasets = setup_datasets(config) - teardown_annotation_classes( - config, [] - ) # Ensure that there are no annotation classes before running tests + # Ensure that there are no annotation classes or item-level properties before running tests + teardown_annotation_classes(config, []) + teardown_item_level_properties(config, []) annotation_classes = setup_annotation_classes(config) + item_level_properties = setup_item_level_properties(config) # pytest.datasets = datasets setattr(pytest, "datasets", datasets) setattr(pytest, "annotation_classes", annotation_classes) + setattr(pytest, "item_level_properties", item_level_properties) setattr(pytest, "config_values", config) # Set the environment variables for running CLI arguments environ["DARWIN_BASE_URL"] = server @@ -83,6 +87,11 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: raise ValueError( "Annotation classes were not created, so could not tear them down" ) + item_level_properties = pytest.item_level_properties + if item_level_properties is None: + raise ValueError( + "Item-level properties were not created, so could not tear them down" + ) server = session.config.cache.get("server", None) api_key = session.config.cache.get("api_key", None) @@ -100,6 +109,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: teardown_tests(config, datasets) assert isinstance(annotation_classes, List) teardown_annotation_classes(config, annotation_classes) + teardown_item_level_properties(config, item_level_properties) @pytest.fixture( diff --git a/e2e_tests/data/import/image_annotations_with_item_level_properties/image_1.json b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_1.json new file mode 100644 index 000000000..4e1b58903 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_1.json @@ -0,0 +1,595 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_1", + "path": "/", + "source_info": { + "item_id": "01920b92-1d5d-94a4-6fbe-8a4d7f9fa15d", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-94a4-6fbe-8a4d7f9fa15d" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/2ec69e41-91b2-4155-9b05-6ed995677b1e/thumbnail", + "source_files": [ + { + "file_name": "image_1", + "storage_key": "darwin-py/images/image_1.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/9dfc5eac-bf16-4380-a148-9fff6e63b9f0" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "b92626e7-a1fd-4ecb-a418-31aa37069bc1", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "40288025-5eea-406c-95fb-fd335df117fe", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "de3a1fd0-bff9-4b4a-bffb-b6b92bff18a3", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "d078cdd0-7ff3-4835-9d5d-f4e0c6a31fa6", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "063cd083-e25a-4f0d-82d9-45d0b3c9dedd", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "a446a6dc-6427-413f-8f5d-5af0121f0923", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "e59b6d18-88dc-417a-a619-513c634e1402", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "4f58cca2-e174-4564-b242-1d17042f1b01", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "8596d3bd-d9f1-4663-8cc7-d444e97abb74", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 503228, + 1, + 8, + 0, + 21, + 1, + 10, + 0, + 1879, + 1, + 12, + 0, + 17, + 1, + 14, + 0, + 1875, + 1, + 16, + 0, + 13, + 1, + 18, + 0, + 1872, + 1, + 18, + 0, + 11, + 1, + 20, + 0, + 1870, + 1, + 20, + 0, + 9, + 1, + 22, + 0, + 7, + 1, + 11, + 0, + 1850, + 1, + 22, + 0, + 7, + 1, + 27, + 0, + 1, + 1, + 18, + 0, + 1845, + 1, + 22, + 0, + 7, + 1, + 48, + 0, + 1842, + 1, + 24, + 0, + 5, + 1, + 51, + 0, + 1840, + 1, + 24, + 0, + 5, + 1, + 52, + 0, + 1838, + 1, + 26, + 0, + 3, + 1, + 54, + 0, + 1837, + 1, + 26, + 0, + 3, + 1, + 55, + 0, + 1836, + 1, + 26, + 0, + 3, + 1, + 55, + 0, + 1836, + 1, + 26, + 0, + 3, + 1, + 56, + 0, + 1835, + 1, + 26, + 0, + 3, + 1, + 56, + 0, + 1835, + 1, + 26, + 0, + 3, + 1, + 57, + 0, + 1834, + 1, + 26, + 0, + 3, + 1, + 57, + 0, + 1834, + 1, + 26, + 0, + 3, + 1, + 57, + 0, + 1834, + 1, + 26, + 0, + 4, + 1, + 56, + 0, + 1834, + 1, + 26, + 0, + 4, + 1, + 56, + 0, + 1835, + 1, + 24, + 0, + 6, + 1, + 55, + 0, + 1835, + 1, + 24, + 0, + 6, + 1, + 55, + 0, + 1836, + 1, + 22, + 0, + 8, + 1, + 54, + 0, + 1836, + 1, + 22, + 0, + 9, + 1, + 52, + 0, + 1838, + 1, + 20, + 0, + 11, + 1, + 51, + 0, + 1839, + 1, + 18, + 0, + 14, + 1, + 48, + 0, + 1841, + 1, + 16, + 0, + 17, + 1, + 46, + 0, + 1843, + 1, + 12, + 0, + 23, + 1, + 41, + 0, + 1846, + 1, + 8, + 0, + 26, + 1, + 39, + 0, + 1882, + 1, + 37, + 0, + 1885, + 1, + 14, + 0, + 1, + 1, + 18, + 0, + 1889, + 1, + 10, + 0, + 8, + 1, + 11, + 0, + 1512704 + ], + "mask_annotation_ids_mapping": { + "4f58cca2-e174-4564-b242-1d17042f1b01": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_with_item_level_properties/image_2.json b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_2.json new file mode 100644 index 000000000..ce84acc81 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_2.json @@ -0,0 +1,487 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_2", + "path": "/", + "source_info": { + "item_id": "01920b92-1d5d-ea77-8fa4-16378bafedb3", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-ea77-8fa4-16378bafedb3" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/5e0b3d9d-9bf8-4166-8949-6ab7392161ad/thumbnail", + "source_files": [ + { + "file_name": "image_2", + "storage_key": "darwin-py/images/image_2.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/4920b12a-1706-47f1-b084-2d2234ed1151" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "81295639-1b29-4681-b3a8-53119bf49036", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "f3b20d5d-987a-44a4-94ac-41acf2ac14fd", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "1220a567-c329-4bca-b955-56387a3f50a2", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "3f01b473-f88f-4f3f-b6f6-559feff994f8", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "4c6e632c-f519-41ef-a918-462e1745f30e", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "d9d250a0-98b9-4b5e-85a2-c1ad7c0e4b2d", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "02d3df0e-8f01-4ba6-a790-954d3c892698", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "c389c7c5-0a92-48a0-b991-cf26b351111d", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "69c45b6b-83c1-4a0a-9210-03754a4e823e", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 470662, + 1, + 12, + 0, + 1906, + 1, + 16, + 0, + 1902, + 1, + 20, + 0, + 1899, + 1, + 22, + 0, + 1896, + 1, + 25, + 0, + 1893, + 1, + 28, + 0, + 1890, + 1, + 30, + 0, + 1888, + 1, + 33, + 0, + 1886, + 1, + 34, + 0, + 1885, + 1, + 36, + 0, + 1883, + 1, + 37, + 0, + 1883, + 1, + 37, + 0, + 1882, + 1, + 38, + 0, + 1881, + 1, + 39, + 0, + 1881, + 1, + 39, + 0, + 1880, + 1, + 40, + 0, + 1880, + 1, + 40, + 0, + 1879, + 1, + 40, + 0, + 1880, + 1, + 40, + 0, + 1879, + 1, + 40, + 0, + 1880, + 1, + 40, + 0, + 1880, + 1, + 39, + 0, + 1881, + 1, + 38, + 0, + 1882, + 1, + 37, + 0, + 1883, + 1, + 35, + 0, + 1885, + 1, + 33, + 0, + 1887, + 1, + 31, + 0, + 1890, + 1, + 28, + 0, + 1892, + 1, + 25, + 0, + 1896, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1899, + 1, + 20, + 0, + 1901, + 1, + 18, + 0, + 1903, + 1, + 16, + 0, + 1906, + 1, + 12, + 0, + 1910, + 1, + 8, + 0, + 1535742 + ], + "mask_annotation_ids_mapping": { + "c389c7c5-0a92-48a0-b991-cf26b351111d": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_with_item_level_properties/image_3.json b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_3.json new file mode 100644 index 000000000..74928f6a2 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_3.json @@ -0,0 +1,531 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_3", + "path": "/dir1", + "source_info": { + "item_id": "01920b92-1d5d-e8ad-986f-ad4942f1bbfc", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-e8ad-986f-ad4942f1bbfc" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/ddd13905-9bbb-4fab-9642-bf4604686fda/thumbnail", + "source_files": [ + { + "file_name": "image_3", + "storage_key": "darwin-py/images/image_3.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/30ec0f13-caaa-4374-be5a-e90b3493fb73" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "6485cc18-0fc6-4a64-9794-474fd7040767", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "f133a6c1-8de2-47f8-a295-e1978e9fda05", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "3088bafc-a6b8-46e1-864a-ea49ce6d46c9", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "a6786798-3d81-4c72-adcb-34a4615c33ce", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "67eb19f3-66cb-4910-8314-8cfdd6bbf34e", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "77cb2ca5-c010-48c9-a68d-d3e08ed78eae", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "318737ce-0f94-40d8-96aa-a616c1e5999c", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "b92d8bad-ace9-4f73-b428-9505eb960016", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "2234893b-eea8-4b90-a2d1-c0ddf32b2918", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 543631, + 1, + 8, + 0, + 1910, + 1, + 12, + 0, + 1906, + 1, + 16, + 0, + 1903, + 1, + 18, + 0, + 1901, + 1, + 20, + 0, + 1899, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1897, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1895, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1895, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1897, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1897, + 1, + 22, + 0, + 1897, + 1, + 22, + 0, + 1897, + 1, + 22, + 0, + 1897, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1897, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1895, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1895, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1897, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1899, + 1, + 20, + 0, + 1901, + 1, + 18, + 0, + 1903, + 1, + 16, + 0, + 1906, + 1, + 12, + 0, + 1910, + 1, + 8, + 0, + 1441645 + ], + "mask_annotation_ids_mapping": { + "b92d8bad-ace9-4f73-b428-9505eb960016": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_with_item_level_properties/image_4.json b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_4.json new file mode 100644 index 000000000..994839a51 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_4.json @@ -0,0 +1,583 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_4", + "path": "/dir1", + "source_info": { + "item_id": "01920b92-1d5d-8b50-17e9-c0f178e6eee6", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-8b50-17e9-c0f178e6eee6" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/3c731d84-7d7f-4ac8-bbd9-0d53f1d47195/thumbnail", + "source_files": [ + { + "file_name": "image_4", + "storage_key": "darwin-py/images/image_4.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/609ba1a4-79da-4743-b331-e57ccd9ee518" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "c7ab6a60-d36d-4fb9-b95d-b46002093b5c", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "02f6e8c9-9ae4-4ff2-a4d2-1d8a15476c04", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "21b82723-e87c-43b5-8eaa-49f1933a6e1a", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "8a7e04f5-76b8-4996-a19e-a864122ad2a0", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "173e6ec8-53d9-4ae5-8a01-5278d4573662", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "2c851bb5-52f1-417b-8aac-23dc2083366f", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "02be609a-e456-4cbf-a704-875a2f0246e3", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "aa057965-2b8b-4c33-931e-3761eda459d6", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "6660192e-76cb-412b-8f8d-fffef3cbb223", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 476334, + 1, + 9, + 0, + 6, + 1, + 8, + 0, + 1895, + 1, + 13, + 0, + 2, + 1, + 12, + 0, + 1891, + 1, + 31, + 0, + 1888, + 1, + 33, + 0, + 1886, + 1, + 35, + 0, + 1884, + 1, + 37, + 0, + 1883, + 1, + 37, + 0, + 1882, + 1, + 39, + 0, + 1881, + 1, + 39, + 0, + 1880, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1880, + 1, + 40, + 0, + 1880, + 1, + 40, + 0, + 1881, + 1, + 38, + 0, + 1882, + 1, + 38, + 0, + 1883, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1885, + 1, + 34, + 0, + 23, + 1, + 8, + 0, + 1856, + 1, + 32, + 0, + 22, + 1, + 12, + 0, + 1855, + 1, + 30, + 0, + 21, + 1, + 16, + 0, + 1855, + 1, + 12, + 0, + 2, + 1, + 12, + 0, + 22, + 1, + 18, + 0, + 1856, + 1, + 8, + 0, + 6, + 1, + 8, + 0, + 23, + 1, + 21, + 0, + 1898, + 1, + 23, + 0, + 1897, + 1, + 24, + 0, + 1895, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1893, + 1, + 28, + 0, + 1892, + 1, + 28, + 0, + 1892, + 1, + 29, + 0, + 1891, + 1, + 29, + 0, + 1891, + 1, + 29, + 0, + 1891, + 1, + 29, + 0, + 1891, + 1, + 29, + 0, + 1891, + 1, + 29, + 0, + 1892, + 1, + 28, + 0, + 1892, + 1, + 28, + 0, + 1892, + 1, + 27, + 0, + 1894, + 1, + 26, + 0, + 1895, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1897, + 1, + 22, + 0, + 1899, + 1, + 20, + 0, + 1901, + 1, + 18, + 0, + 1904, + 1, + 14, + 0, + 1908, + 1, + 10, + 0, + 1501203 + ], + "mask_annotation_ids_mapping": { + "aa057965-2b8b-4c33-931e-3761eda459d6": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_with_item_level_properties/image_5.json b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_5.json new file mode 100644 index 000000000..2195af073 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_5.json @@ -0,0 +1,455 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_5", + "path": "/dir2", + "source_info": { + "item_id": "01920b92-1d5d-55bf-d705-8b39dea7fde6", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-55bf-d705-8b39dea7fde6" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/8f95e81c-def7-4973-9152-6d0fc39e1473/thumbnail", + "source_files": [ + { + "file_name": "image_5", + "storage_key": "darwin-py/images/image_5.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/08448a07-4e23-41f9-abbd-0dc149ef2be4" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "0d920fcc-5ec8-4760-8322-61e4de28aa3e", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "e0d23fd6-2a77-4d14-b178-b42a082cf54d", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "5faa5611-59ba-4d1b-8a1d-e6a451ab5506", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "aa95ba32-2374-4981-93c7-7447d846ca03", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "64d2a78a-0cb7-4849-9202-0901e78caedc", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "657d85c4-7c2a-4874-bcbf-bb15db32aee9", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "a9bf3ac0-8742-43e9-aa7f-46b9618d3cec", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "11cff11e-43a7-4b31-83ab-720fdcc84dd7", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "d07c5cf8-3b3f-489e-be44-927f8c98c7df", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 497558, + 1, + 9, + 0, + 1, + 1, + 8, + 0, + 1900, + 1, + 22, + 0, + 1896, + 1, + 26, + 0, + 1893, + 1, + 28, + 0, + 1891, + 1, + 30, + 0, + 1889, + 1, + 32, + 0, + 1888, + 1, + 32, + 0, + 1887, + 1, + 34, + 0, + 1886, + 1, + 34, + 0, + 1885, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1885, + 1, + 35, + 0, + 1885, + 1, + 34, + 0, + 1887, + 1, + 33, + 0, + 1887, + 1, + 32, + 0, + 1889, + 1, + 31, + 0, + 1890, + 1, + 29, + 0, + 1892, + 1, + 27, + 0, + 1895, + 1, + 24, + 0, + 1898, + 1, + 20, + 0, + 1910, + 1, + 8, + 0, + 1526104 + ], + "mask_annotation_ids_mapping": { + "11cff11e-43a7-4b31-83ab-720fdcc84dd7": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_with_item_level_properties/image_6.json b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_6.json new file mode 100644 index 000000000..446c5d062 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_6.json @@ -0,0 +1,467 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_6", + "path": "/dir2", + "source_info": { + "item_id": "01920b92-1d5d-1832-3a09-1f38557c57b4", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-1832-3a09-1f38557c57b4" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/4950b608-00a1-4e73-b746-bfe1ea0a1ab6/thumbnail", + "source_files": [ + { + "file_name": "image_6", + "storage_key": "darwin-py/images/image_6.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/9e070e8c-03b3-40b7-a3cb-6da6bcc8d4ed" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "8a9dcdc5-e584-45f2-bf71-2f143d11632b", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "cda6055b-cddc-49bb-a322-7a687855518f", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "6f090adc-ecc7-4d01-98c7-89ca688b57f9", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "8f37515b-6cad-4c3e-895b-951431348986", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "4fd320f4-745b-479f-9641-b091f983c393", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "c2a937af-c277-4e80-851b-34c314019aed", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "2f398d01-4099-4b6a-887d-3ac850fd33d9", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "108f0fd2-f6ea-4659-9aa4-1038a8c473ec", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "3334b185-2c1a-4af2-a3eb-619a542c17fe", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 462946, + 1, + 8, + 0, + 1910, + 1, + 12, + 0, + 1906, + 1, + 16, + 0, + 1903, + 1, + 18, + 0, + 1898, + 1, + 27, + 0, + 1891, + 1, + 31, + 0, + 1887, + 1, + 35, + 0, + 1884, + 1, + 37, + 0, + 1882, + 1, + 39, + 0, + 1880, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1878, + 1, + 43, + 0, + 1877, + 1, + 43, + 0, + 1876, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1876, + 1, + 43, + 0, + 1877, + 1, + 43, + 0, + 1878, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1880, + 1, + 39, + 0, + 1882, + 1, + 37, + 0, + 1884, + 1, + 35, + 0, + 1887, + 1, + 31, + 0, + 1891, + 1, + 17, + 0, + 2, + 1, + 8, + 0, + 1554956 + ], + "mask_annotation_ids_mapping": { + "108f0fd2-f6ea-4659-9aa4-1038a8c473ec": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_with_item_level_properties/image_7.json b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_7.json new file mode 100644 index 000000000..042a37133 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_7.json @@ -0,0 +1,507 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_7", + "path": "/dir1/dir3", + "source_info": { + "item_id": "01920b92-1d5d-46ee-5117-53ba0d29d1b0", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-46ee-5117-53ba0d29d1b0" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/1e2f63eb-b7fc-482f-91f3-8caa242e63cb/thumbnail", + "source_files": [ + { + "file_name": "image_7", + "storage_key": "darwin-py/images/image_7.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/20de7c08-20dc-4f16-b559-bbcce2f7b319" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "0a662491-8524-460c-b0a1-463e3def1ce2", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "45b1762d-261b-4af4-a7b3-9250354fbb44", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "5a7b975a-64a6-4eba-89c7-4be161477807", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "0cbe2fe4-989d-420d-a96a-00586ee930dc", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "4024a7ec-a372-48d7-a9ee-604be710548c", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "b76613e5-72ba-4f2b-8bb7-4cfbd8a1b816", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "3b4dc955-bd5e-43d9-bd73-bfcb16e8d5dc", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "d830153a-94e4-434f-8cd1-cbd6c4a717da", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "ccde37ee-00af-4a85-a4ff-0fe8bde1b89b", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 447629, + 1, + 8, + 0, + 1905, + 1, + 17, + 0, + 1901, + 1, + 21, + 0, + 1897, + 1, + 24, + 0, + 1895, + 1, + 26, + 0, + 1893, + 1, + 28, + 0, + 1891, + 1, + 29, + 0, + 1891, + 1, + 30, + 0, + 1889, + 1, + 31, + 0, + 1889, + 1, + 32, + 0, + 1879, + 1, + 41, + 0, + 1877, + 1, + 43, + 0, + 1875, + 1, + 45, + 0, + 1874, + 1, + 46, + 0, + 1873, + 1, + 47, + 0, + 1872, + 1, + 48, + 0, + 1872, + 1, + 48, + 0, + 1871, + 1, + 49, + 0, + 1871, + 1, + 48, + 0, + 1871, + 1, + 49, + 0, + 1871, + 1, + 48, + 0, + 1872, + 1, + 48, + 0, + 1872, + 1, + 47, + 0, + 1873, + 1, + 46, + 0, + 1874, + 1, + 45, + 0, + 1875, + 1, + 43, + 0, + 1877, + 1, + 41, + 0, + 1880, + 1, + 35, + 0, + 1885, + 1, + 33, + 0, + 1887, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1895, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1897, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1899, + 1, + 20, + 0, + 1901, + 1, + 18, + 0, + 1903, + 1, + 16, + 0, + 1906, + 1, + 12, + 0, + 1910, + 1, + 8, + 0, + 1549186 + ], + "mask_annotation_ids_mapping": { + "d830153a-94e4-434f-8cd1-cbd6c4a717da": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_annotations_with_item_level_properties/image_8.json b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_8.json new file mode 100644 index 000000000..f7fe05564 --- /dev/null +++ b/e2e_tests/data/import/image_annotations_with_item_level_properties/image_8.json @@ -0,0 +1,483 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_8", + "path": "/dir1/dir3", + "source_info": { + "item_id": "01920b92-1d5e-908e-7b24-3d339ea72237", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5e-908e-7b24-3d339ea72237" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/ace6c9a2-d39a-43df-9fd2-9f124176810a/thumbnail", + "source_files": [ + { + "file_name": "image_8", + "storage_key": "darwin-py/images/image_8.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/141cdb56-2494-4052-bce2-b22673e6ad68" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "be4fc02d-10ae-489c-871e-15558f1ac58d", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "29c145f2-1ec7-46fd-9bdf-fd68dd82c48b", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "a32b40ec-258e-47ba-b1c5-7508853df665", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "6d4c8c85-827e-49df-b992-c286b2c2a544", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "e54c1454-2347-4a16-8303-8ee5c40071df", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "d9eb4c46-e7ac-4f48-8a2c-04677a761d3f", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "b347d4e5-6914-42c5-9e5a-479afc198671", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "edacf579-187e-4e66-9187-b04579475d0e", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "567ee389-d874-4606-83c9-49af6eb030a7", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 505214, + 1, + 8, + 0, + 1910, + 1, + 12, + 0, + 1906, + 1, + 16, + 0, + 1903, + 1, + 18, + 0, + 1901, + 1, + 20, + 0, + 1899, + 1, + 22, + 0, + 5, + 1, + 8, + 0, + 1885, + 1, + 22, + 0, + 3, + 1, + 12, + 0, + 1882, + 1, + 40, + 0, + 1880, + 1, + 41, + 0, + 1878, + 1, + 43, + 0, + 1877, + 1, + 44, + 0, + 1876, + 1, + 44, + 0, + 1876, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 46, + 0, + 1874, + 1, + 46, + 0, + 1874, + 1, + 46, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1876, + 1, + 44, + 0, + 1876, + 1, + 44, + 0, + 1877, + 1, + 43, + 0, + 1878, + 1, + 41, + 0, + 1880, + 1, + 40, + 0, + 1882, + 1, + 12, + 0, + 3, + 1, + 22, + 0, + 1885, + 1, + 8, + 0, + 5, + 1, + 22, + 0, + 1899, + 1, + 20, + 0, + 1901, + 1, + 18, + 0, + 1903, + 1, + 16, + 0, + 1906, + 1, + 12, + 0, + 1910, + 1, + 8, + 0, + 1510758 + ], + "mask_annotation_ids_mapping": { + "edacf579-187e-4e66-9187-b04579475d0e": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_new_annotations_with_item_level_properties/.v7/metadata.json b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/.v7/metadata.json new file mode 100644 index 000000000..bdbe11d27 --- /dev/null +++ b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/.v7/metadata.json @@ -0,0 +1,94 @@ +{ + "version": "1.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/metadata/1.0/schema.json", + "classes": [ + { + "name": "bbox", + "type": "bounding_box", + "description": null, + "color": "rgba(255,0,255,1.0)", + "sub_types": [ + "inference" + ], + "properties": [ + { + "name": "ann-lvl-ms", + "type": "multi_select", + "description": "property-created-during-annotation-import", + "required": false, + "property_values": [ + { + "value": "3", + "color": "rgba(200,255,0,1.0)" + }, + { + "value": "1", + "color": "rgba(255,46,0,1.0)" + } + ], + "granularity": "annotation" + }, + { + "name": "ann-lvl-ss", + "type": "single_select", + "description": "property-created-during-annotation-import", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(0,102,255,1.0)" + } + ], + "granularity": "annotation" + }, + { + "name": "sec-lvl-ms", + "type": "multi_select", + "description": "property-created-during-annotation-import", + "required": false, + "property_values": [ + { + "value": "2", + "color": "rgba(255,199,0,1.0)" + } + ] + } + ], + "sub_types_settings": { + "inference": {} + } + } + ], + "properties": [ + { + "name": "new_item_level_property_single_select", + "type": "multi_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(255,255,0,1.0)" + }, + { + "value": "2", + "color": "rgba(85,0,255,1.0)" + } + ], + "granularity": "item" + }, + { + "name": "new_item_level_property_multi_select", + "type": "multi_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(255,0,255,1.0)" + } + ], + "granularity": "item" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_1.json b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_1.json new file mode 100644 index 000000000..68977035f --- /dev/null +++ b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_1.json @@ -0,0 +1,595 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_1", + "path": "/", + "source_info": { + "item_id": "01920b92-1d5d-94a4-6fbe-8a4d7f9fa15d", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-94a4-6fbe-8a4d7f9fa15d" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/2ec69e41-91b2-4155-9b05-6ed995677b1e/thumbnail", + "source_files": [ + { + "file_name": "image_1", + "storage_key": "darwin-py/images/image_1.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/9dfc5eac-bf16-4380-a148-9fff6e63b9f0" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "b92626e7-a1fd-4ecb-a418-31aa37069bc1", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "40288025-5eea-406c-95fb-fd335df117fe", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "de3a1fd0-bff9-4b4a-bffb-b6b92bff18a3", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "d078cdd0-7ff3-4835-9d5d-f4e0c6a31fa6", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "063cd083-e25a-4f0d-82d9-45d0b3c9dedd", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "a446a6dc-6427-413f-8f5d-5af0121f0923", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "e59b6d18-88dc-417a-a619-513c634e1402", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "4f58cca2-e174-4564-b242-1d17042f1b01", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "8596d3bd-d9f1-4663-8cc7-d444e97abb74", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 503228, + 1, + 8, + 0, + 21, + 1, + 10, + 0, + 1879, + 1, + 12, + 0, + 17, + 1, + 14, + 0, + 1875, + 1, + 16, + 0, + 13, + 1, + 18, + 0, + 1872, + 1, + 18, + 0, + 11, + 1, + 20, + 0, + 1870, + 1, + 20, + 0, + 9, + 1, + 22, + 0, + 7, + 1, + 11, + 0, + 1850, + 1, + 22, + 0, + 7, + 1, + 27, + 0, + 1, + 1, + 18, + 0, + 1845, + 1, + 22, + 0, + 7, + 1, + 48, + 0, + 1842, + 1, + 24, + 0, + 5, + 1, + 51, + 0, + 1840, + 1, + 24, + 0, + 5, + 1, + 52, + 0, + 1838, + 1, + 26, + 0, + 3, + 1, + 54, + 0, + 1837, + 1, + 26, + 0, + 3, + 1, + 55, + 0, + 1836, + 1, + 26, + 0, + 3, + 1, + 55, + 0, + 1836, + 1, + 26, + 0, + 3, + 1, + 56, + 0, + 1835, + 1, + 26, + 0, + 3, + 1, + 56, + 0, + 1835, + 1, + 26, + 0, + 3, + 1, + 57, + 0, + 1834, + 1, + 26, + 0, + 3, + 1, + 57, + 0, + 1834, + 1, + 26, + 0, + 3, + 1, + 57, + 0, + 1834, + 1, + 26, + 0, + 4, + 1, + 56, + 0, + 1834, + 1, + 26, + 0, + 4, + 1, + 56, + 0, + 1835, + 1, + 24, + 0, + 6, + 1, + 55, + 0, + 1835, + 1, + 24, + 0, + 6, + 1, + 55, + 0, + 1836, + 1, + 22, + 0, + 8, + 1, + 54, + 0, + 1836, + 1, + 22, + 0, + 9, + 1, + 52, + 0, + 1838, + 1, + 20, + 0, + 11, + 1, + 51, + 0, + 1839, + 1, + 18, + 0, + 14, + 1, + 48, + 0, + 1841, + 1, + 16, + 0, + 17, + 1, + 46, + 0, + 1843, + 1, + 12, + 0, + 23, + 1, + 41, + 0, + 1846, + 1, + 8, + 0, + 26, + 1, + 39, + 0, + 1882, + 1, + 37, + 0, + 1885, + 1, + 14, + 0, + 1, + 1, + 18, + 0, + 1889, + 1, + 10, + 0, + 8, + 1, + 11, + 0, + 1512704 + ], + "mask_annotation_ids_mapping": { + "4f58cca2-e174-4564-b242-1d17042f1b01": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "new_item_level_property_multi_select", + "value": "1" + }, + { + "name": "new_item_level_property_multi_select", + "value": "2" + }, + { + "name": "new_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_2.json b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_2.json new file mode 100644 index 000000000..4c493508c --- /dev/null +++ b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_2.json @@ -0,0 +1,487 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_2", + "path": "/", + "source_info": { + "item_id": "01920b92-1d5d-ea77-8fa4-16378bafedb3", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-ea77-8fa4-16378bafedb3" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/5e0b3d9d-9bf8-4166-8949-6ab7392161ad/thumbnail", + "source_files": [ + { + "file_name": "image_2", + "storage_key": "darwin-py/images/image_2.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/4920b12a-1706-47f1-b084-2d2234ed1151" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "81295639-1b29-4681-b3a8-53119bf49036", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "f3b20d5d-987a-44a4-94ac-41acf2ac14fd", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "1220a567-c329-4bca-b955-56387a3f50a2", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "3f01b473-f88f-4f3f-b6f6-559feff994f8", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "4c6e632c-f519-41ef-a918-462e1745f30e", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "d9d250a0-98b9-4b5e-85a2-c1ad7c0e4b2d", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "02d3df0e-8f01-4ba6-a790-954d3c892698", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "c389c7c5-0a92-48a0-b991-cf26b351111d", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "69c45b6b-83c1-4a0a-9210-03754a4e823e", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 470662, + 1, + 12, + 0, + 1906, + 1, + 16, + 0, + 1902, + 1, + 20, + 0, + 1899, + 1, + 22, + 0, + 1896, + 1, + 25, + 0, + 1893, + 1, + 28, + 0, + 1890, + 1, + 30, + 0, + 1888, + 1, + 33, + 0, + 1886, + 1, + 34, + 0, + 1885, + 1, + 36, + 0, + 1883, + 1, + 37, + 0, + 1883, + 1, + 37, + 0, + 1882, + 1, + 38, + 0, + 1881, + 1, + 39, + 0, + 1881, + 1, + 39, + 0, + 1880, + 1, + 40, + 0, + 1880, + 1, + 40, + 0, + 1879, + 1, + 40, + 0, + 1880, + 1, + 40, + 0, + 1879, + 1, + 40, + 0, + 1880, + 1, + 40, + 0, + 1880, + 1, + 39, + 0, + 1881, + 1, + 38, + 0, + 1882, + 1, + 37, + 0, + 1883, + 1, + 35, + 0, + 1885, + 1, + 33, + 0, + 1887, + 1, + 31, + 0, + 1890, + 1, + 28, + 0, + 1892, + 1, + 25, + 0, + 1896, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1899, + 1, + 20, + 0, + 1901, + 1, + 18, + 0, + 1903, + 1, + 16, + 0, + 1906, + 1, + 12, + 0, + 1910, + 1, + 8, + 0, + 1535742 + ], + "mask_annotation_ids_mapping": { + "c389c7c5-0a92-48a0-b991-cf26b351111d": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "new_item_level_property_multi_select", + "value": "1" + }, + { + "name": "new_item_level_property_multi_select", + "value": "2" + }, + { + "name": "new_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_3.json b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_3.json new file mode 100644 index 000000000..4a119ccda --- /dev/null +++ b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_3.json @@ -0,0 +1,531 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_3", + "path": "/dir1", + "source_info": { + "item_id": "01920b92-1d5d-e8ad-986f-ad4942f1bbfc", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-e8ad-986f-ad4942f1bbfc" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/ddd13905-9bbb-4fab-9642-bf4604686fda/thumbnail", + "source_files": [ + { + "file_name": "image_3", + "storage_key": "darwin-py/images/image_3.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/30ec0f13-caaa-4374-be5a-e90b3493fb73" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "6485cc18-0fc6-4a64-9794-474fd7040767", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "f133a6c1-8de2-47f8-a295-e1978e9fda05", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "3088bafc-a6b8-46e1-864a-ea49ce6d46c9", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "a6786798-3d81-4c72-adcb-34a4615c33ce", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "67eb19f3-66cb-4910-8314-8cfdd6bbf34e", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "77cb2ca5-c010-48c9-a68d-d3e08ed78eae", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "318737ce-0f94-40d8-96aa-a616c1e5999c", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "b92d8bad-ace9-4f73-b428-9505eb960016", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "2234893b-eea8-4b90-a2d1-c0ddf32b2918", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 543631, + 1, + 8, + 0, + 1910, + 1, + 12, + 0, + 1906, + 1, + 16, + 0, + 1903, + 1, + 18, + 0, + 1901, + 1, + 20, + 0, + 1899, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1897, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1895, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1895, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1897, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1897, + 1, + 22, + 0, + 1897, + 1, + 22, + 0, + 1897, + 1, + 22, + 0, + 1897, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1897, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1895, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1895, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1897, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1899, + 1, + 20, + 0, + 1901, + 1, + 18, + 0, + 1903, + 1, + 16, + 0, + 1906, + 1, + 12, + 0, + 1910, + 1, + 8, + 0, + 1441645 + ], + "mask_annotation_ids_mapping": { + "b92d8bad-ace9-4f73-b428-9505eb960016": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "new_item_level_property_multi_select", + "value": "1" + }, + { + "name": "new_item_level_property_multi_select", + "value": "2" + }, + { + "name": "new_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_4.json b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_4.json new file mode 100644 index 000000000..449692551 --- /dev/null +++ b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_4.json @@ -0,0 +1,583 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_4", + "path": "/dir1", + "source_info": { + "item_id": "01920b92-1d5d-8b50-17e9-c0f178e6eee6", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-8b50-17e9-c0f178e6eee6" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/3c731d84-7d7f-4ac8-bbd9-0d53f1d47195/thumbnail", + "source_files": [ + { + "file_name": "image_4", + "storage_key": "darwin-py/images/image_4.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/609ba1a4-79da-4743-b331-e57ccd9ee518" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "c7ab6a60-d36d-4fb9-b95d-b46002093b5c", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "02f6e8c9-9ae4-4ff2-a4d2-1d8a15476c04", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "21b82723-e87c-43b5-8eaa-49f1933a6e1a", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "8a7e04f5-76b8-4996-a19e-a864122ad2a0", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "173e6ec8-53d9-4ae5-8a01-5278d4573662", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "2c851bb5-52f1-417b-8aac-23dc2083366f", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "02be609a-e456-4cbf-a704-875a2f0246e3", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "aa057965-2b8b-4c33-931e-3761eda459d6", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "6660192e-76cb-412b-8f8d-fffef3cbb223", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 476334, + 1, + 9, + 0, + 6, + 1, + 8, + 0, + 1895, + 1, + 13, + 0, + 2, + 1, + 12, + 0, + 1891, + 1, + 31, + 0, + 1888, + 1, + 33, + 0, + 1886, + 1, + 35, + 0, + 1884, + 1, + 37, + 0, + 1883, + 1, + 37, + 0, + 1882, + 1, + 39, + 0, + 1881, + 1, + 39, + 0, + 1880, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1880, + 1, + 40, + 0, + 1880, + 1, + 40, + 0, + 1881, + 1, + 38, + 0, + 1882, + 1, + 38, + 0, + 1883, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1885, + 1, + 34, + 0, + 23, + 1, + 8, + 0, + 1856, + 1, + 32, + 0, + 22, + 1, + 12, + 0, + 1855, + 1, + 30, + 0, + 21, + 1, + 16, + 0, + 1855, + 1, + 12, + 0, + 2, + 1, + 12, + 0, + 22, + 1, + 18, + 0, + 1856, + 1, + 8, + 0, + 6, + 1, + 8, + 0, + 23, + 1, + 21, + 0, + 1898, + 1, + 23, + 0, + 1897, + 1, + 24, + 0, + 1895, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1893, + 1, + 28, + 0, + 1892, + 1, + 28, + 0, + 1892, + 1, + 29, + 0, + 1891, + 1, + 29, + 0, + 1891, + 1, + 29, + 0, + 1891, + 1, + 29, + 0, + 1891, + 1, + 29, + 0, + 1891, + 1, + 29, + 0, + 1892, + 1, + 28, + 0, + 1892, + 1, + 28, + 0, + 1892, + 1, + 27, + 0, + 1894, + 1, + 26, + 0, + 1895, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1897, + 1, + 22, + 0, + 1899, + 1, + 20, + 0, + 1901, + 1, + 18, + 0, + 1904, + 1, + 14, + 0, + 1908, + 1, + 10, + 0, + 1501203 + ], + "mask_annotation_ids_mapping": { + "aa057965-2b8b-4c33-931e-3761eda459d6": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "new_item_level_property_multi_select", + "value": "1" + }, + { + "name": "new_item_level_property_multi_select", + "value": "2" + }, + { + "name": "new_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_5.json b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_5.json new file mode 100644 index 000000000..2195af073 --- /dev/null +++ b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_5.json @@ -0,0 +1,455 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_5", + "path": "/dir2", + "source_info": { + "item_id": "01920b92-1d5d-55bf-d705-8b39dea7fde6", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-55bf-d705-8b39dea7fde6" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/8f95e81c-def7-4973-9152-6d0fc39e1473/thumbnail", + "source_files": [ + { + "file_name": "image_5", + "storage_key": "darwin-py/images/image_5.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/08448a07-4e23-41f9-abbd-0dc149ef2be4" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "0d920fcc-5ec8-4760-8322-61e4de28aa3e", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "e0d23fd6-2a77-4d14-b178-b42a082cf54d", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "5faa5611-59ba-4d1b-8a1d-e6a451ab5506", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "aa95ba32-2374-4981-93c7-7447d846ca03", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "64d2a78a-0cb7-4849-9202-0901e78caedc", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "657d85c4-7c2a-4874-bcbf-bb15db32aee9", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "a9bf3ac0-8742-43e9-aa7f-46b9618d3cec", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "11cff11e-43a7-4b31-83ab-720fdcc84dd7", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "d07c5cf8-3b3f-489e-be44-927f8c98c7df", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 497558, + 1, + 9, + 0, + 1, + 1, + 8, + 0, + 1900, + 1, + 22, + 0, + 1896, + 1, + 26, + 0, + 1893, + 1, + 28, + 0, + 1891, + 1, + 30, + 0, + 1889, + 1, + 32, + 0, + 1888, + 1, + 32, + 0, + 1887, + 1, + 34, + 0, + 1886, + 1, + 34, + 0, + 1885, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1884, + 1, + 36, + 0, + 1885, + 1, + 35, + 0, + 1885, + 1, + 34, + 0, + 1887, + 1, + 33, + 0, + 1887, + 1, + 32, + 0, + 1889, + 1, + 31, + 0, + 1890, + 1, + 29, + 0, + 1892, + 1, + 27, + 0, + 1895, + 1, + 24, + 0, + 1898, + 1, + 20, + 0, + 1910, + 1, + 8, + 0, + 1526104 + ], + "mask_annotation_ids_mapping": { + "11cff11e-43a7-4b31-83ab-720fdcc84dd7": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "test_item_level_property_multi_select", + "value": "1" + }, + { + "name": "test_item_level_property_multi_select", + "value": "2" + }, + { + "name": "test_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_6.json b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_6.json new file mode 100644 index 000000000..2b14415b8 --- /dev/null +++ b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_6.json @@ -0,0 +1,467 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_6", + "path": "/dir2", + "source_info": { + "item_id": "01920b92-1d5d-1832-3a09-1f38557c57b4", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-1832-3a09-1f38557c57b4" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/4950b608-00a1-4e73-b746-bfe1ea0a1ab6/thumbnail", + "source_files": [ + { + "file_name": "image_6", + "storage_key": "darwin-py/images/image_6.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/9e070e8c-03b3-40b7-a3cb-6da6bcc8d4ed" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "8a9dcdc5-e584-45f2-bf71-2f143d11632b", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "cda6055b-cddc-49bb-a322-7a687855518f", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "6f090adc-ecc7-4d01-98c7-89ca688b57f9", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "8f37515b-6cad-4c3e-895b-951431348986", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "4fd320f4-745b-479f-9641-b091f983c393", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "c2a937af-c277-4e80-851b-34c314019aed", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "2f398d01-4099-4b6a-887d-3ac850fd33d9", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "108f0fd2-f6ea-4659-9aa4-1038a8c473ec", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "3334b185-2c1a-4af2-a3eb-619a542c17fe", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 462946, + 1, + 8, + 0, + 1910, + 1, + 12, + 0, + 1906, + 1, + 16, + 0, + 1903, + 1, + 18, + 0, + 1898, + 1, + 27, + 0, + 1891, + 1, + 31, + 0, + 1887, + 1, + 35, + 0, + 1884, + 1, + 37, + 0, + 1882, + 1, + 39, + 0, + 1880, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1878, + 1, + 43, + 0, + 1877, + 1, + 43, + 0, + 1876, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1876, + 1, + 43, + 0, + 1877, + 1, + 43, + 0, + 1878, + 1, + 41, + 0, + 1879, + 1, + 41, + 0, + 1880, + 1, + 39, + 0, + 1882, + 1, + 37, + 0, + 1884, + 1, + 35, + 0, + 1887, + 1, + 31, + 0, + 1891, + 1, + 17, + 0, + 2, + 1, + 8, + 0, + 1554956 + ], + "mask_annotation_ids_mapping": { + "108f0fd2-f6ea-4659-9aa4-1038a8c473ec": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "new_item_level_property_multi_select", + "value": "1" + }, + { + "name": "new_item_level_property_multi_select", + "value": "2" + }, + { + "name": "new_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_7.json b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_7.json new file mode 100644 index 000000000..42b1d8a20 --- /dev/null +++ b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_7.json @@ -0,0 +1,507 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_7", + "path": "/dir1/dir3", + "source_info": { + "item_id": "01920b92-1d5d-46ee-5117-53ba0d29d1b0", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5d-46ee-5117-53ba0d29d1b0" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/1e2f63eb-b7fc-482f-91f3-8caa242e63cb/thumbnail", + "source_files": [ + { + "file_name": "image_7", + "storage_key": "darwin-py/images/image_7.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/20de7c08-20dc-4f16-b559-bbcce2f7b319" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "0a662491-8524-460c-b0a1-463e3def1ce2", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "45b1762d-261b-4af4-a7b3-9250354fbb44", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "5a7b975a-64a6-4eba-89c7-4be161477807", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "0cbe2fe4-989d-420d-a96a-00586ee930dc", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "4024a7ec-a372-48d7-a9ee-604be710548c", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "b76613e5-72ba-4f2b-8bb7-4cfbd8a1b816", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "3b4dc955-bd5e-43d9-bd73-bfcb16e8d5dc", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "d830153a-94e4-434f-8cd1-cbd6c4a717da", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "ccde37ee-00af-4a85-a4ff-0fe8bde1b89b", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 447629, + 1, + 8, + 0, + 1905, + 1, + 17, + 0, + 1901, + 1, + 21, + 0, + 1897, + 1, + 24, + 0, + 1895, + 1, + 26, + 0, + 1893, + 1, + 28, + 0, + 1891, + 1, + 29, + 0, + 1891, + 1, + 30, + 0, + 1889, + 1, + 31, + 0, + 1889, + 1, + 32, + 0, + 1879, + 1, + 41, + 0, + 1877, + 1, + 43, + 0, + 1875, + 1, + 45, + 0, + 1874, + 1, + 46, + 0, + 1873, + 1, + 47, + 0, + 1872, + 1, + 48, + 0, + 1872, + 1, + 48, + 0, + 1871, + 1, + 49, + 0, + 1871, + 1, + 48, + 0, + 1871, + 1, + 49, + 0, + 1871, + 1, + 48, + 0, + 1872, + 1, + 48, + 0, + 1872, + 1, + 47, + 0, + 1873, + 1, + 46, + 0, + 1874, + 1, + 45, + 0, + 1875, + 1, + 43, + 0, + 1877, + 1, + 41, + 0, + 1880, + 1, + 35, + 0, + 1885, + 1, + 33, + 0, + 1887, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1894, + 1, + 26, + 0, + 1895, + 1, + 24, + 0, + 1896, + 1, + 24, + 0, + 1897, + 1, + 22, + 0, + 1898, + 1, + 22, + 0, + 1899, + 1, + 20, + 0, + 1901, + 1, + 18, + 0, + 1903, + 1, + 16, + 0, + 1906, + 1, + 12, + 0, + 1910, + 1, + 8, + 0, + 1549186 + ], + "mask_annotation_ids_mapping": { + "d830153a-94e4-434f-8cd1-cbd6c4a717da": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "new_item_level_property_multi_select", + "value": "1" + }, + { + "name": "new_item_level_property_multi_select", + "value": "2" + }, + { + "name": "new_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_8.json b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_8.json new file mode 100644 index 000000000..edf2143dc --- /dev/null +++ b/e2e_tests/data/import/image_new_annotations_with_item_level_properties/image_8.json @@ -0,0 +1,483 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "image_8", + "path": "/dir1/dir3", + "source_info": { + "item_id": "01920b92-1d5e-908e-7b24-3d339ea72237", + "dataset": { + "name": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "slug": "test_dataset_2edf4430-1a35-45a2-8c45-b0325968bee2", + "dataset_management_url": "https://staging.v7labs.com/datasets/339501/dataset-management" + }, + "team": { + "name": "E2E Testing", + "slug": "e2e-testing" + }, + "workview_url": "https://staging.v7labs.com/workview?dataset=339501&item=01920b92-1d5e-908e-7b24-3d339ea72237" + }, + "slots": [ + { + "type": "image", + "slot_name": "0", + "width": 1920, + "height": 1080, + "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/ace6c9a2-d39a-43df-9fd2-9f124176810a/thumbnail", + "source_files": [ + { + "file_name": "image_8", + "storage_key": "darwin-py/images/image_8.jpg", + "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/141cdb56-2494-4052-bce2-b22673e6ad68" + } + ] + } + ] + }, + "annotations": [ + { + "bounding_box": { + "h": 58.941, + "w": 216.693, + "x": 280.449, + "y": 173.355 + }, + "id": "be4fc02d-10ae-489c-871e-15558f1ac58d", + "instance_id": { + "value": 2 + }, + "name": "test_bounding_box_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "ellipse": { + "angle": 0.3985, + "center": { + "x": 133.4425, + "y": 69.1719 + }, + "radius": { + "x": 18.3176, + "y": 18.3176 + } + }, + "id": "29c145f2-1ec7-46fd-9bdf-fd68dd82c48b", + "instance_id": { + "value": 3 + }, + "name": "test_ellipse_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "a32b40ec-258e-47ba-b1c5-7508853df665", + "instance_id": { + "value": 4 + }, + "keypoint": { + "x": 49.6239, + "y": 173.4268 + }, + "name": "test_keypoint_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "6d4c8c85-827e-49df-b992-c286b2c2a544", + "instance_id": { + "value": 5 + }, + "line": { + "path": [ + { + "x": 142.624, + "y": -0.1339 + }, + { + "x": 171.0572, + "y": -45.1531 + }, + { + "x": 199.4903, + "y": 0.4584 + } + ] + }, + "name": "test_line_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "bounding_box": { + "h": 35.85509999999999, + "w": 108.1345, + "x": 275.1268, + "y": 30.7501 + }, + "directional_vector": { + "angle": -1.6494, + "length": 118.3181 + }, + "id": "e54c1454-2347-4a16-8303-8ee5c40071df", + "instance_id": { + "value": 6 + }, + "name": "test_polygon_with_subtypes_and_properties", + "polygon": { + "paths": [ + [ + { + "x": 336.0236, + "y": 30.7501 + }, + { + "x": 304.7215, + "y": 64.8979 + }, + { + "x": 383.2613, + "y": 66.6052 + }, + { + "x": 275.1268, + "y": 34.734 + } + ] + ] + }, + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "2" + } + ], + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "d9eb4c46-e7ac-4f48-8a2c-04677a761d3f", + "instance_id": { + "value": 7 + }, + "name": "test_skeleton_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + } + ], + "skeleton": { + "nodes": [ + { + "name": "node", + "occluded": false, + "x": 301.369, + "y": 175.5217 + }, + { + "name": "2", + "occluded": false, + "x": 341.4465, + "y": 161.2866 + } + ] + }, + "slot_names": [ + "0" + ], + "text": { + "text": "test_text" + } + }, + { + "id": "b347d4e5-6914-42c5-9e5a-479afc198671", + "name": "test_tag_with_subtypes_and_properties", + "properties": [ + { + "frame_index": 0, + "name": "multi_select-1", + "value": "1" + }, + { + "frame_index": 0, + "name": "multi_select-1", + "value": "2" + }, + { + "frame_index": 0, + "name": "single_select-1", + "value": "1" + } + ], + "slot_names": [ + "0" + ], + "tag": {}, + "text": { + "text": "test_text" + } + }, + { + "id": "edacf579-187e-4e66-9187-b04579475d0e", + "mask": {}, + "name": "test_mask_with_subtypes_and_properties", + "properties": [], + "slot_names": [ + "0" + ] + }, + { + "id": "567ee389-d874-4606-83c9-49af6eb030a7", + "name": "__raster_layer__", + "properties": [], + "raster_layer": { + "dense_rle": [ + 0, + 505214, + 1, + 8, + 0, + 1910, + 1, + 12, + 0, + 1906, + 1, + 16, + 0, + 1903, + 1, + 18, + 0, + 1901, + 1, + 20, + 0, + 1899, + 1, + 22, + 0, + 5, + 1, + 8, + 0, + 1885, + 1, + 22, + 0, + 3, + 1, + 12, + 0, + 1882, + 1, + 40, + 0, + 1880, + 1, + 41, + 0, + 1878, + 1, + 43, + 0, + 1877, + 1, + 44, + 0, + 1876, + 1, + 44, + 0, + 1876, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 46, + 0, + 1874, + 1, + 46, + 0, + 1874, + 1, + 46, + 0, + 1875, + 1, + 45, + 0, + 1875, + 1, + 45, + 0, + 1876, + 1, + 44, + 0, + 1876, + 1, + 44, + 0, + 1877, + 1, + 43, + 0, + 1878, + 1, + 41, + 0, + 1880, + 1, + 40, + 0, + 1882, + 1, + 12, + 0, + 3, + 1, + 22, + 0, + 1885, + 1, + 8, + 0, + 5, + 1, + 22, + 0, + 1899, + 1, + 20, + 0, + 1901, + 1, + 18, + 0, + 1903, + 1, + 16, + 0, + 1906, + 1, + 12, + 0, + 1910, + 1, + 8, + 0, + 1510758 + ], + "mask_annotation_ids_mapping": { + "edacf579-187e-4e66-9187-b04579475d0e": 1 + }, + "total_pixels": 2073600 + }, + "slot_names": [ + "0" + ] + } + ], + "properties": [ + { + "name": "new_item_level_property_multi_select", + "value": "1" + }, + { + "name": "new_item_level_property_multi_select", + "value": "2" + }, + { + "name": "new_item_level_property_single_select", + "value": "1" + } + ] +} \ No newline at end of file diff --git a/e2e_tests/objects.py b/e2e_tests/objects.py index cece8b321..c89c89182 100644 --- a/e2e_tests/objects.py +++ b/e2e_tests/objects.py @@ -22,6 +22,14 @@ class E2EAnnotationClass: id: int +class E2EItemLevelProperty: + def __init__(self, name: str, dataset_ids: List[int], type: str, id: str): + self.name = name + self.dataset_ids = dataset_ids + self.type = type + self.id = id + + @dataclass class E2EItem(Exception): name: str diff --git a/e2e_tests/setup_tests.py b/e2e_tests/setup_tests.py index fd45d6f11..e5cc543cb 100644 --- a/e2e_tests/setup_tests.py +++ b/e2e_tests/setup_tests.py @@ -11,7 +11,13 @@ from PIL import Image from e2e_tests.exceptions import DataAlreadyExists, E2EException -from e2e_tests.objects import ConfigValues, E2EAnnotationClass, E2EDataset, E2EItem +from e2e_tests.objects import ( + ConfigValues, + E2EAnnotationClass, + E2EDataset, + E2EItem, + E2EItemLevelProperty, +) def api_call( @@ -247,6 +253,52 @@ def create_annotation_class( ) +def create_item_level_property( + name: str, item_level_property_type: str, config: ConfigValues +) -> E2EItemLevelProperty: + """ + Creates a single item-level property and returns a corresponding E2EItemLevelProperty + + Parameters + ---------- + name : str + The name of the item-level property to create + item_level_property_type: str + The type of item-level property to create. Must be `single_select` or `multi_select` + config : ConfigValues + The config values to use + + Returns + ------- + E2EItemLevelProperty + The minimum info about the created item-level property + """ + url = f"{config.server}/api/v2/teams/{config.team_slug}/properties" + + headers = { + "accept": "application/json", + "content-type": "application/json", + "Authorization": f"ApiKey {config.api_key}", + } + payload = { + "name": name, + "type": item_level_property_type, + "granularity": "item", + "property_values": [ + {"color": "rgba(255,92,0,1.0)", "value": "1"}, + {"color": "rgba(255,92,0,1.0)", "value": "2"}, + ], + } + response = requests.post(url, json=payload, headers=headers) + parsed_response = response.json() + return E2EItemLevelProperty( + name=parsed_response["name"], + dataset_ids=parsed_response["dataset_ids"], + type=parsed_response["type"], + id=parsed_response["id"], + ) + + def delete_annotation_class(id: str, config: ConfigValues) -> None: """ Delete an annotation class on the server @@ -276,6 +328,30 @@ def delete_annotation_class(id: str, config: ConfigValues) -> None: pytest.exit("Test run failed in test setup stage") +def delete_item_level_property(id: str, config: ConfigValues) -> None: + """ + Delete an item-level property class on the server + + Parameters: + ----------- + id : str + The id of the item-level property to delete + config : ConfigValues + The config values to use + """ + host, api_key, team_slug = config.server, config.api_key, config.team_slug + url = f"{host}/api/v2/teams/{team_slug}/properties/{id}" + try: + response = api_call("delete", url, None, api_key) + if not response.ok: + raise E2EException( + f"Failed to delete item-level property {id} - {response.status_code} - {response.text}" + ) + except Exception as e: + print(f"Failed to delete item-level property with ID: {id} - {e}") + pytest.exit("Test run failed in test setup stage") + + def create_item( dataset_slug: str, prefix: str, image: Path, config: ConfigValues ) -> E2EItem: @@ -439,7 +515,7 @@ def setup_datasets(config: ConfigValues) -> List[E2EDataset]: def setup_annotation_classes(config: ConfigValues) -> List[E2EAnnotationClass]: """ - Setup data for End to end test runs + Setup annotation classes for end to end test runs Parameters ---------- @@ -449,7 +525,7 @@ def setup_annotation_classes(config: ConfigValues) -> List[E2EAnnotationClass]: Returns ------- List[E2EAnnotationClass] - The minimal info about the created annotation classes + The minimum info about the created annotation classes """ annotation_classes: List[E2EAnnotationClass] = [] @@ -491,7 +567,7 @@ def setup_annotation_classes(config: ConfigValues) -> List[E2EAnnotationClass]: pass except E2EException as e: print(e) - pytest.exit("Test run failed in test setup stage") + pytest.exit("Test run failed while setting up annotation classes") except Exception as e: print(e) @@ -500,6 +576,45 @@ def setup_annotation_classes(config: ConfigValues) -> List[E2EAnnotationClass]: return annotation_classes +def setup_item_level_properties(config: ConfigValues) -> List[E2EItemLevelProperty]: + """ + Setup item-level properties for end to end test runs + + Parameters + ---------- + config : ConfigValues + The config values to use + + Returns + ------- + List[E2EItemLevelProperty] + The minimal info about the created item-level properties + """ + item_level_properties: List[E2EItemLevelProperty] = [] + + print("Setting up item-level properties") + item_level_property_types = ["single_select", "multi_select"] + try: + for item_level_property_type in item_level_property_types: + try: + item_level_property = create_item_level_property( + f"test_item_level_property_{item_level_property_type}", + item_level_property_type=item_level_property_type, + config=config, + ) + item_level_properties.append(item_level_property) + except DataAlreadyExists: + pass + except E2EException as e: + print(e) + + except Exception as e: + print(e) + pytest.exit("Setup failed - unknown error") + + return item_level_properties + + def teardown_tests(config: ConfigValues, datasets: List[E2EDataset]) -> None: """ Teardown data for End to end test runs @@ -634,3 +749,24 @@ def teardown_annotation_classes( "name" ].startswith("new_"): delete_annotation_class(annotation_class["id"], config) + + +def teardown_item_level_properties( + config: ConfigValues, item_level_properties: List[E2EItemLevelProperty] +) -> None: + for item_level_property in item_level_properties: + delete_item_level_property(str(item_level_property.id), config) + team_slug = config.team_slug + host = config.server + response = api_call( + "get", f"{host}/api/v2/teams/{team_slug}/properties", None, config.api_key + ) + all_properties = response.json()["properties"] + all_item_level_properties = ( + all_properties # Code to filter response for item-level properties + ) + for item_level_property in all_item_level_properties: + if item_level_property["name"].startswith("test_") or item_level_property[ + "name" + ].startswith("new_"): + delete_item_level_property(item_level_property["id"], config) diff --git a/tests/darwin/client_test.py b/tests/darwin/client_test.py index c2f6b4903..5b97fb8db 100644 --- a/tests/darwin/client_test.py +++ b/tests/darwin/client_test.py @@ -375,10 +375,10 @@ def test_get_team_properties(self, darwin_client: Client) -> None: }, ], "required": False, + "granularity": "section", "slug": "property-question", "team_id": 128, "type": "multi_select", - "granularity": "section", }, ] }, @@ -397,7 +397,7 @@ def test_create_property( responses.add( responses.POST, "http://localhost/apiv2/teams/v7-darwin-json-v2/properties", - json=base_property_object.dict(), + json=base_property_object.model_dump(mode="json"), status=200, ) _property = darwin_client.create_property( @@ -413,11 +413,12 @@ def test_create_property_from_json( responses.add( responses.POST, "http://localhost/apiv2/teams/v7-darwin-json-v2/properties", - json=base_property_object.dict(), + json=base_property_object.model_dump(mode="json"), status=200, ) _property = darwin_client.create_property( - team_slug="v7-darwin-json-v2", params=base_property_object.dict() + team_slug="v7-darwin-json-v2", + params=base_property_object.model_dump(mode="json"), ) assert isinstance(_property, FullProperty) assert _property == base_property_object @@ -433,7 +434,7 @@ def test_update_property( responses.add( responses.PUT, f"http://localhost/apiv2/teams/v7-darwin-json-v2/properties/{property_id}", - json=base_property_object.dict(), + json=base_property_object.model_dump(mode="json"), status=200, ) _property = darwin_client.update_property( @@ -450,11 +451,12 @@ def test_update_property_from_json( responses.add( responses.PUT, f"http://localhost/apiv2/teams/v7-darwin-json-v2/properties/{property_id}", - json=base_property_object.dict(), + json=base_property_object.model_dump(mode="json"), status=200, ) _property = darwin_client.update_property( - team_slug="v7-darwin-json-v2", params=base_property_object.dict() + team_slug="v7-darwin-json-v2", + params=base_property_object.model_dump(mode="json"), ) assert isinstance(_property, FullProperty) assert _property == base_property_object diff --git a/tests/darwin/data/metadata.json b/tests/darwin/data/metadata.json index 23e765167..1048d1332 100644 --- a/tests/darwin/data/metadata.json +++ b/tests/darwin/data/metadata.json @@ -28,7 +28,8 @@ "color": "rgba(0, 0, 255, 0)" } ], - "required": true + "required": true, + "granularity": "section" }, { "name": "Shape (expanded format)", @@ -44,7 +45,8 @@ "color": "rgba(150, 150, 150, 0)" } ], - "required": false + "required": false, + "granularity": "section" } ] } diff --git a/tests/darwin/data/metadata_nested_properties.json b/tests/darwin/data/metadata_nested_properties.json index c6639e1ba..3b835a7ce 100644 --- a/tests/darwin/data/metadata_nested_properties.json +++ b/tests/darwin/data/metadata_nested_properties.json @@ -53,6 +53,7 @@ "type": "string" } ], + "granularity": "section", "required": true } ] diff --git a/tests/darwin/importer/importer_test.py b/tests/darwin/importer/importer_test.py index cb4ca3579..ac4492ae8 100644 --- a/tests/darwin/importer/importer_test.py +++ b/tests/darwin/importer/importer_test.py @@ -16,8 +16,10 @@ from darwin import datatypes as dt from darwin.importer import get_importer from darwin.importer.importer import ( + _assign_item_properties_to_dataset, _build_attribute_lookup, _build_main_annotations_lookup_table, + _create_update_item_properties, _display_slot_warnings_and_errors, _find_and_parse, _get_annotation_format, @@ -34,6 +36,45 @@ ) +@pytest.fixture +def mock_client(): + client = Mock() + client.default_team = "test_team" + return client + + +@pytest.fixture +def mock_dataset(mock_client): + dataset = Mock() + dataset.team = mock_client.default_team + dataset.dataset_id = 123456 + return dataset + + +@pytest.fixture +def mock_console(): + return Mock() + + +@pytest.fixture +def annotation_class_ids_map(): + return {("test_class", "bbox"): "1337"} + + +@pytest.fixture +def annotations(): + return [Mock()] + + +@pytest.fixture +def item_properties(): + return [ + {"name": "prop1", "value": "1"}, + {"name": "prop2", "value": "2"}, + {"name": "prop2", "value": "3"}, + ] + + @pytest.fixture def setup_data(request, multiple_annotations=False): granularity = request.param @@ -266,6 +307,7 @@ def test_import_annotations() -> None: [], ) ] + item_properties = [] default_slot_name = "test_slot" append = False delete_for_empty = False @@ -298,6 +340,7 @@ def test_import_annotations() -> None: remote_classes, attributes, annotations, + item_properties, default_slot_name, mock_dataset, append, @@ -453,8 +496,8 @@ def test__handle_reviewers() -> None: m.return_value = "test" - op1 = _handle_reviewers(dt.Annotation("class", {}, [], reviewers=[1, 2, 3]), True) # type: ignore - op2 = _handle_reviewers(dt.Annotation("class", {}, [], reviewers=[1, 2, 3]), False) # type: ignore + op1 = _handle_reviewers(True, annotation=dt.Annotation("class", {}, [], reviewers=[1, 2, 3])) # type: ignore + op2 = _handle_reviewers(False, annotation=dt.Annotation("class", {}, [], reviewers=[1, 2, 3])) # type: ignore assert op1 == "test" assert op2 == [] @@ -466,8 +509,8 @@ def test__handle_annotators() -> None: m.return_value = "test" - op1 = _handle_annotators(dt.Annotation("class", {}, [], annotators=[1, 2, 3]), True) # type: ignore - op2 = _handle_annotators(dt.Annotation("class", {}, [], annotators=[1, 2, 3]), False) # type: ignore + op1 = _handle_annotators(True, annotation=dt.Annotation("class", {}, [], annotators=[1, 2, 3])) # type: ignore + op2 = _handle_annotators(False, annotation=dt.Annotation("class", {}, [], annotators=[1, 2, 3])) # type: ignore assert op1 == "test" assert op2 == [] @@ -823,6 +866,7 @@ def test__import_annotations() -> None: {"bbox": {"test_class": "1337"}}, {}, [annotation], + [], "test_slot", mock_dataset, "test_append_in", # type: ignore @@ -841,8 +885,8 @@ def test__import_annotations() -> None: assert mock_hs.call_count == 1 assert mock_gov.call_args_list[0][0][0] == "test_append_in" - assert mock_ha.call_args_list[0][0][1] == "test_import_annotators" - assert mock_hr.call_args_list[0][0][1] == "test_import_reviewers" + assert mock_ha.call_args_list[0][0][0] == "test_import_annotators" + assert mock_hr.call_args_list[0][0][0] == "test_import_reviewers" # Assert handle slot names assert mock_hsn.call_args_list[0][0][0] == annotation @@ -994,6 +1038,1010 @@ def test_overwrite_warning_aborts_import(): assert result is False +import pytest + + +@pytest.mark.skip(reason="Skipping while properties refactor is taking place") +class TestImportItemLevelProperties: + @pytest.mark.skip(reason="Skipping while properties refactor is taking place") + def test_import_properties_creates_missing_item_level_properties_from_annotations_no_manifest( + self, + mock_client, + mock_dataset, + annotations, + annotation_class_ids_map, + item_properties, + ): + with patch( + "darwin.importer.importer._get_team_properties_annotation_lookup" + ) as mock_get_team_props, patch( + "darwin.importer.importer._create_update_item_properties" + ) as mock_create_update_props: + + metadata_path = False + mock_get_team_props.side_effect = [ + ({}, {}), + ({}, {}), + ({}, {}), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="multi_select", + description="property-created-during-annotation-import", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="property-created-during-annotation-import", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="2"), + PropertyValue(type="string", value="3"), + ], + ), + }, + ), + ] + mock_create_update_props.side_effect = _create_update_item_properties + + _import_properties( + metadata_path=metadata_path, + item_properties=item_properties, + client=mock_client, + annotations=annotations, + annotation_class_ids_map=annotation_class_ids_map, + dataset=mock_dataset, + ) + + create_properties_first_call, update_properties_first_call = ( + mock_create_update_props.call_args_list[0][0][0:2] + ) + create_properties_second_call, update_properties_second_call = ( + mock_create_update_props.call_args_list[1][0][0:2] + ) + + assert len(create_properties_first_call) == 0 + assert len(update_properties_first_call) == 0 + assert len(create_properties_second_call) == 2 + assert len(update_properties_second_call) == 0 + + assert create_properties_second_call[0] == FullProperty( + name="prop1", + type="multi_select", + description="property-created-during-annotation-import", + required=False, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + ], + ) + assert create_properties_second_call[1] == FullProperty( + name="prop2", + type="multi_select", + description="property-created-during-annotation-import", + required=False, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="2"), + PropertyValue(type="string", value="3"), + ], + ) + + @pytest.mark.skip(reason="Skipping while properties refactor is taking place") + def test_import_properties_creates_missing_item_level_properties_from_manifest_no_annotations( + self, + mock_client, + mock_dataset, + annotation_class_ids_map, + item_properties, + ): + with patch( + "darwin.importer.importer._get_team_properties_annotation_lookup" + ) as mock_get_team_props, patch( + "darwin.importer.importer._create_update_item_properties" + ) as mock_create_update_props: + + metadata_path = Path( + "darwin/future/tests/data/.v7/metadata_with_item_level_properties.json" + ) + mock_get_team_props.side_effect = [ + ({}, {}), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ] + mock_create_update_props.side_effect = _create_update_item_properties + + _import_properties( + metadata_path=metadata_path, + item_properties={}, + client=mock_client, + annotations=[], + annotation_class_ids_map=annotation_class_ids_map, + dataset=mock_dataset, + ) + + create_properties_first_call, update_properties_first_call = ( + mock_create_update_props.call_args_list[0][0][2:4] + ) + create_properties_second_call, update_properties_second_call = ( + mock_create_update_props.call_args_list[1][0][2:4] + ) + + assert len(create_properties_first_call) == 2 + assert len(update_properties_first_call) == 0 + assert len(create_properties_second_call) == 0 + assert len(update_properties_second_call) == 0 + + assert create_properties_first_call[0] == FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ) + assert create_properties_first_call[1] == FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ) + + @pytest.mark.skip(reason="Skipping while properties refactor is taking place") + def test_import_properties_creates_missing_item_level_properties_from_manifest_and_annotations( + self, + mock_client, + mock_dataset, + annotations, + annotation_class_ids_map, + item_properties, + ): + with patch( + "darwin.importer.importer._get_team_properties_annotation_lookup" + ) as mock_get_team_props, patch( + "darwin.importer.importer._create_update_item_properties" + ) as mock_create_update_props: + + metadata_path = Path( + "darwin/future/tests/data/.v7/metadata_with_item_level_properties.json" + ) + mock_get_team_props.side_effect = [ + ({}, {}), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + PropertyValue(type="string", value="3"), + ], + ), + }, + ), + ({}, {}), + ] + mock_create_update_props.side_effect = _create_update_item_properties + + _import_properties( + metadata_path=metadata_path, + item_properties=item_properties, + client=mock_client, + annotations=annotations, + annotation_class_ids_map=annotation_class_ids_map, + dataset=mock_dataset, + ) + + create_properties_first_call, update_properties_first_call = ( + mock_create_update_props.call_args_list[0][0][2:4] + ) + create_properties_second_call, update_properties_second_call = ( + mock_create_update_props.call_args_list[1][0][2:4] + ) + + assert len(create_properties_first_call) == 2 + assert len(update_properties_first_call) == 0 + assert len(create_properties_second_call) == 0 + assert len(update_properties_second_call) == 1 + + assert create_properties_first_call[0] == FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ) + assert create_properties_first_call[1] == FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ) + assert update_properties_second_call[0] == FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="3"), + ], + ) + + @pytest.mark.skip(reason="Skipping while properties refactor is taking place") + def test_import_properties_creates_missing_item_level_property_values_from_manifest_no_annotations( + self, + mock_client, + mock_dataset, + annotation_class_ids_map, + item_properties, + ): + with patch( + "darwin.importer.importer._get_team_properties_annotation_lookup" + ) as mock_get_team_props, patch( + "darwin.importer.importer._create_update_item_properties" + ) as mock_create_update_props: + + metadata_path = Path( + "darwin/future/tests/data/.v7/metadata_with_item_level_properties.json" + ) + mock_get_team_props.side_effect = [ + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ] + mock_create_update_props.side_effect = _create_update_item_properties + + _import_properties( + metadata_path=metadata_path, + item_properties={}, + client=mock_client, + annotations=[], + annotation_class_ids_map=annotation_class_ids_map, + dataset=mock_dataset, + ) + + create_properties_first_call, update_properties_first_call = ( + mock_create_update_props.call_args_list[0][0][2:4] + ) + create_properties_second_call, update_properties_second_call = ( + mock_create_update_props.call_args_list[1][0][2:4] + ) + + assert len(create_properties_first_call) == 0 + assert len(update_properties_first_call) == 2 + assert len(create_properties_second_call) == 0 + assert len(update_properties_second_call) == 0 + + assert update_properties_first_call[0] == FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="2"), + ], + ) + assert update_properties_first_call[1] == FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + ], + ) + + @pytest.mark.skip(reason="Skipping while properties refactor is taking place") + def test_import_properties_creates_missing_item_level_property_values_from_annotations_no_manifest( + self, + mock_client, + mock_dataset, + annotations, + annotation_class_ids_map, + item_properties, + ): + with patch( + "darwin.importer.importer._get_team_properties_annotation_lookup" + ) as mock_get_team_props, patch( + "darwin.importer.importer._create_update_item_properties" + ) as mock_create_update_props: + + metadata_path = False + mock_get_team_props.side_effect = [ + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + PropertyValue(type="string", value="3"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + PropertyValue(type="string", value="3"), + ], + ), + }, + ), + ] + mock_create_update_props.side_effect = _create_update_item_properties + + _import_properties( + metadata_path=metadata_path, + item_properties=item_properties, + client=mock_client, + annotations=annotations, + annotation_class_ids_map=annotation_class_ids_map, + dataset=mock_dataset, + ) + + create_properties_first_call, update_properties_first_call = ( + mock_create_update_props.call_args_list[0][0][2:4] + ) + create_properties_second_call, update_properties_second_call = ( + mock_create_update_props.call_args_list[1][0][2:4] + ) + + assert len(create_properties_first_call) == 0 + assert len(update_properties_first_call) == 0 + assert len(create_properties_second_call) == 0 + assert len(update_properties_second_call) == 1 + + assert update_properties_second_call[0] == FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="3"), + ], + ) + + @pytest.mark.skip(reason="Skipping while properties refactor is taking place") + def test_import_properties_creates_missing_item_level_property_values_from_manifest_and_annotations( + self, + mock_client, + mock_dataset, + annotations, + annotation_class_ids_map, + item_properties, + ): + with patch( + "darwin.importer.importer._get_team_properties_annotation_lookup" + ) as mock_get_team_props, patch( + "darwin.importer.importer._create_update_item_properties" + ) as mock_create_update_props: + + metadata_path = Path( + "darwin/future/tests/data/.v7/metadata_with_item_level_properties.json" + ) + mock_get_team_props.side_effect = [ + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + }, + ), + ( + {}, + { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + PropertyValue(type="string", value="2"), + PropertyValue(type="string", value="3"), + ], + ), + }, + ), + ] + mock_create_update_props.side_effect = _create_update_item_properties + + _import_properties( + metadata_path=metadata_path, + item_properties=item_properties, + client=mock_client, + annotations=annotations, + annotation_class_ids_map=annotation_class_ids_map, + dataset=mock_dataset, + ) + + create_properties_first_call, update_properties_first_call = ( + mock_create_update_props.call_args_list[0][0][2:4] + ) + create_properties_second_call, update_properties_second_call = ( + mock_create_update_props.call_args_list[1][0][2:4] + ) + + assert len(create_properties_first_call) == 0 + assert len(update_properties_first_call) == 2 + assert len(create_properties_second_call) == 0 + assert len(update_properties_second_call) == 1 + + assert update_properties_first_call[0] == FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="2"), + ], + ) + assert update_properties_first_call[1] == FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="2"), + ], + ) + assert update_properties_second_call[0] == FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="3"), + ], + ) + + +def test__assign_item_properties_to_dataset(mock_client, mock_dataset, mock_console): + item_properties = [ + {"name": "prop1", "value": "1"}, + {"name": "prop2", "value": "2"}, + ] + + team_item_properties_lookup = { + "prop1": FullProperty( + name="prop1", + type="single_select", + description="", + required=True, + slug="test_team", + dataset_ids=[123], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="1"), + ], + ), + "prop2": FullProperty( + name="prop2", + type="multi_select", + description="", + required=False, + slug="test_team", + dataset_ids=[456], + granularity=PropertyGranularity("item"), + property_values=[ + PropertyValue(type="string", value="2"), + ], + ), + } + + with patch( + "darwin.importer.importer._get_team_properties_annotation_lookup" + ) as mock_get_team_props, patch.object( + mock_client, "update_property" + ) as mock_update_property: + mock_get_team_props.return_value = ({}, team_item_properties_lookup) + + _assign_item_properties_to_dataset( + item_properties, + team_item_properties_lookup, + mock_client, + mock_dataset, + mock_console, + ) + + assert mock_update_property.call_count == 2 + + updated_props = [call[0][1] for call in mock_update_property.call_args_list] + + for updated_prop in updated_props: + assert mock_dataset.dataset_id in updated_prop.dataset_ids + assert 123456 in updated_prop.dataset_ids + if 123 in updated_prop.dataset_ids: + assert "prop1" == updated_prop.name + elif 456 in updated_prop.dataset_ids: + assert "prop2" == updated_prop.name + + def test__get_annotation_format(): assert _get_annotation_format(get_importer("coco")) == "coco" assert _get_annotation_format(get_importer("csv_tags_video")) == "csv_tags_video" @@ -1547,6 +2595,7 @@ def test_does_not_raise_error_for_darwin_format_with_warnings(): @pytest.mark.parametrize("setup_data", ["section"], indirect=True) def test_import_existing_section_level_property_values_without_manifest( mock_get_team_properties, + mock_dataset, setup_data, ): client, team_slug, annotation_class_ids_map, annotations = setup_data @@ -1559,6 +2608,7 @@ def test_import_existing_section_level_property_values_without_manifest( property_values=[ PropertyValue(value="1", id="property_value_id_1"), ], + granularity=PropertyGranularity.section, ), ("existing_property_multi_select", 123): FullProperty( id="property_id_2", @@ -1569,11 +2619,34 @@ def test_import_existing_section_level_property_values_without_manifest( PropertyValue(value="1", id="property_value_id_2"), PropertyValue(value="2", id="property_value_id_3"), ], + granularity=PropertyGranularity.section, + ), + }, { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_1"), + ], + granularity=PropertyGranularity.section, + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + PropertyValue(value="2", id="property_value_id_3"), + ], + granularity=PropertyGranularity.section, ), } metadata_path = False result = _import_properties( - metadata_path, client, annotations, annotation_class_ids_map, team_slug + metadata_path, [], client, annotations, annotation_class_ids_map, mock_dataset ) assert result["annotation_id_1"]["0"]["property_id_1"] == { "property_value_id_1", @@ -1590,6 +2663,7 @@ def test_import_existing_section_level_property_values_without_manifest( @pytest.mark.parametrize("setup_data", ["section"], indirect=True) def test_import_new_section_level_property_values_with_manifest( mock_get_team_properties, + mock_dataset, setup_data, ): client, team_slug, annotation_class_ids_map, annotations = setup_data @@ -1600,6 +2674,7 @@ def test_import_new_section_level_property_values_with_manifest( type="single_select", required=False, property_values=[], + granularity=PropertyGranularity.section, ), ("existing_property_multi_select", 123): FullProperty( id="property_id_2", @@ -1609,6 +2684,27 @@ def test_import_new_section_level_property_values_with_manifest( property_values=[ PropertyValue(value="1", id="property_value_id_2"), ], + granularity=PropertyGranularity.section, + ), + }, { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[PropertyValue(value="1", id="property_value_id_1")], + granularity=PropertyGranularity.section, + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + PropertyValue(value="2", id="property_value_id_3"), + ], + granularity=PropertyGranularity.section, ), } metadata_path = ( @@ -1618,7 +2714,12 @@ def test_import_new_section_level_property_values_with_manifest( ) with patch.object(client, "update_property") as mock_update_property: result = _import_properties( - metadata_path, client, annotations, annotation_class_ids_map, team_slug + metadata_path, + [], + client, + annotations, + annotation_class_ids_map, + mock_dataset, ) assert result["annotation_id_1"]["0"]["property_id_2"] == { "property_value_id_2", @@ -1634,6 +2735,7 @@ def test_import_new_section_level_property_values_with_manifest( property_values=[ PropertyValue(value="1", color="rgba(255,46,0,1.0)"), ], + granularity=PropertyGranularity.section, ) assert mock_update_property.call_args_list[1].kwargs["params"] == FullProperty( id="property_id_2", @@ -1646,16 +2748,17 @@ def test_import_new_section_level_property_values_with_manifest( property_values=[ PropertyValue(value="2", color="rgba(255,199,0,1.0)"), ], + granularity=PropertyGranularity.section, ) @patch("darwin.importer.importer._get_team_properties_annotation_lookup") @pytest.mark.parametrize("setup_data", ["section"], indirect=True) def test_import_identical_properties_to_different_classes( - mock_get_team_properties, setup_data + mock_get_team_properties, mock_dataset, setup_data ): client, team_slug, _, _ = setup_data - # This test requires 2 annotations annotation + # This test requires 2 annotation classes annotation_class_ids_map = { ("test_class_1", "polygon"): 1, ("test_class_2", "polygon"): 2, @@ -1696,7 +2799,7 @@ def test_import_identical_properties_to_different_classes( ) ), ] - mock_get_team_properties.return_value = {} + mock_get_team_properties.return_value = {}, {} metadata_path = ( Path(__file__).parents[1] / "data" @@ -1736,7 +2839,12 @@ def test_import_identical_properties_to_different_classes( ), ] annotation_property_map = _import_properties( - metadata_path, client, annotations, annotation_class_ids_map, team_slug + metadata_path, + [], + client, + annotations, + annotation_class_ids_map, + mock_dataset, ) assert annotation_property_map["1"]["0"]["prop_id_1"] == {"prop_val_id_1"} assert annotation_property_map["2"]["0"]["prop_id_2"] == {"prop_val_id_2"} @@ -1746,10 +2854,11 @@ def test_import_identical_properties_to_different_classes( @pytest.mark.parametrize("setup_data", ["section"], indirect=True) def test_import_new_section_level_properties_with_manifest( mock_get_team_properties, + mock_dataset, setup_data, ): client, team_slug, annotation_class_ids_map, annotations = setup_data - mock_get_team_properties.return_value = {} + mock_get_team_properties.return_value = {}, {} metadata_path = ( Path(__file__).parents[1] / "data" @@ -1757,7 +2866,12 @@ def test_import_new_section_level_properties_with_manifest( ) with patch.object(client, "create_property") as mock_create_property: _import_properties( - metadata_path, client, annotations, annotation_class_ids_map, team_slug + metadata_path, + [], + client, + annotations, + annotation_class_ids_map, + mock_dataset, ) assert mock_create_property.call_args_list[0].kwargs["params"] == FullProperty( id=None, @@ -1786,6 +2900,7 @@ def test_import_new_section_level_properties_with_manifest( PropertyValue(value="1", color="rgba(173,255,0,1.0)"), PropertyValue(value="2", color="rgba(255,199,0,1.0)"), ], + granularity=PropertyGranularity.section, ) @@ -1793,6 +2908,7 @@ def test_import_new_section_level_properties_with_manifest( @pytest.mark.parametrize("setup_data", ["annotation"], indirect=True) def test_import_existing_annotation_level_property_values_without_manifest( mock_get_team_properties, + mock_dataset, setup_data, ): client, team_slug, annotation_class_ids_map, annotations = setup_data @@ -1805,6 +2921,29 @@ def test_import_existing_annotation_level_property_values_without_manifest( property_values=[ PropertyValue(value="1", id="property_value_id_1"), ], + granularity=PropertyGranularity.annotation, + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + PropertyValue(value="2", id="property_value_id_3"), + ], + granularity=PropertyGranularity.annotation, + ), + }, { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_1"), + ], + granularity=PropertyGranularity.annotation, ), ("existing_property_multi_select", 123): FullProperty( id="property_id_2", @@ -1815,11 +2954,12 @@ def test_import_existing_annotation_level_property_values_without_manifest( PropertyValue(value="1", id="property_value_id_2"), PropertyValue(value="2", id="property_value_id_3"), ], + granularity=PropertyGranularity.annotation, ), } metadata_path = False result = _import_properties( - metadata_path, client, annotations, annotation_class_ids_map, team_slug + metadata_path, [], client, annotations, annotation_class_ids_map, mock_dataset ) assert result["annotation_id_1"]["None"]["property_id_1"] == { "property_value_id_1", @@ -1833,28 +2973,52 @@ def test_import_existing_annotation_level_property_values_without_manifest( @patch("darwin.importer.importer._get_team_properties_annotation_lookup") @pytest.mark.parametrize("setup_data", ["annotation"], indirect=True) def test_import_new_annotation_level_property_values_with_manifest( - mock_get_team_properties, - setup_data, + mock_get_team_properties, setup_data, mock_dataset ): client, team_slug, annotation_class_ids_map, annotations = setup_data - mock_get_team_properties.return_value = { - ("existing_property_single_select", 123): FullProperty( - id="property_id_1", - name="existing_property_single_select", - type="single_select", - required=False, - property_values=[], - ), - ("existing_property_multi_select", 123): FullProperty( - id="property_id_2", - name="existing_property_multi_select", - type="multi_select", - required=False, - property_values=[ - PropertyValue(value="1", id="property_value_id_2"), - ], - ), - } + mock_get_team_properties.return_value = ( + { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[], + granularity=PropertyGranularity.annotation, + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + ], + granularity=PropertyGranularity.annotation, + ), + }, + { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[PropertyValue(value="1", id="property_value_id_1")], + granularity=PropertyGranularity.annotation, + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + PropertyValue(value="2", id="property_value_id_3"), + ], + granularity=PropertyGranularity.annotation, + ), + }, + ) metadata_path = ( Path(__file__).parents[1] / "data" @@ -1862,7 +3026,12 @@ def test_import_new_annotation_level_property_values_with_manifest( ) with patch.object(client, "update_property") as mock_update_property: result = _import_properties( - metadata_path, client, annotations, annotation_class_ids_map, team_slug + metadata_path, + [], + client, + annotations, + annotation_class_ids_map, + mock_dataset, ) assert result["annotation_id_1"]["None"]["property_id_2"] == { "property_value_id_2", @@ -1878,6 +3047,7 @@ def test_import_new_annotation_level_property_values_with_manifest( property_values=[ PropertyValue(value="1", color="rgba(255,46,0,1.0)"), ], + granularity=PropertyGranularity.annotation, ) assert mock_update_property.call_args_list[1].kwargs["params"] == FullProperty( id="property_id_2", @@ -1890,6 +3060,7 @@ def test_import_new_annotation_level_property_values_with_manifest( property_values=[ PropertyValue(value="2", color="rgba(255,199,0,1.0)"), ], + granularity=PropertyGranularity.annotation, ) @@ -1897,10 +3068,11 @@ def test_import_new_annotation_level_property_values_with_manifest( @pytest.mark.parametrize("setup_data", ["annotation"], indirect=True) def test_import_new_annotation_level_properties_with_manifest( mock_get_team_properties, + mock_dataset, setup_data, ): client, team_slug, annotation_class_ids_map, annotations = setup_data - mock_get_team_properties.return_value = {} + mock_get_team_properties.return_value = {}, {} metadata_path = ( Path(__file__).parents[1] / "data" @@ -1908,7 +3080,12 @@ def test_import_new_annotation_level_properties_with_manifest( ) with patch.object(client, "create_property") as mock_create_property: _import_properties( - metadata_path, client, annotations, annotation_class_ids_map, team_slug + metadata_path, + [], + client, + annotations, + annotation_class_ids_map, + mock_dataset, ) assert mock_create_property.call_args_list[0].kwargs["params"] == FullProperty( name="existing_property_single_select",