[DAR-3771][External] Import of annotation-level properties (#925)

* Added  to the  class

* Import of annotation-level properties

* Added unit tests for _import_properties()

* Fix for albumentations transform

* Prevent 422s by not performing the same property update/create command multiple times

darwin/datatypes.py

@@ -24,7 +24,11 @@
 except ImportError:
     NDArray = Any  # type:ignore
 
-from darwin.future.data_objects.properties import PropertyType, SelectedProperty
+from darwin.future.data_objects.properties import (
+    PropertyType,
+    SelectedProperty,
+    PropertyGranularity,
+)
 from darwin.path_utils import construct_full_path, is_properties_enabled, parse_metadata
 
 # Utility types
@@ -422,6 +426,9 @@ class Property:
     # Description of the property
     description: Optional[str] = None
 
+    # Granularity of the property
+    granularity: PropertyGranularity = PropertyGranularity("section")
+
 
 @dataclass
 class PropertyClass:
@@ -454,17 +461,27 @@ def parse_property_classes(metadata: dict[str, Any]) -> list[PropertyClass]:
         assert (
             "properties" in metadata_cls
         ), "Metadata class does not contain properties"
+        properties = [
+            Property(
+                name=p["name"],
+                type=p["type"],
+                required=p["required"],
+                property_values=p["property_values"],
+                description=p.get("description"),
+                granularity=PropertyGranularity(p.get("granularity", "section")),
+            )
+            for p in metadata_cls["properties"]
+        ]
         classes.append(
             PropertyClass(
                 name=metadata_cls["name"],
                 type=metadata_cls["type"],
                 description=metadata_cls.get("description"),
                 color=metadata_cls.get("color"),
                 sub_types=metadata_cls.get("sub_types"),
-                properties=[Property(**p) for p in metadata_cls["properties"]],
+                properties=properties,
             )
         )
-
     return classes
 
 

darwin/future/data_objects/properties.py

@@ -3,7 +3,8 @@
 import json
 import os
 from pathlib import Path
-from typing import List, Literal, Optional, Tuple
+from typing import List, Literal, Optional, Tuple, Union
+from enum import Enum
 
 from pydantic import field_validator
 
@@ -19,6 +20,12 @@
 ]
 
 
+class PropertyGranularity(str, Enum):
+    section = "section"
+    annotation = "annotation"
+    item = "item"
+
+
 class PropertyValue(DefaultDarwin):
     """
     Describes a single option for a property
@@ -60,6 +67,8 @@ class FullProperty(DefaultDarwin):
         type (str): Type of the property
         required (bool): If the property is required
         options (List[PropertyOption]): List of all options for the property
+        granularity (PropertyGranularity): Granularity of the property
+
     """
 
     id: Optional[str] = None
@@ -73,6 +82,7 @@ class FullProperty(DefaultDarwin):
     annotation_class_id: Optional[int] = None
     property_values: Optional[List[PropertyValue]] = None
     options: Optional[List[PropertyValue]] = None
+    granularity: PropertyGranularity = PropertyGranularity("section")
 
     def to_create_endpoint(
         self,
@@ -87,14 +97,16 @@ def to_create_endpoint(
                 "annotation_class_id": True,
                 "property_values": {"__all__": {"value", "color"}},
                 "description": True,
+                "granularity": True,
             }
         )
 
     def to_update_endpoint(self) -> Tuple[str, dict]:
         if self.id is None:
             raise ValueError("id must be set")
         updated_base = self.to_create_endpoint()
-        del updated_base["annotation_class_id"]  # can't update this field
+        del updated_base["annotation_class_id"]  # Can't update this field
+        del updated_base["granularity"]  # Can't update this field
         return self.id, updated_base
 
 
@@ -110,6 +122,7 @@ class MetaDataClass(DefaultDarwin):
         description (Optional[str]): Description of the class
         color (Optional[str]): Color of the class in the UI
         sub_types (Optional[List[str]]): Sub types of the class
+        granularity:(PropertyGranularity): Granularity of the property
         properties (List[FullProperty]): List of all properties for the class with all options
     """
 
@@ -118,6 +131,7 @@ class MetaDataClass(DefaultDarwin):
     description: Optional[str] = None
     color: Optional[str] = None
     sub_types: Optional[List[str]] = None
+    granularity: PropertyGranularity = PropertyGranularity("section")
     properties: List[FullProperty]
 
     @classmethod
@@ -141,13 +155,14 @@ class SelectedProperty(DefaultDarwin):
     Selected property for an annotation found inside a darwin annotation
 
     Attributes:
-        frame_index (int): Frame index of the annotation
+        frame_index (int | str): Frame index of the annotation
+        int for section-level properties, and "global" for annotation-level properties
         name (str): Name of the property
         type (str | None): Type of the property (if it exists)
         value (str): Value of the property
     """
 
-    frame_index: Optional[int] = None
+    frame_index: Optional[Union[int, str]] = None
     name: str
     type: Optional[str] = None
     value: Optional[str] = None

darwin/future/tests/core/fixtures.py

@@ -8,7 +8,11 @@
 from darwin.future.core.client import ClientCore, DarwinConfig
 from darwin.future.data_objects.dataset import DatasetCore
 from darwin.future.data_objects.item import ItemCore, ItemLayout, ItemSlot
-from darwin.future.data_objects.properties import FullProperty, PropertyValue
+from darwin.future.data_objects.properties import (
+    FullProperty,
+    PropertyValue,
+    PropertyGranularity,
+)
 from darwin.future.data_objects.team import TeamCore, TeamMemberCore
 from darwin.future.data_objects.team_member_role import TeamMemberRole
 from darwin.future.data_objects.workflow import WorkflowCore
@@ -38,6 +42,7 @@ def base_property_object(base_property_value: PropertyValue) -> FullProperty:
         annotation_class_id=0,
         property_values=[base_property_value],
         options=[base_property_value],
+        granularity=PropertyGranularity("section"),
     )
 
 

darwin/importer/importer.py

@@ -26,6 +26,7 @@
     PropertyType,
     PropertyValue,
     SelectedProperty,
+    PropertyGranularity,
 )
 from darwin.item import DatasetItem
 from darwin.path_utils import is_properties_enabled, parse_metadata
@@ -412,6 +413,7 @@ def _import_properties(
                     # if property value is None, update annotation_property_map with empty set
                     if a_prop.value is None:
                         assert t_prop.id is not None
+
                         annotation_property_map[annotation_id][str(a_prop.frame_index)][
                             t_prop.id
                         ] = set()
@@ -516,8 +518,11 @@ def _import_properties(
                         slug=client.default_team,
                         annotation_class_id=int(annotation_class_id),
                         property_values=property_values,
+                        granularity=PropertyGranularity(m_prop.granularity.value),
                     )
-                    create_properties.append(full_property)
+                    # Don't attempt the same propery creation multiple times
+                    if full_property not in create_properties:
+                        create_properties.append(full_property)
                 continue
 
             # check if property value is different in m_prop (.v7/metadata.json) options
@@ -565,7 +570,9 @@ def _import_properties(
                         )
                     ],
                 )
-                update_properties.append(full_property)
+                # Don't attempt the same propery update multiple times
+                if full_property not in update_properties:
+                    update_properties.append(full_property)
                 continue
 
             assert t_prop.id is not None
@@ -649,6 +656,7 @@ def _import_properties(
                 slug=client.default_team,
                 annotation_class_id=t_prop.annotation_class_id,
                 property_values=extra_property_values,
+                granularity=PropertyGranularity(t_prop.granularity.value),
             )
             console.print(
                 f"Updating property {full_property.name} ({full_property.type}) with extra metadata values {extra_values}",

darwin/torch/transforms.py

@@ -368,8 +368,11 @@ def _pre_process(self, image: np.ndarray, annotation: dict) -> dict:
         if (
             masks is not None and masks.numel() > 0
         ):  # using numel() to check if tensor is non-empty
-            print("WE GOT MASKS")
-            albumentation_dict["masks"] = masks.numpy()
+            if isinstance(masks, torch.Tensor):
+                masks = masks.numpy()
+            if masks.ndim == 3:  # Ensure masks is a list of numpy arrays
+                masks = [masks[i] for i in range(masks.shape[0])]
+            albumentation_dict["masks"] = masks
 
         return albumentation_dict
 

darwin/utils/utils.py

@@ -1149,9 +1149,10 @@ def _parse_properties(
 ) -> Optional[List[SelectedProperty]]:
     selected_properties = []
     for property in properties:
+        frame_index = property.get("frame_index")
         selected_properties.append(
             SelectedProperty(
-                frame_index=property.get("frame_index", None),
+                frame_index=frame_index if frame_index is not None else "global",
                 name=property.get("name", None),
                 value=property.get("value", None),
             )

tests/darwin/client_test.py

@@ -378,6 +378,7 @@ def test_get_team_properties(self, darwin_client: Client) -> None:
                         "slug": "property-question",
                         "team_id": 128,
                         "type": "multi_select",
+                        "granularity": "section",
                     },
                 ]
             },

tests/darwin/data/metadata_missing_annotation_property_values.json

@@ -0,0 +1,51 @@
+{
+    "version": "1.0",
+    "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/metadata/1.0/schema.json",
+    "classes": [
+      {
+        "name": "test_class",
+        "type": "bounding_box",
+        "description": null,
+        "color": "rgba(255,46,0,1.0)",
+        "sub_types": [
+          "inference"
+        ],
+        "properties": [
+          {
+            "name": "existing_property_single_select",
+            "type": "single_select",
+            "description": "",
+            "required": false,
+            "property_values": [
+              {
+                "value": "1",
+                "color": "rgba(255,46,0,1.0)"
+              }
+            ],
+            "granularity": "annotation"
+          },
+          {
+            "name": "existing_property_multi_select",
+            "type": "multi_select",
+            "description": "",
+            "required": false,
+            "property_values": [
+              {
+                "value": "1",
+                "color": "rgba(173,255,0,1.0)"
+              },
+              {
+                "value": "2",
+                "color": "rgba(255,199,0,1.0)"
+              }
+            ],
+            "granularity": "annotation"
+          }
+        ],
+        "sub_types_settings": {
+          "inference": {}
+        }
+      }
+    ],
+    "properties": []
+  }

tests/darwin/data/metadata_missing_section_property_values.json

@@ -0,0 +1,49 @@
+{
+    "version": "1.0",
+    "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/metadata/1.0/schema.json",
+    "classes": [
+      {
+        "name": "test_class",
+        "type": "bounding_box",
+        "description": null,
+        "color": "rgba(255,46,0,1.0)",
+        "sub_types": [
+          "inference"
+        ],
+        "properties": [
+          {
+            "name": "existing_property_single_select",
+            "type": "single_select",
+            "description": "",
+            "required": false,
+            "property_values": [
+              {
+                "value": "1",
+                "color": "rgba(255,46,0,1.0)"
+              }
+            ]
+          },
+          {
+            "name": "existing_property_multi_select",
+            "type": "multi_select",
+            "description": "",
+            "required": false,
+            "property_values": [
+              {
+                "value": "1",
+                "color": "rgba(173,255,0,1.0)"
+              },
+              {
+                "value": "2",
+                "color": "rgba(255,199,0,1.0)"
+              }
+            ]
+          }
+        ],
+        "sub_types_settings": {
+          "inference": {}
+        }
+      }
+    ],
+    "properties": []
+  }