Merge branch 'master' into multi-file-push

darwin/dataset/remote_dataset.py

@@ -692,6 +692,8 @@ def fetch_remote_classes(self, team_wide=False) -> List[Dict[str, Any]]:
             cls["available"] = belongs_to_current_dataset
             if team_wide or belongs_to_current_dataset:
                 classes_to_return.append(cls)
+            elif cls["annotation_types"] == ["raster_layer"]:
+                classes_to_return.append(cls)
         return classes_to_return
 
     def fetch_remote_attributes(self) -> List[Dict[str, Any]]:

darwin/dataset/utils.py

@@ -6,6 +6,7 @@
 
 import numpy as np
 from PIL import Image as PILImage
+from PIL import ImageOps
 from rich.live import Live
 from rich.progress import ProgressBar, track
 
@@ -675,6 +676,7 @@ def load_pil_image(path: Path, to_rgb: Optional[bool] = True) -> PILImage.Image:
         The loaded image.
     """
     pic = PILImage.open(path)
+    pic = ImageOps.exif_transpose(pic)
     if to_rgb:
         pic = convert_to_rgb(pic)
     return pic

darwin/datatypes.py

@@ -24,7 +24,11 @@
 except ImportError:
     NDArray = Any  # type:ignore
 
-from darwin.future.data_objects.properties import PropertyType, SelectedProperty
+from darwin.future.data_objects.properties import (
+    PropertyType,
+    SelectedProperty,
+    PropertyGranularity,
+)
 from darwin.path_utils import construct_full_path, is_properties_enabled, parse_metadata
 
 # Utility types
@@ -422,6 +426,9 @@ class Property:
     # Description of the property
     description: Optional[str] = None
 
+    # Granularity of the property
+    granularity: PropertyGranularity = PropertyGranularity("section")
+
 
 @dataclass
 class PropertyClass:
@@ -454,17 +461,27 @@ def parse_property_classes(metadata: dict[str, Any]) -> list[PropertyClass]:
         assert (
             "properties" in metadata_cls
         ), "Metadata class does not contain properties"
+        properties = [
+            Property(
+                name=p["name"],
+                type=p["type"],
+                required=p["required"],
+                property_values=p["property_values"],
+                description=p.get("description"),
+                granularity=PropertyGranularity(p.get("granularity", "section")),
+            )
+            for p in metadata_cls["properties"]
+        ]
         classes.append(
             PropertyClass(
                 name=metadata_cls["name"],
                 type=metadata_cls["type"],
                 description=metadata_cls.get("description"),
                 color=metadata_cls.get("color"),
                 sub_types=metadata_cls.get("sub_types"),
-                properties=[Property(**p) for p in metadata_cls["properties"]],
+                properties=properties,
             )
         )
-
     return classes
 
 

darwin/future/data_objects/properties.py

@@ -3,7 +3,8 @@
 import json
 import os
 from pathlib import Path
-from typing import List, Literal, Optional, Tuple
+from typing import List, Literal, Optional, Tuple, Union
+from enum import Enum
 
 from pydantic import field_validator
 
@@ -19,6 +20,12 @@
 ]
 
 
+class PropertyGranularity(str, Enum):
+    section = "section"
+    annotation = "annotation"
+    item = "item"
+
+
 class PropertyValue(DefaultDarwin):
     """
     Describes a single option for a property
@@ -60,6 +67,8 @@ class FullProperty(DefaultDarwin):
         type (str): Type of the property
         required (bool): If the property is required
         options (List[PropertyOption]): List of all options for the property
+        granularity (PropertyGranularity): Granularity of the property
+
     """
 
     id: Optional[str] = None
@@ -73,6 +82,7 @@ class FullProperty(DefaultDarwin):
     annotation_class_id: Optional[int] = None
     property_values: Optional[List[PropertyValue]] = None
     options: Optional[List[PropertyValue]] = None
+    granularity: PropertyGranularity = PropertyGranularity("section")
 
     def to_create_endpoint(
         self,
@@ -87,14 +97,16 @@ def to_create_endpoint(
                 "annotation_class_id": True,
                 "property_values": {"__all__": {"value", "color"}},
                 "description": True,
+                "granularity": True,
             }
         )
 
     def to_update_endpoint(self) -> Tuple[str, dict]:
         if self.id is None:
             raise ValueError("id must be set")
         updated_base = self.to_create_endpoint()
-        del updated_base["annotation_class_id"]  # can't update this field
+        del updated_base["annotation_class_id"]  # Can't update this field
+        del updated_base["granularity"]  # Can't update this field
         return self.id, updated_base
 
 
@@ -110,6 +122,7 @@ class MetaDataClass(DefaultDarwin):
         description (Optional[str]): Description of the class
         color (Optional[str]): Color of the class in the UI
         sub_types (Optional[List[str]]): Sub types of the class
+        granularity:(PropertyGranularity): Granularity of the property
         properties (List[FullProperty]): List of all properties for the class with all options
     """
 
@@ -118,6 +131,7 @@ class MetaDataClass(DefaultDarwin):
     description: Optional[str] = None
     color: Optional[str] = None
     sub_types: Optional[List[str]] = None
+    granularity: PropertyGranularity = PropertyGranularity("section")
     properties: List[FullProperty]
 
     @classmethod
@@ -141,13 +155,14 @@ class SelectedProperty(DefaultDarwin):
     Selected property for an annotation found inside a darwin annotation
 
     Attributes:
-        frame_index (int): Frame index of the annotation
+        frame_index (int | str): Frame index of the annotation
+        int for section-level properties, and "global" for annotation-level properties
         name (str): Name of the property
         type (str | None): Type of the property (if it exists)
         value (str): Value of the property
     """
 
-    frame_index: Optional[int] = None
+    frame_index: Optional[Union[int, str]] = None
     name: str
     type: Optional[str] = None
     value: Optional[str] = None

darwin/future/tests/core/fixtures.py

@@ -8,7 +8,11 @@
 from darwin.future.core.client import ClientCore, DarwinConfig
 from darwin.future.data_objects.dataset import DatasetCore
 from darwin.future.data_objects.item import ItemCore, ItemLayout, ItemSlot
-from darwin.future.data_objects.properties import FullProperty, PropertyValue
+from darwin.future.data_objects.properties import (
+    FullProperty,
+    PropertyValue,
+    PropertyGranularity,
+)
 from darwin.future.data_objects.team import TeamCore, TeamMemberCore
 from darwin.future.data_objects.team_member_role import TeamMemberRole
 from darwin.future.data_objects.workflow import WorkflowCore
@@ -38,6 +42,7 @@ def base_property_object(base_property_value: PropertyValue) -> FullProperty:
         annotation_class_id=0,
         property_values=[base_property_value],
         options=[base_property_value],
+        granularity=PropertyGranularity("section"),
     )
 
 

darwin/importer/importer.py

@@ -26,6 +26,7 @@
     PropertyType,
     PropertyValue,
     SelectedProperty,
+    PropertyGranularity,
 )
 from darwin.item import DatasetItem
 from darwin.path_utils import is_properties_enabled, parse_metadata
@@ -412,6 +413,7 @@ def _import_properties(
                     # if property value is None, update annotation_property_map with empty set
                     if a_prop.value is None:
                         assert t_prop.id is not None
+
                         annotation_property_map[annotation_id][str(a_prop.frame_index)][
                             t_prop.id
                         ] = set()
@@ -516,8 +518,11 @@ def _import_properties(
                         slug=client.default_team,
                         annotation_class_id=int(annotation_class_id),
                         property_values=property_values,
+                        granularity=PropertyGranularity(m_prop.granularity.value),
                     )
-                    create_properties.append(full_property)
+                    # Don't attempt the same propery creation multiple times
+                    if full_property not in create_properties:
+                        create_properties.append(full_property)
                 continue
 
             # check if property value is different in m_prop (.v7/metadata.json) options
@@ -565,7 +570,9 @@ def _import_properties(
                         )
                     ],
                 )
-                update_properties.append(full_property)
+                # Don't attempt the same propery update multiple times
+                if full_property not in update_properties:
+                    update_properties.append(full_property)
                 continue
 
             assert t_prop.id is not None
@@ -649,6 +656,7 @@ def _import_properties(
                 slug=client.default_team,
                 annotation_class_id=t_prop.annotation_class_id,
                 property_values=extra_property_values,
+                granularity=PropertyGranularity(t_prop.granularity.value),
             )
             console.print(
                 f"Updating property {full_property.name} ({full_property.type}) with extra metadata values {extra_values}",
@@ -1351,8 +1359,10 @@ def _import_annotations(
         )
 
         if (
-            annotation_type not in remote_classes
-            or annotation_class.name not in remote_classes[annotation_type]
+            (
+                annotation_type not in remote_classes
+                or annotation_class.name not in remote_classes[annotation_type]
+            )
             and annotation_type
             != "raster_layer"  # We do not skip raster layers as they are always available.
         ):
@@ -1518,6 +1528,9 @@ def _get_annotation_format(
     annotation_format : str
         The annotation format of the importer used to parse local files
     """
+    # This `if` block is temporary, but necessary while we migrate NifTI imports between the legacy method & the new method
+    if isinstance(importer, partial):
+        return importer.func.__module__.split(".")[3]
     return importer.__module__.split(".")[3]
 
 

darwin/torch/transforms.py

@@ -368,8 +368,11 @@ def _pre_process(self, image: np.ndarray, annotation: dict) -> dict:
         if (
             masks is not None and masks.numel() > 0
         ):  # using numel() to check if tensor is non-empty
-            print("WE GOT MASKS")
-            albumentation_dict["masks"] = masks.numpy()
+            if isinstance(masks, torch.Tensor):
+                masks = masks.numpy()
+            if masks.ndim == 3:  # Ensure masks is a list of numpy arrays
+                masks = [masks[i] for i in range(masks.shape[0])]
+            albumentation_dict["masks"] = masks
 
         return albumentation_dict
 

darwin/utils/utils.py

@@ -1218,9 +1218,10 @@ def _parse_properties(
 ) -> Optional[List[SelectedProperty]]:
     selected_properties = []
     for property in properties:
+        frame_index = property.get("frame_index")
         selected_properties.append(
             SelectedProperty(
-                frame_index=property.get("frame_index", None),
+                frame_index=frame_index if frame_index is not None else "global",
                 name=property.get("name", None),
                 value=property.get("value", None),
             )

darwin/version/__init__.py

@@ -1 +1 @@
-__version__ = "1.0.6"
+__version__ = "1.0.8"