Skip to content

Commit

Permalink
[DAR-2860][External] Import of item-level properties (#936)
Browse files Browse the repository at this point in the history 
* Automatic creation of item-level properties & values + dataset assignment from metadata & imported files

* Test coverage for automatic property creation & property value creation

* Extended overwrite warning to account for item-level properties

* Introducing compatibility with annotation-level properties (WIP)

* Making item-level properties import fit with annotation-level

* Import of item-level property values

* E2E tests for import of item-level properties

* Light refactoring of

* Style improvements
  • Loading branch information
@JBWilkie
JBWilkie authored Oct 10, 2024
1 parent acb371d commit a8f1a90
Show file tree
Hide file tree
Showing 38 changed files with 10,565 additions and 162 deletions.
15 changes: 15 additions & 0 deletions darwin/backend_v2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,3 +293,18 @@ def _get_remote_annotations(
The team slug.
"""
return self._client._get(f"v2/teams/{team_slug}/items/{item_id}/annotations")

def _get_properties_state_for_item(
self, item_id: str, team_slug: str
) -> Dict[str, List[Dict[str, str]]]:
"""
Returns the state of property values for the specified item.
Parameters
----------
item_id: str
The ID of the item to get properties for.
team_slug: str
The slug of the team to get.
"""
return self._client._get(f"/v2/teams/{team_slug}/items/{item_id}/properties")
6 changes: 6 additions & 0 deletions darwin/datatypes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,9 @@ class Property:
# Property options
property_values: list[dict[str, Any]]

# Granularity of the property
granularity: PropertyGranularity

# Description of the property
description: Optional[str] = None

Expand Down Expand Up @@ -547,6 +550,9 @@ class AnnotationFile:
#: List of ``VideoAnnotation``\s or ``Annotation``\s.
annotations: Sequence[Union[Annotation, VideoAnnotation]]

# Item-level properties
item_properties: Optional[list[dict[str, Any]]] = None

# Deprecated
#: Whether the annotations in the ``annotations`` attribute are ``VideoAnnotation`` or not.
is_video: bool = False
Expand Down
35 changes: 20 additions & 15 deletions darwin/future/data_objects/properties.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@

import json
import os
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional, Tuple, Union
from enum import Enum

from pydantic import field_validator

Expand Down Expand Up @@ -81,31 +81,36 @@ class FullProperty(DefaultDarwin):
team_id: Optional[int] = None
annotation_class_id: Optional[int] = None
property_values: Optional[List[PropertyValue]] = None
granularity: PropertyGranularity
dataset_ids: Optional[List[int]] = None
options: Optional[List[PropertyValue]] = None
granularity: PropertyGranularity = PropertyGranularity("section")

def to_create_endpoint(
self,
) -> dict:
if self.annotation_class_id is None:
raise ValueError("annotation_class_id must be set")
return self.model_dump(
include={
"name": True,
"type": True,
"required": True,
"annotation_class_id": True,
"property_values": {"__all__": {"value", "color"}},
"description": True,
"granularity": True,
}
)
include_fields = {
"name": True,
"type": True,
"required": True,
"property_values": {"__all__": {"value", "color", "type"}},
"description": True,
"granularity": True,
}
if self.granularity != PropertyGranularity.item:
if self.annotation_class_id is None:
raise ValueError("annotation_class_id must be set")
include_fields["annotation_class_id"] = True
if self.dataset_ids is not None:
include_fields["dataset_ids"] = True
return self.model_dump(mode="json", include=include_fields)

def to_update_endpoint(self) -> Tuple[str, dict]:
if self.id is None:
raise ValueError("id must be set")

updated_base = self.to_create_endpoint()
del updated_base["annotation_class_id"] # Can't update this field
updated_base.pop("annotation_class_id", None) # Can't update this field
del updated_base["granularity"] # Can't update this field
return self.id, updated_base

Expand Down
2 changes: 1 addition & 1 deletion darwin/future/tests/core/fixtures.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ def base_property_object(base_property_value: PropertyValue) -> FullProperty:
team_id=0,
annotation_class_id=0,
property_values=[base_property_value],
granularity=PropertyGranularity.section,
options=[base_property_value],
granularity=PropertyGranularity("section"),
)


Expand Down
4 changes: 2 additions & 2 deletions darwin/future/tests/core/properties/test_create.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def test_create_property(
responses.add(
responses.POST,
f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties",
json=base_property_object.model_dump(),
json=base_property_object.model_dump(mode="json"),
status=200,
)
# Call the function being tested
Expand All @@ -38,7 +38,7 @@ def test_create_property_from_json(
responses.add(
responses.POST,
f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties",
json=base_property_object.model_dump(),
json=base_property_object.model_dump(mode="json"),
status=200,
)
# Call the function being tested
Expand Down
6 changes: 3 additions & 3 deletions darwin/future/tests/core/properties/test_get.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def test_get_team_properties(
# Mocking the response using responses library
base_property_object.options = None
base_property_object.property_values = None
response_data = {"properties": [base_property_object.model_dump()]}
response_data = {"properties": [base_property_object.model_dump(mode="json")]}
responses.add(
responses.GET,
f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties",
Expand All @@ -41,7 +41,7 @@ def test_get_team_full_properties(
base_client: ClientCore, base_property_object: FullProperty
) -> None:
# Mocking the response using responses library
response_data = {"properties": [base_property_object.model_dump()]}
response_data = {"properties": [base_property_object.model_dump(mode="json")]}
responses.add(
responses.GET,
f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties",
Expand Down Expand Up @@ -70,7 +70,7 @@ def test_get_property_by_id(
responses.add(
responses.GET,
f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties/{property_id}",
json=base_property_object.model_dump(),
json=base_property_object.model_dump(mode="json"),
status=200,
)

Expand Down
2 changes: 1 addition & 1 deletion darwin/future/tests/core/properties/test_update.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def test_update_property(
responses.add(
responses.PUT,
f"{base_client.config.base_url}api/v2/teams/{base_client.config.default_team}/properties/{base_property_object.id}",
json=base_property_object.model_dump(),
json=base_property_object.model_dump(mode="json"),
status=200,
)
# Call the function being tested
Expand Down
6 changes: 4 additions & 2 deletions darwin/future/tests/data/.v7/metadata.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
{
"name": "Property 1",
"type": "multi_select",
"options": [
"property_values": [
{
"type": "string",
"value": "first value",
Expand All @@ -38,12 +38,13 @@
"color": "rgba(0,0,0,1.0)"
}
],
"granularity": "section",
"required": false
},
{
"name": "Property 2",
"type": "single_select",
"options": [
"property_values": [
{
"type": "string",
"value": "first value",
Expand All @@ -55,6 +56,7 @@
"color": "rgba(255,255,255,1.0)"
}
],
"granularity": "section",
"required": false
}
]
Expand Down
102 changes: 102 additions & 0 deletions darwin/future/tests/data/.v7/metadata_no_item_level_properties.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
{
"classes": [
{
"name": "Test bounding box",
"type": "bounding_box",
"description": null,
"color": "rgba(255,145,82,1)",
"sub_types": [
"inference"
],
"properties": [
{
"name": "Property 1",
"type": "multi_select",
"property_values": [
{
"type": "string",
"value": "first value",
"color": "rgba(255,92,0,1.0)"
},
{
"type": "string",
"value": "second value",
"color": "rgba(0,0,0,1.0)"
}
],
"granularity": "section",
"required": false
},
{
"name": "Property 2",
"type": "single_select",
"property_values": [
{
"type": "string",
"value": "first value",
"color": "rgba(0,194,255,1.0)"
},
{
"type": "string",
"value": "second value",
"color": "rgba(255,255,255,1.0)"
}
],
"granularity": "section",
"required": false
}
]
},
{
"name": "Test Polygon",
"type": "polygon",
"description": null,
"color": "rgba(219,255,0,1.0)",
"sub_types": [
"directional_vector",
"attributes",
"text",
"instance_id",
"inference"
],
"properties": [
{
"name": "Property 1",
"type": "multi_select",
"property_values": [
{
"type": "string",
"value": "first value",
"color": "rgba(255,92,0,1.0)"
},
{
"type": "string",
"value": "second value",
"color": "rgba(0,0,0,1.0)"
}
],
"granularity": "section",
"required": false
},
{
"name": "Property 2",
"type": "single_select",
"property_values": [
{
"type": "string",
"value": "first value",
"color": "rgba(0,194,255,1.0)"
},
{
"type": "string",
"value": "second value",
"color": "rgba(255,255,255,1.0)"
}
],
"granularity": "section",
"required": false
}
]
}
]
}
Loading

0 comments on commit a8f1a90

Please sign in to comment.