diff --git a/.gitignore b/.gitignore index 3dd50932a..5d4b4611b 100644 --- a/.gitignore +++ b/.gitignore @@ -167,6 +167,7 @@ test_output_* !tests/darwin/data/*.png +!darwin/future/tests/data # scripts test.py diff --git a/darwin/future/data_objects/darwinV2.py b/darwin/future/data_objects/darwinV2.py new file mode 100644 index 000000000..2e4d334b7 --- /dev/null +++ b/darwin/future/data_objects/darwinV2.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from typing import List, Literal, Optional, Union + +from pydantic import BaseModel, validator + +from darwin.future.data_objects.properties import SelectedProperty + + +class Point(BaseModel): + x: float + y: float + + def __add__(self, other: Point) -> Point: + return Point(x=self.x + other.x, y=self.y + other.y) + + def __sub__(self, other: Point) -> Point: + return Point(x=self.x - other.x, y=self.y - other.y) + + +PolygonPath = List[Point] + + +class Polygon(BaseModel): + paths: List[PolygonPath] + + def bounding_box(self) -> BoundingBox: + h, w, x, y = 0.0, 0.0, 0.0, 0.0 + for polygon_path in self.paths: + for point in polygon_path: + h = max(h, point.y) + w = max(w, point.x) + x = min(x, point.x) + y = min(y, point.y) + return BoundingBox(h=h, w=w, x=x, y=y) + + @property + def is_complex(self) -> bool: + return len(self.paths) > 1 + + @property + def center(self) -> Point: + return self.bounding_box().center + + +class AnnotationBase(BaseModel): + id: str + name: str + properties: Optional[SelectedProperty] = None + slot_names: Optional[List[str]] = None + + @validator("id", always=True) + def validate_id_is_UUID(cls, v: str) -> str: + assert len(v) == 36 + assert "-" in v + return v + + +class BoundingBox(BaseModel): + h: float + w: float + x: float + y: float + + @property + def center(self) -> Point: + return Point(x=self.x + self.w / 2, y=self.y + self.h / 2) + + +class BoundingBoxAnnotation(AnnotationBase): + bounding_box: BoundingBox + + +class Ellipse(BaseModel): + center: Point + radius: Point + angle: float + + +class EllipseAnnotation(AnnotationBase): + ellipse: Ellipse + + +class PolygonAnnotation(AnnotationBase): + polygon: Polygon + bounding_box: Optional[BoundingBox] = None + + @validator("bounding_box", pre=False, always=True) + def validate_bounding_box( + cls, v: Optional[BoundingBox], values: dict + ) -> BoundingBox: + if v is None: + assert "polygon" in values + assert isinstance(values["polygon"], Polygon) + v = values["polygon"].bounding_box() + return v + + +class FrameAnnotation(AnnotationBase): + frames: List + interpolated: bool + interpolate_algorithm: str + ranges: List[int] + + +AllowedAnnotation = Union[ + PolygonAnnotation, BoundingBoxAnnotation, EllipseAnnotation, FrameAnnotation +] + + +class Item(BaseModel): + name: str + path: str + + +class DarwinV2(BaseModel): + version: Literal["2.0"] = "2.0" + schema_ref: str + item: dict + annotations: List[AllowedAnnotation] + + @validator("schema_ref", always=True) + def validate_schema_ref(cls, v: str) -> str: + assert v.startswith("http") + return v diff --git a/darwin/future/data_objects/properties.py b/darwin/future/data_objects/properties.py new file mode 100644 index 000000000..3e9443f5f --- /dev/null +++ b/darwin/future/data_objects/properties.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Dict, List, Optional, Union + +from pydantic import validator + +from darwin.future.data_objects.pydantic_base import DefaultDarwin + + +class PropertyOption(DefaultDarwin): + """ + Describes a single option for a property + + Attributes: + value (str): Value of the option + color (Optional[str]): Color of the option + type (Optional[str]): Type of the option + + Validators: + color (validator): Validates that the color is in rgba format + """ + + id: Optional[str] + position: Optional[int] + type: str + value: Union[Dict[str, str], str] + color: str + + @validator("color") + def validate_rgba(cls, v: str) -> str: + if not v.startswith("rgba"): + raise ValueError("Color must be in rgba format") + return v + + +class FullProperty(DefaultDarwin): + """ + Describes the property and all of the potential options that are associated with it + + Attributes: + name (str): Name of the property + type (str): Type of the property + required (bool): If the property is required + options (List[PropertyOption]): List of all options for the property + """ + + id: Optional[str] + name: str + type: str + description: Optional[str] + required: bool + slug: Optional[str] + team_id: Optional[int] + annotation_class_id: Optional[int] + property_values: Optional[List[PropertyOption]] + options: Optional[List[PropertyOption]] + + +class MetaDataClass(DefaultDarwin): + """ + Metadata.json -> property mapping. Contains all properties for a class contained + in the metadata.json file. Along with all options for each property that is associated + with the class. + + Attributes: + name (str): Name of the class + type (str): Type of the class + description (Optional[str]): Description of the class + color (Optional[str]): Color of the class in the UI + sub_types (Optional[List[str]]): Sub types of the class + properties (List[FullProperty]): List of all properties for the class with all options + """ + + name: str + type: str + description: Optional[str] + color: Optional[str] + sub_types: Optional[List[str]] + properties: List[FullProperty] + + @classmethod + def from_path(cls, path: Path) -> List[MetaDataClass]: + if not path.exists(): + raise FileNotFoundError(f"File {path} does not exist") + if os.path.isdir(path): + if os.path.exists(path / ".v7" / "metadata.json"): + path = path / ".v7" / "metadata.json" + else: + raise FileNotFoundError("File metadata.json does not exist in path") + if path.suffix != ".json": + raise ValueError(f"File {path} must be a json file") + with open(path, "r") as f: + data = json.load(f) + return [cls(**d) for d in data["classes"]] + + +class SelectedProperty(DefaultDarwin): + """ + Selected property for an annotation found inside a darwin annotation + + Attributes: + frame_index (int): Frame index of the annotation + name (str): Name of the property + type (str): Type of the property + value (str): Value of the property + """ + + frame_index: int + name: str + type: str + value: str diff --git a/darwin/future/tests/data/.v7/metadata.json b/darwin/future/tests/data/.v7/metadata.json new file mode 100644 index 000000000..561711c61 --- /dev/null +++ b/darwin/future/tests/data/.v7/metadata.json @@ -0,0 +1,63 @@ +{ + "classes": [ + { + "name": "Test bounding box", + "type": "bounding_box", + "description": null, + "color": "rgba(255,145,82,1)", + "sub_types": [ + "inference" + ], + "properties": [] + }, + { + "name": "Test Polygon", + "type": "polygon", + "description": null, + "color": "rgba(219,255,0,1.0)", + "sub_types": [ + "directional_vector", + "attributes", + "text", + "instance_id", + "inference" + ], + "properties": [ + { + "name": "Property 1", + "type": "multi_select", + "options": [ + { + "type": "string", + "value": "first value", + "color": "rgba(255,92,0,1.0)" + }, + { + "type": "string", + "value": "second value", + "color": "rgba(0,0,0,1.0)" + } + ], + "required": false + }, + { + "name": "Property 2", + "type": "single_select", + "options": [ + { + "type": "string", + "value": "first value", + "color": "rgba(0,194,255,1.0)" + }, + { + "type": "string", + "value": "second value", + "color": "rgba(255,255,255,1.0)" + } + ], + "required": false + } + ] + } + ] +} \ No newline at end of file diff --git a/darwin/future/tests/data/base_annotation.json b/darwin/future/tests/data/base_annotation.json new file mode 100644 index 000000000..b6b6e2d3c --- /dev/null +++ b/darwin/future/tests/data/base_annotation.json @@ -0,0 +1,67 @@ +{ + "version": "2.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json", + "item": { + "name": "", + "path": "/" + }, + "annotations": [ + { + "bounding_box": { + "h": 1.0, + "w": 1.0, + "x": 0.0, + "y": 0.0 + }, + "id": "007882ff-99c4-4c6f-b71b-79cfc147fef6", + "name": "test_bb" + }, + { + "ellipse": { + "angle": 0.0, + "center": { + "x": 1.0, + "y": 1.0 + }, + "radius": { + "x": 1.0, + "y": 1.0 + } + }, + "id": "320a60f2-643b-4d74-a117-0ea2fdfe7a61", + "name": "test_ellipse" + }, + { + "bounding_box": { + "h": 1.0, + "w": 1.0, + "x": 0.0, + "y": 0.0 + }, + "id": "012dcc6c-5b77-406b-8cd7-d9567c8b00b7", + "name": "test_poly", + "polygon": { + "paths": [ + [ + { + "x": 0.0, + "y": 0.0 + }, + { + "x": 1.0, + "y": 0.0 + }, + { + "x": 1.0, + "y": 1.0 + }, + { + "x": 0.0, + "y": 1.0 + } + ] + ] + } + } + ] +} \ No newline at end of file diff --git a/darwin/future/tests/data_objects/test_darwin.py b/darwin/future/tests/data_objects/test_darwin.py new file mode 100644 index 000000000..cda91ad10 --- /dev/null +++ b/darwin/future/tests/data_objects/test_darwin.py @@ -0,0 +1,52 @@ +import json + +import pytest + +from darwin.future.data_objects.darwinV2 import ( + BoundingBoxAnnotation, + DarwinV2, + EllipseAnnotation, + PolygonAnnotation, +) + + +@pytest.fixture +def raw_json() -> dict: + with open("./darwin/future/tests/data/base_annotation.json") as f: + raw_json = json.load(f) + return raw_json + + +def test_loads_base_darwin_v2(raw_json: dict): + test = DarwinV2.parse_obj(raw_json) + assert len(test.annotations) == 3 + assert isinstance(test.annotations[0], BoundingBoxAnnotation) + assert isinstance(test.annotations[1], EllipseAnnotation) + assert isinstance(test.annotations[2], PolygonAnnotation) + + +def test_bbox_annotation(raw_json: dict): + bounds_annotation = raw_json["annotations"][0] + BoundingBoxAnnotation.parse_obj(bounds_annotation) + + +def test_ellipse_annotation(raw_json: dict): + ellipse_annotation = raw_json["annotations"][1] + EllipseAnnotation.parse_obj(ellipse_annotation) + + +def test_polygon_annotation(raw_json: dict): + polygon_annotation = raw_json["annotations"][2] + PolygonAnnotation.parse_obj(polygon_annotation) + + +def test_polygon_bbx_vaidator(raw_json: dict): + polygon_annotation = raw_json["annotations"][2] + without_bbx = polygon_annotation.copy() + del without_bbx["bounding_box"] + without_bb_annotation = PolygonAnnotation.parse_obj(without_bbx) + with_bb_annotation = PolygonAnnotation.parse_obj(polygon_annotation) + + assert without_bb_annotation.bounding_box is not None + assert with_bb_annotation.bounding_box is not None + assert without_bb_annotation == with_bb_annotation diff --git a/darwin/future/tests/data_objects/test_properties.py b/darwin/future/tests/data_objects/test_properties.py new file mode 100644 index 000000000..36c570c61 --- /dev/null +++ b/darwin/future/tests/data_objects/test_properties.py @@ -0,0 +1,37 @@ +from pathlib import Path + +import pytest + +from darwin.future.data_objects.properties import MetaDataClass + + +@pytest.fixture +def path_to_metadata_folder() -> Path: + return Path("darwin/future/tests/data") + + +@pytest.fixture +def path_to_metadata(path_to_metadata_folder: Path) -> Path: + return path_to_metadata_folder / ".v7" / "metadata.json" + + +def test_properties_metadata_loads_folder(path_to_metadata: Path) -> None: + metadata = MetaDataClass.from_path(path_to_metadata) + assert metadata is not None + assert len(metadata) == 2 + + +def test_properties_metadata_loads_file(path_to_metadata: Path) -> None: + metadata = MetaDataClass.from_path(path_to_metadata) + assert metadata is not None + assert len(metadata) == 2 + + +def test_properties_metadata_fails() -> None: + path = Path("darwin/future/tests/data/does_not_exist.json") + with pytest.raises(FileNotFoundError): + MetaDataClass.from_path(path) + + path = Path("darwin/future/tests/data/does_not_exist") + with pytest.raises(FileNotFoundError): + MetaDataClass.from_path(path) diff --git a/darwin/importer/formats/csv_tags_video.py b/darwin/importer/formats/csv_tags_video.py index d972653cd..a6885ac0c 100644 --- a/darwin/importer/formats/csv_tags_video.py +++ b/darwin/importer/formats/csv_tags_video.py @@ -51,9 +51,9 @@ def parse_path(path: Path) -> Optional[List[dt.AnnotationFile]]: file_annotation_map[filename].append(annotation) for filename in file_annotation_map: annotations = file_annotation_map[filename] - annotation_classes = set( + annotation_classes = { annotation.annotation_class for annotation in annotations - ) + } filename_path = Path(filename) remote_path = str(filename_path.parent) if not remote_path.startswith("/"):