[Ai-1477]-darwin-v2-export-fix

.vscode/settings.json

@@ -20,4 +20,8 @@
   "python.analysis.autoImportCompletions": true,
   "python.testing.pytestEnabled": true,
   "python.analysis.typeCheckingMode": "basic",
+  "python.testing.pytestArgs": [
+    "e2e_tests"
+  ],
+  "python.testing.unittestEnabled": false,
 }

darwin/dataset/local_dataset.py

@@ -67,6 +67,7 @@ def __init__(
         split: str = "default",
         split_type: str = "random",
         release_name: Optional[str] = None,
+        keep_empty_annotations: bool = False,
     ):
         self.dataset_path = dataset_path
         self.annotation_type = annotation_type
@@ -75,6 +76,7 @@ def __init__(
         self.original_classes = None
         self.original_images_path: Optional[List[Path]] = None
         self.original_annotations_path: Optional[List[Path]] = None
+        self.keep_empty_annotations = keep_empty_annotations
 
         release_path, annotations_dir, images_dir = self._initial_setup(
             dataset_path, release_name
@@ -101,6 +103,7 @@ def __init__(
             split,
             partition,
             split_type,
+            keep_empty_annotations,
         )
 
         if len(self.images_path) == 0:
@@ -130,12 +133,15 @@ def _setup_annotations_and_images(
         split,
         partition,
         split_type,
+        keep_empty_annotations: bool = False,
     ):
         # Find all the annotations and their corresponding images
         for annotation_path in sorted(annotations_dir.glob("**/*.json")):
             darwin_json = stream_darwin_json(annotation_path)
             image_path = get_image_path_from_stream(darwin_json, images_dir)
             if image_path.exists():
+                if not keep_empty_annotations and len(darwin_json["annotations"]) < 1:
+                    continue
                 self.images_path.append(image_path)
                 self.annotations_path.append(annotation_path)
                 continue

darwin/dataset/remote_dataset.py

@@ -161,7 +161,7 @@ def split_video_annotations(self, release_name: str = "latest") -> None:
 
             frame_annotations = split_video_annotation(darwin_annotation)
             for frame_annotation in frame_annotations:
-                annotation = build_image_annotation(frame_annotation)
+                annotation = self._build_image_annotation(frame_annotation)
 
                 video_frame_annotations_path = annotations_path / annotation_file.stem
                 video_frame_annotations_path.mkdir(exist_ok=True, parents=True)
@@ -947,3 +947,8 @@ def local_images_path(self) -> Path:
     def identifier(self) -> DatasetIdentifier:
         """The ``DatasetIdentifier`` of this ``RemoteDataset``."""
         return DatasetIdentifier(team_slug=self.team, dataset_slug=self.slug)
+
+    def _build_image_annotation(
+        self, annotation_file: AnnotationFile
+    ) -> Dict[str, Any]:
+        return build_image_annotation(annotation_file)

darwin/dataset/remote_dataset_v1.py

@@ -12,8 +12,9 @@
     UploadHandlerV1,
 )
 from darwin.dataset.utils import is_relative_to
-from darwin.datatypes import ItemId, PathLike
+from darwin.datatypes import AnnotationFile, ItemId, PathLike
 from darwin.exceptions import NotFound, ValidationError
+from darwin.exporter.formats.darwin_1_0 import build_image_annotation
 from darwin.item import DatasetItem
 from darwin.item_sorter import ItemSorter
 from darwin.utils import find_files, urljoin
@@ -512,3 +513,8 @@ def import_annotation(self, item_id: ItemId, payload: Dict[str, Any]) -> None:
         """
 
         self.client.import_annotation(item_id, payload=payload)
+
+    def _build_image_annotation(
+        self, annotation_file: AnnotationFile
+    ) -> Dict[str, Any]:
+        return build_image_annotation(annotation_file)

darwin/dataset/remote_dataset_v2.py

@@ -22,8 +22,9 @@
     UploadHandlerV2,
 )
 from darwin.dataset.utils import is_relative_to
-from darwin.datatypes import ItemId, PathLike
+from darwin.datatypes import AnnotationFile, ItemId, PathLike
 from darwin.exceptions import NotFound, UnknownExportVersion
+from darwin.exporter.formats.darwin import build_image_annotation
 from darwin.item import DatasetItem
 from darwin.item_sorter import ItemSorter
 from darwin.utils import find_files, urljoin
@@ -543,3 +544,8 @@ def _fetch_stages(self, stage_type):
             workflow_id,
             [stage for stage in workflow["stages"] if stage["type"] == stage_type],
         )
+
+    def _build_image_annotation(
+        self, annotation_file: AnnotationFile
+    ) -> Dict[str, Any]:
+        return build_image_annotation(annotation_file)

darwin/datatypes.py

@@ -537,6 +537,7 @@ def make_polygon(
     bounding_box: Optional[Dict] = None,
     subs: Optional[List[SubAnnotation]] = None,
     slot_names: Optional[List[str]] = None,
+    darwin_v1: bool = False,
 ) -> Annotation:
     """
     Creates and returns a polygon annotation.
@@ -565,9 +566,18 @@ def make_polygon(
     Annotation
         A polygon ``Annotation``.
     """
+
+    if darwin_v1:
+        polygon_data = {"path": point_path}
+    else:
+        # Lets handle darwin V2 datasets
+        if not isinstance(point_path[0], list):
+            point_path = [point_path]
+        polygon_data = {"paths": point_path}
+
     return Annotation(
         AnnotationClass(class_name, "polygon"),
-        _maybe_add_bounding_box_data({"path": point_path}, bounding_box),
+        _maybe_add_bounding_box_data(polygon_data, bounding_box),
         subs or [],
         slot_names=slot_names or [],
     )

darwin/exporter/formats/coco.py

@@ -492,6 +492,7 @@ def _build_annotation(
     categories: Dict[str, int],
 ) -> Optional[Dict[str, Any]]:
     annotation_type = annotation.annotation_class.annotation_type
+
     if annotation_type == "polygon":
         sequences = convert_polygons_to_sequences(
             annotation.data["path"], rounding=False
@@ -561,6 +562,7 @@ def _build_annotation(
         return _build_annotation(
             annotation_file,
             annotation_id,
+            # TODO Update this to V2
             dt.make_polygon(
                 annotation.annotation_class.name,
                 [
@@ -571,6 +573,7 @@ def _build_annotation(
                 ],
                 None,
                 annotation.subs,
+                darwin_v1=True,
             ),
             categories,
         )

darwin/exporter/formats/darwin.py

@@ -16,65 +16,144 @@
 
 def build_image_annotation(annotation_file: dt.AnnotationFile) -> Dict[str, Any]:
     """
-    Builds and returns a dictionary with the annotations present in the given file.
+    Builds and returns a dictionary with the annotations present in the given file in Darwin v2 format.
 
     Parameters
     ----------
-    annotation_file: dt.AnnotationFile
+    annotation_file: AnnotationFile
         File with the image annotations to extract.
+        For schema, see: https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json
 
     Returns
     -------
     Dict[str, Any]
-        A dictionary with the annotation from the given file. Has the following structure:
-
-        .. code-block:: python
-
-            {
-                "annotations": [
-                    {
-                        "annotation_type": { ... }, # annotation_data
-                        "name": "annotation class name",
-                        "bounding_box": { ... } # Optional parameter, only present if the file has a bounding box as well
-                    }
-                ],
-                "image": {
-                    "filename": "a_file_name.json",
-                    "height": 1000,
-                    "width": 2000,
-                    "url": "https://www.darwin.v7labs.com/..."
-                }
-            }
+        A dictionary with the annotations in Darwin v2 format.
     """
-    annotations: List[Dict[str, Any]] = []
-    print(annotations)
+    annotations_list: List[Dict[str, Any]] = []
+
     for annotation in annotation_file.annotations:
-        payload = {
-            annotation.annotation_class.annotation_type: _build_annotation_data(
-                annotation
-            ),
-            "name": annotation.annotation_class.name,
-        }
+        annotation_data = _build_v2_annotation_data(annotation)
+        annotations_list.append(annotation_data)
+
+    slots_data = _build_slots_data(annotation_file.slots)
+    item = _build_item_data(annotation_file)
+    item["slots"] = slots_data
+
+    return {
+        "version": "2.0",
+        "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json",
+        "item": item,
+        "annotations": annotations_list,
+    }
+
 
-        if (
-            annotation.annotation_class.annotation_type == "complex_polygon"
-            or annotation.annotation_class.annotation_type == "polygon"
-        ) and "bounding_box" in annotation.data:
-            payload["bounding_box"] = annotation.data["bounding_box"]
+def _build_v2_annotation_data(annotation: dt.Annotation) -> Dict[str, Any]:
+    annotation_data = {"id": annotation.id, "name": annotation.annotation_class.name}
+    if annotation.annotation_class.annotation_type == "bounding_box":
+        annotation_data["bounding_box"] = _build_bounding_box_data(annotation.data)
+    elif annotation.annotation_class.annotation_type == "tag":
+        annotation_data["tag"] = {}
+    elif annotation.annotation_class.annotation_type == "polygon":
+        polygon_data = _build_polygon_data(annotation.data)
+        annotation_data["polygon"] = polygon_data
+        annotation_data["bounding_box"] = _build_bounding_box_data(annotation.data)
+    return annotation_data
 
-        annotations.append(payload)
 
+def _build_bounding_box_data(data: Dict[str, Any]) -> Dict[str, Any]:
+    if "bounding_box" in data:
+        data = data["bounding_box"]
     return {
-        "annotations": annotations,
-        "image": {
-            "filename": annotation_file.filename,
-            "height": annotation_file.image_height,
-            "width": annotation_file.image_width,
-            "url": annotation_file.image_url,
+        "h": data.get("h"),
+        "w": data.get("w"),
+        "x": data.get("x"),
+        "y": data.get("y"),
+    }
+
+
+def _build_polygon_data(
+    data: Dict[str, Any]
+) -> Dict[str, List[List[Dict[str, float]]]]:
+    """
+    Builds the polygon data for Darwin v2 format.
+
+    Parameters
+    ----------
+    data : Dict[str, Any]
+        The original data for the polygon annotation.
+
+    Returns
+    -------
+    Dict[str, List[List[Dict[str, float]]]]
+        The polygon data in the format required for Darwin v2 annotations.
+    """
+
+    return {"paths": data.get("paths", [])}
+
+
+def _build_item_data(annotation_file: dt.AnnotationFile) -> Dict[str, Any]:
+    """
+    Constructs the 'item' section of the Darwin v2 format annotation.
+
+    Parameters
+    ----------
+    annotation_file: dt.AnnotationFile
+        The AnnotationFile object containing annotation data.
+
+    Returns
+    -------
+    Dict[str, Any]
+        The 'item' section of the Darwin v2 format annotation.
+    """
+    return {
+        "name": annotation_file.filename,
+        "path": annotation_file.remote_path or "/",
+        "source_info": {
+            "dataset": {
+                "name": annotation_file.dataset_name,
+                "slug": annotation_file.dataset_name.lower().replace(" ", "-")
+                if annotation_file.dataset_name
+                else None,
+            },
+            "item_id": annotation_file.item_id,
+            "team": {
+                "name": None,  # TODO Replace with actual team name
+                "slug": None,  # TODO Replace with actual team slug
+            },
+            "workview_url": annotation_file.workview_url,
         },
     }
 
 
+def _build_slots_data(slots: List[dt.Slot]) -> List[Dict[str, Any]]:
+    """
+    Constructs the 'slots' data for the Darwin v2 format annotation.
+
+    Parameters
+    ----------
+    slots: List[Slot]
+        A list of Slot objects from the AnnotationFile.
+
+    Returns
+    -------
+    List[Dict[str, Any]]
+        The 'slots' data for the Darwin v2 format annotation.
+    """
+    slots_data = []
+    for slot in slots:
+        slot_data = {
+            "type": slot.type,
+            "slot_name": slot.name,
+            "width": slot.width,
+            "height": slot.height,
+            "thumbnail_url": slot.thumbnail_url,
+            "source_files": slot.source_files,
+        }
+        slots_data.append(slot_data)
+
+    return slots_data
+
+
 @deprecation.deprecated(
     deprecated_in="0.7.8",
     removed_in="0.8.0",