Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DAR-4064][External] Annotation import E2E test for warning about importing multi-value instance IDs #942

Closed
wants to merge 2 commits into from
Closed

[DAR-4064][External] Annotation import E2E test for warning about importing multi-value instance IDs #942

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0cab936
E2E test for importing annotations to videos
JBWilkie Oct 10, 2024
55ba27b
Annotation import E2E test for multiple instance ID value warning
JBWilkie Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions darwin/importer/importer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2170,6 +2170,7 @@ def _warn_for_annotations_with_multiple_instance_ids(
unique_instance_ids = []
for frame_idx in annotation.frames: # type: ignore
for subannotation in annotation.frames[frame_idx].subs: # type: ignore
print(subannotation)
if subannotation.annotation_type == "instance_id":
instance_id = subannotation.data
if instance_id not in unique_instance_ids:
Expand Down
97 changes: 76 additions & 21 deletions e2e_tests/cli/test_import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@


from e2e_tests.helpers import (
assert_cli,
run_cli_command,
export_and_download_annotations,
delete_annotation_uuids,
Expand All @@ -13,7 +12,7 @@
import tempfile
import zipfile
import darwin.datatypes as dt
from typing import List, Dict, Optional
from typing import List, Dict, Optional, Union


def get_actual_annotation_filename(
Expand All @@ -36,34 +35,56 @@ def get_actual_annotation_filename(


def find_matching_actual_annotation(
expected_annotation: dt.Annotation, actual_annotations: List[dt.Annotation]
):
expected_annotation: dt.Annotation,
actual_annotations: List[Union[dt.Annotation, dt.VideoAnnotation]],
) -> Union[dt.Annotation, dt.VideoAnnotation]:
"""
For a given expected annotation, finds the corresponding actual annotation based on
the `data` field & the annotation class type
For a given expected annotation, finds the corresponding actual annotation
"""
expected_annotation_data = expected_annotation.data
expected_annotation_type = expected_annotation.annotation_class.annotation_type
actual_annotation = next(
(
annotation
for annotation in actual_annotations
if annotation.data == expected_annotation_data
and annotation.annotation_class.annotation_type == expected_annotation_type
),
None,
)
if isinstance(expected_annotation, dt.Annotation):
expected_annotation_data = expected_annotation.data
expected_annotation_type = expected_annotation.annotation_class.annotation_type
actual_annotation = next(
(
annotation
for annotation in actual_annotations
if annotation.data == expected_annotation_data
and annotation.annotation_class.annotation_type
== expected_annotation_type
),
None,
)
elif isinstance(expected_annotation, dt.VideoAnnotation):
expected_annotation_frame_data = expected_annotation.frames
expected_annotation_type = expected_annotation.annotation_class.annotation_type
actual_annotation = next(
(
annotation
for annotation in actual_annotations
if annotation.frames == expected_annotation_frame_data
and annotation.annotation_class.annotation_type
== expected_annotation_type
)
)
assert actual_annotation is not None, "Annotation not found in actual annotations"
return actual_annotation


def assert_same_annotation_data(
expected_annotation: dt.Annotation, actual_annotation: dt.Annotation
expected_annotation: Union[dt.Annotation, dt.VideoAnnotation],
actual_annotation: Union[dt.Annotation, dt.VideoAnnotation],
) -> None:
"""
Ensures that `expected_annotation.data` is equivalent to `actual_annotation.data`
For `dt.Annotation` objects:
Ensures that `expected_annotation.data` is equivalent to `actual_annotation.data`

For `dt.VideoAnnotation` objects:
Ensures that `expected_annotation.frames` is equivalent to `actual_annotation.frames`
"""
assert expected_annotation.data == actual_annotation.data
if isinstance(expected_annotation, dt.Annotation):
assert expected_annotation.data == actual_annotation.data
elif isinstance(expected_annotation, dt.VideoAnnotation):
assert expected_annotation.frames == actual_annotation.frames


def assert_same_annotation_properties(
Expand Down Expand Up @@ -216,7 +237,6 @@ def run_import_test(
result = run_cli_command(
f"darwin dataset import {local_dataset.name} darwin {expected_annotations_dir} {additional_flags}"
)
assert_cli(result, exit_code)

if expect_warning:
assert expect_warning in result.stdout
Expand Down Expand Up @@ -421,3 +441,38 @@ def test_import_annotations_to_multi_channel_item_non_base_slot(
annotations_subdir="multi_channel_annotations_aligned_with_non_base_slot",
expect_error="WARNING: 1 file(s) have the following blocking issues and will not be imported",
)


def test_import_basic_annotations_to_videos(
local_dataset: E2EDataset, config_values: ConfigValues
) -> None:
run_import_test(
local_dataset,
config_values,
item_type="single_slotted_video",
annotations_subdir="video_annotations_without_subtypes",
)


def test_import_annotations_with_subtypes_to_videos(
local_dataset: E2EDataset, config_values: ConfigValues
) -> None:
run_import_test(
local_dataset,
config_values,
item_type="single_slotted_video",
annotations_subdir="video_annotations_with_subtypes",
)


# For this test to succeed, the `e2e-testing` Darwin team must have the `STATIC_INSTANCE_ID` feature flag enabled
def test_importing_multiple_instance_id_values_per_annotation_generates_a_warning(
local_dataset: E2EDataset, config_values: ConfigValues
) -> None:
run_import_test(
local_dataset,
config_values,
item_type="single_slotted_video",
annotations_subdir="video_annotation_with_multiple_instance_ids",
expect_error="multiple instance ID values",
)
133 changes: 133 additions & 0 deletions e2e_tests/data/import/video_annotation_with_multiple_instance_ids/mini_uct.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
{
"version": "2.0",
"schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json",
"item": {
"name": "mini_uct.mp4",
"path": "/",
"source_info": {
"item_id": "0192788d-f84c-3e47-3ed9-5801e3073ae1",
"dataset": {
"name": "test_dataset_50ea2a66-d526-4212-928a-eac3745da30a",
"slug": "test_dataset_50ea2a66-d526-4212-928a-eac3745da30a",
"dataset_management_url": "https://staging.v7labs.com/datasets/386274/dataset-management"
},
"team": {
"name": "E2E Testing",
"slug": "e2e-testing"
},
"workview_url": "https://staging.v7labs.com/workview?dataset=386274&item=0192788d-f84c-3e47-3ed9-5801e3073ae1"
},
"slots": [
{
"type": "video",
"slot_name": "0",
"width": 1920,
"height": 1080,
"fps": 1,
"thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/thumbnail",
"source_files": [
{
"file_name": "mini_uct.mp4",
"storage_key": "darwin-py/videos/mini_uct.mp4",
"url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/78f2a1ac-210d-43f9-bb48-d0d0c9ca17b8"
}
],
"frame_count": 10,
"frame_urls": [
"https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/sections/0",
"https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/sections/1",
"https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/sections/2",
"https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/sections/3",
"https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/sections/4",
"https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/sections/5",
"https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/sections/6",
"https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/sections/7",
"https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/sections/8",
"https://staging.v7labs.com/api/v2/teams/e2e-testing/files/65750b55-ccfc-4a01-b9a5-ab3f1d6a0a64/sections/9"
]
}
]
},
"annotations": [
{
"frames": {
"0": {
"bounding_box": {
"h": 3.8235,
"w": 6.1275,
"x": 7.9412,
"y": 8.6765
},
"instance_id": {
"value": 20
},
"keyframe": true
},
"1": {
"bounding_box": {
"h": 3.8235,
"w": 6.1275,
"x": 7.9412,
"y": 8.6765
},
"instance_id": {
"value": 21
},
"keyframe": false
},
"2": {
"bounding_box": {
"h": 3.8235,
"w": 6.1275,
"x": 7.9412,
"y": 8.6765
},
"instance_id": {
"value": 22
},
"keyframe": false
},
"3": {
"bounding_box": {
"h": 3.8235,
"w": 6.1275,
"x": 7.9412,
"y": 8.6765
},
"instance_id": {
"value": 23
},
"keyframe": false
},
"4": {
"bounding_box": {
"h": 3.8235,
"w": 6.1275,
"x": 7.9412,
"y": 8.6765
},
"instance_id": {
"value": 24
},
"keyframe": true
}
},
"global_sub_types": {},
"id": "c92972c6-41a5-48b6-8c4a-9e8062d06d61",
"interpolate_algorithm": "linear-1.1",
"interpolated": true,
"name": "test_bounding_box_with_subtypes_and_properties",
"properties": [],
"ranges": [
[
0,
5
]
],
"slot_names": [
"0"
]
}
],
"properties": []
}
Loading