From 00fd6a96520d5cbe5ebca9c9e509b270eb5f1c9f Mon Sep 17 00:00:00 2001 From: John Wilkie <124276291+JBWilkie@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:20:52 +0100 Subject: [PATCH 1/7] Resolved legacy NifTI annotation import blocker (#917) --- darwin/importer/importer.py | 3 +++ tests/darwin/importer/importer_test.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index f28c87db7..d56a3eedd 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -1518,6 +1518,9 @@ def _get_annotation_format( annotation_format : str The annotation format of the importer used to parse local files """ + # This `if` block is temporary, but necessary while we migrate NifTI imports between the legacy method & the new method + if isinstance(importer, partial): + return importer.func.__module__.split(".")[3] return importer.__module__.split(".")[3] diff --git a/tests/darwin/importer/importer_test.py b/tests/darwin/importer/importer_test.py index a290a0203..70cf197df 100644 --- a/tests/darwin/importer/importer_test.py +++ b/tests/darwin/importer/importer_test.py @@ -1,5 +1,6 @@ import json import tempfile +from functools import partial from pathlib import Path from typing import List, Tuple from unittest.mock import MagicMock, Mock, _patch, patch @@ -929,6 +930,12 @@ def test__get_annotation_format(): assert _get_annotation_format(get_importer("superannotate")) == "superannotate" +def test__get_annotation_format_with_partial(): + nifti_importer = get_importer("nifti") + legacy_nifti_importer = partial(nifti_importer, legacy=True) + assert _get_annotation_format(legacy_nifti_importer) == "nifti" + + def test_no_verify_warning_for_single_slotted_items(): bounding_box_class = dt.AnnotationClass( name="class1", annotation_type="bounding_box" From fe582c8b9848a7dcef39765344e32f3de946f7a8 Mon Sep 17 00:00:00 2001 From: John Wilkie <124276291+JBWilkie@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:43:03 +0100 Subject: [PATCH 2/7] [DAR-3333][External] Import raster layer annotations even if classes are created or updated (#918) * Always return when fetching remote classes * Unit tests * More concise tests --- darwin/dataset/remote_dataset.py | 2 + darwin/importer/importer.py | 6 +- tests/darwin/dataset/remote_dataset_test.py | 73 +++++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/darwin/dataset/remote_dataset.py b/darwin/dataset/remote_dataset.py index 154c26c85..e95cc00e9 100644 --- a/darwin/dataset/remote_dataset.py +++ b/darwin/dataset/remote_dataset.py @@ -691,6 +691,8 @@ def fetch_remote_classes(self, team_wide=False) -> List[Dict[str, Any]]: cls["available"] = belongs_to_current_dataset if team_wide or belongs_to_current_dataset: classes_to_return.append(cls) + elif cls["annotation_types"] == ["raster_layer"]: + classes_to_return.append(cls) return classes_to_return def fetch_remote_attributes(self) -> List[Dict[str, Any]]: diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index d56a3eedd..138769520 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -1351,8 +1351,10 @@ def _import_annotations( ) if ( - annotation_type not in remote_classes - or annotation_class.name not in remote_classes[annotation_type] + ( + annotation_type not in remote_classes + or annotation_class.name not in remote_classes[annotation_type] + ) and annotation_type != "raster_layer" # We do not skip raster layers as they are always available. ): diff --git a/tests/darwin/dataset/remote_dataset_test.py b/tests/darwin/dataset/remote_dataset_test.py index c974ace0e..7ed48a448 100644 --- a/tests/darwin/dataset/remote_dataset_test.py +++ b/tests/darwin/dataset/remote_dataset_test.py @@ -583,6 +583,79 @@ def test_fetches_files_with_commas( ) +@pytest.mark.usefixtures("file_read_write_test") +class TestFetchRemoteClasses: + def setup_method(self): + self.mock_classes = [ + { + "name": "class1", + "datasets": [{"id": 1}], + "annotation_types": ["type1"], + }, + { + "name": "class2", + "datasets": [{"id": 2}], + "annotation_types": ["type2"], + }, + { + "name": "raster_class", + "datasets": [], + "annotation_types": ["raster_layer"], + }, + ] + + def create_remote_dataset( + self, darwin_client, dataset_name, dataset_slug, team_slug_darwin_json_v2 + ): + return RemoteDatasetV2( + client=darwin_client, + team=team_slug_darwin_json_v2, + name=dataset_name, + slug=dataset_slug, + dataset_id=1, + ) + + @responses.activate + def test_fetch_remote_classes_team_wide( + self, + darwin_client: Client, + dataset_name: str, + dataset_slug: str, + team_slug_darwin_json_v2: str, + ): + remote_dataset = self.create_remote_dataset( + darwin_client, dataset_name, dataset_slug, team_slug_darwin_json_v2 + ) + with patch.object( + remote_dataset.client, + "fetch_remote_classes", + return_value=self.mock_classes, + ): + result = remote_dataset.fetch_remote_classes(team_wide=True) + assert len(result) == 3 + assert any(cls["name"] == "raster_class" for cls in result) + + @responses.activate + def test_fetch_remote_classes_local_to_dataset( + self, + darwin_client: Client, + dataset_name: str, + dataset_slug: str, + team_slug_darwin_json_v2: str, + ): + remote_dataset = self.create_remote_dataset( + darwin_client, dataset_name, dataset_slug, team_slug_darwin_json_v2 + ) + with patch.object( + remote_dataset.client, + "fetch_remote_classes", + return_value=self.mock_classes, + ): + result = remote_dataset.fetch_remote_classes(team_wide=False) + assert len(result) == 2 + assert any(cls["name"] == "raster_class" for cls in result) + + @pytest.fixture def remote_dataset( darwin_client: Client, From 38478142ca46d33745b75928fe5ba521816949ac Mon Sep 17 00:00:00 2001 From: John Wilkie <124276291+JBWilkie@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:55:23 +0100 Subject: [PATCH 3/7] version bump to 1.0.7 (#919) --- darwin/version/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/darwin/version/__init__.py b/darwin/version/__init__.py index 382021f30..9e604c040 100644 --- a/darwin/version/__init__.py +++ b/darwin/version/__init__.py @@ -1 +1 @@ -__version__ = "1.0.6" +__version__ = "1.0.7" diff --git a/pyproject.toml b/pyproject.toml index 10c69fc3e..cd9a21ebe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ license = "MIT" name = "darwin-py" readme = "README.md" repository = "https://github.com/v7labs/darwin-py" -version = "1.0.6" +version = "1.0.7" [[tool.poetry.packages]] include = "darwin" From 3b238700a6ed8e05352c1ccf1417da82ed704c55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 10:17:23 +0100 Subject: [PATCH 4/7] Bump the python-requirements group with 3 updates (#921) Bumps the python-requirements group with 3 updates: [rich](https://github.com/Textualize/rich), [mypy](https://github.com/python/mypy) and [ruff](https://github.com/astral-sh/ruff). Updates `rich` from 13.7.1 to 13.8.0 - [Release notes](https://github.com/Textualize/rich/releases) - [Changelog](https://github.com/Textualize/rich/blob/master/CHANGELOG.md) - [Commits](https://github.com/Textualize/rich/compare/v13.7.1...v13.8.0) Updates `mypy` from 1.11.1 to 1.11.2 - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.11.1...v1.11.2) Updates `ruff` from 0.6.1 to 0.6.3 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.6.1...0.6.3) --- updated-dependencies: - dependency-name: rich dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python-requirements - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-requirements - dependency-name: ruff dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-requirements ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 100 ++++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9161c0ff2..2ba1b4dde 100644 --- a/poetry.lock +++ b/poetry.lock @@ -857,38 +857,38 @@ tests = ["pytest (>=4.6)"] [[package]] name = "mypy" -version = "1.11.1" +version = "1.11.2" description = "Optional static typing for Python" optional = true python-versions = ">=3.8" files = [ - {file = "mypy-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a32fc80b63de4b5b3e65f4be82b4cfa362a46702672aa6a0f443b4689af7008c"}, - {file = "mypy-1.11.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1952f5ea8a5a959b05ed5f16452fddadbaae48b5d39235ab4c3fc444d5fd411"}, - {file = "mypy-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1e30dc3bfa4e157e53c1d17a0dad20f89dc433393e7702b813c10e200843b03"}, - {file = "mypy-1.11.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2c63350af88f43a66d3dfeeeb8d77af34a4f07d760b9eb3a8697f0386c7590b4"}, - {file = "mypy-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:a831671bad47186603872a3abc19634f3011d7f83b083762c942442d51c58d58"}, - {file = "mypy-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7b6343d338390bb946d449677726edf60102a1c96079b4f002dedff375953fc5"}, - {file = "mypy-1.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e4fe9f4e5e521b458d8feb52547f4bade7ef8c93238dfb5bbc790d9ff2d770ca"}, - {file = "mypy-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:886c9dbecc87b9516eff294541bf7f3655722bf22bb898ee06985cd7269898de"}, - {file = "mypy-1.11.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca4a60e1dd9fd0193ae0067eaeeb962f2d79e0d9f0f66223a0682f26ffcc809"}, - {file = "mypy-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:0bd53faf56de9643336aeea1c925012837432b5faf1701ccca7fde70166ccf72"}, - {file = "mypy-1.11.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f39918a50f74dc5969807dcfaecafa804fa7f90c9d60506835036cc1bc891dc8"}, - {file = "mypy-1.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bc71d1fb27a428139dd78621953effe0d208aed9857cb08d002280b0422003a"}, - {file = "mypy-1.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b868d3bcff720dd7217c383474008ddabaf048fad8d78ed948bb4b624870a417"}, - {file = "mypy-1.11.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a707ec1527ffcdd1c784d0924bf5cb15cd7f22683b919668a04d2b9c34549d2e"}, - {file = "mypy-1.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:64f4a90e3ea07f590c5bcf9029035cf0efeae5ba8be511a8caada1a4893f5525"}, - {file = "mypy-1.11.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:749fd3213916f1751fff995fccf20c6195cae941dc968f3aaadf9bb4e430e5a2"}, - {file = "mypy-1.11.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b639dce63a0b19085213ec5fdd8cffd1d81988f47a2dec7100e93564f3e8fb3b"}, - {file = "mypy-1.11.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c956b49c5d865394d62941b109728c5c596a415e9c5b2be663dd26a1ff07bc0"}, - {file = "mypy-1.11.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45df906e8b6804ef4b666af29a87ad9f5921aad091c79cc38e12198e220beabd"}, - {file = "mypy-1.11.1-cp38-cp38-win_amd64.whl", hash = "sha256:d44be7551689d9d47b7abc27c71257adfdb53f03880841a5db15ddb22dc63edb"}, - {file = "mypy-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2684d3f693073ab89d76da8e3921883019ea8a3ec20fa5d8ecca6a2db4c54bbe"}, - {file = "mypy-1.11.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79c07eb282cb457473add5052b63925e5cc97dfab9812ee65a7c7ab5e3cb551c"}, - {file = "mypy-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11965c2f571ded6239977b14deebd3f4c3abd9a92398712d6da3a772974fad69"}, - {file = "mypy-1.11.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a2b43895a0f8154df6519706d9bca8280cda52d3d9d1514b2d9c3e26792a0b74"}, - {file = "mypy-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:1a81cf05975fd61aec5ae16501a091cfb9f605dc3e3c878c0da32f250b74760b"}, - {file = "mypy-1.11.1-py3-none-any.whl", hash = "sha256:0624bdb940255d2dd24e829d99a13cfeb72e4e9031f9492148f410ed30bcab54"}, - {file = "mypy-1.11.1.tar.gz", hash = "sha256:f404a0b069709f18bbdb702eb3dcfe51910602995de00bd39cea3050b5772d08"}, + {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"}, + {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"}, + {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"}, + {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"}, + {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"}, + {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"}, + {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"}, + {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"}, + {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"}, + {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"}, + {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"}, + {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"}, + {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"}, + {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"}, + {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"}, + {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"}, + {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"}, + {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"}, + {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"}, + {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"}, + {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"}, + {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"}, + {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"}, ] [package.dependencies] @@ -1615,13 +1615,13 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -version = "13.7.1" +version = "13.8.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, - {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, + {file = "rich-13.8.0-py3-none-any.whl", hash = "sha256:2e85306a063b9492dffc86278197a60cbece75bcb766022f3436f567cae11bdc"}, + {file = "rich-13.8.0.tar.gz", hash = "sha256:a5ac1f1cd448ade0d59cc3356f7db7a7ccda2c8cbae9c7a90c28ff463d3e91f4"}, ] [package.dependencies] @@ -1746,29 +1746,29 @@ files = [ [[package]] name = "ruff" -version = "0.6.1" +version = "0.6.3" description = "An extremely fast Python linter and code formatter, written in Rust." optional = true python-versions = ">=3.7" files = [ - {file = "ruff-0.6.1-py3-none-linux_armv6l.whl", hash = "sha256:b4bb7de6a24169dc023f992718a9417380301b0c2da0fe85919f47264fb8add9"}, - {file = "ruff-0.6.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:45efaae53b360c81043e311cdec8a7696420b3d3e8935202c2846e7a97d4edae"}, - {file = "ruff-0.6.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:bc60c7d71b732c8fa73cf995efc0c836a2fd8b9810e115be8babb24ae87e0850"}, - {file = "ruff-0.6.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c7477c3b9da822e2db0b4e0b59e61b8a23e87886e727b327e7dcaf06213c5cf"}, - {file = "ruff-0.6.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a0af7ab3f86e3dc9f157a928e08e26c4b40707d0612b01cd577cc84b8905cc9"}, - {file = "ruff-0.6.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:392688dbb50fecf1bf7126731c90c11a9df1c3a4cdc3f481b53e851da5634fa5"}, - {file = "ruff-0.6.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5278d3e095ccc8c30430bcc9bc550f778790acc211865520f3041910a28d0024"}, - {file = "ruff-0.6.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe6d5f65d6f276ee7a0fc50a0cecaccb362d30ef98a110f99cac1c7872df2f18"}, - {file = "ruff-0.6.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2e0dd11e2ae553ee5c92a81731d88a9883af8db7408db47fc81887c1f8b672e"}, - {file = "ruff-0.6.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d812615525a34ecfc07fd93f906ef5b93656be01dfae9a819e31caa6cfe758a1"}, - {file = "ruff-0.6.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:faaa4060f4064c3b7aaaa27328080c932fa142786f8142aff095b42b6a2eb631"}, - {file = "ruff-0.6.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:99d7ae0df47c62729d58765c593ea54c2546d5de213f2af2a19442d50a10cec9"}, - {file = "ruff-0.6.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9eb18dfd7b613eec000e3738b3f0e4398bf0153cb80bfa3e351b3c1c2f6d7b15"}, - {file = "ruff-0.6.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:c62bc04c6723a81e25e71715aa59489f15034d69bf641df88cb38bdc32fd1dbb"}, - {file = "ruff-0.6.1-py3-none-win32.whl", hash = "sha256:9fb4c4e8b83f19c9477a8745e56d2eeef07a7ff50b68a6998f7d9e2e3887bdc4"}, - {file = "ruff-0.6.1-py3-none-win_amd64.whl", hash = "sha256:c2ebfc8f51ef4aca05dad4552bbcf6fe8d1f75b2f6af546cc47cc1c1ca916b5b"}, - {file = "ruff-0.6.1-py3-none-win_arm64.whl", hash = "sha256:3bc81074971b0ffad1bd0c52284b22411f02a11a012082a76ac6da153536e014"}, - {file = "ruff-0.6.1.tar.gz", hash = "sha256:af3ffd8c6563acb8848d33cd19a69b9bfe943667f0419ca083f8ebe4224a3436"}, + {file = "ruff-0.6.3-py3-none-linux_armv6l.whl", hash = "sha256:97f58fda4e309382ad30ede7f30e2791d70dd29ea17f41970119f55bdb7a45c3"}, + {file = "ruff-0.6.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3b061e49b5cf3a297b4d1c27ac5587954ccb4ff601160d3d6b2f70b1622194dc"}, + {file = "ruff-0.6.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:34e2824a13bb8c668c71c1760a6ac7d795ccbd8d38ff4a0d8471fdb15de910b1"}, + {file = "ruff-0.6.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bddfbb8d63c460f4b4128b6a506e7052bad4d6f3ff607ebbb41b0aa19c2770d1"}, + {file = "ruff-0.6.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ced3eeb44df75353e08ab3b6a9e113b5f3f996bea48d4f7c027bc528ba87b672"}, + {file = "ruff-0.6.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47021dff5445d549be954eb275156dfd7c37222acc1e8014311badcb9b4ec8c1"}, + {file = "ruff-0.6.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:7d7bd20dc07cebd68cc8bc7b3f5ada6d637f42d947c85264f94b0d1cd9d87384"}, + {file = "ruff-0.6.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:500f166d03fc6d0e61c8e40a3ff853fa8a43d938f5d14c183c612df1b0d6c58a"}, + {file = "ruff-0.6.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42844ff678f9b976366b262fa2d1d1a3fe76f6e145bd92c84e27d172e3c34500"}, + {file = "ruff-0.6.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70452a10eb2d66549de8e75f89ae82462159855e983ddff91bc0bce6511d0470"}, + {file = "ruff-0.6.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:65a533235ed55f767d1fc62193a21cbf9e3329cf26d427b800fdeacfb77d296f"}, + {file = "ruff-0.6.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d2e2c23cef30dc3cbe9cc5d04f2899e7f5e478c40d2e0a633513ad081f7361b5"}, + {file = "ruff-0.6.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d8a136aa7d228975a6aee3dd8bea9b28e2b43e9444aa678fb62aeb1956ff2351"}, + {file = "ruff-0.6.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f92fe93bc72e262b7b3f2bba9879897e2d58a989b4714ba6a5a7273e842ad2f8"}, + {file = "ruff-0.6.3-py3-none-win32.whl", hash = "sha256:7a62d3b5b0d7f9143d94893f8ba43aa5a5c51a0ffc4a401aa97a81ed76930521"}, + {file = "ruff-0.6.3-py3-none-win_amd64.whl", hash = "sha256:746af39356fee2b89aada06c7376e1aa274a23493d7016059c3a72e3b296befb"}, + {file = "ruff-0.6.3-py3-none-win_arm64.whl", hash = "sha256:14a9528a8b70ccc7a847637c29e56fd1f9183a9db743bbc5b8e0c4ad60592a82"}, + {file = "ruff-0.6.3.tar.gz", hash = "sha256:183b99e9edd1ef63be34a3b51fee0a9f4ab95add123dbf89a71f7b1f0c991983"}, ] [[package]] From c0888e1f70fbcda11edd993316cc45b7fa56c77c Mon Sep 17 00:00:00 2001 From: dorfmanrobert <108150810+dorfmanrobert@users.noreply.github.com> Date: Tue, 3 Sep 2024 11:38:28 +0100 Subject: [PATCH 5/7] handle exif data when loading images during training (#924) --- darwin/dataset/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index e948c11b2..978c6e431 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -6,6 +6,7 @@ import numpy as np from PIL import Image as PILImage +from PIL import ImageOps from rich.live import Live from rich.progress import ProgressBar, track @@ -676,6 +677,7 @@ def load_pil_image(path: Path, to_rgb: Optional[bool] = True) -> PILImage.Image: The loaded image. """ pic = PILImage.open(path) + pic = ImageOps.exif_transpose(pic) if to_rgb: pic = convert_to_rgb(pic) return pic From 9f4530d1290db54009ca58ebf9be2880e550149c Mon Sep 17 00:00:00 2001 From: John Wilkie <124276291+JBWilkie@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:34:40 +0100 Subject: [PATCH 6/7] [DAR-3771][External] Import of annotation-level properties (#925) * Added to the class * Import of annotation-level properties * Added unit tests for _import_properties() * Fix for albumentations transform * Prevent 422s by not performing the same property update/create command multiple times --- darwin/datatypes.py | 23 +- darwin/future/data_objects/properties.py | 23 +- darwin/future/tests/core/fixtures.py | 7 +- darwin/importer/importer.py | 12 +- darwin/torch/transforms.py | 7 +- darwin/utils/utils.py | 3 +- tests/darwin/client_test.py | 1 + ...ta_missing_annotation_property_values.json | 51 +++ ...adata_missing_section_property_values.json | 49 +++ tests/darwin/importer/importer_test.py | 347 ++++++++++++++++++ 10 files changed, 510 insertions(+), 13 deletions(-) create mode 100644 tests/darwin/data/metadata_missing_annotation_property_values.json create mode 100644 tests/darwin/data/metadata_missing_section_property_values.json diff --git a/darwin/datatypes.py b/darwin/datatypes.py index 5b79c7958..c2ef99d26 100644 --- a/darwin/datatypes.py +++ b/darwin/datatypes.py @@ -24,7 +24,11 @@ except ImportError: NDArray = Any # type:ignore -from darwin.future.data_objects.properties import PropertyType, SelectedProperty +from darwin.future.data_objects.properties import ( + PropertyType, + SelectedProperty, + PropertyGranularity, +) from darwin.path_utils import construct_full_path, is_properties_enabled, parse_metadata # Utility types @@ -422,6 +426,9 @@ class Property: # Description of the property description: Optional[str] = None + # Granularity of the property + granularity: PropertyGranularity = PropertyGranularity("section") + @dataclass class PropertyClass: @@ -454,6 +461,17 @@ def parse_property_classes(metadata: dict[str, Any]) -> list[PropertyClass]: assert ( "properties" in metadata_cls ), "Metadata class does not contain properties" + properties = [ + Property( + name=p["name"], + type=p["type"], + required=p["required"], + property_values=p["property_values"], + description=p.get("description"), + granularity=PropertyGranularity(p.get("granularity", "section")), + ) + for p in metadata_cls["properties"] + ] classes.append( PropertyClass( name=metadata_cls["name"], @@ -461,10 +479,9 @@ def parse_property_classes(metadata: dict[str, Any]) -> list[PropertyClass]: description=metadata_cls.get("description"), color=metadata_cls.get("color"), sub_types=metadata_cls.get("sub_types"), - properties=[Property(**p) for p in metadata_cls["properties"]], + properties=properties, ) ) - return classes diff --git a/darwin/future/data_objects/properties.py b/darwin/future/data_objects/properties.py index b12ca0a29..f60b917f3 100644 --- a/darwin/future/data_objects/properties.py +++ b/darwin/future/data_objects/properties.py @@ -3,7 +3,8 @@ import json import os from pathlib import Path -from typing import List, Literal, Optional, Tuple +from typing import List, Literal, Optional, Tuple, Union +from enum import Enum from pydantic import field_validator @@ -19,6 +20,12 @@ ] +class PropertyGranularity(str, Enum): + section = "section" + annotation = "annotation" + item = "item" + + class PropertyValue(DefaultDarwin): """ Describes a single option for a property @@ -60,6 +67,8 @@ class FullProperty(DefaultDarwin): type (str): Type of the property required (bool): If the property is required options (List[PropertyOption]): List of all options for the property + granularity (PropertyGranularity): Granularity of the property + """ id: Optional[str] = None @@ -73,6 +82,7 @@ class FullProperty(DefaultDarwin): annotation_class_id: Optional[int] = None property_values: Optional[List[PropertyValue]] = None options: Optional[List[PropertyValue]] = None + granularity: PropertyGranularity = PropertyGranularity("section") def to_create_endpoint( self, @@ -87,6 +97,7 @@ def to_create_endpoint( "annotation_class_id": True, "property_values": {"__all__": {"value", "color"}}, "description": True, + "granularity": True, } ) @@ -94,7 +105,8 @@ def to_update_endpoint(self) -> Tuple[str, dict]: if self.id is None: raise ValueError("id must be set") updated_base = self.to_create_endpoint() - del updated_base["annotation_class_id"] # can't update this field + del updated_base["annotation_class_id"] # Can't update this field + del updated_base["granularity"] # Can't update this field return self.id, updated_base @@ -110,6 +122,7 @@ class MetaDataClass(DefaultDarwin): description (Optional[str]): Description of the class color (Optional[str]): Color of the class in the UI sub_types (Optional[List[str]]): Sub types of the class + granularity:(PropertyGranularity): Granularity of the property properties (List[FullProperty]): List of all properties for the class with all options """ @@ -118,6 +131,7 @@ class MetaDataClass(DefaultDarwin): description: Optional[str] = None color: Optional[str] = None sub_types: Optional[List[str]] = None + granularity: PropertyGranularity = PropertyGranularity("section") properties: List[FullProperty] @classmethod @@ -141,13 +155,14 @@ class SelectedProperty(DefaultDarwin): Selected property for an annotation found inside a darwin annotation Attributes: - frame_index (int): Frame index of the annotation + frame_index (int | str): Frame index of the annotation + int for section-level properties, and "global" for annotation-level properties name (str): Name of the property type (str | None): Type of the property (if it exists) value (str): Value of the property """ - frame_index: Optional[int] = None + frame_index: Optional[Union[int, str]] = None name: str type: Optional[str] = None value: Optional[str] = None diff --git a/darwin/future/tests/core/fixtures.py b/darwin/future/tests/core/fixtures.py index 882f0d411..b824232b2 100644 --- a/darwin/future/tests/core/fixtures.py +++ b/darwin/future/tests/core/fixtures.py @@ -8,7 +8,11 @@ from darwin.future.core.client import ClientCore, DarwinConfig from darwin.future.data_objects.dataset import DatasetCore from darwin.future.data_objects.item import ItemCore, ItemLayout, ItemSlot -from darwin.future.data_objects.properties import FullProperty, PropertyValue +from darwin.future.data_objects.properties import ( + FullProperty, + PropertyValue, + PropertyGranularity, +) from darwin.future.data_objects.team import TeamCore, TeamMemberCore from darwin.future.data_objects.team_member_role import TeamMemberRole from darwin.future.data_objects.workflow import WorkflowCore @@ -38,6 +42,7 @@ def base_property_object(base_property_value: PropertyValue) -> FullProperty: annotation_class_id=0, property_values=[base_property_value], options=[base_property_value], + granularity=PropertyGranularity("section"), ) diff --git a/darwin/importer/importer.py b/darwin/importer/importer.py index 138769520..871160cfe 100644 --- a/darwin/importer/importer.py +++ b/darwin/importer/importer.py @@ -26,6 +26,7 @@ PropertyType, PropertyValue, SelectedProperty, + PropertyGranularity, ) from darwin.item import DatasetItem from darwin.path_utils import is_properties_enabled, parse_metadata @@ -412,6 +413,7 @@ def _import_properties( # if property value is None, update annotation_property_map with empty set if a_prop.value is None: assert t_prop.id is not None + annotation_property_map[annotation_id][str(a_prop.frame_index)][ t_prop.id ] = set() @@ -516,8 +518,11 @@ def _import_properties( slug=client.default_team, annotation_class_id=int(annotation_class_id), property_values=property_values, + granularity=PropertyGranularity(m_prop.granularity.value), ) - create_properties.append(full_property) + # Don't attempt the same propery creation multiple times + if full_property not in create_properties: + create_properties.append(full_property) continue # check if property value is different in m_prop (.v7/metadata.json) options @@ -565,7 +570,9 @@ def _import_properties( ) ], ) - update_properties.append(full_property) + # Don't attempt the same propery update multiple times + if full_property not in update_properties: + update_properties.append(full_property) continue assert t_prop.id is not None @@ -649,6 +656,7 @@ def _import_properties( slug=client.default_team, annotation_class_id=t_prop.annotation_class_id, property_values=extra_property_values, + granularity=PropertyGranularity(t_prop.granularity.value), ) console.print( f"Updating property {full_property.name} ({full_property.type}) with extra metadata values {extra_values}", diff --git a/darwin/torch/transforms.py b/darwin/torch/transforms.py index fd60cb3c5..e783d4947 100644 --- a/darwin/torch/transforms.py +++ b/darwin/torch/transforms.py @@ -368,8 +368,11 @@ def _pre_process(self, image: np.ndarray, annotation: dict) -> dict: if ( masks is not None and masks.numel() > 0 ): # using numel() to check if tensor is non-empty - print("WE GOT MASKS") - albumentation_dict["masks"] = masks.numpy() + if isinstance(masks, torch.Tensor): + masks = masks.numpy() + if masks.ndim == 3: # Ensure masks is a list of numpy arrays + masks = [masks[i] for i in range(masks.shape[0])] + albumentation_dict["masks"] = masks return albumentation_dict diff --git a/darwin/utils/utils.py b/darwin/utils/utils.py index 5f57449ad..b7b13c9f3 100644 --- a/darwin/utils/utils.py +++ b/darwin/utils/utils.py @@ -1149,9 +1149,10 @@ def _parse_properties( ) -> Optional[List[SelectedProperty]]: selected_properties = [] for property in properties: + frame_index = property.get("frame_index") selected_properties.append( SelectedProperty( - frame_index=property.get("frame_index", None), + frame_index=frame_index if frame_index is not None else "global", name=property.get("name", None), value=property.get("value", None), ) diff --git a/tests/darwin/client_test.py b/tests/darwin/client_test.py index 1652c2f58..c2f6b4903 100644 --- a/tests/darwin/client_test.py +++ b/tests/darwin/client_test.py @@ -378,6 +378,7 @@ def test_get_team_properties(self, darwin_client: Client) -> None: "slug": "property-question", "team_id": 128, "type": "multi_select", + "granularity": "section", }, ] }, diff --git a/tests/darwin/data/metadata_missing_annotation_property_values.json b/tests/darwin/data/metadata_missing_annotation_property_values.json new file mode 100644 index 000000000..740f57601 --- /dev/null +++ b/tests/darwin/data/metadata_missing_annotation_property_values.json @@ -0,0 +1,51 @@ +{ + "version": "1.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/metadata/1.0/schema.json", + "classes": [ + { + "name": "test_class", + "type": "bounding_box", + "description": null, + "color": "rgba(255,46,0,1.0)", + "sub_types": [ + "inference" + ], + "properties": [ + { + "name": "existing_property_single_select", + "type": "single_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(255,46,0,1.0)" + } + ], + "granularity": "annotation" + }, + { + "name": "existing_property_multi_select", + "type": "multi_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(173,255,0,1.0)" + }, + { + "value": "2", + "color": "rgba(255,199,0,1.0)" + } + ], + "granularity": "annotation" + } + ], + "sub_types_settings": { + "inference": {} + } + } + ], + "properties": [] + } \ No newline at end of file diff --git a/tests/darwin/data/metadata_missing_section_property_values.json b/tests/darwin/data/metadata_missing_section_property_values.json new file mode 100644 index 000000000..5adc44c5f --- /dev/null +++ b/tests/darwin/data/metadata_missing_section_property_values.json @@ -0,0 +1,49 @@ +{ + "version": "1.0", + "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/metadata/1.0/schema.json", + "classes": [ + { + "name": "test_class", + "type": "bounding_box", + "description": null, + "color": "rgba(255,46,0,1.0)", + "sub_types": [ + "inference" + ], + "properties": [ + { + "name": "existing_property_single_select", + "type": "single_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(255,46,0,1.0)" + } + ] + }, + { + "name": "existing_property_multi_select", + "type": "multi_select", + "description": "", + "required": false, + "property_values": [ + { + "value": "1", + "color": "rgba(173,255,0,1.0)" + }, + { + "value": "2", + "color": "rgba(255,199,0,1.0)" + } + ] + } + ], + "sub_types_settings": { + "inference": {} + } + } + ], + "properties": [] + } \ No newline at end of file diff --git a/tests/darwin/importer/importer_test.py b/tests/darwin/importer/importer_test.py index 70cf197df..be717d589 100644 --- a/tests/darwin/importer/importer_test.py +++ b/tests/darwin/importer/importer_test.py @@ -6,6 +6,12 @@ from unittest.mock import MagicMock, Mock, _patch, patch from zipfile import ZipFile +from darwin.future.data_objects.properties import ( + PropertyGranularity, + SelectedProperty, + FullProperty, + PropertyValue, +) import pytest from darwin import datatypes as dt @@ -24,9 +30,49 @@ _parse_empty_masks, _resolve_annotation_classes, _verify_slot_annotation_alignment, + _import_properties, ) +@pytest.fixture +def setup_data(request): + granularity = request.param + client = Mock() + client.default_team = "test_team" + team_slug = "test_team" + annotation_class_ids_map = {("test_class", "polygon"): "123"} + annotations = [ + dt.Annotation( + dt.AnnotationClass("test_class", "polygon"), + {"paths": [[1, 2, 3, 4, 5]]}, + [], + [], + id="annotation_id_1", + properties=[ + SelectedProperty( + frame_index=None if granularity == "annotation" else "0", + name="existing_property_single_select", + type="single_select", + value="1", + ), + SelectedProperty( + frame_index=None if granularity == "annotation" else "0", + name="existing_property_multi_select", + type="multi_select", + value="1", + ), + SelectedProperty( + frame_index=None if granularity == "annotation" else "1", + name="existing_property_multi_select", + type="multi_select", + value="2", + ), + ], + ) + ] + return client, team_slug, annotation_class_ids_map, annotations + + def root_path(x: str) -> str: return f"darwin.importer.importer.{x}" @@ -1465,3 +1511,304 @@ def test_does_not_raise_error_for_darwin_format_with_warnings(): _display_slot_warnings_and_errors(slot_errors, slot_warnings, "darwin", console) assert not slot_errors + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["section"], indirect=True) +def test_import_existing_section_level_property_values_without_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_1"), + ], + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + PropertyValue(value="2", id="property_value_id_3"), + ], + ), + } + metadata_path = False + result = _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert result["annotation_id_1"]["0"]["property_id_1"] == { + "property_value_id_1", + } + assert result["annotation_id_1"]["0"]["property_id_2"] == { + "property_value_id_2", + } + assert result["annotation_id_1"]["1"]["property_id_2"] == { + "property_value_id_3", + } + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["section"], indirect=True) +def test_import_new_section_level_property_values_with_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[], + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + ], + ), + } + metadata_path = ( + Path(__file__).parents[1] + / "data" + / "metadata_missing_section_property_values.json" + ) + with patch.object(client, "update_property") as mock_update_property: + result = _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert result["annotation_id_1"]["0"]["property_id_2"] == { + "property_value_id_2", + } + assert mock_update_property.call_args_list[0].kwargs["params"] == FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + description="property-updated-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="1", color="rgba(255,46,0,1.0)"), + ], + ) + assert mock_update_property.call_args_list[1].kwargs["params"] == FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + description="property-updated-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="2", color="rgba(255,199,0,1.0)"), + ], + ) + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["section"], indirect=True) +def test_import_new_section_level_properties_with_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = {} + metadata_path = ( + Path(__file__).parents[1] + / "data" + / "metadata_missing_section_property_values.json" + ) + with patch.object(client, "create_property") as mock_create_property: + _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert mock_create_property.call_args_list[0].kwargs["params"] == FullProperty( + id=None, + position=None, + name="existing_property_single_select", + type="single_select", + required=False, + description="property-created-during-annotation-import", + annotation_class_id=123, + slug="test_team", + team_id=None, + property_values=[ + PropertyValue(value="1", color="rgba(255,46,0,1.0)"), + ], + options=None, + granularity=PropertyGranularity.section, + ) + assert mock_create_property.call_args_list[1].kwargs["params"] == FullProperty( + name="existing_property_multi_select", + type="multi_select", + required=False, + description="property-created-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="1", color="rgba(173,255,0,1.0)"), + PropertyValue(value="2", color="rgba(255,199,0,1.0)"), + ], + ) + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["annotation"], indirect=True) +def test_import_existing_annotation_level_property_values_without_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_1"), + ], + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + PropertyValue(value="2", id="property_value_id_3"), + ], + ), + } + metadata_path = False + result = _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert result["annotation_id_1"]["None"]["property_id_1"] == { + "property_value_id_1", + } + assert result["annotation_id_1"]["None"]["property_id_2"] == { + "property_value_id_2", + "property_value_id_3", + } + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["annotation"], indirect=True) +def test_import_new_annotation_level_property_values_with_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = { + ("existing_property_single_select", 123): FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + property_values=[], + ), + ("existing_property_multi_select", 123): FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + property_values=[ + PropertyValue(value="1", id="property_value_id_2"), + ], + ), + } + metadata_path = ( + Path(__file__).parents[1] + / "data" + / "metadata_missing_annotation_property_values.json" + ) + with patch.object(client, "update_property") as mock_update_property: + result = _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert result["annotation_id_1"]["None"]["property_id_2"] == { + "property_value_id_2", + } + assert mock_update_property.call_args_list[0].kwargs["params"] == FullProperty( + id="property_id_1", + name="existing_property_single_select", + type="single_select", + required=False, + description="property-updated-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="1", color="rgba(255,46,0,1.0)"), + ], + ) + assert mock_update_property.call_args_list[1].kwargs["params"] == FullProperty( + id="property_id_2", + name="existing_property_multi_select", + type="multi_select", + required=False, + description="property-updated-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="2", color="rgba(255,199,0,1.0)"), + ], + ) + + +@patch("darwin.importer.importer._get_team_properties_annotation_lookup") +@pytest.mark.parametrize("setup_data", ["annotation"], indirect=True) +def test_import_new_annotation_level_properties_with_manifest( + mock_get_team_properties, + setup_data, +): + client, team_slug, annotation_class_ids_map, annotations = setup_data + mock_get_team_properties.return_value = {} + metadata_path = ( + Path(__file__).parents[1] + / "data" + / "metadata_missing_annotation_property_values.json" + ) + with patch.object(client, "create_property") as mock_create_property: + _import_properties( + metadata_path, client, annotations, annotation_class_ids_map, team_slug + ) + assert mock_create_property.call_args_list[0].kwargs["params"] == FullProperty( + name="existing_property_single_select", + type="single_select", + required=False, + description="property-created-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="1", color="rgba(255,46,0,1.0)"), + ], + granularity=PropertyGranularity.annotation, + ) + assert mock_create_property.call_args_list[1].kwargs["params"] == FullProperty( + name="existing_property_multi_select", + type="multi_select", + required=False, + description="property-created-during-annotation-import", + annotation_class_id=123, + slug="test_team", + property_values=[ + PropertyValue(value="1", color="rgba(173,255,0,1.0)"), + PropertyValue(value="2", color="rgba(255,199,0,1.0)"), + ], + granularity=PropertyGranularity.annotation, + ) From 0b1809e17473f47613c8264484756e49cc74c8fb Mon Sep 17 00:00:00 2001 From: John Wilkie <124276291+JBWilkie@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:15:08 +0100 Subject: [PATCH 7/7] Version bump to 1.0.8 (#926) --- darwin/version/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/darwin/version/__init__.py b/darwin/version/__init__.py index 9e604c040..e13bd590c 100644 --- a/darwin/version/__init__.py +++ b/darwin/version/__init__.py @@ -1 +1 @@ -__version__ = "1.0.7" +__version__ = "1.0.8" diff --git a/pyproject.toml b/pyproject.toml index cd9a21ebe..70f59b36a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ license = "MIT" name = "darwin-py" readme = "README.md" repository = "https://github.com/v7labs/darwin-py" -version = "1.0.7" +version = "1.0.8" [[tool.poetry.packages]] include = "darwin"