Improve transformer mixins (#2039)

cognite/client/data_classes/annotations.py

@@ -10,6 +10,7 @@
     CogniteResource,
     CogniteResourceList,
     CogniteUpdate,
+    IdTransformerMixin,
     PropertySpec,
     WriteableCogniteResource,
     WriteableCogniteResourceList,
@@ -370,7 +371,7 @@ class AnnotationWriteList(CogniteResourceList[AnnotationWrite]):
     _RESOURCE = AnnotationWrite
 
 
-class AnnotationList(WriteableCogniteResourceList[AnnotationWrite, Annotation]):
+class AnnotationList(WriteableCogniteResourceList[AnnotationWrite, Annotation], IdTransformerMixin):
     _RESOURCE = Annotation
 
     def as_write(self) -> AnnotationWriteList:

cognite/client/data_classes/contextualization.py

@@ -17,6 +17,7 @@
     CogniteResource,
     CogniteResourceList,
     CogniteUpdate,
+    IdTransformerMixin,
     PropertySpec,
 )
 from cognite.client.data_classes.annotation_types.images import (
@@ -349,7 +350,7 @@ def _get_update_properties(cls, item: CogniteResource | None = None) -> list[Pro
         ]
 
 
-class EntityMatchingModelList(CogniteResourceList[EntityMatchingModel]):
+class EntityMatchingModelList(CogniteResourceList[EntityMatchingModel], IdTransformerMixin):
     _RESOURCE = EntityMatchingModel
 
 
@@ -1045,7 +1046,7 @@ def __init__(
         self._cognite_client: CogniteClient = cast("CogniteClient", None)  # Read only
 
 
-class ResourceReferenceList(CogniteResourceList[ResourceReference]):
+class ResourceReferenceList(CogniteResourceList[ResourceReference], IdTransformerMixin):
     _RESOURCE = ResourceReference
 
 

cognite/client/data_classes/extractionpipelines.py

@@ -558,7 +558,9 @@ class ExtractionPipelineRunWriteList(CogniteResourceList[ExtractionPipelineRunWr
     _RESOURCE = ExtractionPipelineRunWrite
 
 
-class ExtractionPipelineRunList(WriteableCogniteResourceList[ExtractionPipelineRunWrite, ExtractionPipelineRun]):
+class ExtractionPipelineRunList(
+    WriteableCogniteResourceList[ExtractionPipelineRunWrite, ExtractionPipelineRun], IdTransformerMixin
+):
     _RESOURCE = ExtractionPipelineRun
 
     def as_write(self) -> ExtractionPipelineRunWriteList:
@@ -720,16 +722,18 @@ def as_write(self) -> ExtractionPipelineConfigWrite:
         return self
 
 
-class ExtractionPipelineConfigRevisionList(CogniteResourceList[ExtractionPipelineConfigRevision]):
+class ExtractionPipelineConfigRevisionList(
+    CogniteResourceList[ExtractionPipelineConfigRevision], ExternalIDTransformerMixin
+):
     _RESOURCE = ExtractionPipelineConfigRevision
 
 
-class ExtractionPipelineConfigWriteList(CogniteResourceList[ExtractionPipelineConfigWrite]):
+class ExtractionPipelineConfigWriteList(CogniteResourceList[ExtractionPipelineConfigWrite], ExternalIDTransformerMixin):
     _RESOURCE = ExtractionPipelineConfigWrite
 
 
 class ExtractionPipelineConfigList(
-    WriteableCogniteResourceList[ExtractionPipelineConfigWrite, ExtractionPipelineConfig]
+    WriteableCogniteResourceList[ExtractionPipelineConfigWrite, ExtractionPipelineConfig], ExternalIDTransformerMixin
 ):
     _RESOURCE = ExtractionPipelineConfig
 

cognite/client/data_classes/functions.py

@@ -12,6 +12,7 @@
     CogniteResponse,
     ExternalIDTransformerMixin,
     IdTransformerMixin,
+    InternalIdTransformerMixin,
     WriteableCogniteResource,
     WriteableCogniteResourceList,
 )
@@ -528,7 +529,9 @@ class FunctionScheduleWriteList(CogniteResourceList[FunctionScheduleWrite]):
     _RESOURCE = FunctionScheduleWrite
 
 
-class FunctionSchedulesList(WriteableCogniteResourceList[FunctionScheduleWrite, FunctionSchedule]):
+class FunctionSchedulesList(
+    WriteableCogniteResourceList[FunctionScheduleWrite, FunctionSchedule], InternalIdTransformerMixin
+):
     _RESOURCE = FunctionSchedule
 
     def as_write(self) -> FunctionScheduleWriteList:
@@ -631,7 +634,7 @@ def wait(self) -> None:
             time.sleep(1.0)
 
 
-class FunctionCallList(CogniteResourceList[FunctionCall]):
+class FunctionCallList(CogniteResourceList[FunctionCall], InternalIdTransformerMixin):
     _RESOURCE = FunctionCall
 
 

cognite/client/data_classes/geospatial.py

@@ -150,7 +150,7 @@ class FeatureTypeWriteList(CogniteResourceList[FeatureTypeWrite], ExternalIDTran
     _RESOURCE = FeatureTypeWrite
 
 
-class FeatureTypeList(WriteableCogniteResourceList[FeatureTypeWrite, FeatureType]):
+class FeatureTypeList(WriteableCogniteResourceList[FeatureTypeWrite, FeatureType], ExternalIDTransformerMixin):
     _RESOURCE = FeatureType
 
     def as_write(self) -> FeatureTypeWriteList:

cognite/client/data_classes/iam.py

@@ -10,6 +10,7 @@
     CogniteResource,
     CogniteResourceList,
     CogniteResponse,
+    IdTransformerMixin,
     InternalIdTransformerMixin,
     NameTransformerMixin,
     WriteableCogniteResource,
@@ -321,7 +322,11 @@ class SecurityCategoryWriteList(CogniteResourceList[SecurityCategoryWrite], Name
     _RESOURCE = SecurityCategoryWrite
 
 
-class SecurityCategoryList(WriteableCogniteResourceList[SecurityCategoryWrite, SecurityCategory], NameTransformerMixin):
+class SecurityCategoryList(
+    WriteableCogniteResourceList[SecurityCategoryWrite, SecurityCategory],
+    InternalIdTransformerMixin,
+    NameTransformerMixin,
+):
     _RESOURCE = SecurityCategory
 
     def as_write(self) -> SecurityCategoryWriteList:
@@ -459,7 +464,7 @@ def __init__(
         self.client_id = client_id
 
 
-class SessionList(CogniteResourceList[Session]):
+class SessionList(CogniteResourceList[Session], IdTransformerMixin):
     _RESOURCE = Session
 
 

cognite/client/data_classes/sequences.py

@@ -842,7 +842,7 @@ def __init__(
         )
 
 
-class SequenceRowsList(CogniteResourceList[SequenceRows]):
+class SequenceRowsList(CogniteResourceList[SequenceRows], IdTransformerMixin):
     _RESOURCE = SequenceRows
 
     def __str__(self) -> str:

cognite/client/data_classes/templates.py

@@ -9,6 +9,7 @@
     CogniteResource,
     CogniteResourceList,
     CogniteUpdate,
+    ExternalIDTransformerMixin,
     PropertySpec,
     WriteableCogniteResource,
     WriteableCogniteResourceList,
@@ -118,11 +119,11 @@ def as_write(self) -> TemplateGroupWrite:
         return self
 
 
-class TemplateGroupWriteList(CogniteResourceList[TemplateGroupWrite]):
+class TemplateGroupWriteList(CogniteResourceList[TemplateGroupWrite], ExternalIDTransformerMixin):
     _RESOURCE = TemplateGroupWrite
 
 
-class TemplateGroupList(WriteableCogniteResourceList[TemplateGroupWrite, TemplateGroup]):
+class TemplateGroupList(WriteableCogniteResourceList[TemplateGroupWrite, TemplateGroup], ExternalIDTransformerMixin):
     _RESOURCE = TemplateGroup
 
     def as_write(self) -> TemplateGroupWriteList:
@@ -660,22 +661,24 @@ def _load(cls, resource: dict, cognite_client: CogniteClient | None = None) -> G
         )
 
 
-class TemplateInstanceWriteList(CogniteResourceList[TemplateInstanceWrite]):
+class TemplateInstanceWriteList(CogniteResourceList[TemplateInstanceWrite], ExternalIDTransformerMixin):
     _RESOURCE = TemplateInstanceWrite
 
 
-class TemplateInstanceList(WriteableCogniteResourceList[TemplateInstanceWrite, TemplateInstance]):
+class TemplateInstanceList(
+    WriteableCogniteResourceList[TemplateInstanceWrite, TemplateInstance], ExternalIDTransformerMixin
+):
     _RESOURCE = TemplateInstance
 
     def as_write(self) -> TemplateInstanceWriteList:
         return TemplateInstanceWriteList([item.as_write() for item in self], cognite_client=self._get_cognite_client())
 
 
-class ViewWriteList(CogniteResourceList[ViewWrite]):
+class ViewWriteList(CogniteResourceList[ViewWrite], ExternalIDTransformerMixin):
     _RESOURCE = ViewWrite
 
 
-class ViewList(WriteableCogniteResourceList[ViewWrite, View]):
+class ViewList(WriteableCogniteResourceList[ViewWrite, View], ExternalIDTransformerMixin):
     _RESOURCE = View
 
     def as_write(self) -> ViewWriteList:

cognite/client/data_classes/three_d.py

@@ -542,7 +542,7 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]:
         return result
 
 
-class ThreeDNodeList(CogniteResourceList[ThreeDNode]):
+class ThreeDNodeList(CogniteResourceList[ThreeDNode], InternalIdTransformerMixin):
     _RESOURCE = ThreeDNode
 
 

cognite/client/data_classes/transformations/jobs.py

@@ -5,7 +5,12 @@
 from enum import Enum
 from typing import TYPE_CHECKING, Any, cast
 
-from cognite.client.data_classes._base import CogniteFilter, CogniteResource, CogniteResourceList
+from cognite.client.data_classes._base import (
+    CogniteFilter,
+    CogniteResource,
+    CogniteResourceList,
+    InternalIdTransformerMixin,
+)
 from cognite.client.data_classes.transformations.common import TransformationDestination
 
 if TYPE_CHECKING:
@@ -44,7 +49,7 @@ def __init__(
         self._cognite_client = cast("CogniteClient", cognite_client)
 
 
-class TransformationJobMetricList(CogniteResourceList[TransformationJobMetric]):
+class TransformationJobMetricList(CogniteResourceList[TransformationJobMetric], InternalIdTransformerMixin):
     _RESOURCE = TransformationJobMetric
 
 
@@ -268,7 +273,7 @@ def __hash__(self) -> int:
         return hash(self.id)
 
 
-class TransformationJobList(CogniteResourceList[TransformationJob]):
+class TransformationJobList(CogniteResourceList[TransformationJob], InternalIdTransformerMixin):
     _RESOURCE = TransformationJob
 
 

cognite/client/data_classes/workflows.py

@@ -13,6 +13,7 @@
     CogniteResource,
     CogniteResourceList,
     ExternalIDTransformerMixin,
+    InternalIdTransformerMixin,
     UnknownCogniteObject,
     WriteableCogniteResource,
     WriteableCogniteResourceList,
@@ -1041,7 +1042,7 @@ def _load(cls, resource: dict, cognite_client: CogniteClient | None = None) -> W
         )
 
 
-class WorkflowExecutionList(CogniteResourceList[WorkflowExecution]):
+class WorkflowExecutionList(CogniteResourceList[WorkflowExecution], InternalIdTransformerMixin):
     """
     This class represents a list of workflow executions.
     """
@@ -1500,11 +1501,13 @@ def as_write(self) -> WorkflowTriggerUpsert:
         )
 
 
-class WorkflowTriggerUpsertList(CogniteResourceList[WorkflowTriggerUpsert]):
+class WorkflowTriggerUpsertList(CogniteResourceList[WorkflowTriggerUpsert], ExternalIDTransformerMixin):
     _RESOURCE = WorkflowTriggerUpsert
 
 
-class WorkflowTriggerList(WriteableCogniteResourceList[WorkflowTriggerUpsert, WorkflowTrigger]):
+class WorkflowTriggerList(
+    WriteableCogniteResourceList[WorkflowTriggerUpsert, WorkflowTrigger], ExternalIDTransformerMixin
+):
     """
     This class represents a list of workflow triggers.
     """
@@ -1568,7 +1571,7 @@ def _load(cls, resource: dict, cognite_client: CogniteClient | None = None) -> W
         )
 
 
-class WorkflowTriggerRunList(CogniteResourceList[WorkflowTriggerRun]):
+class WorkflowTriggerRunList(CogniteResourceList[WorkflowTriggerRun], ExternalIDTransformerMixin):
     """
     This class represents a list of workflow trigger runs.
     """

tests/tests_unit/test_meta.py

@@ -2,11 +2,17 @@
 from pathlib import Path
 
 import pytest
-import requests
 
 from cognite.client._api_client import APIClient
-from cognite.client.data_classes._base import CogniteResource, CogniteResourceList
-from tests.utils import all_subclasses
+from cognite.client.data_classes._base import (
+    CogniteResource,
+    CogniteResourceList,
+    ExternalIDTransformerMixin,
+    IdTransformerMixin,
+    InternalIdTransformerMixin,
+)
+from cognite.client.data_classes.datapoints import DatapointsArrayList, DatapointsList
+from tests.utils import all_concrete_subclasses, all_subclasses
 
 ALL_FILEPATHS = Path("cognite/client/").rglob("*.py")
 
@@ -95,12 +101,28 @@ def test_all_base_api_paths_have_retry_or_specifically_no_set(
     assert not (has_retry and no_retry_needed)
 
 
-@pytest.mark.xfail  # updated .proto-files not merged yet (StringDatapoint missing status)
-def test_verify_proto_files(tmpdir):
-    proto_url = "https://raw.githubusercontent.com/cognitedata/protobuf-files/master/v1/timeseries"
+@pytest.mark.parametrize("lst_cls", all_concrete_subclasses(CogniteResourceList))
+def test_ensure_identifier_mixins(lst_cls):
+    # TODO: Data Modeling uses "as_ids()" even though existing classes use the same for "integer internal ids"
+    if "data_modeling" in str(lst_cls):
+        return
+    elif lst_cls in {DatapointsList, DatapointsArrayList}:  # May contain duplicates
+        return
+
+    bases = lst_cls.__mro__
+    sig = inspect.signature(lst_cls._RESOURCE).parameters
+
+    missing_id = "id" in sig and not (InternalIdTransformerMixin in bases or IdTransformerMixin in bases)
+    missing_external_id = "external_id" in sig and not (
+        ExternalIDTransformerMixin in bases or IdTransformerMixin in bases
+    )
 
-    remote_file1 = requests.get(f"{proto_url}/data_points.proto").text.splitlines()
-    remote_file2 = requests.get(f"{proto_url}/data_point_list_response.proto").text.splitlines()
+    # TODO: Make an instance ID mixin class, for now, we just ignore:
+    # missing_instance_id = "instance_id" in sig and ...
 
-    assert remote_file1 == Path("cognite/client/_proto/data_points.proto").read_text().splitlines()
-    assert remote_file2 == Path("cognite/client/_proto/data_point_list_response.proto").read_text().splitlines()
+    if missing_id and missing_external_id:
+        pytest.fail(f"List class: '{lst_cls.__name__}' should inherit from IdTransformerMixin (id+external_id)")
+    elif missing_id:
+        pytest.fail(f"List class: '{lst_cls.__name__}' should inherit from InternalIdTransformerMixin")
+    elif missing_external_id:
+        pytest.fail(f"List class: '{lst_cls.__name__}' should inherit from ExternalIDTransformerMixin")