[CDF-22379] Hosted Extractors: Source

CHANGELOG.md

@@ -17,9 +17,26 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
-## [7.55.2] - 2024-08-29
+## [7.58.1] - 2024-09-03
+### Added
+- [Feature Preview - alpha] Support for `client.hosted_extractors.sources`.
+
+## [7.58.0] - 2024-09-03
+### Added
+- Data Workflows: add support for `SubworkflowReferenceParameters` subworkflow task type. Allowing embedding other workflows into a workflow.
+
+## [7.57.0] - 2024-09-03
 ### Added
-- [Feature Preview - beta] Support for `client.hosted_extractors.sources`.
+- Add a `load` method to CogniteClient, ClientConfig, and CredenitalProvider (and all it's subclasses).
+- Add `apply_settings` method to `global_config` to pass in a dict of settings
+
+## [7.56.0] - 2024-09-02
+### Added
+- Support for referencing files by instance id when running diagrams.detect
+
+## [7.55.2] - 2024-08-29
+### Fixed
+- Turn workflow_orchestration into data_workflows and add trigger doc, fix attribute names in data classes
 
 ## [7.55.1] - 2024-08-29
 ### Fixed

cognite/client/_api/annotations.py

@@ -101,7 +101,6 @@ def _convert_resource_to_patch_object(
         resource: CogniteResource,
         update_attributes: list[PropertySpec],
         mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null",
-        identifying_properties: dict[str, Any] | None = None,
     ) -> dict[str, dict[str, dict]]:
         if not isinstance(resource, Annotation):
             return APIClient._convert_resource_to_patch_object(resource, update_attributes)

cognite/client/_api/hosted_extractors/sources.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
 from collections.abc import Iterator
-from typing import TYPE_CHECKING, Any, Sequence, overload
+from typing import TYPE_CHECKING, Any, Literal, Sequence, overload
 
 from cognite.client._api_client import APIClient
 from cognite.client._constants import DEFAULT_LIMIT_READ
+from cognite.client.data_classes._base import CogniteResource, PropertySpec
 from cognite.client.data_classes.hosted_extractors.sources import Source, SourceList, SourceUpdate, SourceWrite
 from cognite.client.utils._experimental import FeaturePreviewWarning
 from cognite.client.utils._identifier import IdentifierSequence
@@ -49,7 +50,7 @@ def __call__(
     ) -> Iterator[Source] | Iterator[SourceList]:
         """Iterate over sources
 
-        Fetches sources as they are iterated over, so you keep a limited number of spaces in memory.
+        Fetches sources as they are iterated over, so you keep a limited number of sources in memory.
 
         Args:
             chunk_size (int | None): Number of sources to return in each chunk. Defaults to yielding one source a time.
@@ -72,7 +73,7 @@ def __call__(
     def __iter__(self) -> Iterator[Source]:
         """Iterate over sources
 
-        Fetches sources as they are iterated over, so you keep a limited number of spaces in memory.
+        Fetches sources as they are iterated over, so you keep a limited number of sources in memory.
 
         Returns:
             Iterator[Source]: yields Source one by one.
@@ -92,7 +93,7 @@ def retrieve(
 
         Args:
             external_ids (str | SequenceNotStr[str]): The external ID provided by the client. Must be unique for the resource type.
-            ignore_unknown_ids (bool): No description.
+            ignore_unknown_ids (bool): Ignore external IDs that are not found rather than throw an exception.
 
         Returns:
             Source | SourceList: Requested sources
@@ -103,11 +104,11 @@ def retrieve(
                 >>> client = CogniteClient()
                 >>> res = client.hosted_extractors.sources.retrieve('myMQTTSource')
 
-            Get multiple spaces by id:
+            Get multiple sources by id:
 
                 >>> from cognite.client import CogniteClient
                 >>> client = CogniteClient()
-                >>> res = client.hosted_extractors.sources.retrieve(["myMQTTSource", "MyEvenHubSource"], ignore_unknown_ids=True)
+                >>> res = client.hosted_extractors.sources.retrieve(["myMQTTSource", "MyEventHubSource"], ignore_unknown_ids=True)
 
         """
         self._warning.warn()
@@ -126,15 +127,15 @@ def delete(
 
         Args:
             external_ids (str | SequenceNotStr[str]): The external ID provided by the client. Must be unique for the resource type.
-            ignore_unknown_ids (bool): No description.
-            force (bool): No description.
+            ignore_unknown_ids (bool): Ignore external IDs that are not found rather than throw an exception.
+            force (bool): Delete any jobs associated with each item.
         Examples:
 
             Delete sources by id::
 
                 >>> from cognite.client import CogniteClient
                 >>> client = CogniteClient()
-                >>> client.hosted_extractors.sources.delete(spaces=["myMQTTSource", "MyEvenHubSource"])
+                >>> client.hosted_extractors.sources.delete(["myMQTTSource", "MyEventHubSource"])
         """
         self._warning.warn()
         extra_body_fields: dict[str, Any] = {}
@@ -146,7 +147,6 @@ def delete(
         self._delete_multiple(
             identifiers=IdentifierSequence.load(external_ids=external_ids),
             wrap_ids=True,
-            returns_items=False,
             headers={"cdf-version": "beta"},
             extra_body_fields=extra_body_fields or None,
         )
@@ -161,7 +161,7 @@ def create(self, items: SourceWrite | Sequence[SourceWrite]) -> Source | SourceL
         """`Create one or more sources. <https://developer.cognite.com/api#tag/Sources/operation/create_sources>`_
 
         Args:
-            items (SourceWrite | Sequence[SourceWrite]): Space | Sequence[Space]): Source(s) to create.
+            items (SourceWrite | Sequence[SourceWrite]): Source(s) to create.
 
         Returns:
             Source | SourceList: Created source(s)
@@ -195,7 +195,7 @@ def update(self, items: SourceWrite | SourceUpdate | Sequence[SourceWrite | Sour
         """`Update one or more sources. <https://developer.cognite.com/api#tag/Sources/operation/update_sources>`_
 
         Args:
-            items (SourceWrite | SourceUpdate | Sequence[SourceWrite | SourceUpdate]): Space | Sequence[Space]): Source(s) to update.
+            items (SourceWrite | SourceUpdate | Sequence[SourceWrite | SourceUpdate]): Source(s) to update.
 
         Returns:
             Source | SourceList: Updated source(s)
@@ -219,6 +219,18 @@ def update(self, items: SourceWrite | SourceUpdate | Sequence[SourceWrite | Sour
             headers={"cdf-version": "beta"},
         )
 
+    @classmethod
+    def _convert_resource_to_patch_object(
+        cls,
+        resource: CogniteResource,
+        update_attributes: list[PropertySpec],
+        mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null",
+    ) -> dict[str, dict[str, dict]]:
+        output = super()._convert_resource_to_patch_object(resource, update_attributes, mode)
+        if hasattr(resource, "type"):
+            output["type"] = resource.type
+        return output
+
     def list(
         self,
         limit: int | None = DEFAULT_LIMIT_READ,
@@ -237,7 +249,7 @@ def list(
 
                 >>> from cognite.client import CogniteClient
                 >>> client = CogniteClient()
-                >>> space_list = client.hosted_extractors.sources.list(limit=5)
+                >>> source_list = client.hosted_extractors.sources.list(limit=5)
 
             Iterate over sources::
 
@@ -251,7 +263,7 @@ def list(
                 >>> from cognite.client import CogniteClient
                 >>> client = CogniteClient()
                 >>> for source_list in client.hosted_extractors.sources(chunk_size=25):
-                ...     source_list # do something with the spaces
+                ...     source_list # do something with the sources
         """
         self._warning.warn()
         return self._list(

cognite/client/_api_client.py

@@ -1043,7 +1043,6 @@ def _update_multiple(
                         item,
                         update_cls._get_update_properties(item),
                         mode,
-                        update_cls._get_extra_identifying_properties(item),
                     )
                 )
             elif isinstance(item, CogniteUpdate):
@@ -1224,7 +1223,6 @@ def _convert_resource_to_patch_object(
         resource: CogniteResource,
         update_attributes: list[PropertySpec],
         mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null",
-        identifying_properties: dict[str, Any] | None = None,
     ) -> dict[str, dict[str, dict]]:
         dumped_resource = resource.dump(camel_case=True)
         has_id = "id" in dumped_resource
@@ -1240,9 +1238,6 @@ def _convert_resource_to_patch_object(
         elif has_external_id:
             patch_object["externalId"] = dumped_resource.pop("externalId")
 
-        if identifying_properties:
-            patch_object.update(identifying_properties)
-
         update: dict[str, dict] = cls._clear_all_attributes(update_attributes) if mode == "replace" else {}
 
         update_attribute_by_name = {prop.name: prop for prop in update_attributes}

cognite/client/_version.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "7.55.2"
+__version__ = "7.58.1"
 __api_subversion__ = "20230101"

cognite/client/data_classes/_base.py

@@ -519,12 +519,6 @@ def dump(self, camel_case: Literal[True] = True) -> dict[str, Any]:
     def _get_update_properties(cls, item: CogniteResource | None = None) -> list[PropertySpec]:
         raise NotImplementedError
 
-    @classmethod
-    def _get_extra_identifying_properties(cls, item: CogniteResource | None = None) -> dict[str, Any]:
-        # This method is used to provide additional identifying properties for the update object.
-        # It is intended to be overridden by subclasses that need to provide additional identifying properties.
-        return {}
-
 
 T_CogniteUpdate = TypeVar("T_CogniteUpdate", bound=CogniteUpdate)
 

cognite/client/data_classes/hosted_extractors/sources.py

@@ -3,7 +3,7 @@
 import itertools
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, NoReturn, cast
 
 from typing_extensions import Self
 
@@ -213,17 +213,8 @@ def __init__(
         self.created_time = created_time
         self.last_updated_time = last_updated_time
 
-    def as_write(self, key_value: str | None = None) -> EventHubSourceWrite:
-        if key_value is None:
-            raise ValueError("key_value must be provided")
-        return EventHubSourceWrite(
-            external_id=self.external_id,
-            host=self.host,
-            event_hub_name=self.event_hub_name,
-            key_name=self.key_name,
-            key_value=key_value,
-            consumer_group=self.consumer_group,
-        )
+    def as_write(self) -> NoReturn:
+        raise TypeError(f"{type(self).__name__} cannot be converted to write as id does not contain the secrets")
 
     @classmethod
     def _load_source(cls, resource: dict[str, Any]) -> Self:
@@ -355,15 +346,15 @@ def __init__(
         host: str,
         port: int | None = None,
         authentication: MQTTAuthenticationWrite | None = None,
-        useTls: bool = False,
+        use_tls: bool = False,
         ca_certificate: CACertificateWrite | None = None,
         auth_certificate: AuthCertificateWrite | None = None,
     ) -> None:
         super().__init__(external_id)
         self.host = host
         self.port = port
         self.authentication = authentication
-        self.useTls = useTls
+        self.use_tls = use_tls
         self.ca_certificate = ca_certificate
         self.auth_certificate = auth_certificate
 
@@ -376,7 +367,7 @@ def _load_source(cls, resource: dict[str, Any]) -> Self:
             authentication=MQTTAuthenticationWrite._load(resource["authentication"])
             if "authentication" in resource
             else None,
-            useTls=resource.get("useTls", False),
+            use_tls=resource.get("useTls", False),
             ca_certificate=CACertificateWrite._load(resource["caCertificate"]) if "caCertificate" in resource else None,
             auth_certificate=AuthCertificateWrite._load(resource["authCertificate"])
             if "authCertificate" in resource
@@ -461,15 +452,15 @@ def __init__(
         last_updated_time: int,
         port: int | None = None,
         authentication: MQTTAuthentication | None = None,
-        useTls: bool = False,
+        use_tls: bool = False,
         ca_certificate: CACertificate | None = None,
         auth_certificate: AuthCertificate | None = None,
     ) -> None:
         super().__init__(external_id)
         self.host = host
         self.port = port
         self.authentication = authentication
-        self.useTls = useTls
+        self.use_tls = use_tls
         self.ca_certificate = ca_certificate
         self.auth_certificate = auth_certificate
         self.created_time = created_time
@@ -484,7 +475,7 @@ def _load_source(cls, resource: dict[str, Any]) -> Self:
             authentication=MQTTAuthentication._load(resource["authentication"])
             if "authentication" in resource
             else None,
-            useTls=resource.get("useTls", False),
+            use_tls=resource.get("useTls", False),
             ca_certificate=CACertificate._load(resource["caCertificate"]) if "caCertificate" in resource else None,
             auth_certificate=AuthCertificate._load(resource["authCertificate"])
             if "authCertificate" in resource
@@ -493,7 +484,7 @@ def _load_source(cls, resource: dict[str, Any]) -> Self:
             last_updated_time=resource["lastUpdatedTime"],
         )
 
-    def as_write(self) -> _MQTTSourceWrite:
+    def as_write(self) -> NoReturn:
         raise TypeError(f"{type(self).__name__} cannot be converted to write as id does not contain the secrets")
 
     def dump(self, camel_case: bool = True) -> dict[str, Any]:
@@ -601,7 +592,7 @@ class SourceList(WriteableCogniteResourceList[SourceWrite, Source], ExternalIDTr
 
     def as_write(
         self,
-    ) -> SourceWriteList:
+    ) -> NoReturn:
         raise TypeError(f"{type(self).__name__} cannot be converted to write")
 
 

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "cognite-sdk"
 
-version = "7.55.2"
+version = "7.58.1"
 description = "Cognite Python SDK"
 readme = "README.md"
 documentation = "https://cognite-sdk-python.readthedocs-hosted.com"

tests/tests_integration/test_api/test_hosted_extractors/test_sources.py

@@ -25,8 +25,7 @@ def one_event_hub_source(cognite_client: CogniteClient) -> SourceList:
     retrieved = cognite_client.hosted_extractors.sources.retrieve(my_hub.external_id, ignore_unknown_ids=True)
     if retrieved:
         return retrieved
-    created = cognite_client.hosted_extractors.sources.create(my_hub)
-    return SourceList([created])
+    return cognite_client.hosted_extractors.sources.create([my_hub])
 
 
 class TestSources:
@@ -69,7 +68,7 @@ def test_list(self, cognite_client: CogniteClient) -> None:
 
     def test_update_using_write_object(self, cognite_client: CogniteClient) -> None:
         my_hub = EventHubSourceWrite(
-            external_id=f"toupdatate-{random_string(10)}",
+            external_id=f"to-update-{random_string(10)}",
             host="myHost",
             key_name="myKeyName",
             key_value="myKey",