From 3f6ca7b739b4f1f71258c7b0e72feb97ef3d8fc0 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 08:36:16 +0200 Subject: [PATCH 01/32] refactor: added source data classes --- .../hosted_extractors/__init__.py | 0 .../data_classes/hosted_extractors/sources.py | 159 ++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 cognite/client/data_classes/hosted_extractors/__init__.py create mode 100644 cognite/client/data_classes/hosted_extractors/sources.py diff --git a/cognite/client/data_classes/hosted_extractors/__init__.py b/cognite/client/data_classes/hosted_extractors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py new file mode 100644 index 000000000..778f60ee5 --- /dev/null +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +from abc import ABC +from typing import TYPE_CHECKING, Any, Literal + +from typing_extensions import Self, TypeAlias + +from cognite.client.data_classes._base import ( + CogniteResourceList, + ExternalIDTransformerMixin, + WriteableCogniteResource, + WriteableCogniteResourceList, +) +from cognite.client.utils.useful_types import SequenceNotStr + +if TYPE_CHECKING: + from cognite.client import CogniteClient + + +SourceType: TypeAlias = Literal["mqtt5", "mqtt3", "eventhub"] + + +class _SourceCore(WriteableCogniteResource["SourceWrite"], ABC): + def __init__( + self, + type: SourceType, + external_id: str, + host: str, + event_hub_name: str, + key_name: str, + consumer_group: str | None = None, + ) -> None: + self.type = type + self.external_id = external_id + self.host = host + self.event_hub_name = event_hub_name + self.key_name = key_name + self.consumer_group = consumer_group + + +class SourceWrite(_SourceCore): + """A hosted extractor source represents an external source system on the internet. + The source resource in CDF contains all the information the extractor needs to + connect to the external source system. + + This is the write/request format of the source resource. + + Args: + type (SourceType): Source type. + external_id (str): The external ID provided by the client. Must be unique for the resource type. + host (str): URL of the event hub consumer endpoint. + event_hub_name (str): Name of the event hub + key_name (str): The name of the Event Hub key to use. + key_value (str): Value of the Event Hub key to use for authentication. + consumer_group (str | None): The event hub consumer group to use. Microsoft recommends having a distinct consumer group for each application consuming data from event hub. If left out, this uses the default consumer group. + """ + + def __init__( + self, + type: SourceType, + external_id: str, + host: str, + event_hub_name: str, + key_name: str, + key_value: str, + consumer_group: str | None = None, + ) -> None: + super().__init__(type, external_id, host, event_hub_name, key_name, consumer_group) + self.key_value = key_value + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> SourceWrite: + return cls( + type=resource["type"], + external_id=resource["externalId"], + host=resource["host"], + event_hub_name=resource["eventHubName"], + key_name=resource["keyName"], + key_value=resource["keyValue"], + consumer_group=resource.get("consumerGroup"), + ) + + def as_write(self) -> SourceWrite: + return self + + +class Source(_SourceCore): + """A hosted extractor source represents an external source system on the internet. + The source resource in CDF contains all the information the extractor needs to + connect to the external source system. + + This is the read/response format of the source resource. + + Args: + type (SourceType): Source type. + external_id (str): The external ID provided by the client. Must be unique for the resource type. + host (str): URL of the event hub consumer endpoint. + event_hub_name (str): Name of the event hub + key_name (str): The name of the Event Hub key to use. + created_time (int): No description. + last_updated_time (int): No description. + consumer_group (str | None): The event hub consumer group to use. Microsoft recommends having a distinct consumer group for each application consuming data from event hub. If left out, this uses the default consumer group. + """ + + def __init__( + self, + type: SourceType, + external_id: str, + host: str, + event_hub_name: str, + key_name: str, + created_time: int, + last_updated_time: int, + consumer_group: str | None = None, + ) -> None: + super().__init__(type, external_id, host, event_hub_name, key_name, consumer_group) + self.created_time = created_time + self.last_updated_time = last_updated_time + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + return cls( + type=resource["type"], + external_id=resource["externalId"], + host=resource["host"], + event_hub_name=resource["eventHubName"], + key_name=resource["keyName"], + created_time=resource["createdTime"], + last_updated_time=resource["lastUpdatedTime"], + consumer_group=resource.get("consumerGroup"), + ) + + def as_write(self, key_value: str | None = None) -> SourceWrite: + if key_value is None: + raise ValueError("key_value must be provided") + return SourceWrite( + type=self.type, + external_id=self.external_id, + host=self.host, + event_hub_name=self.event_hub_name, + key_name=self.key_name, + key_value=key_value, + consumer_group=self.consumer_group, + ) + + +class SourceWriteList(CogniteResourceList[SourceWrite], ExternalIDTransformerMixin): + _RESOURCE = SourceWrite + + +class SourceList(WriteableCogniteResourceList[SourceWrite, Source], ExternalIDTransformerMixin): + _RESOURCE = Source + + def as_write(self, key_values: SequenceNotStr[str] | None = None) -> SourceWriteList: + if key_values is None: + raise ValueError("key_values must be provided") + if len(self) != len(key_values): + raise ValueError("key_values must be the same length as the sources") + return SourceWriteList([source.as_write(key_value) for source, key_value in zip(self, key_values)]) From f03e0060dd2ae5a2d2b11881a0820a3ac42159d8 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 08:53:00 +0200 Subject: [PATCH 02/32] refactor: setup shell for API --- .../client/_api/hosted_extractors/__init__.py | 16 +++++++++++ .../client/_api/hosted_extractors/sources.py | 27 +++++++++++++++++++ cognite/client/_cognite_client.py | 2 ++ cognite/client/data_classes/sequences.py | 2 +- 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 cognite/client/_api/hosted_extractors/__init__.py create mode 100644 cognite/client/_api/hosted_extractors/sources.py diff --git a/cognite/client/_api/hosted_extractors/__init__.py b/cognite/client/_api/hosted_extractors/__init__.py new file mode 100644 index 000000000..6c6a23a93 --- /dev/null +++ b/cognite/client/_api/hosted_extractors/__init__.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from cognite.client._api.hosted_extractors.sources import SourcesAPI +from cognite.client._api_client import APIClient + +if TYPE_CHECKING: + from cognite.client import CogniteClient + from cognite.client.config import ClientConfig + + +class HostedExtractorsAPI(APIClient): + def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: CogniteClient) -> None: + super().__init__(config, api_version, cognite_client) + self.sources = SourcesAPI(config, api_version, cognite_client) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py new file mode 100644 index 000000000..45503ad90 --- /dev/null +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Sequence + +from cognite.client._api_client import APIClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client.data_classes.hosted_extractors.sources import Source, SourceList, SourceWrite +from cognite.client.utils.useful_types import SequenceNotStr + +if TYPE_CHECKING: + pass + + +class SourcesAPI(APIClient): + _RESOURCE_PATH = "/hostedextractors/sources" + + def create(self, item: SourceWrite | Sequence[SourceWrite]) -> Source | SourceList: ... + + def delete( + self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False, force: bool = False + ) -> None: ... + + def retrieve(self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False) -> Source: ... + + def update(self, items: SourceWrite | Sequence[SourceWrite]) -> Source | SourceList: ... + + def list(self, limit=DEFAULT_LIMIT_READ) -> SourceList: ... diff --git a/cognite/client/_cognite_client.py b/cognite/client/_cognite_client.py index f6143cb94..5a0d16e3e 100644 --- a/cognite/client/_cognite_client.py +++ b/cognite/client/_cognite_client.py @@ -16,6 +16,7 @@ from cognite.client._api.files import FilesAPI from cognite.client._api.functions import FunctionsAPI from cognite.client._api.geospatial import GeospatialAPI +from cognite.client._api.hosted_extractors import HostedExtractorsAPI from cognite.client._api.iam import IAMAPI from cognite.client._api.labels import LabelsAPI from cognite.client._api.raw import RawAPI @@ -71,6 +72,7 @@ def __init__(self, config: ClientConfig | None = None) -> None: self.templates = TemplatesAPI(self._config, self._API_VERSION, self) self.vision = VisionAPI(self._config, self._API_VERSION, self) self.extraction_pipelines = ExtractionPipelinesAPI(self._config, self._API_VERSION, self) + self.hosted_extractors = HostedExtractorsAPI(self._config, self._API_VERSION, self) self.transformations = TransformationsAPI(self._config, self._API_VERSION, self) self.diagrams = DiagramsAPI(self._config, self._API_VERSION, self) self.annotations = AnnotationsAPI(self._config, self._API_VERSION, self) diff --git a/cognite/client/data_classes/sequences.py b/cognite/client/data_classes/sequences.py index 9d59e7e5a..c8ad62004 100644 --- a/cognite/client/data_classes/sequences.py +++ b/cognite/client/data_classes/sequences.py @@ -628,7 +628,7 @@ def _load(cls, resource: dict, cognite_client: CogniteClient | None = None) -> S ) -ColumnNames: TypeAlias = Literal[ +ColumnNames: TypeAlias = Literal[ # type: ignore[valid-type] "externalId", "id", "columnExternalId", From c663cd262f87833b006c9ed6e4e1b796eda6220d Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 09:11:27 +0200 Subject: [PATCH 03/32] refactor: fix Source classes --- .../data_classes/hosted_extractors/sources.py | 115 ++++++++---------- 1 file changed, 54 insertions(+), 61 deletions(-) diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index 778f60ee5..0b82be72c 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -1,44 +1,61 @@ from __future__ import annotations from abc import ABC -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, ClassVar, Literal -from typing_extensions import Self, TypeAlias +from typing_extensions import TypeAlias from cognite.client.data_classes._base import ( + CogniteResource, CogniteResourceList, ExternalIDTransformerMixin, + T_WriteClass, WriteableCogniteResource, WriteableCogniteResourceList, ) -from cognite.client.utils.useful_types import SequenceNotStr if TYPE_CHECKING: - from cognite.client import CogniteClient + pass SourceType: TypeAlias = Literal["mqtt5", "mqtt3", "eventhub"] -class _SourceCore(WriteableCogniteResource["SourceWrite"], ABC): - def __init__( - self, - type: SourceType, - external_id: str, - host: str, - event_hub_name: str, - key_name: str, - consumer_group: str | None = None, - ) -> None: - self.type = type +class SourceWrite(CogniteResource, ABC): + """A hosted extractor source represents an external source system on the internet. + The source resource in CDF contains all the information the extractor needs to + connect to the external source system. + + This is the write/request format of the source resource. + + Args: + external_id (str): The external ID provided by the client. Must be unique for the resource type. + """ + + _type: ClassVar[str] + + def __init__(self, external_id: str) -> None: self.external_id = external_id - self.host = host - self.event_hub_name = event_hub_name - self.key_name = key_name - self.consumer_group = consumer_group -class SourceWrite(_SourceCore): +class Source(WriteableCogniteResource[T_WriteClass], ABC): + """A hosted extractor source represents an external source system on the internet. + The source resource in CDF contains all the information the extractor needs to + connect to the external source system. + + This is the read/response format of the source resource. + + Args: + external_id (str): The external ID provided by the client. Must be unique for the resource type. + """ + + _type: ClassVar[str] + + def __init__(self, external_id: str) -> None: + self.external_id = external_id + + +class EventHubSourceWrite(SourceWrite): """A hosted extractor source represents an external source system on the internet. The source resource in CDF contains all the information the extractor needs to connect to the external source system. @@ -46,7 +63,6 @@ class SourceWrite(_SourceCore): This is the write/request format of the source resource. Args: - type (SourceType): Source type. external_id (str): The external ID provided by the client. Must be unique for the resource type. host (str): URL of the event hub consumer endpoint. event_hub_name (str): Name of the event hub @@ -57,7 +73,6 @@ class SourceWrite(_SourceCore): def __init__( self, - type: SourceType, external_id: str, host: str, event_hub_name: str, @@ -65,26 +80,18 @@ def __init__( key_value: str, consumer_group: str | None = None, ) -> None: - super().__init__(type, external_id, host, event_hub_name, key_name, consumer_group) + super().__init__(external_id) + self.host = host + self.event_hub_name = event_hub_name + self.key_name = key_name self.key_value = key_value - - @classmethod - def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> SourceWrite: - return cls( - type=resource["type"], - external_id=resource["externalId"], - host=resource["host"], - event_hub_name=resource["eventHubName"], - key_name=resource["keyName"], - key_value=resource["keyValue"], - consumer_group=resource.get("consumerGroup"), - ) + self.consumer_group = consumer_group def as_write(self) -> SourceWrite: return self -class Source(_SourceCore): +class EventHubSource(Source): """A hosted extractor source represents an external source system on the internet. The source resource in CDF contains all the information the extractor needs to connect to the external source system. @@ -92,7 +99,6 @@ class Source(_SourceCore): This is the read/response format of the source resource. Args: - type (SourceType): Source type. external_id (str): The external ID provided by the client. Must be unique for the resource type. host (str): URL of the event hub consumer endpoint. event_hub_name (str): Name of the event hub @@ -104,7 +110,6 @@ class Source(_SourceCore): def __init__( self, - type: SourceType, external_id: str, host: str, event_hub_name: str, @@ -113,28 +118,18 @@ def __init__( last_updated_time: int, consumer_group: str | None = None, ) -> None: - super().__init__(type, external_id, host, event_hub_name, key_name, consumer_group) + super().__init__(external_id) + self.host = host + self.event_hub_name = event_hub_name + self.key_name = key_name + self.consumer_group = consumer_group self.created_time = created_time self.last_updated_time = last_updated_time - @classmethod - def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: - return cls( - type=resource["type"], - external_id=resource["externalId"], - host=resource["host"], - event_hub_name=resource["eventHubName"], - key_name=resource["keyName"], - created_time=resource["createdTime"], - last_updated_time=resource["lastUpdatedTime"], - consumer_group=resource.get("consumerGroup"), - ) - - def as_write(self, key_value: str | None = None) -> SourceWrite: + def as_write(self, key_value: str | None = None) -> EventHubSourceWrite: if key_value is None: raise ValueError("key_value must be provided") - return SourceWrite( - type=self.type, + return EventHubSourceWrite( external_id=self.external_id, host=self.host, event_hub_name=self.event_hub_name, @@ -151,9 +146,7 @@ class SourceWriteList(CogniteResourceList[SourceWrite], ExternalIDTransformerMix class SourceList(WriteableCogniteResourceList[SourceWrite, Source], ExternalIDTransformerMixin): _RESOURCE = Source - def as_write(self, key_values: SequenceNotStr[str] | None = None) -> SourceWriteList: - if key_values is None: - raise ValueError("key_values must be provided") - if len(self) != len(key_values): - raise ValueError("key_values must be the same length as the sources") - return SourceWriteList([source.as_write(key_value) for source, key_value in zip(self, key_values)]) + def as_write( + self, + ) -> SourceWriteList: + raise TypeError(f"{type(self).__name__} cannot be converted to write") From 19797561dc27749a6ce6c88f951b4cc3c200d85e Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 10:00:11 +0200 Subject: [PATCH 04/32] refactor; MQTT data classes --- .../data_classes/hosted_extractors/sources.py | 181 +++++++++++++++++- 1 file changed, 178 insertions(+), 3 deletions(-) diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index 0b82be72c..dd0ef7d1b 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -1,11 +1,13 @@ from __future__ import annotations from abc import ABC -from typing import TYPE_CHECKING, ClassVar, Literal +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, ClassVar, Literal -from typing_extensions import TypeAlias +from typing_extensions import Self, TypeAlias from cognite.client.data_classes._base import ( + CogniteObject, CogniteResource, CogniteResourceList, ExternalIDTransformerMixin, @@ -15,7 +17,7 @@ ) if TYPE_CHECKING: - pass + from cognite.client import CogniteClient SourceType: TypeAlias = Literal["mqtt5", "mqtt3", "eventhub"] @@ -139,6 +141,179 @@ def as_write(self, key_value: str | None = None) -> EventHubSourceWrite: ) +@dataclass +class MQTTAuthenticationWrite(CogniteObject, ABC): + _type: ClassVar[str] + + +@dataclass +class BasicMQTTAuthenticationWrite(MQTTAuthenticationWrite): + username: str + password: str | None + + +@dataclass +class CACertificateWrite(CogniteObject): + type: Literal["der", "pem"] + certificate: str + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + return cls(type=resource["type"], certificate=resource["certificate"]) + + +@dataclass +class AuthCertificateWrite(CogniteObject): + type: Literal["der", "pem"] + certificate: str + key: str + key_password: str | None + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + return cls( + type=resource["type"], + certificate=resource["certificate"], + key=resource["key"], + key_password=resource.get("keyPassword"), + ) + + +class MQTTSourceWrite(SourceWrite): + def __init__( + self, + external_id: str, + host: str, + port: int | None = None, + authentication: MQTTAuthenticationWrite | None = None, + useTls: bool = False, + ca_certificate: CACertificateWrite | None = None, + auth_certificate: AuthCertificateWrite | None = None, + ) -> None: + super().__init__(external_id) + self.host = host + self.port = port + self.authentication = authentication + self.useTls = useTls + self.ca_certificate = ca_certificate + self.auth_certificate = auth_certificate + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + return cls( + external_id=resource["externalId"], + host=resource["host"], + port=resource.get("port"), + authentication=MQTTAuthenticationWrite._load(resource["authentication"]) + if "authentication" in resource + else None, + useTls=resource.get("useTls", False), + ca_certificate=CACertificateWrite._load(resource["caCertificate"]) if "caCertificate" in resource else None, + auth_certificate=AuthCertificateWrite._load(resource["authCertificate"]) + if "authCertificate" in resource + else None, + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + output = super().dump(camel_case) + if self.authentication: + output["authentication"] = self.authentication.dump(camel_case) + if self.ca_certificate: + output["caCertificate" if camel_case else "ca_certificate"] = self.ca_certificate.dump(camel_case) + if self.auth_certificate: + output["authCertificate" if camel_case else "auth_certificate"] = self.auth_certificate.dump(camel_case) + return output + + +@dataclass +class MQTTAuthentication(CogniteObject, ABC): + _type: ClassVar[str] + + +@dataclass +class BasicMQTTAuthentication(MQTTAuthentication): + username: str + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + return cls(username=resource["username"]) + + +@dataclass +class CACertificate(CogniteObject): + thumbprint: str + expires_at: str + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + return cls(thumbprint=resource["thumbprint"], expires_at=resource["expiresAt"]) + + +@dataclass +class AuthCertificate(CogniteObject): + thumbprint: str + expires_at: str + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + return cls(thumbprint=resource["thumbprint"], expires_at=resource["expiresAt"]) + + +class MQTTSource(Source): + def __init__( + self, + external_id: str, + host: str, + created_time: int, + last_updated_time: int, + port: int | None = None, + authentication: MQTTAuthentication | None = None, + useTls: bool = False, + ca_certificate: CACertificate | None = None, + auth_certificate: AuthCertificate | None = None, + ) -> None: + super().__init__(external_id) + self.host = host + self.port = port + self.authentication = authentication + self.useTls = useTls + self.ca_certificate = ca_certificate + self.auth_certificate = auth_certificate + self.created_time = created_time + self.last_updated_time = last_updated_time + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + return cls( + external_id=resource["externalId"], + host=resource["host"], + port=resource.get("port"), + authentication=MQTTAuthentication._load(resource["authentication"]) + if "authentication" in resource + else None, + useTls=resource.get("useTls", False), + ca_certificate=CACertificate._load(resource["caCertificate"]) if "caCertificate" in resource else None, + auth_certificate=AuthCertificate._load(resource["authCertificate"]) + if "authCertificate" in resource + else None, + created_time=resource["createdTime"], + last_updated_time=resource["lastUpdatedTime"], + ) + + def as_write(self) -> MQTTSourceWrite: + raise TypeError(f"{type(self).__name__} cannot be converted to write as id does not contain the secrets") + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + output = super().dump(camel_case) + if self.authentication: + output["authentication"] = self.authentication.dump(camel_case) + if self.ca_certificate: + output["caCertificate" if camel_case else "ca_certificate"] = self.ca_certificate.dump(camel_case) + if self.auth_certificate: + output["authCertificate" if camel_case else "auth_certificate"] = self.auth_certificate.dump(camel_case) + return output + + class SourceWriteList(CogniteResourceList[SourceWrite], ExternalIDTransformerMixin): _RESOURCE = SourceWrite From 19f6d9deebcc5b7ec85e7053a2e29b97f679112f Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 10:20:57 +0200 Subject: [PATCH 05/32] refactor: robust loading --- .../data_classes/hosted_extractors/sources.py | 168 ++++++++++++++++-- 1 file changed, 156 insertions(+), 12 deletions(-) diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index dd0ef7d1b..09707fb85 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -1,10 +1,11 @@ from __future__ import annotations -from abc import ABC +import itertools +from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Literal +from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast -from typing_extensions import Self, TypeAlias +from typing_extensions import Self from cognite.client.data_classes._base import ( CogniteObject, @@ -12,17 +13,16 @@ CogniteResourceList, ExternalIDTransformerMixin, T_WriteClass, + UnknownCogniteObject, WriteableCogniteResource, WriteableCogniteResourceList, ) +from cognite.client.utils._auxiliary import fast_dict_load if TYPE_CHECKING: from cognite.client import CogniteClient -SourceType: TypeAlias = Literal["mqtt5", "mqtt3", "eventhub"] - - class SourceWrite(CogniteResource, ABC): """A hosted extractor source represents an external source system on the internet. The source resource in CDF contains all the information the extractor needs to @@ -39,6 +39,28 @@ class SourceWrite(CogniteResource, ABC): def __init__(self, external_id: str) -> None: self.external_id = external_id + @classmethod + @abstractmethod + def _load_source(cls, resource: dict[str, Any]) -> Self: + raise NotImplementedError() + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + type_ = resource.get("type") + if type_ is None and hasattr(cls, "_type"): + type_ = cls._type + else: + raise KeyError("type") + source_class = _SOURCE_WRITE_CLASS_BY_TYPE.get(type_) + if source_class is None: + return UnknownCogniteObject(resource) # type: ignore[return-value] + return cast(Self, source_class._load_source(resource)) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + output = super().dump(camel_case) + output["type"] = self._type + return output + class Source(WriteableCogniteResource[T_WriteClass], ABC): """A hosted extractor source represents an external source system on the internet. @@ -56,6 +78,28 @@ class Source(WriteableCogniteResource[T_WriteClass], ABC): def __init__(self, external_id: str) -> None: self.external_id = external_id + @classmethod + @abstractmethod + def _load_source(cls, resource: dict[str, Any]) -> Self: + raise NotImplementedError() + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + type_ = resource.get("type") + if type_ is None and hasattr(cls, "_type"): + type_ = cls._type + else: + raise KeyError("type") + source_class = _SOURCE_CLASS_BY_TYPE.get(type_) + if source_class is None: + return UnknownCogniteObject(resource) # type: ignore[return-value] + return cast(Self, source_class._load(resource)) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + output = super().dump(camel_case) + output["type"] = self._type + return output + class EventHubSourceWrite(SourceWrite): """A hosted extractor source represents an external source system on the internet. @@ -92,6 +136,10 @@ def __init__( def as_write(self) -> SourceWrite: return self + @classmethod + def _load_source(cls, resource: dict[str, Any]) -> Self: + return fast_dict_load(cls, resource, None) + class EventHubSource(Source): """A hosted extractor source represents an external source system on the internet. @@ -140,17 +188,48 @@ def as_write(self, key_value: str | None = None) -> EventHubSourceWrite: consumer_group=self.consumer_group, ) + @classmethod + def _load_source(cls, resource: dict[str, Any]) -> Self: + return fast_dict_load(cls, resource, None) + @dataclass class MQTTAuthenticationWrite(CogniteObject, ABC): _type: ClassVar[str] + @classmethod + @abstractmethod + def _load_authentication(cls, resource: dict[str, Any]) -> Self: + raise NotImplementedError() + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + type_ = resource.get("type") + if type_ is None and hasattr(cls, "_type"): + type_ = cls._type + else: + raise KeyError("type is required") + authentication_class = _MQTTAUTHENTICATION_WRITE_CLASS_BY_TYPE.get(type_) + if authentication_class is None: + return UnknownCogniteObject(resource) # type: ignore[return-value] + return cast(Self, authentication_class._load_authentication(resource)) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + output = super().dump(camel_case) + output["type"] = self._type + return output + @dataclass class BasicMQTTAuthenticationWrite(MQTTAuthenticationWrite): + _type = "basic" username: str password: str | None + @classmethod + def _load_authentication(cls, resource: dict[str, Any]) -> Self: + return fast_dict_load(cls, resource, None) + @dataclass class CACertificateWrite(CogniteObject): @@ -179,7 +258,7 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = ) -class MQTTSourceWrite(SourceWrite): +class _MQTTSourceWrite(SourceWrite, ABC): def __init__( self, external_id: str, @@ -199,7 +278,7 @@ def __init__( self.auth_certificate = auth_certificate @classmethod - def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + def _load_source(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: return cls( external_id=resource["externalId"], host=resource["host"], @@ -229,13 +308,37 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: class MQTTAuthentication(CogniteObject, ABC): _type: ClassVar[str] + @classmethod + @abstractmethod + def _load_authentication(cls, resource: dict[str, Any]) -> Self: + raise NotImplementedError() + + @classmethod + def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + type_ = resource.get("type") + if type_ is None and hasattr(cls, "_type"): + type_ = cls._type + else: + raise KeyError("type") + + authentication_class = _MQTTAUTHENTICATION_CLASS_BY_TYPE.get(type_) + if authentication_class is None: + return UnknownCogniteObject(resource) # type: ignore[return-value] + return cast(Self, authentication_class._load_authentication(resource)) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + output = super().dump(camel_case) + output["type"] = self._type + return output + @dataclass class BasicMQTTAuthentication(MQTTAuthentication): + _type = "basic" username: str @classmethod - def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + def _load_authentication(cls, resource: dict[str, Any]) -> Self: return cls(username=resource["username"]) @@ -259,7 +362,7 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = return cls(thumbprint=resource["thumbprint"], expires_at=resource["expiresAt"]) -class MQTTSource(Source): +class _MQTTSource(Source, ABC): def __init__( self, external_id: str, @@ -283,7 +386,7 @@ def __init__( self.last_updated_time = last_updated_time @classmethod - def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + def _load_source(cls, resource: dict[str, Any]) -> Self: return cls( external_id=resource["externalId"], host=resource["host"], @@ -300,7 +403,7 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = last_updated_time=resource["lastUpdatedTime"], ) - def as_write(self) -> MQTTSourceWrite: + def as_write(self) -> _MQTTSourceWrite: raise TypeError(f"{type(self).__name__} cannot be converted to write as id does not contain the secrets") def dump(self, camel_case: bool = True) -> dict[str, Any]: @@ -314,6 +417,22 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: return output +class MQTT3SourceWrite(_MQTTSourceWrite): + _type = "mqtt3" + + +class MQTT5SourceWrite(_MQTTSourceWrite): + _type = "mqtt5" + + +class MQTT3Source(_MQTTSource): + _type = "mqtt3" + + +class MQTT5Source(_MQTTSource): + _type = "mqtt5" + + class SourceWriteList(CogniteResourceList[SourceWrite], ExternalIDTransformerMixin): _RESOURCE = SourceWrite @@ -325,3 +444,28 @@ def as_write( self, ) -> SourceWriteList: raise TypeError(f"{type(self).__name__} cannot be converted to write") + + +_SOURCE_WRITE_CLASS_BY_TYPE: dict[str, type[SourceWrite]] = { + subclass._type: subclass # type: ignore[type-abstract, misc] + for subclass in itertools.chain(SourceWrite.__subclasses__(), _MQTTSourceWrite.__subclasses__()) + if hasattr(subclass, "_type") +} + +_SOURCE_CLASS_BY_TYPE: dict[str, type[Source]] = { + subclass._type: subclass # type: ignore[type-abstract, misc] + for subclass in itertools.chain(Source.__subclasses__(), _MQTTSource.__subclasses__()) + if hasattr(subclass, "_type") +} + +_MQTTAUTHENTICATION_WRITE_CLASS_BY_TYPE: dict[str, type[MQTTAuthenticationWrite]] = { + subclass._type: subclass # type: ignore[type-abstract] + for subclass in MQTTAuthenticationWrite.__subclasses__() + if hasattr(subclass, "_type") +} + +_MQTTAUTHENTICATION_CLASS_BY_TYPE: dict[str, type[MQTTAuthentication]] = { + subclass._type: subclass # type: ignore[type-abstract] + for subclass in MQTTAuthentication.__subclasses__() + if hasattr(subclass, "_type") +} From 1ebb1fc39687e4e247ebcf9a1493c4d84650403e Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 10:39:15 +0200 Subject: [PATCH 06/32] feat: Implemented CRD + list method --- .../client/_api/hosted_extractors/sources.py | 185 +++++++++++++++++- .../hosted_extractors/__init__.py | 25 +++ 2 files changed, 202 insertions(+), 8 deletions(-) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index 45503ad90..e22681c28 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -1,10 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Sequence +from collections.abc import Iterator +from typing import TYPE_CHECKING, Sequence, overload from cognite.client._api_client import APIClient from cognite.client._constants import DEFAULT_LIMIT_READ from cognite.client.data_classes.hosted_extractors.sources import Source, SourceList, SourceWrite +from cognite.client.utils._identifier import IdentifierSequence from cognite.client.utils.useful_types import SequenceNotStr if TYPE_CHECKING: @@ -14,14 +16,181 @@ class SourcesAPI(APIClient): _RESOURCE_PATH = "/hostedextractors/sources" - def create(self, item: SourceWrite | Sequence[SourceWrite]) -> Source | SourceList: ... + @overload + def __call__( + self, + chunk_size: None = None, + limit: int | None = None, + ) -> Iterator[Source]: ... - def delete( - self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False, force: bool = False - ) -> None: ... + @overload + def __call__( + self, + chunk_size: int, + limit: int | None = None, + ) -> Iterator[SourceList]: ... - def retrieve(self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False) -> Source: ... + def __call__( + self, + chunk_size: int | None = None, + limit: int | None = None, + ) -> Iterator[Source] | Iterator[SourceList]: + """Iterate over sources - def update(self, items: SourceWrite | Sequence[SourceWrite]) -> Source | SourceList: ... + Fetches sources as they are iterated over, so you keep a limited number of spaces in memory. - def list(self, limit=DEFAULT_LIMIT_READ) -> SourceList: ... + Args: + chunk_size (int | None): Number of sources to return in each chunk. Defaults to yielding one source a time. + limit (int | None): Maximum number of sources to return. Defaults to returning all items. + + Returns: + Iterator[Source] | Iterator[SourceList]: yields Source one by one if chunk_size is not specified, else SourceList objects. + """ + return self._list_generator( + list_cls=SourceList, + resource_cls=Source, # type: ignore[type-abstract] + method="GET", + chunk_size=chunk_size, + limit=limit, + ) + + def __iter__(self) -> Iterator[Source]: + """Iterate over sources + + Fetches sources as they are iterated over, so you keep a limited number of spaces in memory. + + Returns: + Iterator[Source]: yields Source one by one. + """ + return self() + + @overload + def retrieve(self, external_ids: str) -> Source: ... + + @overload + def retrieve(self, external_ids: SequenceNotStr[str]) -> SourceList: ... + + def retrieve( + self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bool = False + ) -> Source | SourceList: + """`Retrieve one or more sources. `_ + + Args: + external_ids (str | SequenceNotStr[str]): The external ID provided by the client. Must be unique for the resource type. + ignore_unknown_ids (bool): No description. + + Returns: + Source | SourceList: Requested sources + + Examples: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> res = client.hosted_extractors.sources.retrieve('myMQTTSource') + + Get multiple spaces by id: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> res = client.hosted_extractors.sources.retrieve(["myMQTTSource", "MyEvenHubSource"], ignore_unknown_ids=True) + + """ + return self._retrieve_multiple( + list_cls=SourceList, + resource_cls=Source, # type: ignore[type-abstract] + identifiers=IdentifierSequence.load(external_ids=external_ids), + ) + + def delete(self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bool, force: bool) -> None: + """`Delete one or more sources `_ + + Args: + external_ids (str | SequenceNotStr[str]): The external ID provided by the client. Must be unique for the resource type. + ignore_unknown_ids (bool): No description. + force (bool): No description. + Examples: + + Delete sources by id:: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> client.hosted_extractors.sources.delete(spaces=["myMQTTSource", "MyEvenHubSource"]) + """ + self._delete_multiple( + identifiers=IdentifierSequence.load(external_ids=external_ids), + wrap_ids=True, + returns_items=False, + ) + + @overload + def create(self, items: SourceWrite) -> Source: ... + + @overload + def create(self, items: Sequence[SourceWrite]) -> SourceList: ... + + def create(self, items: SourceWrite | Sequence[SourceWrite]) -> Source | SourceList: + """`Create one or more sources. `_ + + Args: + items (SourceWrite | Sequence[SourceWrite]): Space | Sequence[Space]): Source(s) to create. + + Returns: + Source | SourceList: Created source(s) + + Examples: + + Create new source: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.hosted_extractors import EventHubSourceWrite + >>> client = CogniteClient() + >>> source = EventHubSourceWrite('my_event_hub', 'http://myeventhub.com', "My EventHub", 'my_key', 'my_value') + >>> res = client.hosted_extractors.sources.create(source) + """ + return self._create_multiple( + list_cls=SourceList, + resource_cls=Source, # type: ignore[type-abstract] + items=items, # type: ignore[arg-type] + input_resource_cls=SourceWrite, # type: ignore[arg-type] + ) + + def list( + self, + limit: int | None = DEFAULT_LIMIT_READ, + ) -> SourceList: + """`List sources `_ + + Args: + limit (int | None): Maximum number of sources to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + + Returns: + SourceList: List of requested sources + + Examples: + + List sources: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> space_list = client.hosted_extractors.sources.list(limit=5) + + Iterate over sources:: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> for source in client.hosted_extractors.sources: + ... source # do something with the source + + Iterate over chunks of sources to reduce memory load:: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> for source_list in client.hosted_extractors.sources(chunk_size=25): + ... source_list # do something with the spaces + """ + return self._list( + list_cls=SourceList, + resource_cls=Source, # type: ignore[type-abstract] + method="GET", + limit=limit, + ) diff --git a/cognite/client/data_classes/hosted_extractors/__init__.py b/cognite/client/data_classes/hosted_extractors/__init__.py index e69de29bb..0b6a786e9 100644 --- a/cognite/client/data_classes/hosted_extractors/__init__.py +++ b/cognite/client/data_classes/hosted_extractors/__init__.py @@ -0,0 +1,25 @@ +from cognite.client.data_classes.hosted_extractors.sources import ( + EventHubSource, + EventHubSourceWrite, + MQTT3Source, + MQTT3SourceWrite, + MQTT5Source, + MQTT5SourceWrite, + Source, + SourceList, + SourceWrite, + SourceWriteList, +) + +__all__ = [ + "EventHubSource", + "EventHubSourceWrite", + "MQTT3Source", + "MQTT3SourceWrite", + "MQTT5Source", + "MQTT5SourceWrite", + "Source", + "SourceList", + "SourceWrite", + "SourceWriteList", +] From c6a9b9782a271742e59c3828c47cc25de2aec39c Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 11:04:01 +0200 Subject: [PATCH 07/32] feat: Added update classes --- .../client/_api/hosted_extractors/sources.py | 34 ++++- cognite/client/data_classes/_base.py | 2 +- .../hosted_extractors/__init__.py | 8 ++ .../data_classes/hosted_extractors/sources.py | 126 ++++++++++++++++++ 4 files changed, 168 insertions(+), 2 deletions(-) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index e22681c28..be1a45121 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -5,7 +5,7 @@ from cognite.client._api_client import APIClient from cognite.client._constants import DEFAULT_LIMIT_READ -from cognite.client.data_classes.hosted_extractors.sources import Source, SourceList, SourceWrite +from cognite.client.data_classes.hosted_extractors.sources import Source, SourceList, SourceUpdate, SourceWrite from cognite.client.utils._identifier import IdentifierSequence from cognite.client.utils.useful_types import SequenceNotStr @@ -154,6 +154,38 @@ def create(self, items: SourceWrite | Sequence[SourceWrite]) -> Source | SourceL input_resource_cls=SourceWrite, # type: ignore[arg-type] ) + @overload + def update(self, items: SourceWrite | SourceUpdate) -> Source: ... + + @overload + def update(self, items: Sequence[SourceWrite | SourceUpdate]) -> SourceList: ... + + def update(self, items: SourceWrite | SourceUpdate | Sequence[SourceWrite | SourceUpdate]) -> Source | SourceList: + """`Update one or more sources. `_ + + Args: + items (SourceWrite | SourceUpdate | Sequence[SourceWrite | SourceUpdate]): Space | Sequence[Space]): Source(s) to update. + + Returns: + Source | SourceList: Updated source(s) + + Examples: + + Update source: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.hosted_extractors import EventHubSourceUpdate + >>> client = CogniteClient() + >>> source = EventHubSourceUpdate('my_event_hub').event_hub_name.set("My Updated EventHub") + >>> res = client.hosted_extractors.sources.update(source) + """ + return self._update_multiple( + items=items, # type: ignore[arg-type] + list_cls=SourceList, + resource_cls=Source, # type: ignore[type-abstract] + update_cls=SourceUpdate, # type: ignore[type-abstract] + ) + def list( self, limit: int | None = DEFAULT_LIMIT_READ, diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index 995ad49a6..f35617043 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -534,7 +534,7 @@ def __init__(self, update_object: T_CogniteUpdate, name: str) -> None: self._update_object = update_object self._name = name - def _set(self, value: None | str | int | bool) -> T_CogniteUpdate: + def _set(self, value: None | str | int | bool | dict) -> T_CogniteUpdate: if value is None: self._update_object._set_null(self._name) else: diff --git a/cognite/client/data_classes/hosted_extractors/__init__.py b/cognite/client/data_classes/hosted_extractors/__init__.py index 0b6a786e9..e8a1a1013 100644 --- a/cognite/client/data_classes/hosted_extractors/__init__.py +++ b/cognite/client/data_classes/hosted_extractors/__init__.py @@ -1,12 +1,16 @@ from cognite.client.data_classes.hosted_extractors.sources import ( EventHubSource, + EventHubSourceUpdate, EventHubSourceWrite, MQTT3Source, + MQTT3SourceUpdate, MQTT3SourceWrite, MQTT5Source, + MQTT5SourceUpdate, MQTT5SourceWrite, Source, SourceList, + SourceUpdate, SourceWrite, SourceWriteList, ) @@ -22,4 +26,8 @@ "SourceList", "SourceWrite", "SourceWriteList", + "SourceUpdate", + "MQTT3SourceUpdate", + "MQTT5SourceUpdate", + "EventHubSourceUpdate", ] diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index 09707fb85..dd4548b86 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -9,9 +9,12 @@ from cognite.client.data_classes._base import ( CogniteObject, + CognitePrimitiveUpdate, CogniteResource, CogniteResourceList, + CogniteUpdate, ExternalIDTransformerMixin, + PropertySpec, T_WriteClass, UnknownCogniteObject, WriteableCogniteResource, @@ -101,6 +104,18 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: return output +class SourceUpdate(CogniteUpdate, ABC): + _type: ClassVar[str] + + def __init__(self, external_id: str) -> None: + super().__init__(external_id=external_id) + + def dump(self, camel_case: Literal[True] = True) -> dict[str, Any]: + output = super().dump(camel_case) + output["type"] = self._type + return output + + class EventHubSourceWrite(SourceWrite): """A hosted extractor source represents an external source system on the internet. The source resource in CDF contains all the information the extractor needs to @@ -193,6 +208,48 @@ def _load_source(cls, resource: dict[str, Any]) -> Self: return fast_dict_load(cls, resource, None) +class EventHubSourceUpdate(SourceUpdate): + _type = "eventHub" + + class _PrimitiveEventHubSourceUpdate(CognitePrimitiveUpdate): + def set(self, value: str) -> EventHubSourceUpdate: + return self._set(value) + + class _PrimitiveNullableEventHubSourceUpdate(CognitePrimitiveUpdate): + def set(self, value: str | None) -> EventHubSourceUpdate: + return self._set(value) + + @property + def host(self) -> _PrimitiveEventHubSourceUpdate: + return EventHubSourceUpdate._PrimitiveEventHubSourceUpdate(self, "host") + + @property + def event_hub_name(self) -> _PrimitiveEventHubSourceUpdate: + return EventHubSourceUpdate._PrimitiveEventHubSourceUpdate(self, "eventHubName") + + @property + def key_name(self) -> _PrimitiveEventHubSourceUpdate: + return EventHubSourceUpdate._PrimitiveEventHubSourceUpdate(self, "keyName") + + @property + def key_value(self) -> _PrimitiveEventHubSourceUpdate: + return EventHubSourceUpdate._PrimitiveEventHubSourceUpdate(self, "keyValue") + + @property + def consumer_group(self) -> _PrimitiveNullableEventHubSourceUpdate: + return EventHubSourceUpdate._PrimitiveNullableEventHubSourceUpdate(self, "consumerGroup") + + @classmethod + def _get_update_properties(cls, item: CogniteResource | None = None) -> list[PropertySpec]: + return [ + PropertySpec("host", is_nullable=False), + PropertySpec("event_hub_name", is_nullable=False), + PropertySpec("key_name", is_nullable=False), + PropertySpec("key_value", is_nullable=False), + PropertySpec("consumer_group", is_nullable=True), + ] + + @dataclass class MQTTAuthenticationWrite(CogniteObject, ABC): _type: ClassVar[str] @@ -417,6 +474,67 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: return output +class _MQTTUpdate(SourceUpdate, ABC): + class _HostUpdate(CognitePrimitiveUpdate): + def set(self, value: str) -> _MQTTUpdate: + return self._set(value) + + class _PortUpdate(CognitePrimitiveUpdate): + def set(self, value: int | None) -> _MQTTUpdate: + return self._set(value) + + class _AuthenticationUpdate(CognitePrimitiveUpdate): + def set(self, value: MQTTAuthentication | None) -> _MQTTUpdate: + return self._set(value.dump() if value else None) + + class _UseTlsUpdate(CognitePrimitiveUpdate): + def set(self, value: bool) -> _MQTTUpdate: + return self._set(value) + + class _CACertificateUpdate(CognitePrimitiveUpdate): + def set(self, value: CACertificate | None) -> _MQTTUpdate: + return self._set(value.dump() if value else None) + + class _AuthCertificateUpdate(CognitePrimitiveUpdate): + def set(self, value: AuthCertificate | None) -> _MQTTUpdate: + return self._set(value.dump() if value else None) + + @property + def host(self) -> _HostUpdate: + return _MQTTUpdate._HostUpdate(self, "host") + + @property + def port(self) -> _PortUpdate: + return _MQTTUpdate._PortUpdate(self, "port") + + @property + def authentication(self) -> _AuthenticationUpdate: + return _MQTTUpdate._AuthenticationUpdate(self, "authentication") + + @property + def useTls(self) -> _UseTlsUpdate: + return _MQTTUpdate._UseTlsUpdate(self, "useTls") + + @property + def ca_certificate(self) -> _CACertificateUpdate: + return _MQTTUpdate._CACertificateUpdate(self, "caCertificate") + + @property + def auth_certificate(self) -> _AuthCertificateUpdate: + return _MQTTUpdate._AuthCertificateUpdate(self, "authCertificate") + + @classmethod + def _get_update_properties(cls, item: CogniteResource | None = None) -> list[PropertySpec]: + return [ + PropertySpec("host", is_nullable=False), + PropertySpec("port", is_nullable=True), + PropertySpec("authentication", is_nullable=True, is_container=True), + PropertySpec("useTls", is_nullable=False), + PropertySpec("ca_certificate", is_nullable=True, is_container=True), + PropertySpec("auth_certificate", is_nullable=True, is_container=True), + ] + + class MQTT3SourceWrite(_MQTTSourceWrite): _type = "mqtt3" @@ -433,6 +551,14 @@ class MQTT5Source(_MQTTSource): _type = "mqtt5" +class MQTT3SourceUpdate(_MQTTUpdate): + _type = "mqtt3" + + +class MQTT5SourceUpdate(_MQTTUpdate): + _type = "mqtt5" + + class SourceWriteList(CogniteResourceList[SourceWrite], ExternalIDTransformerMixin): _RESOURCE = SourceWrite From 1800bd2257ad6f30eaf25ed377b01033246970e1 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 11:18:58 +0200 Subject: [PATCH 08/32] fix: bugs in data classes --- .../data_classes/hosted_extractors/sources.py | 41 ++++++++++++++----- cognite/client/testing.py | 5 +++ 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index dd4548b86..80af5ac56 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -20,7 +20,6 @@ WriteableCogniteResource, WriteableCogniteResourceList, ) -from cognite.client.utils._auxiliary import fast_dict_load if TYPE_CHECKING: from cognite.client import CogniteClient @@ -52,7 +51,7 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = type_ = resource.get("type") if type_ is None and hasattr(cls, "_type"): type_ = cls._type - else: + elif type_ is None: raise KeyError("type") source_class = _SOURCE_WRITE_CLASS_BY_TYPE.get(type_) if source_class is None: @@ -91,12 +90,12 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = type_ = resource.get("type") if type_ is None and hasattr(cls, "_type"): type_ = cls._type - else: + elif type_ is None: raise KeyError("type") source_class = _SOURCE_CLASS_BY_TYPE.get(type_) if source_class is None: return UnknownCogniteObject(resource) # type: ignore[return-value] - return cast(Self, source_class._load(resource)) + return cast(Self, source_class._load_source(resource)) def dump(self, camel_case: bool = True) -> dict[str, Any]: output = super().dump(camel_case) @@ -132,6 +131,8 @@ class EventHubSourceWrite(SourceWrite): consumer_group (str | None): The event hub consumer group to use. Microsoft recommends having a distinct consumer group for each application consuming data from event hub. If left out, this uses the default consumer group. """ + _type = "eventHub" + def __init__( self, external_id: str, @@ -153,7 +154,14 @@ def as_write(self) -> SourceWrite: @classmethod def _load_source(cls, resource: dict[str, Any]) -> Self: - return fast_dict_load(cls, resource, None) + return cls( + external_id=resource["externalId"], + host=resource["host"], + event_hub_name=resource["eventHubName"], + key_name=resource["keyName"], + key_value=resource["keyValue"], + consumer_group=resource.get("consumerGroup"), + ) class EventHubSource(Source): @@ -173,6 +181,8 @@ class EventHubSource(Source): consumer_group (str | None): The event hub consumer group to use. Microsoft recommends having a distinct consumer group for each application consuming data from event hub. If left out, this uses the default consumer group. """ + _type = "eventHub" + def __init__( self, external_id: str, @@ -205,7 +215,15 @@ def as_write(self, key_value: str | None = None) -> EventHubSourceWrite: @classmethod def _load_source(cls, resource: dict[str, Any]) -> Self: - return fast_dict_load(cls, resource, None) + return cls( + external_id=resource["externalId"], + host=resource["host"], + event_hub_name=resource["eventHubName"], + key_name=resource["keyName"], + created_time=resource["createdTime"], + last_updated_time=resource["lastUpdatedTime"], + consumer_group=resource.get("consumerGroup"), + ) class EventHubSourceUpdate(SourceUpdate): @@ -264,7 +282,7 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = type_ = resource.get("type") if type_ is None and hasattr(cls, "_type"): type_ = cls._type - else: + elif type_ is None: raise KeyError("type is required") authentication_class = _MQTTAUTHENTICATION_WRITE_CLASS_BY_TYPE.get(type_) if authentication_class is None: @@ -285,7 +303,10 @@ class BasicMQTTAuthenticationWrite(MQTTAuthenticationWrite): @classmethod def _load_authentication(cls, resource: dict[str, Any]) -> Self: - return fast_dict_load(cls, resource, None) + return cls( + username=resource["username"], + password=resource.get("password"), + ) @dataclass @@ -335,7 +356,7 @@ def __init__( self.auth_certificate = auth_certificate @classmethod - def _load_source(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = None) -> Self: + def _load_source(cls, resource: dict[str, Any]) -> Self: return cls( external_id=resource["externalId"], host=resource["host"], @@ -375,7 +396,7 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = type_ = resource.get("type") if type_ is None and hasattr(cls, "_type"): type_ = cls._type - else: + elif type_ is None: raise KeyError("type") authentication_class = _MQTTAUTHENTICATION_CLASS_BY_TYPE.get(type_) diff --git a/cognite/client/testing.py b/cognite/client/testing.py index bfc3f8593..a4bfcb6f3 100644 --- a/cognite/client/testing.py +++ b/cognite/client/testing.py @@ -29,6 +29,8 @@ from cognite.client._api.files import FilesAPI from cognite.client._api.functions import FunctionCallsAPI, FunctionsAPI, FunctionSchedulesAPI from cognite.client._api.geospatial import GeospatialAPI +from cognite.client._api.hosted_extractors import HostedExtractorsAPI +from cognite.client._api.hosted_extractors.sources import SourcesAPI from cognite.client._api.iam import IAMAPI, GroupsAPI, SecurityCategoriesAPI, SessionsAPI, TokenAPI from cognite.client._api.labels import LabelsAPI from cognite.client._api.raw import RawAPI, RawDatabasesAPI, RawRowsAPI, RawTablesAPI @@ -136,6 +138,9 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.sequences = MagicMock(spec=SequencesAPI) self.sequences.data = MagicMock(spec_set=SequencesDataAPI) + self.hosted_extractors = MagicMock(spec=HostedExtractorsAPI) + self.hosted_extractors.sources = MagicMock(spec_set=SourcesAPI) + self.templates = MagicMock(spec=TemplatesAPI) self.templates.groups = MagicMock(spec_set=TemplateGroupsAPI) self.templates.instances = MagicMock(spec_set=TemplateInstancesAPI) From 04311296049a6057581216c04edb1235e970c585 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 11:24:59 +0200 Subject: [PATCH 09/32] tests: updated tests --- tests/tests_unit/test_base.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 1d7232564..317a82945 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -36,10 +36,11 @@ ) from cognite.client.data_classes.datapoints import DatapointsArray from cognite.client.data_classes.events import Event, EventList +from cognite.client.data_classes.hosted_extractors import Source, SourceList from cognite.client.exceptions import CogniteMissingClientError from cognite.client.testing import CogniteClientMock from cognite.client.utils import _json -from tests.utils import FakeCogniteResourceGenerator, all_concrete_subclasses, all_subclasses +from tests.utils import FakeCogniteResourceGenerator, all_concrete_subclasses class MyResource(CogniteResource): @@ -192,7 +193,9 @@ def test_dump_load_only_required( "cognite_writable_cls", [ pytest.param(cls, id=f"{cls.__name__} in {cls.__module__}") + # Hosted extractors does not support the as_write method for cls in all_concrete_subclasses(WriteableCogniteResource) + if not issubclass(cls, Source) ], ) def test_writable_as_write( @@ -210,7 +213,7 @@ def test_writable_as_write( [ pytest.param(cls, id=f"{cls.__name__} in {cls.__module__}") for cls in all_concrete_subclasses(WriteableCogniteResourceList) - if cls not in [EdgeListWithCursor, NodeListWithCursor] + if cls not in [EdgeListWithCursor, NodeListWithCursor, SourceList] ], ) def test_writable_list_as_write( @@ -762,13 +765,20 @@ def test_get_update_properties(self): assert hasattr(MyUpdate, "columns") and "columns" not in props assert {"string", "list", "object", "labels"} == set(props) - @pytest.mark.parametrize("cognite_update_subclass", all_subclasses(CogniteUpdate)) + @pytest.mark.parametrize("cognite_update_subclass", all_concrete_subclasses(CogniteUpdate)) def test_correct_implementation_get_update_properties(self, cognite_update_subclass: CogniteUpdate): expected = sorted( key for key in cognite_update_subclass.__dict__ if not key.startswith("_") and key not in {"columns", "dump"} ) + if not expected: + # Check parent class if there are no attributes in the subclass + expected = sorted( + key + for key in cognite_update_subclass.__bases__[0].__dict__ + if not key.startswith("_") and key != "dump" + ) actual = sorted(prop.name for prop in cognite_update_subclass._get_update_properties()) assert expected == actual From 1757dc82812d790477c12e250dc26402fa5c8bc0 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 11:34:17 +0200 Subject: [PATCH 10/32] refactor: created ingegration tests --- .../client/_api/hosted_extractors/sources.py | 5 +- .../test_hosted_extractors/__init__.py | 0 .../test_hosted_extractors/test_sources.py | 68 +++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 tests/tests_integration/test_api/test_hosted_extractors/__init__.py create mode 100644 tests/tests_integration/test_api/test_hosted_extractors/test_sources.py diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index be1a45121..41b63d2d3 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -65,10 +65,10 @@ def __iter__(self) -> Iterator[Source]: return self() @overload - def retrieve(self, external_ids: str) -> Source: ... + def retrieve(self, external_ids: str, ignore_unknown_ids: bool = False) -> Source: ... @overload - def retrieve(self, external_ids: SequenceNotStr[str]) -> SourceList: ... + def retrieve(self, external_ids: SequenceNotStr[str], ignore_unknown_ids: bool = False) -> SourceList: ... def retrieve( self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bool = False @@ -99,6 +99,7 @@ def retrieve( list_cls=SourceList, resource_cls=Source, # type: ignore[type-abstract] identifiers=IdentifierSequence.load(external_ids=external_ids), + ignore_unknown_ids=ignore_unknown_ids, ) def delete(self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bool, force: bool) -> None: diff --git a/tests/tests_integration/test_api/test_hosted_extractors/__init__.py b/tests/tests_integration/test_api/test_hosted_extractors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py new file mode 100644 index 000000000..4f8114556 --- /dev/null +++ b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import pytest + +from cognite.client import CogniteClient +from cognite.client.data_classes.hosted_extractors import ( + EventHubSource, + EventHubSourceUpdate, + EventHubSourceWrite, + SourceList, +) +from cognite.client.exceptions import CogniteAPIError +from cognite.client.utils._text import random_string + + +@pytest.fixture(scope="session") +def one_event_hub_source(cognite_client: CogniteClient) -> SourceList: + my_hub = EventHubSourceWrite( + external_id=f"myNewHub-{random_string(10)}", + host="myHost", + key_name="myKeyName", + key_value="myKey", + event_hub_name="myEventHub", + ) + retrieved = cognite_client.hosted_extractors.sources.retrieve(my_hub.external_id, ignore_unknown_ids=True) + if retrieved: + return retrieved + created = cognite_client.hosted_extractors.sources.create(my_hub) + return SourceList([created]) + + +class TestSources: + def test_create_update_retrieve_delete(self, cognite_client: CogniteClient) -> None: + my_hub = EventHubSourceWrite( + external_id=f"myNewHub-{random_string(10)}", + host="myHost", + key_name="myKeyName", + key_value="myKey", + event_hub_name="myEventHub", + ) + created: EventHubSource | None = None + try: + created = cognite_client.hosted_extractors.sources.create(my_hub) + assert isinstance(created, EventHubSource) + update = EventHubSourceUpdate(external_id=my_hub.external_id).event_hub_name.set("myNewEventHub") + updated = cognite_client.hosted_extractors.sources.update(update) + assert updated.event_hub_name == "myNewEventHub" + retrieved = cognite_client.hosted_extractors.sources.retrieve(created.external_id) + assert retrieved is not None + assert retrieved.external_id == created.external_id + assert retrieved.event_hub_name == "myNewEventHub" + + cognite_client.hosted_extractors.sources.delete(created.external_id) + + with pytest.raises(CogniteAPIError): + cognite_client.hosted_extractors.sources.retrieve(created.external_id) + + cognite_client.hosted_extractors.sources.retrieve(created.external_id, ignore_unknown_ids=True) + + finally: + if created: + cognite_client.hosted_extractors.sources.delete(created.external_id) + + @pytest.mark.usefixtures("one_event_hub_source") + def test_list(self, cognite_client: CogniteClient) -> None: + res = cognite_client.hosted_extractors.sources.list(limit=1) + assert len(res) == 1 + assert isinstance(res, SourceList) From 18c0e3da1d71a0447cb6f0516dc95da20d0fa44b Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 11:39:07 +0200 Subject: [PATCH 11/32] refactor: added warnings --- .../client/_api/hosted_extractors/sources.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index 41b63d2d3..c43311a76 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -6,16 +6,28 @@ from cognite.client._api_client import APIClient from cognite.client._constants import DEFAULT_LIMIT_READ from cognite.client.data_classes.hosted_extractors.sources import Source, SourceList, SourceUpdate, SourceWrite +from cognite.client.utils._experimental import FeaturePreviewWarning from cognite.client.utils._identifier import IdentifierSequence from cognite.client.utils.useful_types import SequenceNotStr if TYPE_CHECKING: - pass + from cognite.client import ClientConfig, CogniteClient class SourcesAPI(APIClient): _RESOURCE_PATH = "/hostedextractors/sources" + def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: CogniteClient) -> None: + super().__init__(config, api_version, cognite_client) + self._warning = FeaturePreviewWarning( + api_maturity="beta", sdk_maturity="beta", feature_name="Hosted Extractors" + ) + self._CREATE_LIMIT = 100 + self._LIST_LIMIT = 100 + self._RETRIEVE_LIMIT = 100 + self._DELETE_LIMIT = 100 + self._UPDATE_LIMIT = 100 + @overload def __call__( self, @@ -46,12 +58,15 @@ def __call__( Returns: Iterator[Source] | Iterator[SourceList]: yields Source one by one if chunk_size is not specified, else SourceList objects. """ + self._warning.warn() + return self._list_generator( list_cls=SourceList, resource_cls=Source, # type: ignore[type-abstract] method="GET", chunk_size=chunk_size, limit=limit, + headers={"cdf-version": "beta"}, ) def __iter__(self) -> Iterator[Source]: @@ -95,11 +110,13 @@ def retrieve( >>> res = client.hosted_extractors.sources.retrieve(["myMQTTSource", "MyEvenHubSource"], ignore_unknown_ids=True) """ + self._warning.warn() return self._retrieve_multiple( list_cls=SourceList, resource_cls=Source, # type: ignore[type-abstract] identifiers=IdentifierSequence.load(external_ids=external_ids), ignore_unknown_ids=ignore_unknown_ids, + headers={"cdf-version": "beta"}, ) def delete(self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bool, force: bool) -> None: @@ -117,10 +134,12 @@ def delete(self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bo >>> client = CogniteClient() >>> client.hosted_extractors.sources.delete(spaces=["myMQTTSource", "MyEvenHubSource"]) """ + self._warning.warn() self._delete_multiple( identifiers=IdentifierSequence.load(external_ids=external_ids), wrap_ids=True, returns_items=False, + headers={"cdf-version": "beta"}, ) @overload @@ -148,11 +167,13 @@ def create(self, items: SourceWrite | Sequence[SourceWrite]) -> Source | SourceL >>> source = EventHubSourceWrite('my_event_hub', 'http://myeventhub.com', "My EventHub", 'my_key', 'my_value') >>> res = client.hosted_extractors.sources.create(source) """ + self._warning.warn() return self._create_multiple( list_cls=SourceList, resource_cls=Source, # type: ignore[type-abstract] items=items, # type: ignore[arg-type] input_resource_cls=SourceWrite, # type: ignore[arg-type] + headers={"cdf-version": "beta"}, ) @overload @@ -180,11 +201,13 @@ def update(self, items: SourceWrite | SourceUpdate | Sequence[SourceWrite | Sour >>> source = EventHubSourceUpdate('my_event_hub').event_hub_name.set("My Updated EventHub") >>> res = client.hosted_extractors.sources.update(source) """ + self._warning.warn() return self._update_multiple( items=items, # type: ignore[arg-type] list_cls=SourceList, resource_cls=Source, # type: ignore[type-abstract] update_cls=SourceUpdate, # type: ignore[type-abstract] + headers={"cdf-version": "beta"}, ) def list( @@ -221,9 +244,11 @@ def list( >>> for source_list in client.hosted_extractors.sources(chunk_size=25): ... source_list # do something with the spaces """ + self._warning.warn() return self._list( list_cls=SourceList, resource_cls=Source, # type: ignore[type-abstract] method="GET", limit=limit, + headers={"cdf-version": "beta"}, ) From 3699ddea345260546e7100f3edb6311884c8cccb Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 13:45:53 +0200 Subject: [PATCH 12/32] refactor: updated script for adding capability --- scripts/add_capability.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/add_capability.py b/scripts/add_capability.py index b47c8ae21..1240d5190 100644 --- a/scripts/add_capability.py +++ b/scripts/add_capability.py @@ -2,6 +2,7 @@ from pathlib import Path from cognite.client import CogniteClient +from cognite.client.data_classes.capabilities import Capability TMP_DIR = Path(__file__).resolve().parent / "tmp" @@ -9,9 +10,9 @@ def main(client: CogniteClient): new_capabilities = [ { - "experimentAcl": { - "actions": ["USE"], - "scope": {"experimentscope": {"experiments": ["workflowOrchestrator"]}}, + "hostedExtractorsAcl": { + "actions": ["READ", "WRITE"], + "scope": {"all": {}}, } }, ] @@ -33,28 +34,27 @@ def main(client: CogniteClient): (TMP_DIR / f"{selected_group.name}.json").write_text(json.dumps(selected_group.dump(camel_case=True), indent=4)) existing_capability_by_name = { - next(iter(capability.keys())): capability for capability in selected_group.capabilities + capability._capability_name: capability for capability in selected_group.capabilities } added = [] for new_capability in new_capabilities: (capability_name,) = new_capability.keys() if capability_name not in existing_capability_by_name: - selected_group.capabilities.append(new_capability) + selected_group.capabilities.append(Capability.load(new_capability)) added.append(capability_name) elif new_capability[capability_name] != existing_capability_by_name[capability_name]: # Capability exists, but with different scope or actions - selected_group.capabilities.append(new_capability) - added.append(capability_name) + raise NotImplementedError() + # selected_group.capabilities.append(new_capability) + # added.append(capability_name) else: print(f"Capability {capability_name} already exists") if not added: print("All capabilities already exists") return delete_id = selected_group.id - selected_group.id = None - selected_group.is_deleted = None - selected_group.deleted_time = None - client.iam.groups.create(selected_group) + selected_group_write = selected_group.as_write() + client.iam.groups.create(selected_group_write) client.iam.groups.delete(delete_id) From ec5de1eaf84232d9a02b1a6c885bf4e56fdec1dd Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 13:49:50 +0200 Subject: [PATCH 13/32] fix: small bugs --- cognite/client/_api/hosted_extractors/sources.py | 13 +++++++++++-- .../data_classes/hosted_extractors/sources.py | 6 +++--- .../test_api/test_hosted_extractors/test_sources.py | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index c43311a76..f52bd0a07 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Iterator -from typing import TYPE_CHECKING, Sequence, overload +from typing import TYPE_CHECKING, Any, Sequence, overload from cognite.client._api_client import APIClient from cognite.client._constants import DEFAULT_LIMIT_READ @@ -119,7 +119,9 @@ def retrieve( headers={"cdf-version": "beta"}, ) - def delete(self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bool, force: bool) -> None: + def delete( + self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bool = False, force: bool = False + ) -> None: """`Delete one or more sources `_ Args: @@ -135,11 +137,18 @@ def delete(self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bo >>> client.hosted_extractors.sources.delete(spaces=["myMQTTSource", "MyEvenHubSource"]) """ self._warning.warn() + extra_body_fields: dict[str, Any] = {} + if ignore_unknown_ids: + extra_body_fields["ignoreUnknownIds"] = True + if force: + extra_body_fields["force"] = True + self._delete_multiple( identifiers=IdentifierSequence.load(external_ids=external_ids), wrap_ids=True, returns_items=False, headers={"cdf-version": "beta"}, + extra_body_fields=extra_body_fields or None, ) @overload diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index 80af5ac56..daebcedfc 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -131,7 +131,7 @@ class EventHubSourceWrite(SourceWrite): consumer_group (str | None): The event hub consumer group to use. Microsoft recommends having a distinct consumer group for each application consuming data from event hub. If left out, this uses the default consumer group. """ - _type = "eventHub" + _type = "eventhub" def __init__( self, @@ -181,7 +181,7 @@ class EventHubSource(Source): consumer_group (str | None): The event hub consumer group to use. Microsoft recommends having a distinct consumer group for each application consuming data from event hub. If left out, this uses the default consumer group. """ - _type = "eventHub" + _type = "eventhub" def __init__( self, @@ -227,7 +227,7 @@ def _load_source(cls, resource: dict[str, Any]) -> Self: class EventHubSourceUpdate(SourceUpdate): - _type = "eventHub" + _type = "eventhub" class _PrimitiveEventHubSourceUpdate(CognitePrimitiveUpdate): def set(self, value: str) -> EventHubSourceUpdate: diff --git a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py index 4f8114556..611cad3ae 100644 --- a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py +++ b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py @@ -59,7 +59,7 @@ def test_create_update_retrieve_delete(self, cognite_client: CogniteClient) -> N finally: if created: - cognite_client.hosted_extractors.sources.delete(created.external_id) + cognite_client.hosted_extractors.sources.delete(created.external_id, ignore_unknown_ids=True) @pytest.mark.usefixtures("one_event_hub_source") def test_list(self, cognite_client: CogniteClient) -> None: From 1712d8db57cf8053834f7fed9881519fe0d2f756 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 13:51:51 +0200 Subject: [PATCH 14/32] refactor: Only unknowns on read --- .../data_classes/hosted_extractors/sources.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index daebcedfc..29b5fb676 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -53,10 +53,10 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = type_ = cls._type elif type_ is None: raise KeyError("type") - source_class = _SOURCE_WRITE_CLASS_BY_TYPE.get(type_) - if source_class is None: - return UnknownCogniteObject(resource) # type: ignore[return-value] - return cast(Self, source_class._load_source(resource)) + try: + return cast(Self, _SOURCE_WRITE_CLASS_BY_TYPE[type_]._load_source(resource)) + except KeyError: + raise TypeError(f"Unknown source type: {type_}") def dump(self, camel_case: bool = True) -> dict[str, Any]: output = super().dump(camel_case) @@ -284,10 +284,10 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = type_ = cls._type elif type_ is None: raise KeyError("type is required") - authentication_class = _MQTTAUTHENTICATION_WRITE_CLASS_BY_TYPE.get(type_) - if authentication_class is None: - return UnknownCogniteObject(resource) # type: ignore[return-value] - return cast(Self, authentication_class._load_authentication(resource)) + try: + return cast(Self, _MQTTAUTHENTICATION_WRITE_CLASS_BY_TYPE[type_]) + except KeyError: + raise TypeError(f"Unknown authentication type: {type_}") def dump(self, camel_case: bool = True) -> dict[str, Any]: output = super().dump(camel_case) From 9a63e589dc3c6ea1bcfc542f9dd3c858582455f1 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 14:04:53 +0200 Subject: [PATCH 15/32] fix: update with write object --- cognite/client/_api/annotations.py | 1 + cognite/client/_api_client.py | 11 +++++++- cognite/client/data_classes/_base.py | 6 +++++ .../data_classes/hosted_extractors/sources.py | 18 +++++++++++++ .../test_hosted_extractors/test_sources.py | 27 +++++++++++++++++++ 5 files changed, 62 insertions(+), 1 deletion(-) diff --git a/cognite/client/_api/annotations.py b/cognite/client/_api/annotations.py index 953f03ced..ef24ca333 100644 --- a/cognite/client/_api/annotations.py +++ b/cognite/client/_api/annotations.py @@ -101,6 +101,7 @@ def _convert_resource_to_patch_object( resource: CogniteResource, update_attributes: list[PropertySpec], mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null", + identifying_properties: dict[str, Any] | None = None, ) -> dict[str, dict[str, dict]]: if not isinstance(resource, Annotation): return APIClient._convert_resource_to_patch_object(resource, update_attributes) diff --git a/cognite/client/_api_client.py b/cognite/client/_api_client.py index 74f9d4c8e..b33350593 100644 --- a/cognite/client/_api_client.py +++ b/cognite/client/_api_client.py @@ -1038,7 +1038,12 @@ def _update_multiple( for index, item in enumerate(item_list): if isinstance(item, CogniteResource): patch_objects.append( - self._convert_resource_to_patch_object(item, update_cls._get_update_properties(item), mode) + self._convert_resource_to_patch_object( + item, + update_cls._get_update_properties(item), + mode, + update_cls._get_extra_identifying_properties(item), + ) ) elif isinstance(item, CogniteUpdate): patch_objects.append(item.dump(camel_case=True)) @@ -1218,6 +1223,7 @@ def _convert_resource_to_patch_object( resource: CogniteResource, update_attributes: list[PropertySpec], mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null", + identifying_properties: dict[str, Any] | None = None, ) -> dict[str, dict[str, dict]]: dumped_resource = resource.dump(camel_case=True) has_id = "id" in dumped_resource @@ -1233,6 +1239,9 @@ def _convert_resource_to_patch_object( elif has_external_id: patch_object["externalId"] = dumped_resource.pop("externalId") + if identifying_properties: + patch_object.update(identifying_properties) + update: dict[str, dict] = cls._clear_all_attributes(update_attributes) if mode == "replace" else {} update_attribute_by_name = {prop.name: prop for prop in update_attributes} diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index f35617043..cc9cffbcf 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -525,6 +525,12 @@ def dump(self, camel_case: Literal[True] = True) -> dict[str, Any]: def _get_update_properties(cls, item: CogniteResource | None = None) -> list[PropertySpec]: raise NotImplementedError + @classmethod + def _get_extra_identifying_properties(cls, item: CogniteResource | None = None) -> dict[str, Any]: + # This method is used to provide additional identifying properties for the update object. + # It is intended to be overridden by subclasses that need to provide additional identifying properties. + return {} + T_CogniteUpdate = TypeVar("T_CogniteUpdate", bound=CogniteUpdate) diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index 29b5fb676..1a273db27 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -114,6 +114,18 @@ def dump(self, camel_case: Literal[True] = True) -> dict[str, Any]: output["type"] = self._type return output + @classmethod + def _get_update_properties(cls, item: CogniteResource | None = None) -> list[PropertySpec]: + if item is None or not isinstance(item, SourceWrite): + return [] + return _SOURCE_UPDATE_BY_TYPE[item._type]._get_update_properties(item) + + @classmethod + def _get_extra_identifying_properties(cls, item: CogniteResource | None = None) -> dict[str, Any]: + if not isinstance(item, SourceWrite): + return {} + return {"type": item._type} + class EventHubSourceWrite(SourceWrite): """A hosted extractor source represents an external source system on the internet. @@ -605,6 +617,12 @@ def as_write( if hasattr(subclass, "_type") } +_SOURCE_UPDATE_BY_TYPE: dict[str, type[SourceUpdate]] = { + subclass._type: subclass # type: ignore[type-abstract, misc] + for subclass in itertools.chain(SourceUpdate.__subclasses__(), _MQTTUpdate.__subclasses__()) + if hasattr(subclass, "_type") +} + _MQTTAUTHENTICATION_WRITE_CLASS_BY_TYPE: dict[str, type[MQTTAuthenticationWrite]] = { subclass._type: subclass # type: ignore[type-abstract] for subclass in MQTTAuthenticationWrite.__subclasses__() diff --git a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py index 611cad3ae..d7c742f74 100644 --- a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py +++ b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py @@ -66,3 +66,30 @@ def test_list(self, cognite_client: CogniteClient) -> None: res = cognite_client.hosted_extractors.sources.list(limit=1) assert len(res) == 1 assert isinstance(res, SourceList) + + def test_update_using_write_object(self, cognite_client: CogniteClient) -> None: + my_hub = EventHubSourceWrite( + external_id=f"toupdatate-{random_string(10)}", + host="myHost", + key_name="myKeyName", + key_value="myKey", + event_hub_name="myEventHub", + ) + created: EventHubSource | None = None + try: + created = cognite_client.hosted_extractors.sources.create(my_hub) + + my_new_hub = EventHubSourceWrite( + external_id=created.external_id, + host="updatedHost", + key_name="updatedKeyName", + key_value="updatedKey", + event_hub_name="updatedEventHub", + ) + + updated = cognite_client.hosted_extractors.sources.update(my_new_hub) + + assert updated.host == my_new_hub.host + finally: + if created: + cognite_client.hosted_extractors.sources.delete(created.external_id, ignore_unknown_ids=True) From 1cec0fa166c9de0c2d5116133729b1f42e23df39 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 14:13:26 +0200 Subject: [PATCH 16/32] docs; updated docs --- docs/source/hosted_extractors.rst | 24 ++++++++++++++++++++++++ docs/source/index.rst | 1 + 2 files changed, 25 insertions(+) create mode 100644 docs/source/hosted_extractors.rst diff --git a/docs/source/hosted_extractors.rst b/docs/source/hosted_extractors.rst new file mode 100644 index 000000000..afc45101d --- /dev/null +++ b/docs/source/hosted_extractors.rst @@ -0,0 +1,24 @@ +Hosted Extractors +================= + +Sources +------- +Create new source +^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.hosted_extractors.SourcesAPI.create + +Delete source +^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.hosted_extractors.SourcesAPI.delete + +List sources +^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.hosted_extractors.SourcesAPI.list + +Retrieve sources +^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.hosted_extractors.SourcesAPI.retrieve + +Update sources +^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.hosted_extractors.SourcesAPI.update diff --git a/docs/source/index.rst b/docs/source/index.rst index 2c6883410..3d6796e5d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -50,6 +50,7 @@ Contents contextualization documents data_ingestion + hosted_extractors data_organization transformations functions From 9169272d01985e637befa0ccc06fd500abfab859 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 14:14:35 +0200 Subject: [PATCH 17/32] build: changelog --- CHANGELOG.md | 4 ++++ cognite/client/_version.py | 2 +- pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb0d1dc04..39e26b59a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ Changes are grouped as follows - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [7.55.2] - 2024-08-29 +### Added +- [Feature Preview - beta] Support for `client.hosted_extractors.sources`. + ## [7.55.1] - 2024-08-29 ### Fixed - Missing exports for workflow triggers diff --git a/cognite/client/_version.py b/cognite/client/_version.py index edd6657ff..a2f3828ca 100644 --- a/cognite/client/_version.py +++ b/cognite/client/_version.py @@ -1,4 +1,4 @@ from __future__ import annotations -__version__ = "7.55.1" +__version__ = "7.55.2" __api_subversion__ = "20230101" diff --git a/pyproject.toml b/pyproject.toml index a3566ee1b..f2c757338 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "cognite-sdk" -version = "7.55.1" +version = "7.55.2" description = "Cognite Python SDK" readme = "README.md" documentation = "https://cognite-sdk-python.readthedocs-hosted.com" From bd3e18a046a5e4429af4363788673cc02aac9955 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 14:31:23 +0200 Subject: [PATCH 18/32] fix: introduced bug --- cognite/client/_api_client.py | 1 + .../data_classes/hosted_extractors/sources.py | 14 +++++++------- tests/tests_unit/test_meta.py | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cognite/client/_api_client.py b/cognite/client/_api_client.py index b33350593..30c88c896 100644 --- a/cognite/client/_api_client.py +++ b/cognite/client/_api_client.py @@ -106,6 +106,7 @@ class APIClient: "transformations/(filter|byids|jobs/byids|schedules/byids|query/run)", "extpipes/(list|byids|runs/list)", "workflows/.*", + "hostedextractors/.*", ) ) ] diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index 1a273db27..b655f1524 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -297,7 +297,7 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None = elif type_ is None: raise KeyError("type is required") try: - return cast(Self, _MQTTAUTHENTICATION_WRITE_CLASS_BY_TYPE[type_]) + return cast(Self, _MQTTAUTHENTICATION_WRITE_CLASS_BY_TYPE[type_]._load_authentication(resource)) except KeyError: raise TypeError(f"Unknown authentication type: {type_}") @@ -385,11 +385,11 @@ def _load_source(cls, resource: dict[str, Any]) -> Self: def dump(self, camel_case: bool = True) -> dict[str, Any]: output = super().dump(camel_case) - if self.authentication: + if isinstance(self.authentication, MQTTAuthenticationWrite): output["authentication"] = self.authentication.dump(camel_case) - if self.ca_certificate: + if isinstance(self.ca_certificate, CACertificateWrite): output["caCertificate" if camel_case else "ca_certificate"] = self.ca_certificate.dump(camel_case) - if self.auth_certificate: + if isinstance(self.auth_certificate, AuthCertificateWrite): output["authCertificate" if camel_case else "auth_certificate"] = self.auth_certificate.dump(camel_case) return output @@ -498,11 +498,11 @@ def as_write(self) -> _MQTTSourceWrite: def dump(self, camel_case: bool = True) -> dict[str, Any]: output = super().dump(camel_case) - if self.authentication: + if isinstance(self.authentication, MQTTAuthentication): output["authentication"] = self.authentication.dump(camel_case) - if self.ca_certificate: + if isinstance(self.ca_certificate, CACertificate): output["caCertificate" if camel_case else "ca_certificate"] = self.ca_certificate.dump(camel_case) - if self.auth_certificate: + if isinstance(self.auth_certificate, AuthCertificate): output["authCertificate" if camel_case else "auth_certificate"] = self.auth_certificate.dump(camel_case) return output diff --git a/tests/tests_unit/test_meta.py b/tests/tests_unit/test_meta.py index 30d38c12a..df4199468 100644 --- a/tests/tests_unit/test_meta.py +++ b/tests/tests_unit/test_meta.py @@ -79,7 +79,7 @@ def apis_that_should_not_have_post_retry_rule(): ) def test_all_base_api_paths_have_retry_or_specifically_no_set( api, apis_with_post_method_retry_set, apis_that_should_not_have_post_retry_rule -): +) -> None: # So you've added a new API to the SDK, but suddenly this test is failing - what's the deal?! # Answer the following: # Does this new API have POST methods that should be retried automatically? From e407b827fc8145acc98a6fccb1f940140ba00414 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 14:32:47 +0200 Subject: [PATCH 19/32] style: happier mypy --- .../data_classes/hosted_extractors/sources.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index b655f1524..a387e5d4a 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -606,31 +606,27 @@ def as_write( _SOURCE_WRITE_CLASS_BY_TYPE: dict[str, type[SourceWrite]] = { - subclass._type: subclass # type: ignore[type-abstract, misc] + subclass._type: subclass # type: ignore[misc] for subclass in itertools.chain(SourceWrite.__subclasses__(), _MQTTSourceWrite.__subclasses__()) if hasattr(subclass, "_type") } _SOURCE_CLASS_BY_TYPE: dict[str, type[Source]] = { - subclass._type: subclass # type: ignore[type-abstract, misc] + subclass._type: subclass # type: ignore[misc] for subclass in itertools.chain(Source.__subclasses__(), _MQTTSource.__subclasses__()) if hasattr(subclass, "_type") } _SOURCE_UPDATE_BY_TYPE: dict[str, type[SourceUpdate]] = { - subclass._type: subclass # type: ignore[type-abstract, misc] + subclass._type: subclass # type: ignore[misc] for subclass in itertools.chain(SourceUpdate.__subclasses__(), _MQTTUpdate.__subclasses__()) if hasattr(subclass, "_type") } _MQTTAUTHENTICATION_WRITE_CLASS_BY_TYPE: dict[str, type[MQTTAuthenticationWrite]] = { - subclass._type: subclass # type: ignore[type-abstract] - for subclass in MQTTAuthenticationWrite.__subclasses__() - if hasattr(subclass, "_type") + subclass._type: subclass for subclass in MQTTAuthenticationWrite.__subclasses__() if hasattr(subclass, "_type") } _MQTTAUTHENTICATION_CLASS_BY_TYPE: dict[str, type[MQTTAuthentication]] = { - subclass._type: subclass # type: ignore[type-abstract] - for subclass in MQTTAuthentication.__subclasses__() - if hasattr(subclass, "_type") + subclass._type: subclass for subclass in MQTTAuthentication.__subclasses__() if hasattr(subclass, "_type") } From ce959cb0ed1b4350b88fc6bb986123fd6da7626c Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 14:37:35 +0200 Subject: [PATCH 20/32] style: mypy --- cognite/client/_api/hosted_extractors/sources.py | 4 ++-- .../client/data_classes/datapoints_subscriptions.py | 2 +- .../client/data_classes/hosted_extractors/sources.py | 10 +++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index f52bd0a07..b9e7b9020 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -181,7 +181,7 @@ def create(self, items: SourceWrite | Sequence[SourceWrite]) -> Source | SourceL list_cls=SourceList, resource_cls=Source, # type: ignore[type-abstract] items=items, # type: ignore[arg-type] - input_resource_cls=SourceWrite, # type: ignore[arg-type] + input_resource_cls=SourceWrite, headers={"cdf-version": "beta"}, ) @@ -215,7 +215,7 @@ def update(self, items: SourceWrite | SourceUpdate | Sequence[SourceWrite | Sour items=items, # type: ignore[arg-type] list_cls=SourceList, resource_cls=Source, # type: ignore[type-abstract] - update_cls=SourceUpdate, # type: ignore[type-abstract] + update_cls=SourceUpdate, headers={"cdf-version": "beta"}, ) diff --git a/cognite/client/data_classes/datapoints_subscriptions.py b/cognite/client/data_classes/datapoints_subscriptions.py index 273cc1eed..806595d4d 100644 --- a/cognite/client/data_classes/datapoints_subscriptions.py +++ b/cognite/client/data_classes/datapoints_subscriptions.py @@ -185,7 +185,7 @@ def set(self, value: Any) -> DataPointSubscriptionUpdate: class _FilterDataPointSubscriptionUpdate(CognitePrimitiveUpdate): def set(self, value: Filter) -> DataPointSubscriptionUpdate: - return self._set(value.dump()) # type: ignore[arg-type] + return self._set(value.dump()) class _ListDataPointSubscriptionUpdate(CogniteListUpdate): def set(self, value: list) -> DataPointSubscriptionUpdate: diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index a387e5d4a..49b93633a 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -618,15 +618,19 @@ def as_write( } _SOURCE_UPDATE_BY_TYPE: dict[str, type[SourceUpdate]] = { - subclass._type: subclass # type: ignore[misc] + subclass._type: subclass for subclass in itertools.chain(SourceUpdate.__subclasses__(), _MQTTUpdate.__subclasses__()) if hasattr(subclass, "_type") } _MQTTAUTHENTICATION_WRITE_CLASS_BY_TYPE: dict[str, type[MQTTAuthenticationWrite]] = { - subclass._type: subclass for subclass in MQTTAuthenticationWrite.__subclasses__() if hasattr(subclass, "_type") + subclass._type: subclass # type: ignore[type-abstract] + for subclass in MQTTAuthenticationWrite.__subclasses__() + if hasattr(subclass, "_type") } _MQTTAUTHENTICATION_CLASS_BY_TYPE: dict[str, type[MQTTAuthentication]] = { - subclass._type: subclass for subclass in MQTTAuthentication.__subclasses__() if hasattr(subclass, "_type") + subclass._type: subclass # type: ignore[type-abstract] + for subclass in MQTTAuthentication.__subclasses__() + if hasattr(subclass, "_type") } From 3c7f4473252251f98809f1cda624042e7c1cb908 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 14:38:13 +0200 Subject: [PATCH 21/32] styl: mypy --- cognite/client/data_classes/sequences.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cognite/client/data_classes/sequences.py b/cognite/client/data_classes/sequences.py index c8ad62004..9d59e7e5a 100644 --- a/cognite/client/data_classes/sequences.py +++ b/cognite/client/data_classes/sequences.py @@ -628,7 +628,7 @@ def _load(cls, resource: dict, cognite_client: CogniteClient | None = None) -> S ) -ColumnNames: TypeAlias = Literal[ # type: ignore[valid-type] +ColumnNames: TypeAlias = Literal[ "externalId", "id", "columnExternalId", From 09980a50d741c26e069a8a4575d0f4a6bfa48d07 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 14:39:30 +0200 Subject: [PATCH 22/32] refactor: adjust --- cognite/client/_api/hosted_extractors/sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index b9e7b9020..adfe2f971 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -20,7 +20,7 @@ class SourcesAPI(APIClient): def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: CogniteClient) -> None: super().__init__(config, api_version, cognite_client) self._warning = FeaturePreviewWarning( - api_maturity="beta", sdk_maturity="beta", feature_name="Hosted Extractors" + api_maturity="alpha", sdk_maturity="alpha", feature_name="Hosted Extractors" ) self._CREATE_LIMIT = 100 self._LIST_LIMIT = 100 From aa39c78550fc1cc49d85c7f36b66e1c9835067c0 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Thu, 29 Aug 2024 14:41:06 +0200 Subject: [PATCH 23/32] refactor: added missing --- cognite/client/data_classes/hosted_extractors/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cognite/client/data_classes/hosted_extractors/__init__.py b/cognite/client/data_classes/hosted_extractors/__init__.py index e8a1a1013..abadfb1c4 100644 --- a/cognite/client/data_classes/hosted_extractors/__init__.py +++ b/cognite/client/data_classes/hosted_extractors/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from cognite.client.data_classes.hosted_extractors.sources import ( EventHubSource, EventHubSourceUpdate, From 1f2e91460b1074a81993a553ce0c711e38521dff Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 2 Sep 2024 14:46:16 +0200 Subject: [PATCH 24/32] refactor: review feedback --- .../client/_api/hosted_extractors/sources.py | 25 +++++++-------- .../data_classes/hosted_extractors/sources.py | 31 +++++++------------ 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index adfe2f971..7d23f1fdb 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -49,7 +49,7 @@ def __call__( ) -> Iterator[Source] | Iterator[SourceList]: """Iterate over sources - Fetches sources as they are iterated over, so you keep a limited number of spaces in memory. + Fetches sources as they are iterated over, so you keep a limited number of sources in memory. Args: chunk_size (int | None): Number of sources to return in each chunk. Defaults to yielding one source a time. @@ -72,7 +72,7 @@ def __call__( def __iter__(self) -> Iterator[Source]: """Iterate over sources - Fetches sources as they are iterated over, so you keep a limited number of spaces in memory. + Fetches sources as they are iterated over, so you keep a limited number of sources in memory. Returns: Iterator[Source]: yields Source one by one. @@ -92,7 +92,7 @@ def retrieve( Args: external_ids (str | SequenceNotStr[str]): The external ID provided by the client. Must be unique for the resource type. - ignore_unknown_ids (bool): No description. + ignore_unknown_ids (bool): Ignore external IDs that are not found rather than throw an exception. Returns: Source | SourceList: Requested sources @@ -103,11 +103,11 @@ def retrieve( >>> client = CogniteClient() >>> res = client.hosted_extractors.sources.retrieve('myMQTTSource') - Get multiple spaces by id: + Get multiple sources by id: >>> from cognite.client import CogniteClient >>> client = CogniteClient() - >>> res = client.hosted_extractors.sources.retrieve(["myMQTTSource", "MyEvenHubSource"], ignore_unknown_ids=True) + >>> res = client.hosted_extractors.sources.retrieve(["myMQTTSource", "MyEventHubSource"], ignore_unknown_ids=True) """ self._warning.warn() @@ -126,15 +126,15 @@ def delete( Args: external_ids (str | SequenceNotStr[str]): The external ID provided by the client. Must be unique for the resource type. - ignore_unknown_ids (bool): No description. - force (bool): No description. + ignore_unknown_ids (bool): Ignore external IDs that are not found rather than throw an exception. + force (bool): Delete any jobs associated with each item. Examples: Delete sources by id:: >>> from cognite.client import CogniteClient >>> client = CogniteClient() - >>> client.hosted_extractors.sources.delete(spaces=["myMQTTSource", "MyEvenHubSource"]) + >>> client.hosted_extractors.sources.delete(["myMQTTSource", "MyEventHubSource"]) """ self._warning.warn() extra_body_fields: dict[str, Any] = {} @@ -146,7 +146,6 @@ def delete( self._delete_multiple( identifiers=IdentifierSequence.load(external_ids=external_ids), wrap_ids=True, - returns_items=False, headers={"cdf-version": "beta"}, extra_body_fields=extra_body_fields or None, ) @@ -161,7 +160,7 @@ def create(self, items: SourceWrite | Sequence[SourceWrite]) -> Source | SourceL """`Create one or more sources. `_ Args: - items (SourceWrite | Sequence[SourceWrite]): Space | Sequence[Space]): Source(s) to create. + items (SourceWrite | Sequence[SourceWrite]): Source(s) to create. Returns: Source | SourceList: Created source(s) @@ -195,7 +194,7 @@ def update(self, items: SourceWrite | SourceUpdate | Sequence[SourceWrite | Sour """`Update one or more sources. `_ Args: - items (SourceWrite | SourceUpdate | Sequence[SourceWrite | SourceUpdate]): Space | Sequence[Space]): Source(s) to update. + items (SourceWrite | SourceUpdate | Sequence[SourceWrite | SourceUpdate]): Source(s) to update. Returns: Source | SourceList: Updated source(s) @@ -237,7 +236,7 @@ def list( >>> from cognite.client import CogniteClient >>> client = CogniteClient() - >>> space_list = client.hosted_extractors.sources.list(limit=5) + >>> source_list = client.hosted_extractors.sources.list(limit=5) Iterate over sources:: @@ -251,7 +250,7 @@ def list( >>> from cognite.client import CogniteClient >>> client = CogniteClient() >>> for source_list in client.hosted_extractors.sources(chunk_size=25): - ... source_list # do something with the spaces + ... source_list # do something with the sources """ self._warning.warn() return self._list( diff --git a/cognite/client/data_classes/hosted_extractors/sources.py b/cognite/client/data_classes/hosted_extractors/sources.py index 49b93633a..640244db8 100644 --- a/cognite/client/data_classes/hosted_extractors/sources.py +++ b/cognite/client/data_classes/hosted_extractors/sources.py @@ -3,7 +3,7 @@ import itertools from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast +from typing import TYPE_CHECKING, Any, ClassVar, Literal, NoReturn, cast from typing_extensions import Self @@ -213,17 +213,8 @@ def __init__( self.created_time = created_time self.last_updated_time = last_updated_time - def as_write(self, key_value: str | None = None) -> EventHubSourceWrite: - if key_value is None: - raise ValueError("key_value must be provided") - return EventHubSourceWrite( - external_id=self.external_id, - host=self.host, - event_hub_name=self.event_hub_name, - key_name=self.key_name, - key_value=key_value, - consumer_group=self.consumer_group, - ) + def as_write(self) -> NoReturn: + raise TypeError(f"{type(self).__name__} cannot be converted to write as id does not contain the secrets") @classmethod def _load_source(cls, resource: dict[str, Any]) -> Self: @@ -355,7 +346,7 @@ def __init__( host: str, port: int | None = None, authentication: MQTTAuthenticationWrite | None = None, - useTls: bool = False, + use_tls: bool = False, ca_certificate: CACertificateWrite | None = None, auth_certificate: AuthCertificateWrite | None = None, ) -> None: @@ -363,7 +354,7 @@ def __init__( self.host = host self.port = port self.authentication = authentication - self.useTls = useTls + self.use_tls = use_tls self.ca_certificate = ca_certificate self.auth_certificate = auth_certificate @@ -376,7 +367,7 @@ def _load_source(cls, resource: dict[str, Any]) -> Self: authentication=MQTTAuthenticationWrite._load(resource["authentication"]) if "authentication" in resource else None, - useTls=resource.get("useTls", False), + use_tls=resource.get("useTls", False), ca_certificate=CACertificateWrite._load(resource["caCertificate"]) if "caCertificate" in resource else None, auth_certificate=AuthCertificateWrite._load(resource["authCertificate"]) if "authCertificate" in resource @@ -461,7 +452,7 @@ def __init__( last_updated_time: int, port: int | None = None, authentication: MQTTAuthentication | None = None, - useTls: bool = False, + use_tls: bool = False, ca_certificate: CACertificate | None = None, auth_certificate: AuthCertificate | None = None, ) -> None: @@ -469,7 +460,7 @@ def __init__( self.host = host self.port = port self.authentication = authentication - self.useTls = useTls + self.use_tls = use_tls self.ca_certificate = ca_certificate self.auth_certificate = auth_certificate self.created_time = created_time @@ -484,7 +475,7 @@ def _load_source(cls, resource: dict[str, Any]) -> Self: authentication=MQTTAuthentication._load(resource["authentication"]) if "authentication" in resource else None, - useTls=resource.get("useTls", False), + use_tls=resource.get("useTls", False), ca_certificate=CACertificate._load(resource["caCertificate"]) if "caCertificate" in resource else None, auth_certificate=AuthCertificate._load(resource["authCertificate"]) if "authCertificate" in resource @@ -493,7 +484,7 @@ def _load_source(cls, resource: dict[str, Any]) -> Self: last_updated_time=resource["lastUpdatedTime"], ) - def as_write(self) -> _MQTTSourceWrite: + def as_write(self) -> NoReturn: raise TypeError(f"{type(self).__name__} cannot be converted to write as id does not contain the secrets") def dump(self, camel_case: bool = True) -> dict[str, Any]: @@ -601,7 +592,7 @@ class SourceList(WriteableCogniteResourceList[SourceWrite, Source], ExternalIDTr def as_write( self, - ) -> SourceWriteList: + ) -> NoReturn: raise TypeError(f"{type(self).__name__} cannot be converted to write") From 958b41cb897ee1701ba41bc0438fb7ba430f9d8e Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 2 Sep 2024 14:48:58 +0200 Subject: [PATCH 25/32] build: docs --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39e26b59a..8a467407d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ Changes are grouped as follows ## [7.55.2] - 2024-08-29 ### Added -- [Feature Preview - beta] Support for `client.hosted_extractors.sources`. +- [Feature Preview - alpha] Support for `client.hosted_extractors.sources`. ## [7.55.1] - 2024-08-29 ### Fixed From b65d68f80c86ad25de234cff76e367cb06b010b5 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 2 Sep 2024 14:55:23 +0200 Subject: [PATCH 26/32] refactor: review feedback --- .../test_api/test_hosted_extractors/test_sources.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py index d7c742f74..d0e395889 100644 --- a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py +++ b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py @@ -25,8 +25,7 @@ def one_event_hub_source(cognite_client: CogniteClient) -> SourceList: retrieved = cognite_client.hosted_extractors.sources.retrieve(my_hub.external_id, ignore_unknown_ids=True) if retrieved: return retrieved - created = cognite_client.hosted_extractors.sources.create(my_hub) - return SourceList([created]) + return cognite_client.hosted_extractors.sources.create([my_hub]) class TestSources: @@ -69,7 +68,7 @@ def test_list(self, cognite_client: CogniteClient) -> None: def test_update_using_write_object(self, cognite_client: CogniteClient) -> None: my_hub = EventHubSourceWrite( - external_id=f"toupdatate-{random_string(10)}", + external_id=f"to-update-{random_string(10)}", host="myHost", key_name="myKeyName", key_value="myKey", From f11d1b7efc3cf90c90c52e9eed1d738edd6d9c63 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Tue, 3 Sep 2024 12:56:55 +0200 Subject: [PATCH 27/32] refactor: better solution to Update object --- cognite/client/_api/annotations.py | 1 - cognite/client/_api/hosted_extractors/sources.py | 15 ++++++++++++++- cognite/client/_api_client.py | 5 ----- cognite/client/data_classes/_base.py | 6 ------ 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/cognite/client/_api/annotations.py b/cognite/client/_api/annotations.py index ef24ca333..953f03ced 100644 --- a/cognite/client/_api/annotations.py +++ b/cognite/client/_api/annotations.py @@ -101,7 +101,6 @@ def _convert_resource_to_patch_object( resource: CogniteResource, update_attributes: list[PropertySpec], mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null", - identifying_properties: dict[str, Any] | None = None, ) -> dict[str, dict[str, dict]]: if not isinstance(resource, Annotation): return APIClient._convert_resource_to_patch_object(resource, update_attributes) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index 7d23f1fdb..afd6b629a 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -1,10 +1,11 @@ from __future__ import annotations from collections.abc import Iterator -from typing import TYPE_CHECKING, Any, Sequence, overload +from typing import TYPE_CHECKING, Any, Literal, Sequence, overload from cognite.client._api_client import APIClient from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client.data_classes._base import CogniteResource, PropertySpec from cognite.client.data_classes.hosted_extractors.sources import Source, SourceList, SourceUpdate, SourceWrite from cognite.client.utils._experimental import FeaturePreviewWarning from cognite.client.utils._identifier import IdentifierSequence @@ -260,3 +261,15 @@ def list( limit=limit, headers={"cdf-version": "beta"}, ) + + @classmethod + def _convert_resource_to_patch_object( + cls, + resource: CogniteResource, + update_attributes: list[PropertySpec], + mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null", + ) -> dict[str, dict[str, dict]]: + output = super()._convert_resource_to_patch_object(resource, update_attributes, mode) + if hasattr(resource, "type"): + output["type"] = resource.type + return output diff --git a/cognite/client/_api_client.py b/cognite/client/_api_client.py index 30c88c896..f58946a6c 100644 --- a/cognite/client/_api_client.py +++ b/cognite/client/_api_client.py @@ -1043,7 +1043,6 @@ def _update_multiple( item, update_cls._get_update_properties(item), mode, - update_cls._get_extra_identifying_properties(item), ) ) elif isinstance(item, CogniteUpdate): @@ -1224,7 +1223,6 @@ def _convert_resource_to_patch_object( resource: CogniteResource, update_attributes: list[PropertySpec], mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null", - identifying_properties: dict[str, Any] | None = None, ) -> dict[str, dict[str, dict]]: dumped_resource = resource.dump(camel_case=True) has_id = "id" in dumped_resource @@ -1240,9 +1238,6 @@ def _convert_resource_to_patch_object( elif has_external_id: patch_object["externalId"] = dumped_resource.pop("externalId") - if identifying_properties: - patch_object.update(identifying_properties) - update: dict[str, dict] = cls._clear_all_attributes(update_attributes) if mode == "replace" else {} update_attribute_by_name = {prop.name: prop for prop in update_attributes} diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index cc9cffbcf..f35617043 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -525,12 +525,6 @@ def dump(self, camel_case: Literal[True] = True) -> dict[str, Any]: def _get_update_properties(cls, item: CogniteResource | None = None) -> list[PropertySpec]: raise NotImplementedError - @classmethod - def _get_extra_identifying_properties(cls, item: CogniteResource | None = None) -> dict[str, Any]: - # This method is used to provide additional identifying properties for the update object. - # It is intended to be overridden by subclasses that need to provide additional identifying properties. - return {} - T_CogniteUpdate = TypeVar("T_CogniteUpdate", bound=CogniteUpdate) From 020313a816c6135104bba9888ba09973ad1dbbf3 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Tue, 3 Sep 2024 13:04:28 +0200 Subject: [PATCH 28/32] fix: moved to correct location --- .../client/_api/hosted_extractors/sources.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index afd6b629a..edae5e421 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -219,6 +219,18 @@ def update(self, items: SourceWrite | SourceUpdate | Sequence[SourceWrite | Sour headers={"cdf-version": "beta"}, ) + @classmethod + def _convert_resource_to_patch_object( + cls, + resource: CogniteResource, + update_attributes: list[PropertySpec], + mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null", + ) -> dict[str, dict[str, dict]]: + output = super()._convert_resource_to_patch_object(resource, update_attributes, mode) + if hasattr(resource, "type"): + output["type"] = resource.type + return output + def list( self, limit: int | None = DEFAULT_LIMIT_READ, @@ -261,15 +273,3 @@ def list( limit=limit, headers={"cdf-version": "beta"}, ) - - @classmethod - def _convert_resource_to_patch_object( - cls, - resource: CogniteResource, - update_attributes: list[PropertySpec], - mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null", - ) -> dict[str, dict[str, dict]]: - output = super()._convert_resource_to_patch_object(resource, update_attributes, mode) - if hasattr(resource, "type"): - output["type"] = resource.type - return output From 5652bfceadbd65cfef7b27a4180bcff1c256e10f Mon Sep 17 00:00:00 2001 From: anders-albert Date: Tue, 3 Sep 2024 14:27:56 +0200 Subject: [PATCH 29/32] fix: introducde bug --- cognite/client/_api/hosted_extractors/sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cognite/client/_api/hosted_extractors/sources.py b/cognite/client/_api/hosted_extractors/sources.py index edae5e421..066930127 100644 --- a/cognite/client/_api/hosted_extractors/sources.py +++ b/cognite/client/_api/hosted_extractors/sources.py @@ -227,8 +227,8 @@ def _convert_resource_to_patch_object( mode: Literal["replace_ignore_null", "patch", "replace"] = "replace_ignore_null", ) -> dict[str, dict[str, dict]]: output = super()._convert_resource_to_patch_object(resource, update_attributes, mode) - if hasattr(resource, "type"): - output["type"] = resource.type + if hasattr(resource, "_type"): + output["type"] = resource._type return output def list( From 448054e3195f18dfb4955bffefa94ff6be3aea08 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Tue, 3 Sep 2024 14:33:23 +0200 Subject: [PATCH 30/32] fix: stuff --- .../test_api/test_hosted_extractors/test_sources.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py index d0e395889..881b02062 100644 --- a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py +++ b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py @@ -1,5 +1,7 @@ from __future__ import annotations +import platform + import pytest from cognite.client import CogniteClient @@ -68,7 +70,7 @@ def test_list(self, cognite_client: CogniteClient) -> None: def test_update_using_write_object(self, cognite_client: CogniteClient) -> None: my_hub = EventHubSourceWrite( - external_id=f"to-update-{random_string(10)}", + external_id=f"to-update-{platform.system()}-{platform.python_version()}", host="myHost", key_name="myKeyName", key_value="myKey", From 9ce95402ba2dbf45ac7cb4c38483282b138fec2f Mon Sep 17 00:00:00 2001 From: anders-albert Date: Tue, 3 Sep 2024 14:41:58 +0200 Subject: [PATCH 31/32] tests: fix --- .../test_api/test_hosted_extractors/test_sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py index 881b02062..db08162fd 100644 --- a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py +++ b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py @@ -18,7 +18,7 @@ @pytest.fixture(scope="session") def one_event_hub_source(cognite_client: CogniteClient) -> SourceList: my_hub = EventHubSourceWrite( - external_id=f"myNewHub-{random_string(10)}", + external_id=f"myNewHub-{platform.system()}-{platform.python_version()}", host="myHost", key_name="myKeyName", key_value="myKey", @@ -70,7 +70,7 @@ def test_list(self, cognite_client: CogniteClient) -> None: def test_update_using_write_object(self, cognite_client: CogniteClient) -> None: my_hub = EventHubSourceWrite( - external_id=f"to-update-{platform.system()}-{platform.python_version()}", + external_id=f"to-update-{random_string(10)}", host="myHost", key_name="myKeyName", key_value="myKey", From 2d2c72cab2de536e44307cf996b2fdea2c27ac73 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Tue, 3 Sep 2024 14:54:56 +0200 Subject: [PATCH 32/32] build: bump --- cognite/client/_version.py | 2 +- pyproject.toml | 2 +- .../test_api/test_hosted_extractors/test_sources.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cognite/client/_version.py b/cognite/client/_version.py index b4bf6d573..b1cad4426 100644 --- a/cognite/client/_version.py +++ b/cognite/client/_version.py @@ -1,4 +1,4 @@ from __future__ import annotations -__version__ = "7.58.1" +__version__ = "7.58.2" __api_subversion__ = "20230101" diff --git a/pyproject.toml b/pyproject.toml index 70327d822..777549750 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "cognite-sdk" -version = "7.58.1" +version = "7.58.2" description = "Cognite Python SDK" readme = "README.md" documentation = "https://cognite-sdk-python.readthedocs-hosted.com" diff --git a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py index db08162fd..08cba2afb 100644 --- a/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py +++ b/tests/tests_integration/test_api/test_hosted_extractors/test_sources.py @@ -24,7 +24,7 @@ def one_event_hub_source(cognite_client: CogniteClient) -> SourceList: key_value="myKey", event_hub_name="myEventHub", ) - retrieved = cognite_client.hosted_extractors.sources.retrieve(my_hub.external_id, ignore_unknown_ids=True) + retrieved = cognite_client.hosted_extractors.sources.retrieve([my_hub.external_id], ignore_unknown_ids=True) if retrieved: return retrieved return cognite_client.hosted_extractors.sources.create([my_hub])