From 53f9e1f931fd05e338655193ceebd98f06fcbf80 Mon Sep 17 00:00:00 2001 From: Ahmed AlSahaf Date: Mon, 11 Dec 2023 14:06:21 +0300 Subject: [PATCH 1/8] feat(metrics): add option to specify dataset. Add option to specify data set under which metrics timeseries will be created under when pushing metrics to cognite. --- cognite/extractorutils/configtools/elements.py | 6 ++++-- cognite/extractorutils/metrics.py | 17 ++++++++++++++--- schema/metrics_config.schema.json | 13 +++++++++++-- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/cognite/extractorutils/configtools/elements.py b/cognite/extractorutils/configtools/elements.py index 0f24285d..594225c5 100644 --- a/cognite/extractorutils/configtools/elements.py +++ b/cognite/extractorutils/configtools/elements.py @@ -24,11 +24,11 @@ from urllib.parse import urljoin import yaml -from prometheus_client import REGISTRY, start_http_server - from cognite.client import ClientConfig, CogniteClient from cognite.client.credentials import CredentialProvider, OAuthClientCertificate, OAuthClientCredentials from cognite.client.data_classes import Asset, DataSet, ExtractionPipeline +from prometheus_client import REGISTRY, start_http_server + from cognite.extractorutils.configtools._util import _load_certificate_data from cognite.extractorutils.exceptions import InvalidConfigError from cognite.extractorutils.metrics import AbstractMetricsPusher, CognitePusher, PrometheusPusher @@ -436,6 +436,7 @@ class _CogniteMetricsConfig: external_id_prefix: str asset_name: Optional[str] asset_external_id: Optional[str] + data_set: Optional[EitherIdConfig] push_interval: TimeIntervalConfig = TimeIntervalConfig("30s") @@ -485,6 +486,7 @@ def start_pushers(self, cdf_client: CogniteClient, cancellation_token: Event = E external_id_prefix=self.cognite.external_id_prefix, push_interval=self.cognite.push_interval.seconds, asset=asset, + data_set=self.cognite.data_set, thread_name="CogniteMetricsPusher", # There is only one Cognite project as a target cancellation_token=cancellation_token, ) diff --git a/cognite/extractorutils/metrics.py b/cognite/extractorutils/metrics.py index a2c4b10d..d8ace1d2 100644 --- a/cognite/extractorutils/metrics.py +++ b/cognite/extractorutils/metrics.py @@ -48,13 +48,14 @@ def __init__(self): import arrow import psutil +from cognite.client import CogniteClient +from cognite.client.data_classes import Asset, Datapoints, DatapointsArray, TimeSeries +from cognite.client.exceptions import CogniteDuplicatedError from prometheus_client import Gauge, Info, Metric from prometheus_client.core import REGISTRY from prometheus_client.exposition import basic_auth_handler, delete_from_gateway, pushadd_to_gateway -from cognite.client import CogniteClient -from cognite.client.data_classes import Asset, Datapoints, DatapointsArray, TimeSeries -from cognite.client.exceptions import CogniteDuplicatedError +from cognite.extractorutils.configtools.elements import EitherIdConfig from .util import ensure_time_series @@ -330,6 +331,7 @@ class CognitePusher(AbstractMetricsPusher): external_id_prefix: Unique external ID prefix for this pusher. push_interval: Seconds between each upload call asset: Optional contextualization. + data_set: Data set the metrics timeseries created under. thread_name: Name of thread to start. If omitted, a standard name such as Thread-4 will be generated. cancellation_token: Event object to be used as a thread cancelation event """ @@ -340,6 +342,7 @@ def __init__( external_id_prefix: str, push_interval: int, asset: Optional[Asset] = None, + data_set: Optional[EitherIdConfig] = None, thread_name: Optional[str] = None, cancellation_token: Event = Event(), ): @@ -348,6 +351,7 @@ def __init__( self.cdf_client = cdf_client self.asset = asset self.external_id_prefix = external_id_prefix + self.data_set = data_set self._init_cdf() @@ -372,6 +376,12 @@ def _init_cdf(self) -> None: else: asset_id = None + data_set_id = None + if self.data_set: + dataset = self.cdf_client.data_sets.retrieve(id=self.data_set.id, external_id=self.data_set.external_id) + if dataset: + data_set_id = dataset.id + for metric in REGISTRY.collect(): if type(metric) == Metric and metric.type in ["gauge", "counter"]: external_id = self.external_id_prefix + metric.name @@ -383,6 +393,7 @@ def _init_cdf(self) -> None: legacy_name=external_id, description=metric.documentation, asset_id=asset_id, # type: ignore # this is optional. Type hint in SDK is wrong + data_set_id=data_set_id, # type: ignore # this is optional. Type hint in SDK is wrong ) ) diff --git a/schema/metrics_config.schema.json b/schema/metrics_config.schema.json index 654f8f1a..5eb4892d 100644 --- a/schema/metrics_config.schema.json +++ b/schema/metrics_config.schema.json @@ -29,7 +29,10 @@ "description": "PushGateway password" }, "clear-after": { - "type": ["null", "integer"], + "type": [ + "null", + "integer" + ], "description": "Clear metrics after this many seconds when the extractor stops. Default is disabled" }, "push-interval": { @@ -44,7 +47,9 @@ "type": "object", "description": "Push metrics to CDF timeseries. Requires CDF credentials to be configured", "unevaluatedProperties": false, - "required": ["external-id-prefix"], + "required": [ + "external-id-prefix" + ], "properties": { "external-id-prefix": { "type": "string", @@ -62,6 +67,10 @@ "type": "integer", "description": "Interval in seconds between each push to CDF", "default": 30 + }, + "data-set": { + "description": "Data set the metrics timeseries created under", + "$ref": "either_id.schema.json" } } }, From 83d7f0354610f07fffd5d86063a87b052298e958 Mon Sep 17 00:00:00 2001 From: Ahmed AlSahaf Date: Mon, 11 Dec 2023 14:38:27 +0300 Subject: [PATCH 2/8] Bump version to 6.1.0 --- cognite/extractorutils/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cognite/extractorutils/__init__.py b/cognite/extractorutils/__init__.py index f4092f1c..7c10f06f 100644 --- a/cognite/extractorutils/__init__.py +++ b/cognite/extractorutils/__init__.py @@ -16,5 +16,5 @@ Cognite extractor utils is a Python package that simplifies the development of new extractors. """ -__version__ = "6.0.1" +__version__ = "6.1.0" from .base import Extractor diff --git a/pyproject.toml b/pyproject.toml index f63cf01c..70a759bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cognite-extractor-utils" -version = "6.0.1" +version = "6.1.0" description = "Utilities for easier development of extractors for CDF" authors = ["Mathias Lohne "] license = "Apache-2.0" From 4e41ddec0f577291435261eb63c5a3e4381d7ab0 Mon Sep 17 00:00:00 2001 From: Ahmed AlSahaf Date: Tue, 12 Dec 2023 11:38:19 +0300 Subject: [PATCH 3/8] Fix style --- cognite/extractorutils/configtools/elements.py | 4 ++-- cognite/extractorutils/metrics.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cognite/extractorutils/configtools/elements.py b/cognite/extractorutils/configtools/elements.py index 594225c5..945290d0 100644 --- a/cognite/extractorutils/configtools/elements.py +++ b/cognite/extractorutils/configtools/elements.py @@ -24,11 +24,11 @@ from urllib.parse import urljoin import yaml +from prometheus_client import REGISTRY, start_http_server + from cognite.client import ClientConfig, CogniteClient from cognite.client.credentials import CredentialProvider, OAuthClientCertificate, OAuthClientCredentials from cognite.client.data_classes import Asset, DataSet, ExtractionPipeline -from prometheus_client import REGISTRY, start_http_server - from cognite.extractorutils.configtools._util import _load_certificate_data from cognite.extractorutils.exceptions import InvalidConfigError from cognite.extractorutils.metrics import AbstractMetricsPusher, CognitePusher, PrometheusPusher diff --git a/cognite/extractorutils/metrics.py b/cognite/extractorutils/metrics.py index d8ace1d2..f99319b5 100644 --- a/cognite/extractorutils/metrics.py +++ b/cognite/extractorutils/metrics.py @@ -48,13 +48,13 @@ def __init__(self): import arrow import psutil -from cognite.client import CogniteClient -from cognite.client.data_classes import Asset, Datapoints, DatapointsArray, TimeSeries -from cognite.client.exceptions import CogniteDuplicatedError from prometheus_client import Gauge, Info, Metric from prometheus_client.core import REGISTRY from prometheus_client.exposition import basic_auth_handler, delete_from_gateway, pushadd_to_gateway +from cognite.client import CogniteClient +from cognite.client.data_classes import Asset, Datapoints, DatapointsArray, TimeSeries +from cognite.client.exceptions import CogniteDuplicatedError from cognite.extractorutils.configtools.elements import EitherIdConfig from .util import ensure_time_series From e1987426c2f231eff05d8468d249b387e7d6f1d7 Mon Sep 17 00:00:00 2001 From: Ahmed AlSahaf Date: Tue, 12 Dec 2023 11:43:19 +0300 Subject: [PATCH 4/8] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1619be46..8675a7bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,12 @@ Changes are grouped as follows - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [6.1.0] + +### Added + + * Added ability to specify dataset under which metrics timeseries are created + ## [6.0.1] ### Fixed From dfe403f31324ddcfe322324b32fb4b110bfbb342 Mon Sep 17 00:00:00 2001 From: Ahmed AlSahaf Date: Tue, 12 Dec 2023 15:12:30 +0300 Subject: [PATCH 5/8] Replace EitherIdConfig with EitherId --- cognite/extractorutils/configtools/elements.py | 2 +- cognite/extractorutils/metrics.py | 8 +++++--- schema/metrics_config.schema.json | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cognite/extractorutils/configtools/elements.py b/cognite/extractorutils/configtools/elements.py index 945290d0..488534e0 100644 --- a/cognite/extractorutils/configtools/elements.py +++ b/cognite/extractorutils/configtools/elements.py @@ -436,7 +436,7 @@ class _CogniteMetricsConfig: external_id_prefix: str asset_name: Optional[str] asset_external_id: Optional[str] - data_set: Optional[EitherIdConfig] + data_set: Optional[EitherId] push_interval: TimeIntervalConfig = TimeIntervalConfig("30s") diff --git a/cognite/extractorutils/metrics.py b/cognite/extractorutils/metrics.py index f99319b5..cd2955f0 100644 --- a/cognite/extractorutils/metrics.py +++ b/cognite/extractorutils/metrics.py @@ -55,7 +55,7 @@ def __init__(self): from cognite.client import CogniteClient from cognite.client.data_classes import Asset, Datapoints, DatapointsArray, TimeSeries from cognite.client.exceptions import CogniteDuplicatedError -from cognite.extractorutils.configtools.elements import EitherIdConfig +from cognite.extractorutils.util import EitherId from .util import ensure_time_series @@ -342,7 +342,7 @@ def __init__( external_id_prefix: str, push_interval: int, asset: Optional[Asset] = None, - data_set: Optional[EitherIdConfig] = None, + data_set: Optional[EitherId] = None, thread_name: Optional[str] = None, cancellation_token: Event = Event(), ): @@ -378,7 +378,9 @@ def _init_cdf(self) -> None: data_set_id = None if self.data_set: - dataset = self.cdf_client.data_sets.retrieve(id=self.data_set.id, external_id=self.data_set.external_id) + dataset = self.cdf_client.data_sets.retrieve( + id=self.data_set.internal_id, external_id=self.data_set.external_id + ) if dataset: data_set_id = dataset.id diff --git a/schema/metrics_config.schema.json b/schema/metrics_config.schema.json index 5eb4892d..e9c1a65a 100644 --- a/schema/metrics_config.schema.json +++ b/schema/metrics_config.schema.json @@ -69,8 +69,8 @@ "default": 30 }, "data-set": { - "description": "Data set the metrics timeseries created under", - "$ref": "either_id.schema.json" + "type": "string", + "description": "Data set the metrics timeseries created under" } } }, From f07ec95a0c5e7bc101e69835c81ad5e8524fa482 Mon Sep 17 00:00:00 2001 From: Mathias Lohne Date: Tue, 12 Dec 2023 13:41:42 +0100 Subject: [PATCH 6/8] Fix config --- cognite/extractorutils/configtools/elements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cognite/extractorutils/configtools/elements.py b/cognite/extractorutils/configtools/elements.py index 488534e0..ec25be06 100644 --- a/cognite/extractorutils/configtools/elements.py +++ b/cognite/extractorutils/configtools/elements.py @@ -436,7 +436,7 @@ class _CogniteMetricsConfig: external_id_prefix: str asset_name: Optional[str] asset_external_id: Optional[str] - data_set: Optional[EitherId] + data_set: Optional[EitherIdConfig] = None push_interval: TimeIntervalConfig = TimeIntervalConfig("30s") @@ -486,7 +486,7 @@ def start_pushers(self, cdf_client: CogniteClient, cancellation_token: Event = E external_id_prefix=self.cognite.external_id_prefix, push_interval=self.cognite.push_interval.seconds, asset=asset, - data_set=self.cognite.data_set, + data_set=self.cognite.data_set.either_id if self.cognite.data_set else None, thread_name="CogniteMetricsPusher", # There is only one Cognite project as a target cancellation_token=cancellation_token, ) From 00a8c8df20703234014efdf0f4f231515724abe6 Mon Sep 17 00:00:00 2001 From: Babatunde Aromire Date: Wed, 13 Dec 2023 11:01:55 +0100 Subject: [PATCH 7/8] refactor: quick catch --- cognite/extractorutils/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cognite/extractorutils/util.py b/cognite/extractorutils/util.py index c7873fc4..5a21341f 100644 --- a/cognite/extractorutils/util.py +++ b/cognite/extractorutils/util.py @@ -121,7 +121,7 @@ def __init__(self, **kwargs: Union[int, str, None]): raise TypeError("Internal IDs must be integers") if external_id is not None and not isinstance(external_id, str): - raise TypeError("Internal IDs must be integers") + raise TypeError("External IDs must be strings") self.internal_id: Optional[int] = internal_id self.external_id: Optional[str] = external_id From 4a9c29a932cc6889bb7162cb57b92db31080f781 Mon Sep 17 00:00:00 2001 From: Mathias Lohne Date: Wed, 13 Dec 2023 13:48:14 +0100 Subject: [PATCH 8/8] Fix merge mistake --- cognite/extractorutils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cognite/extractorutils/__init__.py b/cognite/extractorutils/__init__.py index 663778d9..7c10f06f 100644 --- a/cognite/extractorutils/__init__.py +++ b/cognite/extractorutils/__init__.py @@ -16,5 +16,5 @@ Cognite extractor utils is a Python package that simplifies the development of new extractors. """ -__version__ = "6.1.0 +__version__ = "6.1.0" from .base import Extractor