From 3048fe032a5245290a34dcf7f296561663ec924b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Sandst=C3=B8?= Date: Tue, 24 Aug 2021 14:31:57 +0200 Subject: [PATCH] Add template views (#850) --- CHANGELOG.md | 4 + cognite/client/_api/templates.py | 184 +++++++++++++++++- cognite/client/_api_client.py | 7 +- cognite/client/_version.py | 2 +- cognite/client/data_classes/templates.py | 116 ++++++++++- docs/source/cognite.rst | 20 ++ .../test_api/test_templates.py | 65 ++++++- 7 files changed, 385 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2599cfc3..3693ec65b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ Changes are grouped as follows - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [2.30.0] - 2021-08-18 +### Added +- Support for Template Views + ## [2.29.0] - 2021-08-16 ### Added - Raw rows are retrieved using parallel cursors when no limit is set. diff --git a/cognite/client/_api/templates.py b/cognite/client/_api/templates.py index cfedaad40..bd7419321 100644 --- a/cognite/client/_api/templates.py +++ b/cognite/client/_api/templates.py @@ -10,6 +10,7 @@ def __init__(self, *args, **kwargs): self.groups = TemplateGroupsAPI(*args, **kwargs) self.versions = TemplateGroupVersionsAPI(*args, **kwargs) self.instances = TemplateInstancesAPI(*args, **kwargs) + self.views = TemplateViewsAPI(*args, **kwargs) def graphql_query(self, external_id: str, version: int, query: str) -> GraphQlResponse: """ @@ -73,7 +74,8 @@ def create( Union[TemplateGroup, TemplateGroupList]: Created template group(s) Examples: - create a new template group: + Create a new template group: + >>> from cognite.client import CogniteClient >>> from cognite.client.data_classes import TemplateGroup >>> c = CogniteClient() @@ -96,7 +98,7 @@ def upsert( Union[TemplateGroup, TemplateGroupList]: Created template group(s) Examples: - create a new template group: + Upsert a template group: >>> from cognite.client import CogniteClient >>> from cognite.client.data_classes import TemplateGroup @@ -308,8 +310,7 @@ def create( >>> from cognite.client import CogniteClient >>> from cognite.client.data_classes import TemplateInstance >>> c = CogniteClient() - >>> template_instance_1 = TemplateInstance( - >>> external_id="norway", + >>> template_instance_1 = TemplateInstance(external_id="norway", >>> template_name="Country", >>> field_resolvers={ >>> "name": ConstantResolver("Norway"), @@ -318,8 +319,7 @@ def create( >>> "confirmed": ConstantResolver("Norway_confirmed"), >>> } >>> ) - >>> template_instance_2 = TemplateInstance( - >>> external_id="norway_demographics", + >>> template_instance_2 = TemplateInstance(external_id="norway_demographics", >>> template_name="Demographics", >>> field_resolvers={ >>> "populationSize": ConstantResolver(5328000), @@ -351,8 +351,7 @@ def upsert( >>> from cognite.client import CogniteClient >>> from cognite.client.data_classes import TemplateInstance >>> c = CogniteClient() - >>> template_instance_1 = TemplateInstance( - >>> external_id="norway", + >>> template_instance_1 = TemplateInstance(external_id="norway", >>> template_name="Country", >>> field_resolvers={ >>> "name": ConstantResolver("Norway"), @@ -494,3 +493,172 @@ def delete(self, external_id: str, version: int, external_ids: List[str], ignore wrap_ids=True, extra_body_fields={"ignoreUnknownIds": ignore_unknown_ids}, ) + + +class TemplateViewsAPI(APIClient): + _RESOURCE_PATH = "/templategroups/{}/versions/{}/views" + _LIST_CLASS = ViewList + + def create(self, external_id: str, version: int, views: Union[View, List[View]]) -> Union[View, ViewList]: + """`Create one or more template views.` + + Args: + external_id (str): The external id of the template group. + version (int): The version of the template group to create views for. + views (Union[View, List[View]]): The views to create. + + Returns: + Union[View, ViewList]: Created view(s). + + Examples: + Create new views: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import View + >>> c = CogniteClient() + >>> view = View(external_id="view", + >>> source=Source( + >>> type: 'events', + >>> filter: { + >>> startTime: { + >>> min: "$startTime" + >>> }, + >>> type: "Test", + >>> } + >>> mappings: { + >>> author: "metadata/author" + >>> } + >>> ) + >>> ) + >>> c.templates.views.create("sdk-test-group", 1, [view]) + """ + resource_path = utils._auxiliary.interpolate_and_url_encode(self._RESOURCE_PATH, external_id, version) + return self._create_multiple(resource_path=resource_path, items=views) + + def upsert(self, external_id: str, version: int, views: Union[View, List[View]]) -> Union[View, ViewList]: + """`Upsert one or more template views.` + + Args: + external_id (str): The external id of the template group. + version (int): The version of the template group to create views for. + views (Union[View, List[View]]): The views to create. + + Returns: + Union[View, ViewList]: Created view(s). + + Examples: + Upsert new views: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import View + >>> c = CogniteClient() + >>> view = View(external_id="view", + >>> source=Source( + >>> type: 'events', + >>> filter: { + >>> startTime: { + >>> min: "$startTime" + >>> }, + >>> type: "Test", + >>> } + >>> mappings: { + >>> author: "metadata/author" + >>> } + >>> ) + >>> ) + >>> c.templates.views.upsert("sdk-test-group", 1, [view]) + """ + if isinstance(views, View): + views = [views] + resource_path = ( + utils._auxiliary.interpolate_and_url_encode(self._RESOURCE_PATH, external_id, version) + "/upsert" + ) + updated = self._post(resource_path, {"items": [view.dump(camel_case=True) for view in views]}).json()["items"] + res = ViewList._load(updated, cognite_client=self._cognite_client) + if len(res) == 1: + return res[0] + return res + + def resolve( + self, external_id: str, version: int, view_external_id: str, input: Optional[Dict[str, any]], limit: int = 25 + ) -> ViewResolveList: + """`Resolves a View.` + It resolves the source specified in a View with the provided input and applies the mapping rules to the response. + + Args: + external_id (str): The external id of the template group. + version (int): The version of the template group. + input (Optional[Dict[str, any]]): The input for the View. + limit (int): Maximum number of views to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + ViewResolveList: The resolved items. + + Examples: + Resolve view: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> c.templates.views.resolve("template-group-ext-id", 1, "view", { "startTime": 10 }, limit=5) + """ + url_path = utils._auxiliary.interpolate_and_url_encode(self._RESOURCE_PATH, external_id, version) + "/resolve" + return self._list( + url_path=url_path, + method="POST", + cls=ViewResolveList, + limit=limit, + other_params={"externalId": view_external_id, "input": input}, + ) + + def list(self, external_id: str, version: int, limit: int = 25) -> ViewList: + """`Lists view in a template group.` + Up to 1000 views can be retrieved in one operation. + + Args: + external_id (str): The external id of the template group. + version (int): The version of the template group. + limit (int): Maximum number of views to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + ViewList: List of requested views + + Examples: + List views: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> c.templates.views.list("template-group-ext-id", 1, limit=5) + """ + resource_path = utils._auxiliary.interpolate_and_url_encode(self._RESOURCE_PATH, external_id, version) + return self._list(resource_path=resource_path, method="POST", limit=limit) + + def delete( + self, external_id: str, version: int, view_external_id: Union[List[str], str], ignore_unknown_ids: bool = False + ) -> None: + """`Delete one or more views.` + + Args: + external_id (Union[str, List[str]]): External ID of the template group. + version (int): The version of the template group. + view_external_id (Union[List[str], str]): The external ids of the views to delete + ignore_unknown_ids (bool): Ignore external IDs that are not found rather than throw an exception. + + Returns: + None + + Examples: + Delete views by external id: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> c.templates.views.delete("sdk-test-group", 1, external_id=["a", "b"]) + """ + resource_path = utils._auxiliary.interpolate_and_url_encode(self._RESOURCE_PATH, external_id, version) + return self._delete_multiple( + resource_path=resource_path, + external_ids=view_external_id, + wrap_ids=True, + extra_body_fields={"ignoreUnknownIds": ignore_unknown_ids}, + ) diff --git a/cognite/client/_api_client.py b/cognite/client/_api_client.py index 2196d0cb2..05c213b5a 100644 --- a/cognite/client/_api_client.py +++ b/cognite/client/_api_client.py @@ -277,6 +277,7 @@ def _list_generator( method: str, cls=None, resource_path: str = None, + url_path: str = None, limit: int = None, chunk_size: int = None, filter: Dict = None, @@ -326,12 +327,12 @@ def _list_generator( params["cursor"] = next_cursor if sort is not None: params["sort"] = sort - res = self._get(url_path=resource_path, params=params, headers=headers) + res = self._get(url_path=url_path or resource_path, params=params, headers=headers) elif method == "POST": body = {"filter": filter, "limit": current_limit, "cursor": next_cursor, **(other_params or {})} if sort is not None: body["sort"] = sort - res = self._post(url_path=resource_path + "/list", json=body, headers=headers) + res = self._post(url_path=url_path or resource_path + "/list", json=body, headers=headers) else: raise ValueError("_list_generator parameter `method` must be GET or POST, not {}".format(method)) last_received_items = res.json()["items"] @@ -398,6 +399,7 @@ def _list( method: str, cls=None, resource_path: str = None, + url_path: str = None, limit: int = None, filter: Dict = None, other_params=None, @@ -427,6 +429,7 @@ def _list( for resource_list in self._list_generator( cls=cls, resource_path=resource_path, + url_path=url_path, method=method, limit=limit, chunk_size=self._LIST_LIMIT, diff --git a/cognite/client/_version.py b/cognite/client/_version.py index 143752543..4fed2c9aa 100644 --- a/cognite/client/_version.py +++ b/cognite/client/_version.py @@ -1,2 +1,2 @@ -__version__ = "2.29.0" +__version__ = "2.30.0" __api_subversion__ = "V20210423" diff --git a/cognite/client/data_classes/templates.py b/cognite/client/data_classes/templates.py index fb2913663..6fe73e06b 100644 --- a/cognite/client/data_classes/templates.py +++ b/cognite/client/data_classes/templates.py @@ -1,4 +1,8 @@ -from typing import List, Optional +from collections import UserDict +from cognite.client.data_classes.assets import AssetFilter +from enum import Enum, auto +from cognite.client.data_classes.events import EventFilter +from typing import List, Optional, Generic from cognite.client.data_classes._base import * @@ -266,6 +270,106 @@ def field_resolvers(self): return TemplateInstanceUpdate._ObjectAssetUpdate(self, "fieldResolvers") +class Source(CogniteResource): + """ + A source defines the data source with filters and a mapping table. + + Args: + type (str): The type of source. Possible values are: "events", "assets", "sequences", "timeSeries", "files". + filter (Dict[str, any]): The filter to apply to the source when resolving the source. A filter also supports binding view input to the filter, by prefixing the input name with '$'. + mappings (Dict[str, str]): The mapping between source result and expected schema. + """ + + def __init__( + self, type: str = None, filter: Dict[str, any] = None, mappings: Dict[str, str] = None, cognite_client=None + ) -> None: + self.type = type + self.filter = filter + self.mappings = mappings + self._cognite_client = cognite_client + + +class View(CogniteResource): + """ + A view is used to map existing data to a type in the template group. A view supports input, that can be bound to the underlying filter. + + Args: + external_id (str): The external ID provided by the client. Must be unique for the resource type. + source (Source): Defines the data source for the view. + """ + + def __init__( + self, + external_id: str = None, + source: Source = None, + created_time: int = None, + last_updated_time: int = None, + cognite_client=None, + ): + self.external_id = external_id + self.source = source + self.created_time = created_time + self.last_updated_time = last_updated_time + self._cognite_client = cognite_client + + def dump(self, camel_case: bool = False) -> Dict[str, Any]: + """Dump the instance into a json serializable Python data type. + + Args: + camel_case (bool): Use camelCase for attribute names. Defaults to False. + + Returns: + Dict[str, Any]: A dictionary representation of the instance. + """ + if camel_case: + return { + utils._auxiliary.to_camel_case(key): View.resolve_nested_classes(value, camel_case) + for key, value in self.__dict__.items() + if value not in EXCLUDE_VALUE and not key.startswith("_") + } + return { + key: View.resolve_nested_classes(value, camel_case) + for key, value in self.__dict__.items() + if value not in EXCLUDE_VALUE and not key.startswith("_") + } + + def resolve_nested_classes(value, camel_case): + if isinstance(value, CogniteResource): + return value.dump(camel_case) + else: + return value + + @classmethod + def _load(cls, resource: Union[Dict, str], cognite_client=None): + if isinstance(resource, str): + return cls._load(json.loads(resource), cognite_client=cognite_client) + elif isinstance(resource, Dict): + instance = cls(cognite_client=cognite_client) + for key, value in resource.items(): + snake_case_key = utils._auxiliary.to_snake_case(key) + if hasattr(instance, snake_case_key): + value = value if key != "source" else Source._load(value, cognite_client) + setattr(instance, snake_case_key, value) + return instance + raise TypeError("Resource must be json str or Dict, not {}".format(type(resource))) + + +class ViewResolveItem(UserDict): + def __init__(self, data: Dict[str, any], cognite_client=None) -> None: + self._cognite_client = cognite_client + super().__init__(data) + + def dump(self, camel_case: bool = False) -> Dict[str, Any]: + return self.data + + @classmethod + def _load(cls, data: Union[Dict, str], cognite_client=None): + if isinstance(data, str): + return cls._load(json.loads(data), cognite_client=cognite_client) + elif isinstance(data, Dict): + return cls(data, cognite_client=cognite_client) + + class GraphQlError(CogniteResource): def __init__( self, message: str = None, path: List[str] = None, locations: List[Dict[str, Any]] = None, cognite_client=None @@ -286,3 +390,13 @@ def __init__(self, data: any = None, errors: List[GraphQlError] = None, cognite_ class TemplateInstanceList(CogniteResourceList): _RESOURCE = TemplateInstance _UPDATE = CogniteUpdate + + +class ViewList(CogniteResourceList): + _RESOURCE = View + _UPDATE = CogniteUpdate + + +class ViewResolveList(CogniteResourceList): + _RESOURCE = ViewResolveItem + _UPDATE = CogniteUpdate diff --git a/docs/source/cognite.rst b/docs/source/cognite.rst index 82cdb21c4..a327f92e3 100644 --- a/docs/source/cognite.rst +++ b/docs/source/cognite.rst @@ -938,6 +938,26 @@ Delete Template instances ^^^^^^^^^^^^^^^^^^^^^^^^^ .. automethod:: cognite.client._api.templates.TemplateInstancesAPI.delete +Create Views +^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.templates.TemplateViewsAPI.create + +Upsert Views +^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.templates.TemplateViewsAPI.upsert + +List Views +^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.templates.TemplateViewsAPI.list + +Resolve View +^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.templates.TemplateViewsAPI.resolve + +Delete Views +^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.templates.TemplateViewsAPI.delete + Data classes ^^^^^^^^^^^^ .. automodule:: cognite.client.data_classes.templates diff --git a/tests/tests_integration/test_api/test_templates.py b/tests/tests_integration/test_api/test_templates.py index 0c2aedf9f..c5b882cd9 100644 --- a/tests/tests_integration/test_api/test_templates.py +++ b/tests/tests_integration/test_api/test_templates.py @@ -1,4 +1,5 @@ -from cognite.client.data_classes.templates import TemplateInstanceUpdate +from cognite.client.data_classes.events import Event +from cognite.client.data_classes.templates import Source, TemplateInstanceUpdate, View import uuid import pytest @@ -19,6 +20,7 @@ API_GROUPS = API.templates.groups API_VERSION = API.templates.versions API_INSTANCES = API.templates.instances +API_VIEWS = API.templates.views @pytest.fixture @@ -77,6 +79,34 @@ def new_template_instance(new_template_group_version): API_INSTANCES.delete(ext_id, new_version.version, instance.external_id) +@pytest.fixture +def new_view(new_template_group_version): + events = [] + for i in range(0, 1001): + events.append(Event(external_id="test_evt_templates_1_" + str(i), type="test_templates_1", start_time=i * 1000)) + try: + API.events.create(events) + except: + # We only generate this data once for a given project, to prevent issues with eventual consistency etc. + None + + new_group, ext_id, new_version = new_template_group_version + view = View( + external_id="test", + source=Source( + type="events", + filter={"startTime": {"min": "$minStartTime"}, "type": "test_templates_1"}, + mappings={"test_type": "type", "startTime": "startTime"}, + ), + ) + view = API_VIEWS.create(ext_id, new_version.version, view) + yield new_group, ext_id, new_version, view + try: + API_VIEWS.delete(ext_id, new_version.version, view.external_id) + except: + None + + class TestTemplatesAPI: def test_groups_get_single(self, new_template_group): new_group, ext_id = new_template_group @@ -178,3 +208,36 @@ def test_query(self, new_template_instance): """ res = API.templates.graphql_query(ext_id, 1, query) assert res.data is not None + + def test_view_list(self, new_view): + new_group, ext_id, new_version, view = new_view + first_element = [ + res for res in API_VIEWS.list(ext_id, new_version.version) if res.external_id == view.external_id + ][0] + assert first_element == view + + def test_view_delete(self, new_view): + new_group, ext_id, new_version, view = new_view + API_VIEWS.delete(ext_id, new_version.version, [view.external_id]) + assert ( + len([res for res in API_VIEWS.list(ext_id, new_version.version) if res.external_id == view.external_id]) + == 0 + ) + + def test_view_resolve(self, new_view): + new_group, ext_id, new_version, view = new_view + res = API_VIEWS.resolve( + ext_id, new_version.version, view.external_id, input={"minStartTime": 10 * 1000}, limit=10 + ) + assert res == [{"startTime": (i + 10) * 1000, "test_type": "test_templates_1"} for i in range(0, 10)] + + def test_view_resolve_pagination(self, new_view): + new_group, ext_id, new_version, view = new_view + res = API_VIEWS.resolve(ext_id, new_version.version, view.external_id, input={"minStartTime": 0}, limit=-1) + assert res == [{"startTime": i * 1000, "test_type": "test_templates_1"} for i in range(0, 1001)] + + def test_view_upsert(self, new_view): + new_group, ext_id, new_version, view = new_view + view.source.mappings["another_type"] = "type" + res = API_VIEWS.upsert(ext_id, new_version.version, [view]) + assert res == view