Merge remote-tracking branch 'origin/master' into hosted-extractors

# Conflicts:
#	CHANGELOG.md
#	cognite/client/_version.py
#	pyproject.toml

.github/workflows/build.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
         with:
-          extras: '-E pandas'
+          extras: "-E pandas"
       - name: Linting and static code checks
         run: pre-commit run --all-files
 
@@ -46,13 +46,13 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: ['3.8', '3.11', '3.12']  # TODO: 3.9, 3.10 (requires a lot of work for FakeCogResGen for tests)
+        python-version: ["3.8", "3.11", "3.12"] # TODO: 3.9, 3.10 (requires a lot of work for FakeCogResGen for tests)
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
         with:
           python_version: ${{ matrix.python-version }}
-          extras: '-E all'
+          extras: "-E all"
 
       - name: Test full
         env:
@@ -64,7 +64,8 @@ jobs:
           COGNITE_PROJECT: python-sdk-test
           COGNITE_BASE_URL: https://greenfield.cognitedata.com
           COGNITE_CLIENT_NAME: python-sdk-integration-tests
-        run: pytest tests --durations=10 --cov --cov-report term --cov-report xml:coverage.xml -n8 --dist loadscope --reruns 2 --maxfail 20
+        # Testpaths are defined in the pytest.ini file:
+        run: pytest --durations=10 --cov --cov-report term --cov-report xml:coverage.xml -n8 --dist loadscope --reruns 2 --maxfail 20
 
       - uses: codecov/codecov-action@v4
         if: matrix.os == 'windows-latest' && matrix.python-version == '3.8'

.github/workflows/release.yaml

@@ -25,7 +25,8 @@ jobs:
           COGNITE_PROJECT: python-sdk-test
           COGNITE_BASE_URL: https://greenfield.cognitedata.com
           COGNITE_CLIENT_NAME: python-sdk-integration-tests
-        run: pytest tests --durations=10 --cov --cov-report term --cov-report xml:coverage.xml -n8 --dist loadscope --reruns 2 --maxfail 20
+        # Testpaths are defined in the pytest.ini file:
+        run: pytest --durations=10 --cov --cov-report term --cov-report xml:coverage.xml -n8 --dist loadscope --reruns 2 --maxfail 20
 
       - uses: codecov/codecov-action@v4
         with:

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.2
+    rev: v0.6.3
     hooks:
       - id: ruff
         args:

CHANGELOG.md

@@ -17,18 +17,35 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
-## [7.55.2] - 2024-08-29
+## [7.58.1] - 2024-09-03
 ### Added
 - [Feature Preview - alpha] Support for `client.hosted_extractors.sources`.
 
+## [7.58.0] - 2024-09-03
+### Added
+- Data Workflows: add support for `SubworkflowReferenceParameters` subworkflow task type. Allowing embedding other workflows into a workflow.
+
+## [7.57.0] - 2024-09-03
+### Added
+- Add a `load` method to CogniteClient, ClientConfig, and CredenitalProvider (and all it's subclasses).
+- Add `apply_settings` method to `global_config` to pass in a dict of settings
+
+## [7.56.0] - 2024-09-02
+### Added
+- Support for referencing files by instance id when running diagrams.detect
+
+## [7.55.2] - 2024-08-29
+### Fixed
+- Turn workflow_orchestration into data_workflows and add trigger doc, fix attribute names in data classes
+
 ## [7.55.1] - 2024-08-29
 ### Fixed
-- Missing exports for workflow triggers 
+- Missing exports for workflow triggers
 
 ## [7.55.0] - 2024-08-23
 ### Added
 - Support for creating a session using a one-shot token in the `client.iam.session.create` method.
-- Parameter `nonce` to the `client.functions.call()` and `client.workflow.executions.run()` methods to allow passing 
+- Parameter `nonce` to the `client.functions.call()` and `client.workflow.executions.run()` methods to allow passing
   a custom nonce instead of letting the SDK generate it from your current credentials.
 
 ## [7.54.19] - 2024-08-23

CONTRIBUTING.md

@@ -95,6 +95,16 @@ To speed up test runs pass the following arguments (this will parallelize across
 pytest -n4 --dist loadscope tests
 ```
 
+#### Unit Tests for Examples in Documentation
+
+For code examples defined in *docstrings* the doctest library is used and docstring tests are defined in `tests/tests_unit/test_docstring_examples.py`. Some docstring code examples may require patching which should be done here.
+
+For any code examples written directly in `docs/source` we are using the [sphinx doctest extension](https://www.sphinx-doc.org/en/master/usage/extensions/doctest.html) with pytest. See the `docs/source/quickstart.rst` for an example of a unit test that is setup to use some fixtures defined through pytest (`docs/source/conftest.py`). To run all the tests defined in docs run:
+
+```
+pytest docs
+```
+
 ### Documentation
 
 Build html files of documentation locally by running

cognite/client/_api/diagrams.py

@@ -15,6 +15,7 @@
     FileReference,
     T_ContextualizationJob,
 )
+from cognite.client.data_classes.data_modeling import NodeId
 from cognite.client.exceptions import CogniteAPIError, CogniteMissingClientError
 from cognite.client.utils._experimental import FeaturePreviewWarning
 from cognite.client.utils._text import to_camel_case
@@ -92,24 +93,37 @@ def _list_from_instance_or_list(
     def _process_file_ids(
         ids: Sequence[int] | int | None,
         external_ids: SequenceNotStr[str] | str | None,
+        instance_ids: Sequence[NodeId] | NodeId | None,
         file_references: Sequence[FileReference] | FileReference | None,
-    ) -> list[dict[str, int | str | dict[str, int]] | dict[str, str] | dict[str, int]]:
+    ) -> list[
+        dict[str, int | str | dict[str, str] | dict[str, int]]
+        | dict[str, dict[str, str]]
+        | dict[str, str]
+        | dict[str, int]
+    ]:
         ids = DiagramsAPI._list_from_instance_or_list(ids, int, "ids must be int or list of int")
         external_ids = cast(
             SequenceNotStr[str],
             DiagramsAPI._list_from_instance_or_list(external_ids, str, "external_ids must be str or list of str"),
         )
+        instance_ids = DiagramsAPI._list_from_instance_or_list(
+            instance_ids, NodeId, "instance_ids must be NodeId or list of NodeId"
+        )
         file_references = DiagramsAPI._list_from_instance_or_list(
             file_references, FileReference, "file_references must be FileReference or list of FileReference"
         )
         # Handle empty lists
-        if not (external_ids or ids or file_references):
+        if not (external_ids or ids or instance_ids or file_references):
             raise ValueError("No ids, external ids or file references specified")
 
         id_objs = [{"fileId": id} for id in ids]
         external_id_objs = [{"fileExternalId": external_id} for external_id in external_ids]
+        instance_id_objs = [
+            {"fileInstanceId": instance_id.dump(camel_case=True, include_instance_type=False)}
+            for instance_id in instance_ids
+        ]
         file_reference_objects = [file_reference.to_api_item() for file_reference in file_references]
-        return [*id_objs, *external_id_objs, *file_reference_objects]
+        return [*id_objs, *external_id_objs, *instance_id_objs, *file_reference_objects]
 
     @overload
     def detect(
@@ -120,6 +134,7 @@ def detect(
         min_tokens: int = 2,
         file_ids: int | Sequence[int] | None = None,
         file_external_ids: str | SequenceNotStr[str] | None = None,
+        file_instance_ids: NodeId | Sequence[NodeId] | None = None,
         file_references: list[FileReference] | FileReference | None = None,
         pattern_mode: bool = False,
         configuration: dict[str, Any] | None = None,
@@ -136,6 +151,7 @@ def detect(
         min_tokens: int = 2,
         file_ids: int | Sequence[int] | None = None,
         file_external_ids: str | SequenceNotStr[str] | None = None,
+        file_instance_ids: NodeId | Sequence[NodeId] | None = None,
         file_references: list[FileReference] | FileReference | None = None,
         pattern_mode: bool = False,
         configuration: DiagramDetectConfig | dict[str, Any] | None = None,
@@ -152,6 +168,7 @@ def detect(
         min_tokens: int = 2,
         file_ids: int | Sequence[int] | None = None,
         file_external_ids: str | SequenceNotStr[str] | None = None,
+        file_instance_ids: NodeId | Sequence[NodeId] | None = None,
         file_references: list[FileReference] | FileReference | None = None,
         pattern_mode: bool = False,
         configuration: DiagramDetectConfig | dict[str, Any] | None = None,
@@ -165,6 +182,7 @@ def detect(
         min_tokens: int = 2,
         file_ids: int | Sequence[int] | None = None,
         file_external_ids: str | SequenceNotStr[str] | None = None,
+        file_instance_ids: NodeId | Sequence[NodeId] | None = None,
         file_references: list[FileReference] | FileReference | None = None,
         pattern_mode: bool | None = None,
         configuration: DiagramDetectConfig | dict[str, Any] | None = None,
@@ -184,7 +202,8 @@ def detect(
             min_tokens (int): Minimal number of tokens a match must be based on
             file_ids (int | Sequence[int] | None): ID of the files, should already be uploaded in the same tenant.
             file_external_ids (str | SequenceNotStr[str] | None): File external ids, alternative to file_ids and file_references.
-            file_references (list[FileReference] | FileReference | None): File references (id or external_id), and first_page and last_page to specify page ranges per file. Each reference can specify up to 50 pages. Providing a page range will also make the page count of the document a part of the response.
+            file_instance_ids (NodeId | Sequence[NodeId] | None): Files to detect in, specified by instance id.
+            file_references (list[FileReference] | FileReference | None): File references (id, external_id or instance_id), and first_page and last_page to specify page ranges per file. Each reference can specify up to 50 pages. Providing a page range will also make the page count of the document a part of the response.
             pattern_mode (bool | None): If True, entities must be provided with a sample field. This enables detecting tags that are similar to the sample, but not necessarily identical. Defaults to None.
             configuration (DiagramDetectConfig | dict[str, Any] | None): Additional configuration for the detect algorithm. See `DiagramDetectConfig` class documentation and `beta API docs <https://api-docs.cognite.com/20230101-beta/tag/Engineering-diagrams/operation/diagramDetect/#!path=configuration&t=request>`_.
             multiple_jobs (bool): Enables you to publish multiple jobs. If True the method returns a tuple of DetectJobBundle and list of potentially unposted files. If False it will return a single DiagramDetectResults. Defaults to False.
@@ -258,7 +277,7 @@ def detect(
 
             Check the documentation for `DiagramDetectConfig` for more information on the available options.
         """
-        items = self._process_file_ids(file_ids, file_external_ids, file_references)
+        items = self._process_file_ids(file_ids, file_external_ids, file_instance_ids, file_references)
         entities = [
             entity.dump(camel_case=True) if isinstance(entity, CogniteResource) else entity for entity in entities
         ]

cognite/client/_api/raw.py

@@ -398,7 +398,7 @@ def __call__(
 
         Note:
             When iterating using partitions > 1, the memory usage is bounded at 2 x partitions x chunk_size. This is implemented
-            by halting retrival speed when the callers code can't keep up.
+            by halting retrieval speed when the callers code can't keep up.
         """
         if partitions is None or _RUNNING_IN_BROWSER:
             return self._list_generator(

cognite/client/_cognite_client.py

@@ -32,7 +32,7 @@
 from cognite.client._api_client import APIClient
 from cognite.client.config import ClientConfig, global_config
 from cognite.client.credentials import CredentialProvider, OAuthClientCredentials, OAuthInteractive
-from cognite.client.utils._auxiliary import get_current_sdk_version
+from cognite.client.utils._auxiliary import get_current_sdk_version, load_resource_to_dict
 
 
 class CogniteClient:
@@ -215,3 +215,37 @@ def default_oauth_interactive(
         """
         credentials = OAuthInteractive.default_for_azure_ad(tenant_id, client_id, cdf_cluster)
         return cls.default(project, cdf_cluster, credentials, client_name)
+
+    @classmethod
+    def load(cls, config: dict[str, Any] | str) -> CogniteClient:
+        """Load a cognite client object from a YAML/JSON string or dict.
+
+        Args:
+            config (dict[str, Any] | str): A dictionary or YAML/JSON string containing configuration values defined in the CogniteClient class.
+
+        Returns:
+            CogniteClient: A cognite client object.
+
+        Examples:
+
+            Create a cognite client object from a dictionary input:
+
+                >>> from cognite.client import CogniteClient
+                >>> import os
+                >>> config = {
+                ...     "client_name": "abcd",
+                ...     "project": "cdf-project",
+                ...     "base_url": "https://api.cognitedata.com/",
+                ...     "credentials": {
+                ...         "client_credentials": {
+                ...             "client_id": "abcd",
+                ...             "client_secret": os.environ["OAUTH_CLIENT_SECRET"],
+                ...             "token_url": "https://login.microsoftonline.com/xyz/oauth2/v2.0/token",
+                ...             "scopes": ["https://api.cognitedata.com/.default"],
+                ...         },
+                ...     },
+                ... }
+                >>> client = CogniteClient.load(config)
+        """
+        loaded = load_resource_to_dict(config)
+        return cls(config=ClientConfig.load(loaded))

cognite/client/_version.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "7.55.2"
+__version__ = "7.58.1"
 __api_subversion__ = "20230101"