diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f5db236c..3f9000fe0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add initial support for accessing [Federation Extension](https://github.com/Open-EO/openeo-api/tree/master/extensions/federation) related metadata ([#668](https://github.com/Open-EO/openeo-python-client/issues/668)) + ### Changed - Improved tracking of metadata changes with `resample_spatial` and `resample_cube_spatial` ([#690](https://github.com/Open-EO/openeo-python-client/issues/690)) diff --git a/docs/api.rst b/docs/api.rst index 91b4e0cbd..cf61ba115 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -70,6 +70,16 @@ openeo.rest.capabilities :members: OpenEoCapabilities +openeo.rest.models +------------------- + +.. automodule:: openeo.rest.models.general + :members: + +.. automodule:: openeo.rest.models.logs + :members: LogEntry, normalize_log_level + + openeo.api.process -------------------- @@ -77,12 +87,6 @@ openeo.api.process :members: Parameter -openeo.api.logs ------------------ - -.. automodule:: openeo.api.logs - :members: LogEntry, normalize_log_level - openeo.rest.connection ---------------------- diff --git a/docs/federation-extension.rst b/docs/federation-extension.rst new file mode 100644 index 000000000..0f08d7581 --- /dev/null +++ b/docs/federation-extension.rst @@ -0,0 +1,70 @@ + +.. _federation-extension: + +=========================== +openEO Federation Extension +=========================== + + +The `openEO Federation extension `_ +is a set of additional specifications, +on top of the standard openEO API specification, +to address the need for extra metadata in the context +of federated openEO processing, +where multiple (separately operated) openEO services are bundled together +behind a single API endpoint. + + +Accessing federation extension metadata +======================================== + +The openEO Python client library provides access to this additional metadata +in a couple of resources. + +.. versionadded:: 0.38.0 + initial support to access federation extension related metadata. + +.. warning:: this API is experimental and subject to change. + + +Backend details +--------------- + +Participating backends in a federation are listed under the ``federation`` field +of the capabilities document (``GET /``) and can be inspected +using :py:meth:`OpenEoCapabilities.ext_federation_backend_details() `: + +.. code-block:: python + + import openeo + connection = openeo.connect(url=...) + capabilities = connection.capabilities() + print("Federated backends:", capabilities.ext_federation_backend_details()) + + +Unavailable backends (``federation:missing``) +---------------------------------------------- + +When listing resources like +collections (with :py:meth:`Connection.list_collections() `), +processes (with :py:meth:`Connection.list_processes() `), +jobs (with :py:meth:`Connection.list_jobs() `), +etc., +there might be items missing due to federation participants being temporarily unavailable. +These missing federation components are listed in the response under the ``federation:missing`` field +and can be inspected as follows: + +.. code-block:: python + + import openeo + connection = openeo.connect(url=...) + collections = connection.list_collections() + print("Number of collections:", len(collections)) + print("Missing federation components:", collections.ext_federation_missing()) + + +Note that the ``collections`` object in this example, returned by +:py:meth:`Connection.list_collections() `, +acts at the surface as a simple list of dictionaries with collection metadata, +but also provides additional properties/methods like +:py:attr:`ext_federation_missing() `. diff --git a/docs/index.rst b/docs/index.rst index b2c1ba643..aab3358eb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -64,6 +64,7 @@ Table of contents process_mapping development best_practices + Federation extension changelog diff --git a/openeo/api/logs.py b/openeo/api/logs.py index 5a7ae02d5..f3a7b9b8c 100644 --- a/openeo/api/logs.py +++ b/openeo/api/logs.py @@ -1,99 +1,13 @@ -import logging -from typing import Optional, Union +import warnings +from openeo.internal.warnings import UserDeprecationWarning +from openeo.rest.models.logs import LogEntry, log_level_name, normalize_log_level -class LogEntry(dict): - """ - Log message and info for jobs and services +warnings.warn( + message="Submodule `openeo.api.logs` is deprecated in favor of `openeo.rest.models.logs`.", + category=UserDeprecationWarning, + stacklevel=2, +) - Fields: - - ``id``: Unique ID for the log, string, REQUIRED - - ``code``: Error code, string, optional - - ``level``: Severity level, string (error, warning, info or debug), REQUIRED - - ``message``: Error message, string, REQUIRED - - ``time``: Date and time of the error event as RFC3339 date-time, string, available since API 1.1.0 - - ``path``: A "stack trace" for the process, array of dicts - - ``links``: Related links, array of dicts - - ``usage``: Usage metrics available as property 'usage', dict, available since API 1.1.0 - May contain the following metrics: cpu, memory, duration, network, disk, storage and other custom ones - Each of the metrics is also a dict with the following parts: value (numeric) and unit (string) - - ``data``: Arbitrary data the user wants to "log" for debugging purposes. - Please note that this property may not exist as there's a difference - between None and non-existing. None for example refers to no-data in - many cases while the absence of the property means that the user did - not provide any data for debugging. - """ - _required = {"id", "level", "message"} - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # Check required fields - missing = self._required.difference(self.keys()) - if missing: - raise ValueError("Missing required fields: {m}".format(m=sorted(missing))) - - @property - def id(self): - return self["id"] - - # Legacy alias - log_id = id - - @property - def message(self): - return self["message"] - - @property - def level(self): - return self["level"] - - # TODO: add properties for "code", "time", "path", "links" and "data" with sensible defaults? - - -def normalize_log_level( - log_level: Union[int, str, None], default: int = logging.DEBUG -) -> int: - """ - Helper function to convert a openEO API log level (e.g. string "error") - to the integer constants defined in Python's standard library ``logging`` module (e.g. ``logging.ERROR``). - - :param log_level: log level to normalize: a log level string in the style of - the openEO API ("error", "warning", "info", or "debug"), - an integer value (e.g. a ``logging`` constant), or ``None``. - - :param default: fallback log level to return on unknown log level strings or ``None`` input. - - :raises TypeError: when log_level is any other type than str, an int or None. - :return: One of the following log level constants from the standard module ``logging``: - ``logging.ERROR``, ``logging.WARNING``, ``logging.INFO``, or ``logging.DEBUG`` . - """ - if isinstance(log_level, str): - log_level = log_level.upper() - if log_level in ["CRITICAL", "ERROR", "FATAL"]: - return logging.ERROR - elif log_level in ["WARNING", "WARN"]: - return logging.WARNING - elif log_level == "INFO": - return logging.INFO - elif log_level == "DEBUG": - return logging.DEBUG - else: - return default - elif isinstance(log_level, int): - return log_level - elif log_level is None: - return default - else: - raise TypeError( - f"Value for log_level is not an int or str: type={type(log_level)}, value={log_level!r}" - ) - - -def log_level_name(log_level: Union[int, str, None]) -> str: - """ - Get the name of a normalized log level. - This value conforms to log level names used in the openEO API. - """ - return logging.getLevelName(normalize_log_level(log_level)).lower() +__all__ = ["LogEntry", "normalize_log_level", "log_level_name"] diff --git a/openeo/rest/auth/testing.py b/openeo/rest/auth/testing.py index 651abd21f..25ad334a8 100644 --- a/openeo/rest/auth/testing.py +++ b/openeo/rest/auth/testing.py @@ -3,12 +3,11 @@ """ import base64 -import contextlib +import dataclasses import json import urllib.parse import uuid from typing import List, Optional, Union -from unittest import mock import requests import requests_mock.request @@ -290,3 +289,37 @@ def get_request_history( for r in self.requests_mock.request_history if (method is None or method.lower() == r.method.lower()) and (url is None or url == r.url) ] + + +def build_basic_auth_header(username: str, password: str) -> str: + """Generate basic auth header (per RFC 7617) from given username and password.""" + credentials = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + return f"Basic {credentials}" + + +@dataclasses.dataclass(frozen=True) +class SimpleBasicAuthMocker: + """ + Helper to create a test fixture for simple basic auth handling in openEO context: + set up `/credentials/basic` handling with a fixed username/password/access_token combo. + """ + + username: str = "john" + password: str = "j0hn!" + access_token: str = "6cc3570k3n" + + def expected_auth_header(self) -> str: + return build_basic_auth_header(username=self.username, password=self.password) + + def setup_credentials_basic_handler(self, *, api_root: str, requests_mock): + """Set up `requests_mock` handler for `/credentials/basic` endpoint.""" + expected_auth_header = self.expected_auth_header() + + def credentials_basic_handler(request, context): + assert request.headers["Authorization"] == expected_auth_header + return json.dumps({"access_token": self.access_token}) + + return requests_mock.get( + url_join(api_root, "/credentials/basic"), + text=credentials_basic_handler, + ) diff --git a/openeo/rest/capabilities.py b/openeo/rest/capabilities.py index 08f6435a7..768093f6f 100644 --- a/openeo/rest/capabilities.py +++ b/openeo/rest/capabilities.py @@ -1,6 +1,7 @@ from typing import Dict, List, Optional, Union from openeo.internal.jupyter import render_component +from openeo.rest.models import federation_extension from openeo.util import deep_get from openeo.utils.version import ApiVersionException, ComparableVersion @@ -54,12 +55,13 @@ def list_plans(self) -> List[dict]: def _repr_html_(self): return render_component("capabilities", data=self.capabilities, parameters={"url": self.url}) - def get_federation(self) -> Union[Dict[str, dict], None]: + def ext_federation_backend_details(self) -> Union[Dict[str, dict], None]: """ Lists all back-ends (with details, such as URL) that are part of the federation if this backend acts as a federated backend, as specified in the openEO Federation Extension. - Returns ``None`` otherwise + Returns ``None`` otherwise. + + .. versionadded:: 0.38.0 """ - # TODO: also check related conformance class in `/conformance`? - return self.get("federation") + return federation_extension.get_backend_details(data=self.capabilities) diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index 73a594110..92f32a2f9 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -66,6 +66,11 @@ from openeo.rest.graph_building import CollectionProperty from openeo.rest.job import BatchJob, RESTJob from openeo.rest.mlmodel import MlModel +from openeo.rest.models.general import ( + CollectionListingResponse, + JobListingResponse, + ProcessListingResponse, +) from openeo.rest.service import Service from openeo.rest.udp import Parameter, RESTUserDefinedProcess from openeo.rest.userfile import UserFile @@ -671,7 +676,7 @@ def describe_account(self) -> dict: def user_jobs(self) -> List[dict]: return self.list_jobs() - def list_collections(self) -> List[dict]: + def list_collections(self) -> CollectionListingResponse: """ List basic metadata of all collections provided by the back-end. @@ -682,10 +687,15 @@ def list_collections(self) -> List[dict]: it is recommended to use :py:meth:`~openeo.rest.connection.Connection.describe_collection` instead. :return: list of dictionaries with basic collection metadata. + + .. versionchanged:: 0.38.0 + Returns a :py:class:`~openeo.rest.models.general.CollectionListingResponse` object + instead of a simple ``List[dict]``. """ # TODO: add caching #383, but reset cache on auth change #254 - data = self.get('/collections', expected_status=200).json()["collections"] - return VisualList("collections", data=data) + # TODO #677 add pagination support? + data = self.get("/collections", expected_status=200).json() + return CollectionListingResponse(response_data=data) def list_collection_ids(self) -> List[str]: """ @@ -822,23 +832,27 @@ def collection_metadata(self, name) -> CollectionMetadata: # TODO: duplication with `Connection.describe_collection`: deprecate one or the other? return CollectionMetadata(metadata=self.describe_collection(name)) - def list_processes(self, namespace: Optional[str] = None) -> List[dict]: - # TODO: Maybe format the result dictionary so that the process_id is the key of the dictionary. + def list_processes(self, namespace: Optional[str] = None) -> ProcessListingResponse: """ Loads all available processes of the back end. :param namespace: The namespace for which to list processes. - :return: processes_dict: Dict All available processes of the back end. + :return: listing of available processes + + .. versionchanged:: 0.38.0 + Returns a :py:class:`~openeo.rest.models.general.ProcessListingResponse` object + instead of a simple ``List[dict]``. """ + # TODO: Maybe format the result dictionary so that the process_id is the key of the dictionary. + # TODO #677 add pagination support? if namespace is None: - processes = self._capabilities_cache.get( - key=("processes", "backend"), - load=lambda: self.get('/processes', expected_status=200).json()["processes"] + response = self._capabilities_cache.get( + key=("processes", "backend"), load=lambda: self.get("/processes", expected_status=200).json() ) else: - processes = self.get('/processes/' + namespace, expected_status=200).json()["processes"] - return VisualList("processes", data=processes, parameters={'show-graph': True, 'provide-download': False}) + response = self.get("/processes/" + namespace, expected_status=200).json() + return ProcessListingResponse(response_data=response) def describe_process(self, id: str, namespace: Optional[str] = None) -> dict: """ @@ -857,26 +871,25 @@ def describe_process(self, id: str, namespace: Optional[str] = None) -> dict: raise OpenEoClientException("Process does not exist.") - def list_jobs(self, limit: Union[int, None] = None) -> List[dict]: + def list_jobs(self, limit: Union[int, None] = None) -> JobListingResponse: """ - Lists all jobs of the authenticated user. + Lists (batch) jobs metadata of the authenticated user. :param limit: maximum number of jobs to return. Setting this limit enables pagination. :return: job_list: Dict of all jobs of the user. - .. versionadded:: 0.36.0 + .. versionchanged:: 0.36.0 Added ``limit`` argument + + .. versionchanged:: 0.38.0 + Returns a :py:class:`~openeo.rest.models.general.JobListingResponse` object + instead of simple ``List[dict]``. """ # TODO: Parse the result so that Job classes returned? - resp = self.get("/jobs", params={"limit": limit}, expected_status=200).json() - if resp.get("federation:missing"): - _log.warning("Partial user job listing due to missing federation components: {c}".format( - c=",".join(resp["federation:missing"]) - )) # TODO: when pagination is enabled: how to expose link to next page? - jobs = resp["jobs"] - return VisualList("data-table", data=jobs, parameters={'columns': 'jobs'}) + resp = self.get("/jobs", params={"limit": limit}, expected_status=200).json() + return JobListingResponse(response_data=resp) def assert_user_defined_process_support(self): """ @@ -928,13 +941,18 @@ def save_user_defined_process( ) return udp - def list_user_defined_processes(self) -> List[dict]: + def list_user_defined_processes(self) -> ProcessListingResponse: """ Lists all user-defined processes of the authenticated user. + + .. versionchanged:: 0.38.0 + Returns a :py:class:`~openeo.rest.models.general.ProcessListingResponse` object + instead of a simple ``List[dict]``. """ + # TODO #677 add pagination support? self.assert_user_defined_process_support() - data = self.get("/process_graphs", expected_status=200).json()["processes"] - return VisualList("processes", data=data, parameters={'show-graph': True, 'provide-download': False}) + data = self.get("/process_graphs", expected_status=200).json() + return ProcessListingResponse(response_data=data) def user_defined_process(self, user_defined_process_id: str) -> RESTUserDefinedProcess: """ diff --git a/openeo/rest/job.py b/openeo/rest/job.py index 63bce4a8a..2680cb053 100644 --- a/openeo/rest/job.py +++ b/openeo/rest/job.py @@ -10,7 +10,6 @@ import requests -from openeo.api.logs import LogEntry, log_level_name, normalize_log_level from openeo.internal.documentation import openeo_endpoint from openeo.internal.jupyter import ( VisualDict, @@ -26,6 +25,8 @@ OpenEoApiPlainError, OpenEoClientException, ) +from openeo.rest.models.general import LogsResponse +from openeo.rest.models.logs import LogEntry, log_level_name, normalize_log_level from openeo.util import ensure_dir if typing.TYPE_CHECKING: @@ -184,9 +185,7 @@ def get_results(self) -> JobResults: """ return JobResults(job=self) - def logs( - self, offset: Optional[str] = None, level: Optional[Union[str, int]] = None - ) -> List[LogEntry]: + def logs(self, offset: Optional[str] = None, level: Union[str, int, None] = None) -> LogsResponse: """Retrieve job logs. :param offset: The last identifier (property ``id`` of a LogEntry) the client has received. @@ -217,22 +216,8 @@ def logs( params["offset"] = offset if level is not None: params["level"] = log_level_name(level) - response = self.connection.get(url, params=params, expected_status=200) - logs = response.json()["logs"] - - # Only filter logs when specified. - # We should still support client-side log_level filtering because not all backends - # support the minimum log level parameter. - if level is not None: - log_level = normalize_log_level(level) - logs = ( - log - for log in logs - if normalize_log_level(log.get("level")) >= log_level - ) - - entries = [LogEntry(log) for log in logs] - return VisualList("logs", data=entries) + response_data = self.connection.get(url, params=params, expected_status=200).json() + return LogsResponse(response_data=response_data, log_level=level) def run_synchronous( self, diff --git a/openeo/rest/models/__init__.py b/openeo/rest/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openeo/rest/models/federation_extension.py b/openeo/rest/models/federation_extension.py new file mode 100644 index 000000000..9360d1ba4 --- /dev/null +++ b/openeo/rest/models/federation_extension.py @@ -0,0 +1,28 @@ +import logging +from typing import Dict, List, Union + +_log = logging.getLogger(__name__) + + +def get_backend_details(data: dict) -> Union[Dict[str, dict], None]: + """ + Get federated backend details from capabilities document (``GET /``) + at "federation" field + """ + # TODO: return a richer object instead of raw dicts? + return data.get("federation", None) + + +def get_federation_missing(data: dict, *, resource_name: str, auto_warn: bool = False) -> Union[List[str], None]: + """ + Get "federation:missing" field from response data, if present. + + :param data: response data + :param resource_name: name of the requested resource (listing) + :param auto_warn: whether to automatically log a warning if missing federation components are detected. + """ + # TODO: options to return richer objects (e.g. resolve backend id to URL/title) + missing = data.get("federation:missing", None) + if auto_warn and missing: + _log.warning(f"Partial {resource_name}: missing federation components: {missing!r}.") + return missing diff --git a/openeo/rest/models/general.py b/openeo/rest/models/general.py new file mode 100644 index 000000000..f98a40f01 --- /dev/null +++ b/openeo/rest/models/general.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +import functools +from dataclasses import dataclass +from typing import List, Optional, Union + +from openeo.internal.jupyter import render_component +from openeo.rest.models import federation_extension +from openeo.rest.models.logs import LogEntry, normalize_log_level + + +@dataclass(frozen=True) +class Link: + """ + Container for (web) link data, used throughout the openEO API, + to point to alternate representations, a license, extra detailed information, and more. + """ + + rel: str + href: str + type: Optional[str] = None + title: Optional[str] = None + + @classmethod + def from_dict(cls, data: dict) -> Link: + """Build :py:class:`Link` from dictionary (e.g. parsed JSON representation).""" + return cls(rel=data["rel"], href=data["href"], type=data.get("type"), title=data.get("title")) + + # TODO: add _html_repr_ for Jupyter integration + # TODO: also provide container for list of links with methods to easily look up by `rel` or `type` + + +class CollectionListingResponse(list): + """ + Container for collection metadata listing received + from a ``GET /collections`` request. + + .. note:: + This object mimics, for backward compatibility reasons, + the interface of simple list of collection metadata dictionaries (``List[dict]``), + which was the original return API of + :py:meth:`~openeo.rest.connection.Connection.list_collections()`, + but now also provides methods/properties to access additional response data. + + :param response_data: response data from a ``GET /collections`` request + + .. seealso:: :py:meth:`openeo.rest.connection.Connection.list_collections()` + + .. versionadded:: 0.38.0 + """ + + __slots__ = ["_data"] + + def __init__(self, response_data: dict): + self._data = response_data + # Mimic original list of collection metadata dictionaries + super().__init__(response_data["collections"]) + + self.ext_federation_missing(auto_warn=True) + + def _repr_html_(self): + return render_component(component="collections", data=self) + + @property + def links(self) -> List[Link]: + """Get links related to this resource.""" + return [Link.from_dict(d) for d in self._data.get("links", [])] + + def ext_federation_missing(self, *, auto_warn: bool = False) -> Union[None, List[str]]: + """ + List of backends IDs (from the federation) + that were not available during the resource listing request. + + :param auto_warn: whether to automatically log a warning if missing federation components are detected. + + .. seealso:: :ref:`federation-extension` + + .. warning:: this API is experimental and subject to change. + """ + return federation_extension.get_federation_missing( + data=self._data, resource_name="collection listing", auto_warn=auto_warn + ) + + +class ProcessListingResponse(list): + """ + Container for process metadata listing received + from a ``GET /processes`` request. + + .. note:: + This object mimics, for backward compatibility reasons, + the interface of simple list of process metadata dictionaries (``List[dict]``), + :py:meth:`~openeo.rest.connection.Connection.list_processes()`, + but now also provides methods/properties to access additional response data. + + :param response_data: response data from a ``GET /processes`` request + + .. seealso:: :py:meth:`openeo.rest.connection.Connection.list_processes()` + + .. versionadded:: 0.38.0 + """ + + __slots__ = ["_data"] + + def __init__(self, response_data: dict): + self._data = response_data + # Mimic original list of process metadata dictionaries + super().__init__(response_data["processes"]) + + self.ext_federation_missing(auto_warn=True) + + def _repr_html_(self): + return render_component( + component="processes", data=self, parameters={"show-graph": True, "provide-download": False} + ) + + @property + def links(self) -> List[Link]: + """Get links related to this resource.""" + return [Link.from_dict(d) for d in self._data.get("links", [])] + + def ext_federation_missing(self, *, auto_warn: bool = False) -> Union[None, List[str]]: + """ + List of backends IDs (from the federation) + that were not available during the resource listing request. + + :param auto_warn: whether to automatically log a warning if missing federation components are detected. + + .. seealso:: :ref:`federation-extension` + + .. warning:: this API is experimental and subject to change. + """ + return federation_extension.get_federation_missing( + data=self._data, resource_name="process listing", auto_warn=auto_warn + ) + + +class JobListingResponse(list): + """ + Container for job metadata listing received + from a ``GET /jobs`` request. + + .. note:: + This object mimics, for backward compatibility reasons, + the interface of simple list of job metadata dictionaries (``List[dict]``), + which was the original return API of + :py:meth:`~openeo.rest.connection.Connection.list_jobs()`, + but now also provides methods/properties to access additional response data. + + :param response_data: response data from a ``GET /jobs`` request + + .. seealso:: :py:meth:`openeo.rest.connection.Connection.list_jobs()` + + .. versionadded:: 0.38.0 + """ + + __slots__ = ["_data"] + + def __init__(self, response_data: dict): + self._data = response_data + # Mimic original list of process metadata dictionaries + super().__init__(response_data["jobs"]) + + self.ext_federation_missing(auto_warn=True) + + def _repr_html_(self): + return render_component(component="data-table", data=self, parameters={"columns": "jobs"}) + + @property + def links(self) -> List[Link]: + """Get links related to this resource.""" + return [Link.from_dict(d) for d in self._data.get("links", [])] + + def ext_federation_missing(self, *, auto_warn: bool = False) -> Union[None, List[str]]: + """ + List of backends IDs (from the federation) + that were not available during the resource listing request. + + :param auto_warn: whether to automatically log a warning if missing federation components are detected. + + .. seealso:: :ref:`federation-extension` + + .. warning:: this API is experimental and subject to change. + """ + return federation_extension.get_federation_missing( + data=self._data, resource_name="job listing", auto_warn=auto_warn + ) + + +class LogsResponse(list): + """ + Container for job/service logs as received + from a ``GET /jobs/{job_id}/logs`` or ``GET /services/{service_id}/logs`` request. + + .. note:: + This object mimics, for backward compatibility reasons, + the interface of a simple list (``List[LogEntry]``) + which was the original return API of + :py:meth:`~openeo.rest.job.BatchJob.logs()` + and :py:meth:`~openeo.rest.service.Service.logs()`, + but now also provides methods/properties to access additional response data. + + :param response_data: response data from a ``GET /jobs/{job_id}/logs`` + or ``GET /services/{service_id}/logs`` request. + + .. seealso:: :py:meth:`~openeo.rest.job.BatchJob.logs()` + and :py:meth:`~openeo.rest.service.Service.logs()` + + .. versionadded:: 0.38.0 + """ + + __slots__ = ["_data"] + + def __init__(self, response_data: dict, *, log_level: Optional[str] = None): + self._data = response_data + + logs = response_data.get("logs", []) + # Extra client-side level filtering (in case the back-end does not support that) + if log_level: + + @functools.lru_cache + def accept_level(level: str) -> bool: + return normalize_log_level(level) >= normalize_log_level(log_level) + + if ( + # Backend does not list effective lowest level + "level" not in response_data + # Or effective lowest level is still too low + or not accept_level(response_data["level"]) + ): + logs = (log for log in logs if accept_level(log.get("level"))) + logs = [LogEntry(log) for log in logs] + + # Mimic original list of process metadata dictionaries + super().__init__(logs) + + self.ext_federation_missing(auto_warn=True) + + def _repr_html_(self): + return render_component(component="logs", data=self) + + @property + def logs(self) -> List[LogEntry]: + """Get the log entries.""" + return self + + @property + def links(self) -> List[Link]: + """Get links related to this resource.""" + return [Link.from_dict(d) for d in self._data.get("links", [])] + + def ext_federation_missing(self, *, auto_warn: bool = False) -> Union[None, List[str]]: + """ + List of backends IDs (from the federation) + that were not available during the resource listing request. + + :param auto_warn: whether to automatically log a warning if missing federation components are detected. + + .. seealso:: :ref:`federation-extension` + + .. warning:: this API is experimental and subject to change. + """ + return federation_extension.get_federation_missing( + data=self._data, resource_name="log listing", auto_warn=auto_warn + ) diff --git a/openeo/rest/models/logs.py b/openeo/rest/models/logs.py new file mode 100644 index 000000000..d836b7a28 --- /dev/null +++ b/openeo/rest/models/logs.py @@ -0,0 +1,95 @@ +import logging +from typing import Optional, Union + + +class LogEntry(dict): + """ + Log message and info for jobs and services + + Fields: + - ``id``: Unique ID for the log, string, REQUIRED + - ``code``: Error code, string, optional + - ``level``: Severity level, string (error, warning, info or debug), REQUIRED + - ``message``: Error message, string, REQUIRED + - ``time``: Date and time of the error event as RFC3339 date-time, string, available since API 1.1.0 + - ``path``: A "stack trace" for the process, array of dicts + - ``links``: Related links, array of dicts + - ``usage``: Usage metrics available as property 'usage', dict, available since API 1.1.0 + May contain the following metrics: cpu, memory, duration, network, disk, storage and other custom ones + Each of the metrics is also a dict with the following parts: value (numeric) and unit (string) + - ``data``: Arbitrary data the user wants to "log" for debugging purposes. + Please note that this property may not exist as there's a difference + between None and non-existing. None for example refers to no-data in + many cases while the absence of the property means that the user did + not provide any data for debugging. + """ + + _required = {"id", "level", "message"} + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Check required fields + missing = self._required.difference(self.keys()) + if missing: + raise ValueError("Missing required fields: {m}".format(m=sorted(missing))) + + @property + def id(self): + return self["id"] + + # Legacy alias + log_id = id + + @property + def message(self): + return self["message"] + + @property + def level(self): + return self["level"] + + # TODO: add properties for "code", "time", "path", "links" and "data" with sensible defaults? + + +def normalize_log_level(log_level: Union[int, str, None], default: int = logging.DEBUG) -> int: + """ + Helper function to convert a openEO API log level (e.g. string "error") + to the integer constants defined in Python's standard library ``logging`` module (e.g. ``logging.ERROR``). + + :param log_level: log level to normalize: a log level string in the style of + the openEO API ("error", "warning", "info", or "debug"), + an integer value (e.g. a ``logging`` constant), or ``None``. + + :param default: fallback log level to return on unknown log level strings or ``None`` input. + + :raises TypeError: when log_level is any other type than str, an int or None. + :return: One of the following log level constants from the standard module ``logging``: + ``logging.ERROR``, ``logging.WARNING``, ``logging.INFO``, or ``logging.DEBUG`` . + """ + if isinstance(log_level, str): + log_level = log_level.upper() + if log_level in ["CRITICAL", "ERROR", "FATAL"]: + return logging.ERROR + elif log_level in ["WARNING", "WARN"]: + return logging.WARNING + elif log_level == "INFO": + return logging.INFO + elif log_level == "DEBUG": + return logging.DEBUG + else: + return default + elif isinstance(log_level, int): + return log_level + elif log_level is None: + return default + else: + raise TypeError(f"Value for log_level is not an int or str: type={type(log_level)}, value={log_level!r}") + + +def log_level_name(log_level: Union[int, str, None]) -> str: + """ + Get the name of a normalized log level. + This value conforms to log level names used in the openEO API. + """ + return logging.getLevelName(normalize_log_level(log_level)).lower() diff --git a/openeo/rest/service.py b/openeo/rest/service.py index a12383695..c49a89ec9 100644 --- a/openeo/rest/service.py +++ b/openeo/rest/service.py @@ -3,8 +3,9 @@ import typing from typing import List, Optional, Union -from openeo.api.logs import LogEntry, log_level_name -from openeo.internal.jupyter import VisualDict, VisualList +from openeo.internal.jupyter import VisualDict +from openeo.rest.models.general import LogsResponse +from openeo.rest.models.logs import LogEntry, log_level_name if typing.TYPE_CHECKING: # Imports for type checking only (circular import issue at runtime). @@ -42,9 +43,7 @@ def delete_service(self): # DELETE /services/{service_id} self.connection.delete("/services/{}".format(self.service_id), expected_status=204) - def logs( - self, offset: Optional[str] = None, level: Optional[Union[str, int]] = None - ) -> List[LogEntry]: + def logs(self, offset: Optional[str] = None, level: Union[str, int, None] = None) -> List[LogEntry]: """Retrieve service logs.""" url = f"/service/{self.service_id}/logs" params = {} @@ -52,7 +51,5 @@ def logs( params["offset"] = offset if level is not None: params["level"] = log_level_name(level) - resp = self.connection.get(url, params=params, expected_status=200) - logs = resp.json()["logs"] - entries = [LogEntry(log) for log in logs] - return VisualList("logs", data=entries) + response_data = self.connection.get(url, params=params, expected_status=200).json() + return LogsResponse(response_data=response_data, log_level=level) diff --git a/tests/internal/test_jupyter.py b/tests/internal/test_jupyter.py index 7e95b4af8..5f1c1aefc 100644 --- a/tests/internal/test_jupyter.py +++ b/tests/internal/test_jupyter.py @@ -1,5 +1,5 @@ -from openeo.api.logs import LogEntry from openeo.internal.jupyter import render_component +from openeo.rest.models.logs import LogEntry def test_render_component_logs(): diff --git a/tests/rest/auth/test_testing.py b/tests/rest/auth/test_testing.py index 9b50d30a2..f10e58445 100644 --- a/tests/rest/auth/test_testing.py +++ b/tests/rest/auth/test_testing.py @@ -3,7 +3,7 @@ OidcClientInfo, OidcProviderInfo, ) -from openeo.rest.auth.testing import OidcMock +from openeo.rest.auth.testing import OidcMock, build_basic_auth_header class TestOidcMock: @@ -33,3 +33,7 @@ def test_request_history(self, requests_mock): assert [r.url for r in oidc_mock.get_request_history("/token")] == [ "https://oidc.test/token" ] + + +def test_build_basic_auth_header(): + assert build_basic_auth_header("john", "56(r61!?") == "Basic am9objo1NihyNjEhPw==" diff --git a/tests/rest/conftest.py b/tests/rest/conftest.py index 5879929c5..13849fb7b 100644 --- a/tests/rest/conftest.py +++ b/tests/rest/conftest.py @@ -1,4 +1,6 @@ import contextlib +import dataclasses +import json import re import typing from unittest import mock @@ -7,7 +9,9 @@ import time_machine from openeo.rest._testing import DummyBackend, build_capabilities +from openeo.rest.auth.testing import SimpleBasicAuthMocker, build_basic_auth_header from openeo.rest.connection import Connection +from openeo.util import url_join API_URL = "https://oeo.test/" @@ -119,3 +123,14 @@ def another_dummy_backend(requests_mock) -> DummyBackend: another_dummy_backend.setup_file_format("GTiff") another_dummy_backend.setup_file_format("netCDF") return another_dummy_backend + + +@pytest.fixture +def basic_auth(requests_mock) -> SimpleBasicAuthMocker: + """ + Fixture for simple basic auth handling in openEO context: + set up /credentials/basic handling with a fixed username/password/access_token combo. + """ + fixture = SimpleBasicAuthMocker() + fixture.setup_credentials_basic_handler(api_root=API_URL, requests_mock=requests_mock) + return fixture diff --git a/tests/rest/models/__init__.py b/tests/rest/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/rest/models/test_federation_extension.py b/tests/rest/models/test_federation_extension.py new file mode 100644 index 000000000..a138b2681 --- /dev/null +++ b/tests/rest/models/test_federation_extension.py @@ -0,0 +1,57 @@ +import pytest + +from openeo.rest.models.federation_extension import ( + get_backend_details, + get_federation_missing, +) + + +def test_get_backend_details(): + assert get_backend_details({}) is None + assert get_backend_details( + { + "api_version": "1.2.0", + "backend_version": "1.1.2", + "stac_version": "1.0.0", + "type": "Catalog", + "id": "cool-eo-cloud", + "endpoints": [ + {"path": "/collections", "methods": ["GET"]}, + ], + "federation": { + "eoa": {"title": "EO Answers", "url": "https://eoe.test/go"}, + "eob": {"title": "Beyond EO", "url": "https://eoeb.example.com"}, + }, + } + ) == { + "eoa": {"title": "EO Answers", "url": "https://eoe.test/go"}, + "eob": {"title": "Beyond EO", "url": "https://eoeb.example.com"}, + } + + +def test_get_federation_missing(): + assert get_federation_missing({}, resource_name="things") is None + assert get_federation_missing( + { + "things": ["apple", "banana"], + "federation:missing": ["veggies"], + }, + resource_name="things", + ) == ["veggies"] + + +@pytest.mark.parametrize(["auto_warn"], [[True], [False]]) +def test_get_federation_missing_auto_warn(auto_warn, caplog): + assert get_federation_missing( + { + "things": ["apple", "banana"], + "federation:missing": ["veggies"], + }, + resource_name="things", + auto_warn=auto_warn, + ) == ["veggies"] + + if auto_warn: + assert "Partial things: missing federation components: ['veggies']." in caplog.text + else: + assert caplog.text == "" diff --git a/tests/rest/models/test_general.py b/tests/rest/models/test_general.py new file mode 100644 index 000000000..1390218ee --- /dev/null +++ b/tests/rest/models/test_general.py @@ -0,0 +1,189 @@ +import pytest + +from openeo.rest.models.general import ( + CollectionListingResponse, + JobListingResponse, + Link, + LogsResponse, + ProcessListingResponse, +) +from openeo.rest.models.logs import LogEntry + + +class TestLink: + def test_basic(self): + link = Link(rel="about", href="https://example.com/about") + assert link.rel == "about" + assert link.href == "https://example.com/about" + assert link.title is None + assert link.type is None + + def test_full(self): + link = Link(rel="about", href="https://example.com/about", type="text/html", title="About example") + assert link.rel == "about" + assert link.href == "https://example.com/about" + assert link.title == "About example" + assert link.type == "text/html" + + def test_repr(self): + link = Link(rel="about", href="https://example.com/about") + assert repr(link) == "Link(rel='about', href='https://example.com/about', type=None, title=None)" + + +class TestCollectionListingResponse: + def test_basic(self): + data = {"collections": [{"id": "S2"}, {"id": "S3"}]} + collections = CollectionListingResponse(data) + assert collections == [{"id": "S2"}, {"id": "S3"}] + assert repr(collections) == "[{'id': 'S2'}, {'id': 'S3'}]" + + def test_links(self): + data = { + "collections": [{"id": "S2"}, {"id": "S3"}], + "links": [ + {"rel": "self", "href": "https://openeo.test/collections"}, + {"rel": "next", "href": "https://openeo.test/collections?page=2"}, + ], + } + collections = CollectionListingResponse(data) + assert collections.links == [ + Link(rel="self", href="https://openeo.test/collections"), + Link(rel="next", href="https://openeo.test/collections?page=2"), + ] + + @pytest.mark.parametrize( + ["data", "expected"], + [ + ( + {"collections": [{"id": "S2"}], "federation:missing": ["wwu"]}, + ["wwu"], + ), + ( + {"collections": [{"id": "S2"}]}, + None, + ), + ], + ) + def test_federation_missing(self, data, expected): + collections = CollectionListingResponse(data) + assert collections.ext_federation_missing() == expected + + +class TestProcessListingResponse: + def test_basic(self): + data = {"processes": [{"id": "ndvi"}, {"id": "s2mask"}]} + processes = ProcessListingResponse(data) + assert processes == [{"id": "ndvi"}, {"id": "s2mask"}] + assert repr(processes) == "[{'id': 'ndvi'}, {'id': 's2mask'}]" + + def test_links(self): + data = { + "processes": [{"id": "ndvi"}, {"id": "s2mask"}], + "links": [ + {"rel": "self", "href": "https://openeo.test/processes"}, + {"rel": "next", "href": "https://openeo.test/processes?page=2"}, + ], + } + processes = ProcessListingResponse(data) + assert processes.links == [ + Link(rel="self", href="https://openeo.test/processes"), + Link(rel="next", href="https://openeo.test/processes?page=2"), + ] + + @pytest.mark.parametrize( + ["data", "expected"], + [ + ( + {"processes": [{"id": "ndvi"}], "federation:missing": ["wow"]}, + ["wow"], + ), + ( + {"processes": [{"id": "ndvi"}]}, + None, + ), + ], + ) + def test_federation_missing(self, data, expected): + processes = ProcessListingResponse(data) + assert processes.ext_federation_missing() == expected + + +class TestJobListingResponse: + def test_basic(self): + data = {"jobs": [{"id": "job-01"}, {"id": "job-02"}]} + jobs = JobListingResponse(data) + assert jobs == [{"id": "job-01"}, {"id": "job-02"}] + assert repr(jobs) == "[{'id': 'job-01'}, {'id': 'job-02'}]" + + def test_links(self): + data = { + "jobs": [{"id": "job-01"}, {"id": "job-02"}], + "links": [ + {"rel": "self", "href": "https://openeo.test/jobs"}, + {"rel": "next", "href": "https://openeo.test/jobs?page=2"}, + ], + } + jobs = JobListingResponse(data) + assert jobs.links == [ + Link(rel="self", href="https://openeo.test/jobs"), + Link(rel="next", href="https://openeo.test/jobs?page=2"), + ] + + @pytest.mark.parametrize( + ["data", "expected"], + [ + ( + {"jobs": [{"id": "job-01"}], "federation:missing": ["wow"]}, + ["wow"], + ), + ( + {"jobs": [{"id": "job-01"}]}, + None, + ), + ], + ) + def test_federation_missing(self, data, expected): + jobs = JobListingResponse(data) + assert jobs.ext_federation_missing() == expected + + +class TestLogsResponse: + def test_basic(self): + data = {"logs": [{"id": "log-01", "level": "info", "message": "hello"}]} + logs = LogsResponse(data) + assert logs == [{"id": "log-01", "level": "info", "message": "hello"}] + assert logs == [LogEntry(id="log-01", level="info", message="hello")] + assert logs.logs == [{"id": "log-01", "level": "info", "message": "hello"}] + assert logs.logs == [LogEntry(id="log-01", level="info", message="hello")] + assert repr(logs) == "[{'id': 'log-01', 'level': 'info', 'message': 'hello'}]" + + def test_links(self): + data = { + "logs": [{"id": "log-01", "level": "info", "message": "hello"}], + "links": [ + {"rel": "self", "href": "https://openeo.test/logs"}, + {"rel": "next", "href": "https://openeo.test/logs?page=2"}, + ], + } + logs = LogsResponse(data) + assert logs.links == [ + Link(rel="self", href="https://openeo.test/logs"), + Link(rel="next", href="https://openeo.test/logs?page=2"), + ] + + @pytest.mark.parametrize( + ["data", "expected"], + [ + ( + {"logs": [{"id": "log-01", "level": "info", "message": "hello"}], "federation:missing": ["wow"]}, + ["wow"], + ), + ( + {"logs": [{"id": "log-01", "level": "info", "message": "hello"}]}, + None, + ), + ], + ) + def test_federation_missing(self, data, expected): + logs = LogsResponse(data) + assert logs.ext_federation_missing() == expected diff --git a/tests/api/test_logs.py b/tests/rest/models/test_logs.py similarity index 97% rename from tests/api/test_logs.py rename to tests/rest/models/test_logs.py index 5d228a42f..48312d38a 100644 --- a/tests/api/test_logs.py +++ b/tests/rest/models/test_logs.py @@ -2,7 +2,7 @@ import pytest -from openeo.api.logs import LogEntry, log_level_name, normalize_log_level +from openeo.rest.models.logs import LogEntry, log_level_name, normalize_log_level def test_log_entry_empty(): diff --git a/tests/rest/test_capabilities.py b/tests/rest/test_capabilities.py index bacd31b20..08c61a4d8 100644 --- a/tests/rest/test_capabilities.py +++ b/tests/rest/test_capabilities.py @@ -51,7 +51,7 @@ def test_list_plans(self): assert OpenEoCapabilities({"billing": {"plans": [{"name": "free"}]}}).list_plans() == [{"name": "free"}] def test_federation_absent(self): - assert OpenEoCapabilities({}).get_federation() is None + assert OpenEoCapabilities({}).ext_federation_backend_details() is None def test_federation_present(self): data = { @@ -62,7 +62,7 @@ def test_federation_present(self): }, } capabilities = OpenEoCapabilities(data) - assert capabilities.get_federation() == { + assert capabilities.ext_federation_backend_details() == { "a": {"url": "https://a.test/openeo/v2", "title": "A backend"}, "bb": {"url": "https://openeo.b.test/v9"}, } diff --git a/tests/rest/test_connection.py b/tests/rest/test_connection.py index ed60e9337..ea48e5bc4 100644 --- a/tests/rest/test_connection.py +++ b/tests/rest/test_connection.py @@ -11,7 +11,7 @@ from pathlib import Path import pytest -import requests.auth +import requests import requests_mock import shapely.geometry @@ -30,7 +30,7 @@ from openeo.rest._testing import DummyBackend, build_capabilities from openeo.rest.auth.auth import BearerAuth, NullAuth from openeo.rest.auth.oidc import OidcException -from openeo.rest.auth.testing import ABSENT, OidcMock +from openeo.rest.auth.testing import ABSENT, OidcMock, SimpleBasicAuthMocker from openeo.rest.connection import ( DEFAULT_TIMEOUT, DEFAULT_TIMEOUT_SYNCHRONOUS_EXECUTE, @@ -40,6 +40,7 @@ extract_connections, paginate, ) +from openeo.rest.models.general import Link from openeo.rest.vectorcube import VectorCube from openeo.testing.stac import StacDummyBuilder from openeo.util import ContextTimer, deep_get, dict_no_none @@ -563,10 +564,8 @@ def _get_capabilities_auth_dependent(request, context): return capabilities -def test_capabilities_caching_after_authenticate_basic(requests_mock): - user, pwd = "john262", "J0hndo3" +def test_capabilities_caching_after_authenticate_basic(requests_mock, basic_auth): get_capabilities_mock = requests_mock.get(API_URL, json=_get_capabilities_auth_dependent) - requests_mock.get(API_URL + 'credentials/basic', text=_credentials_basic_handler(user, pwd)) con = Connection(API_URL) assert con.capabilities().capabilities["endpoints"] == [ @@ -577,7 +576,7 @@ def test_capabilities_caching_after_authenticate_basic(requests_mock): con.capabilities() assert get_capabilities_mock.call_count == 1 - con.authenticate_basic(username=user, password=pwd) + con.authenticate_basic(username=basic_auth.username, password=basic_auth.password) assert get_capabilities_mock.call_count == 1 assert con.capabilities().capabilities["endpoints"] == [ {"methods": ["GET"], "path": "/credentials/basic"}, @@ -714,30 +713,17 @@ def test_api_error_non_json(requests_mock): assert exc.message == "olapola" -def _credentials_basic_handler(username, password, access_token="w3lc0m3"): - # TODO: better reuse of this helper - expected_auth = requests.auth._basic_auth_str(username=username, password=password) - - def handler(request, context): - assert request.headers["Authorization"] == expected_auth - return json.dumps({"access_token": access_token}) - - return handler - - -def test_create_connection_lazy_auth_config(requests_mock, api_version): - user, pwd = "john262", "J0hndo3" +def test_create_connection_lazy_auth_config(requests_mock, api_version, basic_auth): requests_mock.get(API_URL, json={"api_version": api_version, "endpoints": BASIC_ENDPOINTS}) - requests_mock.get(API_URL + 'credentials/basic', text=_credentials_basic_handler(user, pwd)) with mock.patch('openeo.rest.connection.AuthConfig') as AuthConfig: # Don't create default AuthConfig when not necessary conn = Connection(API_URL) assert AuthConfig.call_count == 0 - conn.authenticate_basic(user, pwd) + conn.authenticate_basic(basic_auth.username, basic_auth.password) assert AuthConfig.call_count == 0 # call `authenticate_basic` so that fallback AuthConfig is created/used lazily - AuthConfig.return_value.get_basic_auth.return_value = (user, pwd) + AuthConfig.return_value.get_basic_auth.return_value = (basic_auth.username, basic_auth.password) conn.authenticate_basic() assert AuthConfig.call_count == 1 conn.authenticate_basic() @@ -784,29 +770,25 @@ def test_authenticate_basic_no_support(requests_mock, api_version): assert isinstance(conn.auth, NullAuth) -def test_authenticate_basic(requests_mock, api_version): - user, pwd = "john262", "J0hndo3" +def test_authenticate_basic(requests_mock, api_version, basic_auth): requests_mock.get(API_URL, json={"api_version": api_version, "endpoints": BASIC_ENDPOINTS}) - requests_mock.get(API_URL + 'credentials/basic', text=_credentials_basic_handler(user, pwd)) conn = Connection(API_URL) assert isinstance(conn.auth, NullAuth) - conn.authenticate_basic(username=user, password=pwd) + conn.authenticate_basic(username=basic_auth.username, password=basic_auth.password) assert isinstance(conn.auth, BearerAuth) - assert conn.auth.bearer == "basic//w3lc0m3" + assert conn.auth.bearer == "basic//6cc3570k3n" -def test_authenticate_basic_from_config(requests_mock, api_version, auth_config): - user, pwd = "john281", "J0hndo3" +def test_authenticate_basic_from_config(requests_mock, api_version, auth_config, basic_auth): requests_mock.get(API_URL, json={"api_version": api_version, "endpoints": BASIC_ENDPOINTS}) - requests_mock.get(API_URL + 'credentials/basic', text=_credentials_basic_handler(user, pwd)) - auth_config.set_basic_auth(backend=API_URL, username=user, password=pwd) + auth_config.set_basic_auth(backend=API_URL, username=basic_auth.username, password=basic_auth.password) conn = Connection(API_URL) assert isinstance(conn.auth, NullAuth) conn.authenticate_basic() assert isinstance(conn.auth, BearerAuth) - assert conn.auth.bearer == "basic//w3lc0m3" + assert conn.auth.bearer == "basic//6cc3570k3n" @pytest.mark.slow @@ -3329,6 +3311,24 @@ def test_list_collections(requests_mock): assert con.list_collections() == collections +def test_list_collections_extra_metadata(requests_mock, caplog): + requests_mock.get(API_URL, json={"api_version": "1.0.0"}) + requests_mock.get( + API_URL + "collections", + json={ + "collections": [{"id": "S2"}, {"id": "NDVI"}], + "links": [{"rel": "next", "href": "https://oeo.test/collections?page=2"}], + "federation:missing": ["oeob"], + }, + ) + con = Connection(API_URL) + collections = con.list_collections() + assert collections == [{"id": "S2"}, {"id": "NDVI"}] + assert collections.links == [Link(rel="next", href="https://oeo.test/collections?page=2", type=None, title=None)] + assert collections.ext_federation_missing() == ["oeob"] + assert "Partial collection listing: missing federation components: ['oeob']." in caplog.text + + def test_describe_collection(requests_mock): requests_mock.get(API_URL, json={"api_version": "1.0.0"}) requests_mock.get( @@ -3389,6 +3389,24 @@ def test_list_processes_namespace(requests_mock): assert m.call_count == 1 +def test_list_processes_extra_metadata(requests_mock, caplog): + requests_mock.get(API_URL, json={"api_version": "1.0.0"}) + m = requests_mock.get( + API_URL + "processes", + json={ + "processes": [{"id": "add"}, {"id": "mask"}], + "links": [{"rel": "next", "href": "https://oeo.test/processes?page=2"}], + "federation:missing": ["oeob"], + }, + ) + conn = Connection(API_URL) + processes = conn.list_processes() + assert processes == [{"id": "add"}, {"id": "mask"}] + assert processes.links == [Link(rel="next", href="https://oeo.test/processes?page=2", type=None, title=None)] + assert processes.ext_federation_missing() == ["oeob"] + assert "Partial process listing: missing federation components: ['oeob']." in caplog.text + + def test_get_job(requests_mock): requests_mock.get(API_URL, json={"api_version": "1.0.0"}) conn = Connection(API_URL) @@ -3675,16 +3693,30 @@ def test_create_udp_unsupported(self, requests_mock, test_data): def test_list_udps(self, requests_mock, test_data): requests_mock.get(API_URL, json=build_capabilities(udp=True)) - conn = Connection(API_URL) - udp = test_data.load_json("1.0.0/udp_details.json") - requests_mock.get(API_URL + "process_graphs", json={"processes": [udp]}) + conn = Connection(API_URL) user_udps = conn.list_user_defined_processes() + assert user_udps == [udp] + + def test_list_udps_extra_metadata(self, requests_mock, test_data, caplog): + requests_mock.get(API_URL, json=build_capabilities(udp=True)) + requests_mock.get( + API_URL + "process_graphs", + json={ + "processes": [{"id": "myevi"}], + "links": [{"rel": "about", "href": "https://oeo.test/my-evi"}], + "federation:missing": ["oeob"], + }, + ) - assert len(user_udps) == 1 - assert user_udps[0] == udp + conn = Connection(API_URL) + udps = conn.list_user_defined_processes() + assert udps == [{"id": "myevi"}] + assert udps.links == [Link(rel="about", href="https://oeo.test/my-evi")] + assert udps.ext_federation_missing() == ["oeob"] + assert "Partial process listing: missing federation components: ['oeob']." in caplog.text def test_list_udps_unsupported(self, requests_mock): @@ -3915,9 +3947,10 @@ def test_connect_auto_auth_from_config_basic( """)) user, pwd = "john", "j0hn" for u, a in [(default, "Hell0!"), (other, "Wazz6!")]: - auth_config.set_basic_auth(backend=u, username=user, password=pwd) + basic_auth_mocker = SimpleBasicAuthMocker(username=user, password=pwd, access_token=a) + auth_config.set_basic_auth(backend=u, username=basic_auth_mocker.username, password=basic_auth_mocker.password) requests_mock.get(u, json={"api_version": "1.0.0", "endpoints": BASIC_ENDPOINTS}) - requests_mock.get(f"{u}/credentials/basic", text=_credentials_basic_handler(user, pwd, access_token=a)) + basic_auth_mocker.setup_credentials_basic_handler(api_root=u, requests_mock=requests_mock) if use_default: # Without arguments: use default diff --git a/tests/rest/test_job.py b/tests/rest/test_job.py index 4990d9180..7d1465471 100644 --- a/tests/rest/test_job.py +++ b/tests/rest/test_job.py @@ -1,7 +1,9 @@ +import itertools import json import logging import re from pathlib import Path +from typing import Optional from unittest import mock import pytest @@ -11,8 +13,8 @@ import openeo.rest.job from openeo.rest import JobFailedException, OpenEoApiPlainError, OpenEoClientException from openeo.rest.job import BatchJob, ResultAsset - -from .test_connection import _credentials_basic_handler +from openeo.rest.models.general import Link +from openeo.rest.models.logs import LogEntry API_URL = "https://oeo.test" @@ -308,109 +310,197 @@ def test_execute_batch_with_excessive_soft_errors(con100, requests_mock, tmpdir, ] -def test_get_job_logs(con100, requests_mock): - requests_mock.get(API_URL + "/jobs/f00ba5/logs", json={ - 'logs': [{ - 'id': "123abc", - 'level': 'error', - 'message': "error processing batch job" - }] - }) +class LogGenerator: + """Helper to generate log entry (dicts) with auto-generated ids, messages, etc.""" + + def __init__(self): + self._auto_id = itertools.count().__next__ + + def _auto_message(self, id: str, level: str) -> str: + greeting = {"debug": "Yo", "info": "Hello", "warning": "Beware", "error": "Halt!"}.get(level, "Greetings") + return f"{greeting} {id}" + + def log(self, message: Optional[str] = None, *, id: Optional[str] = None, level: str = "info") -> dict: + id = id or f"abc{self._auto_id():03d}" + message = message or self._auto_message(id=id, level=level) + return {"id": id, "level": level, "message": message} + + def debug(self, **kwargs) -> dict: + return self.log(level="debug", **kwargs) - log_entries = con100.job("f00ba5").logs(offset="123abc") + def info(self, **kwargs) -> dict: + return self.log(level="info", **kwargs) - assert log_entries[0].message == "error processing batch job" + def warning(self, **kwargs) -> dict: + return self.log(level="warning", **kwargs) + def error(self, **kwargs) -> dict: + return self.log(level="error", **kwargs) -def test_get_job_logs_returns_debug_loglevel_by_default(con100, requests_mock): + def __call__(self, **kwargs): + return self.log(**kwargs) + + +@pytest.fixture +def log_generator() -> LogGenerator: + return LogGenerator() + + +def test_get_job_logs_basic(con100, requests_mock, log_generator): requests_mock.get( API_URL + "/jobs/f00ba5/logs", json={ "logs": [ - { - "id": "123abc", - "level": "error", - "message": "error processing batch job", - }, - { - "id": "234abc", - "level": "debug", - "message": "Some debug info we want to filter out", - }, - { - "id": "345abc", - "level": "info", - "message": "Some general info we want to filter out", - }, - { - "id": "345abc", - "level": "warning", - "message": "Some warning we want to filter out", - }, + log_generator.info(message="Starting"), + log_generator.error(message="Nope!"), ] }, ) - log_entries = con100.job("f00ba5").logs() + logs = con100.job("f00ba5").logs(offset="TODO") + # Original interface + assert logs == [ + {"id": "abc000", "level": "info", "message": "Starting"}, + {"id": "abc001", "level": "error", "message": "Nope!"}, + ] + assert logs == [ + LogEntry(id="abc000", level="info", message="Starting"), + LogEntry(id="abc001", level="error", message="Nope!"), + ] + # Explicit property to get log entry listing + assert logs.logs == [ + LogEntry(id="abc000", level="info", message="Starting"), + LogEntry(id="abc001", level="error", message="Nope!"), + ] + + +def test_get_job_logs_extra_metadata(con100, requests_mock, log_generator): + requests_mock.get( + API_URL + "/jobs/f00ba5/logs", + json={ + "logs": [log_generator.info(message="Hello world")], + "links": [ + {"rel": "next", "href": "https://oeo.test/jobs/f00ba5/logs?offset=123abc"}, + ], + "federation:missing": ["eoeb"], + }, + ) + + logs = con100.job("f00ba5").logs() + assert logs.logs == [ + LogEntry(id="abc000", level="info", message="Hello world"), + ] + assert logs.links == [ + Link(rel="next", href="https://oeo.test/jobs/f00ba5/logs?offset=123abc"), + ] + assert logs.ext_federation_missing() == ["eoeb"] + - assert len(log_entries) == 4 - assert log_entries[0].level == "error" - assert log_entries[1].level == "debug" - assert log_entries[2].level == "info" - assert log_entries[3].level == "warning" +def test_get_job_logs_level_handling_default(con100, requests_mock, log_generator): + requests_mock.get( + API_URL + "/jobs/f00ba5/logs", + json={ + "logs": [ + log_generator.error(), + log_generator.debug(), + log_generator.info(), + log_generator.warning(), + ] + }, + ) + logs = con100.job("f00ba5").logs() + assert logs == [ + {"id": "abc000", "level": "error", "message": "Halt! abc000"}, + {"id": "abc001", "level": "debug", "message": "Yo abc001"}, + {"id": "abc002", "level": "info", "message": "Hello abc002"}, + {"id": "abc003", "level": "warning", "message": "Beware abc003"}, + ] @pytest.mark.parametrize( - ["log_level", "exp_num_messages"], + ["levels", "expected"], [ - (None, 4), # Default is DEBUG / show all log levels. - (logging.ERROR, 1), - ("error", 1), - ("ERROR", 1), - (logging.WARNING, 2), - ("warning", 2), - ("WARNING", 2), - (logging.INFO, 3), - ("INFO", 3), - ("info", 3), - (logging.DEBUG, 4), - ("DEBUG", 4), - ("debug", 4), + ([logging.ERROR, "error", "ERROR"], [{"id": "abc005", "level": "error", "message": "Halt! abc005"}]), + ( + [logging.WARNING, "warning", "WARNING"], + [ + {"id": "abc004", "level": "warning", "message": "Beware abc004"}, + {"id": "abc005", "level": "error", "message": "Halt! abc005"}, + ], + ), + ( + [logging.INFO, "INFO", "info"], + [ + {"id": "abc000", "level": "info", "message": "Hello abc000"}, + {"id": "abc003", "level": "info", "message": "Hello abc003"}, + {"id": "abc004", "level": "warning", "message": "Beware abc004"}, + {"id": "abc005", "level": "error", "message": "Halt! abc005"}, + ], + ), + ( + [logging.DEBUG, "DEBUG", "debug", None, 0], + [ + {"id": "abc000", "level": "info", "message": "Hello abc000"}, + {"id": "abc001", "level": "debug", "message": "Yo abc001"}, + {"id": "abc002", "level": "weird", "message": "Greetings abc002"}, + {"id": "abc003", "level": "info", "message": "Hello abc003"}, + {"id": "abc004", "level": "warning", "message": "Beware abc004"}, + {"id": "abc005", "level": "error", "message": "Halt! abc005"}, + ], + ), ], ) -def test_get_job_logs_keeps_loglevel_that_is_higher_or_equal( - con100, requests_mock, log_level, exp_num_messages -): +def test_get_job_logs_level_handling_custom(con100, requests_mock, log_generator, levels, expected): requests_mock.get( API_URL + "/jobs/f00ba5/logs", json={ "logs": [ - { - "id": "123abc", - "level": "error", - "message": "error processing batch job", - }, - { - "id": "234abc", - "level": "debug", - "message": "Some debug info we want to filter out", - }, - { - "id": "345abc", - "level": "info", - "message": "Some general info we want to filter out", - }, - { - "id": "345abc", - "level": "warning", - "message": "Some warning we want to filter out", - }, + log_generator.info(), + log_generator.debug(), + log_generator.log(level="weird"), + log_generator.info(), + log_generator.warning(), + log_generator.error(), ] }, ) - log_entries = con100.job("f00ba5").logs(level=log_level) - assert len(log_entries) == exp_num_messages + for level in levels: + logs = con100.job("f00ba5").logs(level=level) + assert logs == expected + + +@pytest.mark.parametrize( + ["response_extra", "expected"], + [ + ( + {}, + [{"id": "abc001", "level": "error", "message": "Halt! abc001"}], + ), + ( + {"level": "warning"}, + [ + {"id": "abc000", "level": "info", "message": "Not a warning"}, + {"id": "abc001", "level": "error", "message": "Halt! abc001"}, + ], + ), + ], +) +def test_get_job_logs_level_handling_custom_with_backend_level( + con100, requests_mock, log_generator, response_extra, expected +): + """If backend response includes a "level": trust it (no client-side filtering).""" + requests_mock.get( + API_URL + "/jobs/f00ba5/logs", + json={ + "logs": [ + log_generator.info(message="Not a warning"), + log_generator.error(), + ], + **response_extra, + }, + ) + assert con100.job("f00ba5").logs(level="warning") == expected def test_create_job_100(con100, requests_mock): @@ -718,6 +808,7 @@ def download_tiff(request, context): assert f.read() == TIFF_CONTENT + @pytest.mark.parametrize( ["list_jobs_kwargs", "expected_qs"], [ @@ -725,19 +816,10 @@ def download_tiff(request, context): ({"limit": 123}, {"limit": ["123"]}), ], ) -def test_list_jobs(con100, requests_mock, list_jobs_kwargs, expected_qs): - username = "john" - password = "j0hn!" - access_token = "6cc35!" - requests_mock.get( - API_URL + "/credentials/basic", - text=_credentials_basic_handler( - username=username, password=password, access_token=access_token - ), - ) +def test_list_jobs(con100, requests_mock, list_jobs_kwargs, expected_qs, basic_auth): def get_jobs(request, context): - assert request.headers["Authorization"] == f"Bearer basic//{access_token}" + assert request.headers["Authorization"] == f"Bearer basic//{basic_auth.access_token}" assert request.qs == expected_qs return { "jobs": [ @@ -756,9 +838,45 @@ def get_jobs(request, context): requests_mock.get(API_URL + "/jobs", json=get_jobs) - con100.authenticate_basic(username, password) + con100.authenticate_basic(basic_auth.username, basic_auth.password) jobs = con100.list_jobs(**list_jobs_kwargs) assert jobs == [ {"id": "job123", "status": "running", "created": "2021-02-22T09:00:00Z"}, {"id": "job456", "status": "created", "created": "2021-03-22T10:00:00Z"}, ] + + +def test_list_jobs_extra_metadata(con100, requests_mock, caplog, basic_auth): + + def get_jobs(request, context): + assert request.headers["Authorization"] == f"Bearer basic//{basic_auth.access_token}" + return { + "jobs": [ + { + "id": "job123", + "status": "running", + "created": "2021-02-22T09:00:00Z", + }, + { + "id": "job456", + "status": "created", + "created": "2021-03-22T10:00:00Z", + }, + ], + "links": [ + {"rel": "next", "href": API_URL + "/jobs?limit=2&offset=2"}, + ], + "federation:missing": ["oeob"], + } + + requests_mock.get(API_URL + "/jobs", json=get_jobs) + + con100.authenticate_basic(basic_auth.username, basic_auth.password) + jobs = con100.list_jobs() + assert jobs == [ + {"id": "job123", "status": "running", "created": "2021-02-22T09:00:00Z"}, + {"id": "job456", "status": "created", "created": "2021-03-22T10:00:00Z"}, + ] + assert jobs.links == [Link(rel="next", href="https://oeo.test/jobs?limit=2&offset=2")] + assert jobs.ext_federation_missing() == ["oeob"] + assert "Partial job listing: missing federation components: ['oeob']." in caplog.text