Release timezone and calendar features in DatapointsAPI (beta)

.github/workflows/build.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
         with:
-          extras: '-E numpy'
+          extras: '-E pandas'
       - name: Linting and static code checks
         run: pre-commit run --all-files
 

CHANGELOG.md

@@ -17,13 +17,21 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [7.45.0] - 2024-05-28
+### Added
+- DatapointsAPI now support `timezone` and new calendar-based granularities like `month`, `quarter` and `year`.
+  These API features are in beta, and the SDK implementation in alpha, meaning breaking changes can
+  occur without warning. Set beta header to avoid warning. Users of `retrieve_dataframe_in_tz` should
+  consider preparing to upgrade as soon as the features reach general availability (GA).
+
 ## [7.44.1] - 2024-05-24
 ### Added
-- Missing parameter `timeout` to `client.transformations.preview`. 
+- Missing parameter `timeout` to `client.transformations.preview`.
 
 ## [7.44.0] - 2024-05-24
 ### Added
-- New utility function `datetime_to_ms_iso_timestamp` in `cognite.client.utils` to convert a datetime object to a string representing a timestamp in the format expected by the Cognite GraphQL API.
+- New utility function `datetime_to_ms_iso_timestamp` in `cognite.client.utils` to convert a datetime object
+  to a string representing a timestamp in the format expected by the Cognite GraphQL API.
 
 ## [7.43.6] - 2024-05-27
 ### Improved

cognite/client/_version.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "7.44.1"
+__version__ = "7.45.0"
 __api_subversion__ = "20230101"

cognite/client/utils/__init__.py

@@ -1,24 +1,15 @@
 from __future__ import annotations
 
-import contextlib
-import sys
-
 from cognite.client.utils._time import (
     MAX_TIMESTAMP_MS,
     MIN_TIMESTAMP_MS,
+    ZoneInfo,
     datetime_to_ms,
     datetime_to_ms_iso_timestamp,
     ms_to_datetime,
     timestamp_to_ms,
 )
 
-# Needed for doctest to pass.
-with contextlib.suppress(ImportError):
-    if sys.version_info >= (3, 9):
-        from zoneinfo import ZoneInfo
-    else:
-        from backports.zoneinfo import ZoneInfo
-
 __all__ = [
     "ZoneInfo",
     "MAX_TIMESTAMP_MS",

cognite/client/utils/_importing.py

@@ -7,6 +7,7 @@
 if TYPE_CHECKING:
     from concurrent.futures import Future
 
+
 _T = TypeVar("_T")
 
 

cognite/client/utils/_pandas_helpers.py

@@ -2,19 +2,21 @@
 
 import re
 import warnings
+from datetime import timezone
 from inspect import signature
 from itertools import chain
 from numbers import Integral
-from typing import TYPE_CHECKING, Any, Sequence
+from typing import TYPE_CHECKING, Any, Literal, Sequence
 
 from cognite.client.exceptions import CogniteImportError
 from cognite.client.utils._importing import local_import
 from cognite.client.utils._text import to_camel_case
-from cognite.client.utils._time import TIME_ATTRIBUTES
+from cognite.client.utils._time import TIME_ATTRIBUTES, ZoneInfo
 
 if TYPE_CHECKING:
     import pandas as pd
 
+    from cognite.client.data_classes import DatapointsArrayList, DatapointsList
     from cognite.client.data_classes._base import T_CogniteResource, T_CogniteResourceList
 
 
@@ -43,6 +45,48 @@ def pandas_major_version() -> int:
     return int(__version__.split(".")[0])
 
 
+def convert_tz_for_pandas(tz: str | timezone | ZoneInfo | None) -> str | timezone | None:
+    if tz is None or isinstance(tz, (str, timezone)):
+        return tz
+    if isinstance(tz, ZoneInfo):
+        # pandas is not happy about ZoneInfo :shrug:
+        if tz.key is not None:
+            return tz.key
+        raise ValueError("timezone of type ZoneInfo does not have the required 'key' attribute set")
+    raise ValueError(f"'timezone' not understood, expected one of: [None, datetime.timezone, ZoneInfo], got {type(tz)}")
+
+
+def concat_dps_dataframe_list(
+    dps_lst: DatapointsList | DatapointsArrayList,
+    column_names: Literal["id", "external_id"],
+    include_aggregate_name: bool,
+    include_granularity_name: bool,
+    include_status: bool,
+) -> pd.DataFrame:
+    pd = local_import("pandas")
+    dfs = [
+        dps.to_pandas(
+            column_names=column_names,
+            include_aggregate_name=include_aggregate_name,
+            include_granularity_name=include_granularity_name,
+            include_status=include_status,
+        )
+        for dps in dps_lst
+    ]
+    if not dfs:
+        return pd.DataFrame(index=pd.to_datetime([]))
+    timezones = set(dps.timezone for dps in dps_lst) - {None}
+    # If attempting to join naive & aware, pandas will raise (so we don't need to):
+    # TypeError: Cannot join tz-naive with tz-aware DatetimeIndex
+    if len(timezones) > 1:
+        warnings.warn(
+            f"When concatenating datapoints localized to different timezones ({sorted(map(str, timezones))}), the "
+            "final dataframe index (timestamps) will be a union of the UTC converted timestamps.",
+            UserWarning,
+        )
+    return concat_dataframes_with_nullable_int_cols(dfs)
+
+
 def notebook_display_with_fallback(inst: T_CogniteResource | T_CogniteResourceList, **kwargs: Any) -> str:
     if "camel_case" in signature(inst.to_pandas).parameters:
         # Default of False enforced (when accepted by method):