Datetime units (#782)

* ENH: support units for numpy datetime types

* MNT: removed unnecessary imports

* ENH: add default value for units

* ENH: update BuiltinDtype in pydantic

* ENH: update BuiltinDtype in pydantic

* ENH: add default value for units

* TST: datetime dtypes in test_array

* MNT: Update changelog

* FIX: typo in comment

* MNT: fix changelog

* FIX: default value of units to empty string.

* FIX: use None as the sentinel for the units kwarg

* ENH: use np.datetime_data to extract units

* TST: Fix failing authorization test -- empty password

* MNT: format and lint

CHANGELOG.md

@@ -3,6 +3,12 @@ Write the date in place of the "Unreleased" in the case a new version is release
 
 # Changelog
 
+## Unreleased
+
+### Added
+
+- Added support for explicit units in numpy datetime64 dtypes.
+
 ## v0.1.0b8 (2024-09-06)
 
 ### Fixed

tiled/_tests/test_array.py

@@ -23,11 +23,9 @@
     "uint64": numpy.arange(10, dtype="uint64"),
     "f": numpy.arange(10, dtype="f"),
     "c": (numpy.arange(10) * 1j).astype("c"),
-    # "m": (
-    #     numpy.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64') -
-    #     numpy.datetime64('2008-01-01'),
-    # )
-    # "M": numpy.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64'),
+    "m": numpy.array(["2007-07-13", "2006-01-13", "2010-08-13"], dtype="datetime64[D]")
+    - numpy.datetime64("2008-01-01"),
+    "M": numpy.array(["2007-07-13", "2006-01-13", "2010-08-13"], dtype="datetime64[D]"),
     "S": numpy.array([letter * 3 for letter in string.ascii_letters], dtype="S3"),
     "U": numpy.array([letter * 3 for letter in string.ascii_letters], dtype="U3"),
 }

tiled/_tests/test_authentication.py

@@ -7,11 +7,7 @@
 
 import numpy
 import pytest
-from starlette.status import (
-    HTTP_400_BAD_REQUEST,
-    HTTP_401_UNAUTHORIZED,
-    HTTP_422_UNPROCESSABLE_ENTITY,
-)
+from starlette.status import HTTP_400_BAD_REQUEST, HTTP_401_UNAUTHORIZED
 
 from ..adapters.array import ArrayAdapter
 from ..adapters.mapping import MapAdapter
@@ -93,7 +89,7 @@ def test_password_auth(enter_password, config):
                 from_context(context, username="alice")
 
         # Empty password should not work.
-        with fail_with_status_code(HTTP_422_UNPROCESSABLE_ENTITY):
+        with fail_with_status_code(HTTP_401_UNAUTHORIZED):
             with enter_password(""):
                 from_context(context, username="alice")
 

tiled/server/pydantic_array.py

@@ -27,6 +27,7 @@ class BuiltinDtype(BaseModel):
     endianness: Endianness
     kind: Kind
     itemsize: int
+    dt_units: Optional[str] = None
 
     __endianness_map = {
         ">": "big",
@@ -39,10 +40,18 @@ class BuiltinDtype(BaseModel):
 
     @classmethod
     def from_numpy_dtype(cls, dtype) -> "BuiltinDtype":
+        # Extract datetime units from the dtype string representation,
+        # e.g. `'<M8[ns]'` has `dt_units = '[ns]'`. Count determines the number of base units in a step.
+        dt_units = None
+        if dtype.kind in ("m", "M"):
+            unit, count = numpy.datetime_data(dtype)
+            dt_units = f"[{count if count > 1 else ''}{unit}]"
+
         return cls(
             endianness=cls.__endianness_map[dtype.byteorder],
             kind=Kind(dtype.kind),
             itemsize=dtype.itemsize,
+            dt_units=dt_units,
         )
 
     def to_numpy_dtype(self):
@@ -60,14 +69,15 @@ def to_numpy_str(self):
         # so the reported itemsize is 4x the char count.  To get back to the string
         # we need to divide by 4.
         size = self.itemsize if self.kind != Kind.unicode else self.itemsize // 4
-        return f"{endianness}{self.kind.value}{size}"
+        return f"{endianness}{self.kind.value}{size}{self.dt_units or ''}"
 
     @classmethod
     def from_json(cls, structure):
         return cls(
             kind=Kind(structure["kind"]),
             itemsize=structure["itemsize"],
             endianness=Endianness(structure["endianness"]),
+            units=structure.get("dt_units"),
         )
 
 

tiled/structures/array.py

@@ -76,6 +76,7 @@ class BuiltinDtype:
     endianness: Endianness
     kind: Kind
     itemsize: int
+    dt_units: Optional[str] = None
 
     __endianness_map = {
         ">": "big",
@@ -88,15 +89,21 @@ class BuiltinDtype:
 
     @classmethod
     def from_numpy_dtype(cls, dtype) -> "BuiltinDtype":
+        # Extract datetime units from the dtype string representation,
+        # e.g. `'<M8[ns]'` has `dt_units = '[ns]'`. Count determines the number of base units in a step.
+        dt_units = None
+        if dtype.kind in ("m", "M"):
+            unit, count = numpy.datetime_data(dtype)
+            dt_units = f"[{count if count > 1 else ''}{unit}]"
+
         return cls(
             endianness=cls.__endianness_map[dtype.byteorder],
             kind=Kind(dtype.kind),
             itemsize=dtype.itemsize,
+            dt_units=dt_units,
         )
 
     def to_numpy_dtype(self) -> numpy.dtype:
-        import numpy
-
         return numpy.dtype(self.to_numpy_str())
 
     def to_numpy_str(self):
@@ -111,14 +118,15 @@ def to_numpy_str(self):
         # so the reported itemsize is 4x the char count.  To get back to the string
         # we need to divide by 4.
         size = self.itemsize if self.kind != Kind.unicode else self.itemsize // 4
-        return f"{endianness}{self.kind.value}{size}"
+        return f"{endianness}{self.kind.value}{size}{self.dt_units or ''}"
 
     @classmethod
     def from_json(cls, structure):
         return cls(
             kind=Kind(structure["kind"]),
             itemsize=structure["itemsize"],
             endianness=Endianness(structure["endianness"]),
+            dt_units=structure.get("dt_units"),
         )
 
 
@@ -130,8 +138,6 @@ class Field:
 
     @classmethod
     def from_numpy_descr(cls, field):
-        import numpy
-
         name, *rest = field
         if name == "":
             raise ValueError(
@@ -189,8 +195,6 @@ def from_numpy_dtype(cls, dtype):
         )
 
     def to_numpy_dtype(self):
-        import numpy
-
         return numpy.dtype(self.to_numpy_descr())
 
     def to_numpy_descr(self):
@@ -241,8 +245,6 @@ def from_array(cls, array, shape=None, chunks=None, dims=None) -> "ArrayStructur
 
         if not hasattr(array, "__array__"):
             # may be a list of something; convert to array
-            import numpy
-
             array = numpy.asanyarray(array)
 
         # Why would shape ever be different from array.shape, you ask?