From 35cf1a85cd77ac6ad974cc9b8311bbbd94bb9ca1 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Thu, 5 Sep 2024 15:06:28 -0400 Subject: [PATCH 01/15] ENH: support units for numpy datetime types --- tiled/structures/array.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tiled/structures/array.py b/tiled/structures/array.py index 9581763b1..73ca3cda0 100644 --- a/tiled/structures/array.py +++ b/tiled/structures/array.py @@ -1,11 +1,10 @@ import enum import os import sys +import re from dataclasses import dataclass from typing import List, Optional, Tuple, Union -import numpy - class Endianness(str, enum.Enum): """ @@ -76,6 +75,7 @@ class BuiltinDtype: endianness: Endianness kind: Kind itemsize: int + units: str __endianness_map = { ">": "big", @@ -88,13 +88,22 @@ class BuiltinDtype: @classmethod def from_numpy_dtype(cls, dtype) -> "BuiltinDtype": + + # Extract datetime units from the dtype string representation, + # e.g. `' numpy.dtype: + def to_numpy_dtype(self): import numpy return numpy.dtype(self.to_numpy_str()) @@ -111,7 +120,8 @@ def to_numpy_str(self): # so the reported itemsize is 4x the char count. To get back to the string # we need to divide by 4. size = self.itemsize if self.kind != Kind.unicode else self.itemsize // 4 - return f"{endianness}{self.kind.value}{size}" + units = f"[{self.units}]" if self.units else '' + return f"{endianness}{self.kind.value}{size}{units}" @classmethod def from_json(cls, structure): @@ -119,6 +129,7 @@ def from_json(cls, structure): kind=Kind(structure["kind"]), itemsize=structure["itemsize"], endianness=Endianness(structure["endianness"]), + units=structure.get('units', '') ) From a31bd9436ae8afe932c25feb5918115c9222c8bf Mon Sep 17 00:00:00 2001 From: Eugene M Date: Thu, 5 Sep 2024 15:10:20 -0400 Subject: [PATCH 02/15] MNT: removed unnecessary imports --- tiled/structures/array.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/tiled/structures/array.py b/tiled/structures/array.py index 73ca3cda0..8453111dc 100644 --- a/tiled/structures/array.py +++ b/tiled/structures/array.py @@ -1,10 +1,12 @@ import enum import os -import sys import re +import sys from dataclasses import dataclass from typing import List, Optional, Tuple, Union +import numpy + class Endianness(str, enum.Enum): """ @@ -88,24 +90,21 @@ class BuiltinDtype: @classmethod def from_numpy_dtype(cls, dtype) -> "BuiltinDtype": - # Extract datetime units from the dtype string representation, # e.g. `' numpy.dtype: return numpy.dtype(self.to_numpy_str()) def to_numpy_str(self): @@ -120,7 +119,7 @@ def to_numpy_str(self): # so the reported itemsize is 4x the char count. To get back to the string # we need to divide by 4. size = self.itemsize if self.kind != Kind.unicode else self.itemsize // 4 - units = f"[{self.units}]" if self.units else '' + units = f"[{self.units}]" if self.units else "" return f"{endianness}{self.kind.value}{size}{units}" @classmethod @@ -129,7 +128,7 @@ def from_json(cls, structure): kind=Kind(structure["kind"]), itemsize=structure["itemsize"], endianness=Endianness(structure["endianness"]), - units=structure.get('units', '') + units=structure.get("units", ""), ) @@ -141,8 +140,6 @@ class Field: @classmethod def from_numpy_descr(cls, field): - import numpy - name, *rest = field if name == "": raise ValueError( @@ -200,8 +197,6 @@ def from_numpy_dtype(cls, dtype): ) def to_numpy_dtype(self): - import numpy - return numpy.dtype(self.to_numpy_descr()) def to_numpy_descr(self): @@ -252,8 +247,6 @@ def from_array(cls, array, shape=None, chunks=None, dims=None) -> "ArrayStructur if not hasattr(array, "__array__"): # may be a list of something; convert to array - import numpy - array = numpy.asanyarray(array) # Why would shape ever be different from array.shape, you ask? From b36c6b451a6226bb9ba61e5af8000826ac0d881e Mon Sep 17 00:00:00 2001 From: Eugene M Date: Thu, 5 Sep 2024 17:19:43 -0400 Subject: [PATCH 03/15] ENH: add default value for units --- tiled/structures/array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiled/structures/array.py b/tiled/structures/array.py index 8453111dc..cf1736900 100644 --- a/tiled/structures/array.py +++ b/tiled/structures/array.py @@ -77,7 +77,7 @@ class BuiltinDtype: endianness: Endianness kind: Kind itemsize: int - units: str + units: str = '' __endianness_map = { ">": "big", From 260e83ee71ba006001e7b5f755997aa0cc6da655 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Thu, 5 Sep 2024 17:20:25 -0400 Subject: [PATCH 04/15] ENH: update BuiltinDtype in pydantic --- tiled/server/pydantic_array.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tiled/server/pydantic_array.py b/tiled/server/pydantic_array.py index 6bc2090e8..661c1543c 100644 --- a/tiled/server/pydantic_array.py +++ b/tiled/server/pydantic_array.py @@ -14,6 +14,7 @@ it. """ +import re import sys from typing import List, Optional, Tuple, Union @@ -27,6 +28,7 @@ class BuiltinDtype(BaseModel): endianness: Endianness kind: Kind itemsize: int + units: str = "" __endianness_map = { ">": "big", @@ -39,10 +41,18 @@ class BuiltinDtype(BaseModel): @classmethod def from_numpy_dtype(cls, dtype) -> "BuiltinDtype": + # Extract datetime units from the dtype string representation, + # e.g. `' Date: Thu, 5 Sep 2024 17:21:22 -0400 Subject: [PATCH 05/15] ENH: update BuiltinDtype in pydantic --- tiled/server/pydantic_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiled/server/pydantic_array.py b/tiled/server/pydantic_array.py index 661c1543c..08a258bf2 100644 --- a/tiled/server/pydantic_array.py +++ b/tiled/server/pydantic_array.py @@ -79,7 +79,7 @@ def from_json(cls, structure): kind=Kind(structure["kind"]), itemsize=structure["itemsize"], endianness=Endianness(structure["endianness"]), - units=structure.get('units', 's') + units=structure.get("units", "s"), ) From 4ffa2bfdf01c92739560d6e48d373f85a9262a33 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Thu, 5 Sep 2024 17:21:28 -0400 Subject: [PATCH 06/15] ENH: add default value for units --- tiled/structures/array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiled/structures/array.py b/tiled/structures/array.py index cf1736900..ae606af75 100644 --- a/tiled/structures/array.py +++ b/tiled/structures/array.py @@ -77,7 +77,7 @@ class BuiltinDtype: endianness: Endianness kind: Kind itemsize: int - units: str = '' + units: str = "" __endianness_map = { ">": "big", From 917319d2eb28a5019cb878d8c5197aaa47ec2abd Mon Sep 17 00:00:00 2001 From: Eugene M Date: Thu, 5 Sep 2024 17:21:45 -0400 Subject: [PATCH 07/15] TST: datetime dtypes in test_array --- tiled/_tests/test_array.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tiled/_tests/test_array.py b/tiled/_tests/test_array.py index 6cb0d21e1..ed1a55463 100644 --- a/tiled/_tests/test_array.py +++ b/tiled/_tests/test_array.py @@ -23,11 +23,9 @@ "uint64": numpy.arange(10, dtype="uint64"), "f": numpy.arange(10, dtype="f"), "c": (numpy.arange(10) * 1j).astype("c"), - # "m": ( - # numpy.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64') - - # numpy.datetime64('2008-01-01'), - # ) - # "M": numpy.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64'), + "m": numpy.array(["2007-07-13", "2006-01-13", "2010-08-13"], dtype="datetime64[D]") + - numpy.datetime64("2008-01-01"), + "M": numpy.array(["2007-07-13", "2006-01-13", "2010-08-13"], dtype="datetime64[D]"), "S": numpy.array([letter * 3 for letter in string.ascii_letters], dtype="S3"), "U": numpy.array([letter * 3 for letter in string.ascii_letters], dtype="U3"), } From 91f0f98dc4a51b9514c143508616144d242356d9 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Thu, 5 Sep 2024 17:24:25 -0400 Subject: [PATCH 08/15] MNT: Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e68e29f4..926b39f96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Write the date in place of the "Unreleased" in the case a new version is release ### Added - Add method to `TableAdapter` which accepts a Python dictionary. - Added an `Arrow` adapter which supports reading/writing arrow tables via `RecordBatchFileReader`/`RecordBatchFileWriter`. +- Added support for explicit units in numpy datetime64 dtypes. ### Changed - Make `tiled.client` accept a Python dictionary when fed to `write_dataframe()`. From e68e0e59e5a183fa084e5f5f42dc6088144f6d94 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Fri, 6 Sep 2024 10:34:37 -0400 Subject: [PATCH 09/15] FIX: typo in comment --- tiled/server/pydantic_array.py | 2 +- tiled/structures/array.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tiled/server/pydantic_array.py b/tiled/server/pydantic_array.py index 08a258bf2..a85f54e4e 100644 --- a/tiled/server/pydantic_array.py +++ b/tiled/server/pydantic_array.py @@ -42,7 +42,7 @@ class BuiltinDtype(BaseModel): @classmethod def from_numpy_dtype(cls, dtype) -> "BuiltinDtype": # Extract datetime units from the dtype string representation, - # e.g. `' "BuiltinDtype": # Extract datetime units from the dtype string representation, - # e.g. `' Date: Fri, 6 Sep 2024 11:42:51 -0400 Subject: [PATCH 10/15] MNT: fix changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 912a5acaa..347761673 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ Write the date in place of the "Unreleased" in the case a new version is release # Changelog +## Unreleased + +### Added + +- Added support for explicit units in numpy datetime64 dtypes. + ## v0.1.0b8 (2024-09-06) ### Fixed @@ -15,7 +21,6 @@ Write the date in place of the "Unreleased" in the case a new version is release - Add method to `TableAdapter` which accepts a Python dictionary. - Added an `Arrow` adapter which supports reading/writing arrow tables via `RecordBatchFileReader`/`RecordBatchFileWriter`. -- Added support for explicit units in numpy datetime64 dtypes. ### Changed From 484c6a0ef9871bbbe9ced2a151bc289ab1df031f Mon Sep 17 00:00:00 2001 From: Eugene Date: Tue, 10 Sep 2024 14:26:53 -0400 Subject: [PATCH 11/15] FIX: default value of units to empty string. --- tiled/server/pydantic_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiled/server/pydantic_array.py b/tiled/server/pydantic_array.py index a85f54e4e..952a5c8ba 100644 --- a/tiled/server/pydantic_array.py +++ b/tiled/server/pydantic_array.py @@ -79,7 +79,7 @@ def from_json(cls, structure): kind=Kind(structure["kind"]), itemsize=structure["itemsize"], endianness=Endianness(structure["endianness"]), - units=structure.get("units", "s"), + units=structure.get("units", ""), ) From c9fa03e34212bb8f3528aa1e937ac72a558089aa Mon Sep 17 00:00:00 2001 From: Eugene M Date: Tue, 10 Sep 2024 20:29:05 -0400 Subject: [PATCH 12/15] FIX: use None as the sentinel for the units kwarg --- tiled/server/pydantic_array.py | 6 +++--- tiled/structures/array.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tiled/server/pydantic_array.py b/tiled/server/pydantic_array.py index 952a5c8ba..4e466f7fb 100644 --- a/tiled/server/pydantic_array.py +++ b/tiled/server/pydantic_array.py @@ -28,7 +28,7 @@ class BuiltinDtype(BaseModel): endianness: Endianness kind: Kind itemsize: int - units: str = "" + units: Optional[str] = None __endianness_map = { ">": "big", @@ -43,7 +43,7 @@ class BuiltinDtype(BaseModel): def from_numpy_dtype(cls, dtype) -> "BuiltinDtype": # Extract datetime units from the dtype string representation, # e.g. `'": "big", @@ -92,7 +92,7 @@ class BuiltinDtype: def from_numpy_dtype(cls, dtype) -> "BuiltinDtype": # Extract datetime units from the dtype string representation, # e.g. `' Date: Wed, 11 Sep 2024 16:45:31 -0400 Subject: [PATCH 13/15] ENH: use np.datetime_data to extract units --- tiled/server/pydantic_array.py | 18 ++++++++---------- tiled/structures/array.py | 18 ++++++++---------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/tiled/server/pydantic_array.py b/tiled/server/pydantic_array.py index 4e466f7fb..2c37a51dc 100644 --- a/tiled/server/pydantic_array.py +++ b/tiled/server/pydantic_array.py @@ -14,7 +14,6 @@ it. """ -import re import sys from typing import List, Optional, Tuple, Union @@ -28,7 +27,7 @@ class BuiltinDtype(BaseModel): endianness: Endianness kind: Kind itemsize: int - units: Optional[str] = None + dt_units: Optional[str] = None __endianness_map = { ">": "big", @@ -42,17 +41,17 @@ class BuiltinDtype(BaseModel): @classmethod def from_numpy_dtype(cls, dtype) -> "BuiltinDtype": # Extract datetime units from the dtype string representation, - # e.g. `' 1 else ''}{unit}]" return cls( endianness=cls.__endianness_map[dtype.byteorder], kind=Kind(dtype.kind), itemsize=dtype.itemsize, - units=units, + dt_units=dt_units, ) def to_numpy_dtype(self): @@ -70,8 +69,7 @@ def to_numpy_str(self): # so the reported itemsize is 4x the char count. To get back to the string # we need to divide by 4. size = self.itemsize if self.kind != Kind.unicode else self.itemsize // 4 - units = f"[{self.units}]" if self.units else "" - return f"{endianness}{self.kind.value}{size}{units}" + return f"{endianness}{self.kind.value}{size}{self.dt_units or ''}" @classmethod def from_json(cls, structure): @@ -79,7 +77,7 @@ def from_json(cls, structure): kind=Kind(structure["kind"]), itemsize=structure["itemsize"], endianness=Endianness(structure["endianness"]), - units=structure.get("units"), + units=structure.get("dt_units"), ) diff --git a/tiled/structures/array.py b/tiled/structures/array.py index e8e06e491..23b625d51 100644 --- a/tiled/structures/array.py +++ b/tiled/structures/array.py @@ -1,6 +1,5 @@ import enum import os -import re import sys from dataclasses import dataclass from typing import List, Optional, Tuple, Union @@ -77,7 +76,7 @@ class BuiltinDtype: endianness: Endianness kind: Kind itemsize: int - units: Optional[str] = None + dt_units: Optional[str] = None __endianness_map = { ">": "big", @@ -91,17 +90,17 @@ class BuiltinDtype: @classmethod def from_numpy_dtype(cls, dtype) -> "BuiltinDtype": # Extract datetime units from the dtype string representation, - # e.g. `' 1 else ''}{unit}]" return cls( endianness=cls.__endianness_map[dtype.byteorder], kind=Kind(dtype.kind), itemsize=dtype.itemsize, - units=units, + dt_units=dt_units, ) def to_numpy_dtype(self) -> numpy.dtype: @@ -119,8 +118,7 @@ def to_numpy_str(self): # so the reported itemsize is 4x the char count. To get back to the string # we need to divide by 4. size = self.itemsize if self.kind != Kind.unicode else self.itemsize // 4 - units = f"[{self.units}]" if self.units else "" - return f"{endianness}{self.kind.value}{size}{units}" + return f"{endianness}{self.kind.value}{size}{self.dt_units or ''}" @classmethod def from_json(cls, structure): @@ -128,7 +126,7 @@ def from_json(cls, structure): kind=Kind(structure["kind"]), itemsize=structure["itemsize"], endianness=Endianness(structure["endianness"]), - units=structure.get("units"), + dt_units=structure.get("dt_units"), ) From a742bd99a8ef1fa009b12213368c57ac0951d42a Mon Sep 17 00:00:00 2001 From: Eugene M Date: Fri, 13 Sep 2024 14:11:44 -0400 Subject: [PATCH 14/15] TST: Fix failing authorization test -- empty password --- tiled/_tests/test_authentication.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tiled/_tests/test_authentication.py b/tiled/_tests/test_authentication.py index 519ec5261..cd5b60520 100644 --- a/tiled/_tests/test_authentication.py +++ b/tiled/_tests/test_authentication.py @@ -10,7 +10,6 @@ from starlette.status import ( HTTP_400_BAD_REQUEST, HTTP_401_UNAUTHORIZED, - HTTP_422_UNPROCESSABLE_ENTITY, ) from ..adapters.array import ArrayAdapter @@ -93,7 +92,7 @@ def test_password_auth(enter_password, config): from_context(context, username="alice") # Empty password should not work. - with fail_with_status_code(HTTP_422_UNPROCESSABLE_ENTITY): + with fail_with_status_code(HTTP_401_UNAUTHORIZED): with enter_password(""): from_context(context, username="alice") From 73b2b40ae23ac780d0b472106c665fae04324e61 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Fri, 13 Sep 2024 14:15:23 -0400 Subject: [PATCH 15/15] MNT: format and lint --- tiled/_tests/test_authentication.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tiled/_tests/test_authentication.py b/tiled/_tests/test_authentication.py index cd5b60520..1bc2cfb6c 100644 --- a/tiled/_tests/test_authentication.py +++ b/tiled/_tests/test_authentication.py @@ -7,10 +7,7 @@ import numpy import pytest -from starlette.status import ( - HTTP_400_BAD_REQUEST, - HTTP_401_UNAUTHORIZED, -) +from starlette.status import HTTP_400_BAD_REQUEST, HTTP_401_UNAUTHORIZED from ..adapters.array import ArrayAdapter from ..adapters.mapping import MapAdapter