From 2210bebeab7c87a4659f4c9fc5afaec5a2d9244f Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Tue, 5 Sep 2023 15:55:51 +0200 Subject: [PATCH 01/17] KDP-1761: Initial commit after running bump-pydantic --- src/covjson_pydantic/base_models.py | 25 +++++++------ src/covjson_pydantic/coverage.py | 14 +++---- src/covjson_pydantic/domain.py | 45 ++++++++++++----------- src/covjson_pydantic/ndarray.py | 8 ++-- src/covjson_pydantic/observed_property.py | 8 ++-- src/covjson_pydantic/parameter.py | 25 +++++++------ src/covjson_pydantic/reference_system.py | 22 ++++++----- src/covjson_pydantic/unit.py | 10 +++-- 8 files changed, 84 insertions(+), 73 deletions(-) diff --git a/src/covjson_pydantic/base_models.py b/src/covjson_pydantic/base_models.py index 3836e7b..06997a7 100644 --- a/src/covjson_pydantic/base_models.py +++ b/src/covjson_pydantic/base_models.py @@ -1,6 +1,6 @@ import orjson from pydantic import BaseModel as PydanticBaseModel -from pydantic import Extra +from pydantic import ConfigDict def orjson_dumps(v, *, default, indent=None, sort_keys=False): @@ -16,17 +16,18 @@ def orjson_dumps(v, *, default, indent=None, sort_keys=False): class BaseModel(PydanticBaseModel): - class Config: - anystr_strip_whitespace = True - min_anystr_length = 1 - extra = Extra.forbid - validate_all = True - validate_assignment = True - - json_loads = orjson.loads - json_dumps = orjson_dumps + # TODO[pydantic]: The following keys were removed: `json_loads`, `json_dumps`. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. + model_config = ConfigDict( + str_strip_whitespace=True, + str_min_length=1, + extra="forbid", + validate_default=True, + validate_assignment=True, + json_loads=orjson.loads, + json_dumps=orjson_dumps, + ) class CovJsonBaseModel(BaseModel): - class Config: - extra = Extra.allow # allow custom members + model_config = ConfigDict(extra="allow") diff --git a/src/covjson_pydantic/coverage.py b/src/covjson_pydantic/coverage.py index 11d5fd4..2bcb9dc 100644 --- a/src/covjson_pydantic/coverage.py +++ b/src/covjson_pydantic/coverage.py @@ -17,18 +17,18 @@ class Coverage(CovJsonBaseModel): - id: Optional[str] + id: Optional[str] = None type: Literal["Coverage"] = "Coverage" domain: Domain - parameters: Optional[Dict[str, Parameter]] - parameterGroups: Optional[List[ParameterGroup]] # noqa: N815 + parameters: Optional[Dict[str, Parameter]] = None + parameterGroups: Optional[List[ParameterGroup]] = None # noqa: N815 ranges: Dict[str, Union[NdArray, TiledNdArray, AnyUrl]] class CoverageCollection(CovJsonBaseModel): type: Literal["CoverageCollection"] = "CoverageCollection" - domainType: Optional[DomainType] # noqa: N815 + domainType: Optional[DomainType] = None # noqa: N815 coverages: List[Coverage] - parameters: Optional[Dict[str, Parameter]] - parameterGroups: Optional[List[ParameterGroup]] # noqa: N815 - referencing: Optional[List[ReferenceSystemConnectionObject]] + parameters: Optional[Dict[str, Parameter]] = None + parameterGroups: Optional[List[ParameterGroup]] = None # noqa: N815 + referencing: Optional[List[ReferenceSystemConnectionObject]] = None diff --git a/src/covjson_pydantic/domain.py b/src/covjson_pydantic/domain.py index 8141393..5a4b635 100644 --- a/src/covjson_pydantic/domain.py +++ b/src/covjson_pydantic/domain.py @@ -8,10 +8,11 @@ from typing import TypeVar from typing import Union -from pydantic import Extra +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import model_validator from pydantic import PositiveInt from pydantic.class_validators import root_validator -from pydantic.generics import GenericModel from .base_models import BaseModel from .base_models import CovJsonBaseModel @@ -23,7 +24,8 @@ class CompactAxis(BaseModel): stop: float num: PositiveInt - @root_validator(skip_on_failure=True) + @model_validator(skip_on_failure=True) + @classmethod def single_value_case(cls, values): if values["num"] == 1 and values["start"] != values["stop"]: raise ValueError("If the value of 'num' is 1, then 'start' and 'stop' MUST have identical values.") @@ -33,20 +35,17 @@ def single_value_case(cls, values): ValuesT = TypeVar("ValuesT") -class ValuesAxis(GenericModel, Generic[ValuesT]): +class ValuesAxis(BaseModel, Generic[ValuesT]): dataType: Optional[str] # noqa: N815 coordinates: Optional[List[str]] values: List[ValuesT] bounds: Optional[List[ValuesT]] + model_config = ConfigDict( + str_strip_whitespace=True, str_min_length=1, extra="allow", validate_default=True, validate_assignment=True + ) - class Config: - anystr_strip_whitespace = True - min_anystr_length = 1 - extra = Extra.allow # allow custom members - validate_all = True - validate_assignment = True - - @root_validator(skip_on_failure=True) + @model_validator(skip_on_failure=True) + @classmethod def bounds_length(cls, values): if values["bounds"] is not None and len(values["bounds"]) != 2 * len(values["values"]): raise ValueError("If provided, the length of 'bounds' should be twice that of 'values'.") @@ -63,13 +62,14 @@ class DomainType(str, Enum): class Axes(BaseModel): - x: Optional[Union[ValuesAxis[float], CompactAxis]] - y: Optional[Union[ValuesAxis[float], CompactAxis]] - z: Optional[Union[ValuesAxis[float], CompactAxis]] - t: Optional[ValuesAxis[datetime]] - composite: Optional[ValuesAxis[Tuple]] - - @root_validator(skip_on_failure=True) + x: Optional[Union[ValuesAxis[float], CompactAxis]] = None + y: Optional[Union[ValuesAxis[float], CompactAxis]] = None + z: Optional[Union[ValuesAxis[float], CompactAxis]] = None + t: Optional[ValuesAxis[datetime]] = None + composite: Optional[ValuesAxis[Tuple]] = None + + @model_validator(skip_on_failure=True) + @classmethod def at_least_one_axes(cls, values): if ( values["x"] is None @@ -84,9 +84,9 @@ def at_least_one_axes(cls, values): class Domain(CovJsonBaseModel): type: Literal["Domain"] = "Domain" - domainType: Optional[DomainType] # noqa: N815 + domainType: Optional[DomainType] = None # noqa: N815 axes: Axes - referencing: Optional[List[ReferenceSystemConnectionObject]] + referencing: Optional[List[ReferenceSystemConnectionObject]] = None @staticmethod def check_axis(domain_type, axes, required_axes, allowed_axes, single_value_axes): @@ -119,7 +119,8 @@ def check_axis(domain_type, axes, required_axes, allowed_axes, single_value_axes f"of a '{domain_type.value}' domain must contain a single value." ) - @root_validator(skip_on_failure=True) + @model_validator(skip_on_failure=True) + @classmethod def check_domain_consistent(cls, values): domain_type = values.get("domainType") axes = values.get("axes") diff --git a/src/covjson_pydantic/ndarray.py b/src/covjson_pydantic/ndarray.py index 068bb39..ac6127e 100644 --- a/src/covjson_pydantic/ndarray.py +++ b/src/covjson_pydantic/ndarray.py @@ -5,6 +5,7 @@ from typing import Optional from pydantic import AnyUrl +from pydantic import model_validator from pydantic.class_validators import root_validator from .base_models import BaseModel @@ -19,11 +20,12 @@ class DataType(str, Enum): class NdArray(CovJsonBaseModel): type: Literal["NdArray"] = "NdArray" dataType: DataType = DataType.float # noqa: N815 - axisNames: Optional[List[str]] # noqa: N815 - shape: Optional[List[int]] + axisNames: Optional[List[str]] = None # noqa: N815 + shape: Optional[List[int]] = None values: List[Optional[float]] - @root_validator(skip_on_failure=True) + @model_validator(skip_on_failure=True) + @classmethod def check_field_dependencies(cls, values): if len(values["values"]) > 1 and (values.get("axisNames") is None or len(values.get("axisNames")) == 0): raise ValueError("'axisNames' must to be provided if array is not 0D") diff --git a/src/covjson_pydantic/observed_property.py b/src/covjson_pydantic/observed_property.py index f2f57f1..71a44e3 100644 --- a/src/covjson_pydantic/observed_property.py +++ b/src/covjson_pydantic/observed_property.py @@ -8,11 +8,11 @@ class Category(BaseModel): id: str label: i18n - description: Optional[i18n] + description: Optional[i18n] = None class ObservedProperty(BaseModel): - id: Optional[str] + id: Optional[str] = None label: i18n - description: Optional[i18n] - categories: Optional[List[Category]] + description: Optional[i18n] = None + categories: Optional[List[Category]] = None diff --git a/src/covjson_pydantic/parameter.py b/src/covjson_pydantic/parameter.py index 63c0769..ea9b498 100644 --- a/src/covjson_pydantic/parameter.py +++ b/src/covjson_pydantic/parameter.py @@ -5,6 +5,7 @@ from typing import Union from pydantic import Extra +from pydantic import model_validator from pydantic.class_validators import root_validator from .base_models import BaseModel @@ -15,14 +16,15 @@ class Parameter(BaseModel, extra=Extra.allow): type: Literal["Parameter"] = "Parameter" - id: Optional[str] - label: Optional[i18n] - description: Optional[i18n] + id: Optional[str] = None + label: Optional[i18n] = None + description: Optional[i18n] = None observedProperty: ObservedProperty # noqa: N815 - categoryEncoding: Optional[Dict[str, Union[int, List[int]]]] # noqa: N815 - unit: Optional[Unit] + categoryEncoding: Optional[Dict[str, Union[int, List[int]]]] = None # noqa: N815 + unit: Optional[Unit] = None - @root_validator(skip_on_failure=True) + @model_validator(skip_on_failure=True) + @classmethod def must_not_have_unit_if_observed_property_has_categories(cls, values): if ( values.get("unit") is not None @@ -39,13 +41,14 @@ def must_not_have_unit_if_observed_property_has_categories(cls, values): class ParameterGroup(BaseModel, extra=Extra.allow): type: Literal["ParameterGroup"] = "ParameterGroup" - id: Optional[str] - label: Optional[i18n] - description: Optional[i18n] - observedProperty: Optional[ObservedProperty] # noqa: N815 + id: Optional[str] = None + label: Optional[i18n] = None + description: Optional[i18n] = None + observedProperty: Optional[ObservedProperty] = None # noqa: N815 members: List[str] - @root_validator(skip_on_failure=True) + @model_validator(skip_on_failure=True) + @classmethod def must_have_label_and_or_observed_property(cls, values): if values.get("label") is None and values.get("observedProperty") is None: raise ValueError( diff --git a/src/covjson_pydantic/reference_system.py b/src/covjson_pydantic/reference_system.py index ba6c795..eb16257 100644 --- a/src/covjson_pydantic/reference_system.py +++ b/src/covjson_pydantic/reference_system.py @@ -6,6 +6,7 @@ from pydantic import AnyUrl from pydantic import Extra +from pydantic import model_validator from pydantic.class_validators import root_validator from .base_models import BaseModel @@ -13,26 +14,27 @@ class TargetConcept(BaseModel): - id: Optional[str] # Not in spec, but needed for example in spec for 'Identifier-based Reference Systems' + id: Optional[str] = None # Not in spec, but needed for example in spec for 'Identifier-based Reference Systems' label: i18n - description: Optional[i18n] + description: Optional[i18n] = None class ReferenceSystem(BaseModel, extra=Extra.allow): type: Literal["GeographicCRS", "ProjectedCRS", "VerticalCRS", "TemporalRS", "IdentifierRS"] - id: Optional[str] - description: Optional[i18n] + id: Optional[str] = None + description: Optional[i18n] = None # Only for TemporalRS - calendar: Optional[Union[Literal["Gregorian"], AnyUrl]] - timeScale: Optional[AnyUrl] # noqa: N815 + calendar: Optional[Union[Literal["Gregorian"], AnyUrl]] = None + timeScale: Optional[AnyUrl] = None # noqa: N815 # Only for IdentifierRS - label: Optional[i18n] - targetConcept: Optional[TargetConcept] # noqa: N815 - identifiers: Optional[Dict[str, TargetConcept]] + label: Optional[i18n] = None + targetConcept: Optional[TargetConcept] = None # noqa: N815 + identifiers: Optional[Dict[str, TargetConcept]] = None - @root_validator(skip_on_failure=True) + @model_validator(skip_on_failure=True) + @classmethod def check_type_specific_fields(cls, values): if values["type"] != "TemporalRS" and ( values.get("calendar") is not None or values.get("timeScale") is not None diff --git a/src/covjson_pydantic/unit.py b/src/covjson_pydantic/unit.py index f4a8a44..5c7ab8b 100644 --- a/src/covjson_pydantic/unit.py +++ b/src/covjson_pydantic/unit.py @@ -1,6 +1,7 @@ from typing import Optional from typing import Union +from pydantic import model_validator from pydantic.class_validators import root_validator from .base_models import BaseModel @@ -13,11 +14,12 @@ class Symbol(BaseModel): class Unit(BaseModel): - id: Optional[str] - label: Optional[i18n] - symbol: Optional[Union[str, Symbol]] + id: Optional[str] = None + label: Optional[i18n] = None + symbol: Optional[Union[str, Symbol]] = None - @root_validator(skip_on_failure=True) + @model_validator(skip_on_failure=True) + @classmethod def check_either_label_or_symbol(cls, values): if values.get("label") is None and values.get("symbol") is None: raise ValueError("Either 'label' or 'symbol' should be set") From c7701429375b876458ad6ffba173c328cf990fe9 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 8 Sep 2023 13:45:33 +0200 Subject: [PATCH 02/17] KDP-1761: Fix validators. Remove orjson. --- example.py | 2 +- pyproject.toml | 2 +- src/covjson_pydantic/base_models.py | 16 -------- src/covjson_pydantic/domain.py | 52 +++++++++--------------- src/covjson_pydantic/ndarray.py | 24 ++++------- src/covjson_pydantic/parameter.py | 23 ++++------- src/covjson_pydantic/reference_system.py | 20 ++++----- src/covjson_pydantic/unit.py | 10 ++--- 8 files changed, 50 insertions(+), 99 deletions(-) diff --git a/example.py b/example.py index 238b4a2..d5d96f6 100644 --- a/example.py +++ b/example.py @@ -16,4 +16,4 @@ ranges={"temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])}, ) -print(c.json(exclude_none=True, indent=True)) +print(c.model_dump_json(exclude_none=True, indent=4)) diff --git a/pyproject.toml b/pyproject.toml index 0f2e3eb..5e4e132 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Typing :: Typed", ] version = "0.1.0" -dependencies = ["pydantic>=1,<2", "orjson>=3"] +dependencies = ["pydantic>=2,<3"] [project.optional-dependencies] test = ["pytest", "pytest-cov"] diff --git a/src/covjson_pydantic/base_models.py b/src/covjson_pydantic/base_models.py index 06997a7..d010d11 100644 --- a/src/covjson_pydantic/base_models.py +++ b/src/covjson_pydantic/base_models.py @@ -1,22 +1,8 @@ -import orjson from pydantic import BaseModel as PydanticBaseModel from pydantic import ConfigDict -def orjson_dumps(v, *, default, indent=None, sort_keys=False): - options = orjson.OPT_NON_STR_KEYS | orjson.OPT_UTC_Z | orjson.OPT_NAIVE_UTC - if indent: - options |= orjson.OPT_INDENT_2 - - if sort_keys: - options |= orjson.OPT_SORT_KEYS - - # orjson.dumps returns bytes, to match standard json.dumps we need to decode - return orjson.dumps(v, default=default, option=options).decode() - - class BaseModel(PydanticBaseModel): - # TODO[pydantic]: The following keys were removed: `json_loads`, `json_dumps`. # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. model_config = ConfigDict( str_strip_whitespace=True, @@ -24,8 +10,6 @@ class BaseModel(PydanticBaseModel): extra="forbid", validate_default=True, validate_assignment=True, - json_loads=orjson.loads, - json_dumps=orjson_dumps, ) diff --git a/src/covjson_pydantic/domain.py b/src/covjson_pydantic/domain.py index 5a4b635..a735f0e 100644 --- a/src/covjson_pydantic/domain.py +++ b/src/covjson_pydantic/domain.py @@ -8,11 +8,9 @@ from typing import TypeVar from typing import Union -from pydantic import BaseModel from pydantic import ConfigDict from pydantic import model_validator from pydantic import PositiveInt -from pydantic.class_validators import root_validator from .base_models import BaseModel from .base_models import CovJsonBaseModel @@ -24,32 +22,30 @@ class CompactAxis(BaseModel): stop: float num: PositiveInt - @model_validator(skip_on_failure=True) - @classmethod - def single_value_case(cls, values): - if values["num"] == 1 and values["start"] != values["stop"]: + @model_validator(mode="after") + def single_value_case(self): + if self.num == 1 and self.start != self.stop: raise ValueError("If the value of 'num' is 1, then 'start' and 'stop' MUST have identical values.") - return values + return self ValuesT = TypeVar("ValuesT") class ValuesAxis(BaseModel, Generic[ValuesT]): - dataType: Optional[str] # noqa: N815 - coordinates: Optional[List[str]] + dataType: Optional[str] = None # noqa: N815 + coordinates: Optional[List[str]] = None values: List[ValuesT] - bounds: Optional[List[ValuesT]] + bounds: Optional[List[ValuesT]] = None model_config = ConfigDict( str_strip_whitespace=True, str_min_length=1, extra="allow", validate_default=True, validate_assignment=True ) - @model_validator(skip_on_failure=True) - @classmethod - def bounds_length(cls, values): - if values["bounds"] is not None and len(values["bounds"]) != 2 * len(values["values"]): + @model_validator(mode="after") + def bounds_length(self): + if self.bounds is not None and len(self.bounds) != 2 * len(self.values): raise ValueError("If provided, the length of 'bounds' should be twice that of 'values'.") - return values + return self class DomainType(str, Enum): @@ -68,18 +64,11 @@ class Axes(BaseModel): t: Optional[ValuesAxis[datetime]] = None composite: Optional[ValuesAxis[Tuple]] = None - @model_validator(skip_on_failure=True) - @classmethod - def at_least_one_axes(cls, values): - if ( - values["x"] is None - and values["y"] is None - and values["z"] is None - and values["t"] is None - and values["composite"] is None - ): + @model_validator(mode="after") + def at_least_one_axes(self): + if self.x is None and self.y is None and self.z is None and self.t is None and self.composite is None: raise ValueError("At least one axis of x,y,z,t or composite must be given.") - return values + return self class Domain(CovJsonBaseModel): @@ -119,11 +108,10 @@ def check_axis(domain_type, axes, required_axes, allowed_axes, single_value_axes f"of a '{domain_type.value}' domain must contain a single value." ) - @model_validator(skip_on_failure=True) - @classmethod - def check_domain_consistent(cls, values): - domain_type = values.get("domainType") - axes = values.get("axes") + @model_validator(mode="after") + def check_domain_consistent(self): + domain_type = self.domainType + axes = self.axes if domain_type == DomainType.grid: Domain.check_axis( @@ -159,4 +147,4 @@ def check_domain_consistent(cls, values): domain_type, axes, required_axes={"composite"}, allowed_axes={"t"}, single_value_axes={"t"} ) - return values + return self diff --git a/src/covjson_pydantic/ndarray.py b/src/covjson_pydantic/ndarray.py index ac6127e..a65e79c 100644 --- a/src/covjson_pydantic/ndarray.py +++ b/src/covjson_pydantic/ndarray.py @@ -6,7 +6,6 @@ from pydantic import AnyUrl from pydantic import model_validator -from pydantic.class_validators import root_validator from .base_models import BaseModel from .base_models import CovJsonBaseModel @@ -24,31 +23,26 @@ class NdArray(CovJsonBaseModel): shape: Optional[List[int]] = None values: List[Optional[float]] - @model_validator(skip_on_failure=True) - @classmethod - def check_field_dependencies(cls, values): - if len(values["values"]) > 1 and (values.get("axisNames") is None or len(values.get("axisNames")) == 0): + @model_validator(mode="after") + def check_field_dependencies(self): + if len(self.values) > 1 and (self.axisNames is None or len(self.axisNames) == 0): raise ValueError("'axisNames' must to be provided if array is not 0D") - if len(values["values"]) > 1 and (values.get("shape") is None or len(values.get("shape")) == 0): + if len(self.values) > 1 and (self.shape is None or len(self.shape) == 0): raise ValueError("'shape' must to be provided if array is not 0D") - if ( - values.get("axisNames") is not None - and values.get("shape") is not None - and len(values.get("axisNames")) != len(values.get("shape")) - ): + if self.axisNames is not None and self.shape is not None and len(self.axisNames) != len(self.shape): raise ValueError("'axisNames' and 'shape' should have equal length") - if values.get("shape") is not None and len(values.get("shape")) >= 1: - prod = math.prod(values["shape"]) - if len(values["values"]) != prod: + if self.shape is not None and len(self.shape) >= 1: + prod = math.prod(self.shape) + if len(self.values) != prod: raise ValueError( "Where 'shape' is present and non-empty, the product of its values MUST equal " "the number of elements in the 'values' array." ) - return values + return self class TileSet(BaseModel): diff --git a/src/covjson_pydantic/parameter.py b/src/covjson_pydantic/parameter.py index ea9b498..7b578ed 100644 --- a/src/covjson_pydantic/parameter.py +++ b/src/covjson_pydantic/parameter.py @@ -6,7 +6,6 @@ from pydantic import Extra from pydantic import model_validator -from pydantic.class_validators import root_validator from .base_models import BaseModel from .i18n import i18n @@ -23,20 +22,15 @@ class Parameter(BaseModel, extra=Extra.allow): categoryEncoding: Optional[Dict[str, Union[int, List[int]]]] = None # noqa: N815 unit: Optional[Unit] = None - @model_validator(skip_on_failure=True) - @classmethod - def must_not_have_unit_if_observed_property_has_categories(cls, values): - if ( - values.get("unit") is not None - and values.get("observedProperty") is not None - and values.get("observedProperty").categories is not None - ): + @model_validator(mode="after") + def must_not_have_unit_if_observed_property_has_categories(self): + if self.unit is not None and self.observedProperty is not None and self.observedProperty.categories is not None: raise ValueError( "A parameter object MUST NOT have a 'unit' member " "if the 'observedProperty' member has a 'categories' member." ) - return values + return self class ParameterGroup(BaseModel, extra=Extra.allow): @@ -47,11 +41,10 @@ class ParameterGroup(BaseModel, extra=Extra.allow): observedProperty: Optional[ObservedProperty] = None # noqa: N815 members: List[str] - @model_validator(skip_on_failure=True) - @classmethod - def must_have_label_and_or_observed_property(cls, values): - if values.get("label") is None and values.get("observedProperty") is None: + @model_validator(mode="after") + def must_have_label_and_or_observed_property(self): + if self.label is None and self.observedProperty is None: raise ValueError( "A parameter group object MUST have either or both the members 'label' or/and 'observedProperty'" ) - return values + return self diff --git a/src/covjson_pydantic/reference_system.py b/src/covjson_pydantic/reference_system.py index eb16257..9aaca94 100644 --- a/src/covjson_pydantic/reference_system.py +++ b/src/covjson_pydantic/reference_system.py @@ -7,7 +7,6 @@ from pydantic import AnyUrl from pydantic import Extra from pydantic import model_validator -from pydantic.class_validators import root_validator from .base_models import BaseModel from .i18n import i18n @@ -33,27 +32,22 @@ class ReferenceSystem(BaseModel, extra=Extra.allow): targetConcept: Optional[TargetConcept] = None # noqa: N815 identifiers: Optional[Dict[str, TargetConcept]] = None - @model_validator(skip_on_failure=True) - @classmethod - def check_type_specific_fields(cls, values): - if values["type"] != "TemporalRS" and ( - values.get("calendar") is not None or values.get("timeScale") is not None - ): + @model_validator(mode="after") + def check_type_specific_fields(self): + if self.type != "TemporalRS" and (self.calendar is not None or self.timeScale is not None): raise ValueError("'calendar' and 'timeScale' fields can only be used for type 'TemporalRS'") - if values["type"] != "IdentifierRS" and ( - values.get("label") is not None - or values.get("targetConcept") is not None - or values.get("identifiers") is not None + if self.type != "IdentifierRS" and ( + self.label is not None or self.targetConcept is not None or self.identifiers is not None ): raise ValueError( "'label', 'targetConcept' and 'identifiers' fields can only be used for type 'IdentifierRS'" ) - if values["type"] == "IdentifierRS" and values.get("targetConcept") is None: + if self.type == "IdentifierRS" and self.targetConcept is None: raise ValueError("An identifier RS object MUST have a member 'targetConcept'") - return values + return self class ReferenceSystemConnectionObject(BaseModel): diff --git a/src/covjson_pydantic/unit.py b/src/covjson_pydantic/unit.py index 5c7ab8b..63ac5a7 100644 --- a/src/covjson_pydantic/unit.py +++ b/src/covjson_pydantic/unit.py @@ -2,7 +2,6 @@ from typing import Union from pydantic import model_validator -from pydantic.class_validators import root_validator from .base_models import BaseModel from .i18n import i18n @@ -18,10 +17,9 @@ class Unit(BaseModel): label: Optional[i18n] = None symbol: Optional[Union[str, Symbol]] = None - @model_validator(skip_on_failure=True) - @classmethod - def check_either_label_or_symbol(cls, values): - if values.get("label") is None and values.get("symbol") is None: + @model_validator(mode="after") + def check_either_label_or_symbol(self): + if self.label is None and self.symbol is None: raise ValueError("Either 'label' or 'symbol' should be set") - return values + return self From d92ccfe32ff67a689c82429d653bf15ac1de8740 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 8 Sep 2023 13:46:46 +0200 Subject: [PATCH 03/17] KDP-1761: Remove comment --- src/covjson_pydantic/base_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/covjson_pydantic/base_models.py b/src/covjson_pydantic/base_models.py index d010d11..7511afe 100644 --- a/src/covjson_pydantic/base_models.py +++ b/src/covjson_pydantic/base_models.py @@ -3,7 +3,6 @@ class BaseModel(PydanticBaseModel): - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. model_config = ConfigDict( str_strip_whitespace=True, str_min_length=1, From 1c995fd00cbd54b0cb0b3617d31856098aa3f8a4 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 8 Sep 2023 14:08:48 +0200 Subject: [PATCH 04/17] KDP-1761: Fix testing --- src/covjson_pydantic/ndarray.py | 3 +-- src/covjson_pydantic/parameter.py | 5 ++--- src/covjson_pydantic/reference_system.py | 3 +-- tests/test_coverage.py | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/covjson_pydantic/ndarray.py b/src/covjson_pydantic/ndarray.py index a65e79c..274a30f 100644 --- a/src/covjson_pydantic/ndarray.py +++ b/src/covjson_pydantic/ndarray.py @@ -4,7 +4,6 @@ from typing import Literal from typing import Optional -from pydantic import AnyUrl from pydantic import model_validator from .base_models import BaseModel @@ -47,7 +46,7 @@ def check_field_dependencies(self): class TileSet(BaseModel): tileShape: List[Optional[int]] # noqa: N815 - urlTemplate: AnyUrl # noqa: N815 + urlTemplate: str # noqa: N815 # TODO: Validation of field dependencies diff --git a/src/covjson_pydantic/parameter.py b/src/covjson_pydantic/parameter.py index 7b578ed..25b5645 100644 --- a/src/covjson_pydantic/parameter.py +++ b/src/covjson_pydantic/parameter.py @@ -4,7 +4,6 @@ from typing import Optional from typing import Union -from pydantic import Extra from pydantic import model_validator from .base_models import BaseModel @@ -13,7 +12,7 @@ from .unit import Unit -class Parameter(BaseModel, extra=Extra.allow): +class Parameter(BaseModel, extra="allow"): type: Literal["Parameter"] = "Parameter" id: Optional[str] = None label: Optional[i18n] = None @@ -33,7 +32,7 @@ def must_not_have_unit_if_observed_property_has_categories(self): return self -class ParameterGroup(BaseModel, extra=Extra.allow): +class ParameterGroup(BaseModel, extra="allow"): type: Literal["ParameterGroup"] = "ParameterGroup" id: Optional[str] = None label: Optional[i18n] = None diff --git a/src/covjson_pydantic/reference_system.py b/src/covjson_pydantic/reference_system.py index 9aaca94..570610a 100644 --- a/src/covjson_pydantic/reference_system.py +++ b/src/covjson_pydantic/reference_system.py @@ -5,7 +5,6 @@ from typing import Union from pydantic import AnyUrl -from pydantic import Extra from pydantic import model_validator from .base_models import BaseModel @@ -18,7 +17,7 @@ class TargetConcept(BaseModel): description: Optional[i18n] = None -class ReferenceSystem(BaseModel, extra=Extra.allow): +class ReferenceSystem(BaseModel, extra="allow"): type: Literal["GeographicCRS", "ProjectedCRS", "VerticalCRS", "TemporalRS", "IdentifierRS"] id: Optional[str] = None description: Optional[i18n] = None diff --git a/tests/test_coverage.py b/tests/test_coverage.py index fdfc296..139872b 100644 --- a/tests/test_coverage.py +++ b/tests/test_coverage.py @@ -47,7 +47,7 @@ def test_happy_cases(file_name, object_type): json_string = json.dumps(data, separators=(",", ":")) # Round-trip - assert object_type.parse_raw(json_string).json(exclude_none=True) == json_string + assert object_type.model_validate_json(json_string).model_dump_json(exclude_none=True) == json_string error_cases = [ @@ -70,4 +70,4 @@ def test_error_cases(file_name, object_type, error_message): json_string = json.dumps(data, separators=(",", ":")) with pytest.raises(ValidationError, match=error_message): - object_type.parse_raw(json_string) + object_type.model_validate_json(json_string) From 3c02d03ef2956b175d103c7af2812f6407d0c3ec Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 8 Sep 2023 16:10:47 +0200 Subject: [PATCH 05/17] KDP-1761: Ignored warning for i18n type --- src/covjson_pydantic/i18n.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/covjson_pydantic/i18n.py b/src/covjson_pydantic/i18n.py index 2cc6e0b..87035f6 100644 --- a/src/covjson_pydantic/i18n.py +++ b/src/covjson_pydantic/i18n.py @@ -9,4 +9,9 @@ class LanguageTag(str, Enum): undefined = "und" -i18n = Dict[LanguageTag, str] +# TODO: This was throwing warning: +# Expected `definition-ref` but got `LanguageTag` - serialized value may not be as expected +# This may be a bug in Pydantic: https://github.com/pydantic/pydantic/issues/6467 +# So, for now, reverted to a less strict type +# i18n = Dict[LanguageTag, str] +i18n = Dict[str, str] From 5907c4e5464155e0e6d7028fb9648dd0afbd702d Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 8 Sep 2023 16:25:23 +0200 Subject: [PATCH 06/17] KDP-1761: Upgrade mypy --- .pre-commit-config.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 40eda99..ed5aa23 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: additional_dependencies: [ 'pep8-naming==0.12.1' ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.971 + rev: v1.5.1 hooks: - id: mypy language_version: python @@ -64,5 +64,4 @@ repos: # Don't pass it the individual filenames because it is already doing the whole folder. pass_filenames: false additional_dependencies: - - orjson - pydantic From c8e70a39868578082a6fcc593ae0bf6d5e610e97 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 8 Sep 2023 16:25:52 +0200 Subject: [PATCH 07/17] KDP-1761: Force t-axis to be timezone aware --- example.py | 5 +++-- src/covjson_pydantic/domain.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/example.py b/example.py index d5d96f6..32140ae 100644 --- a/example.py +++ b/example.py @@ -1,4 +1,5 @@ -import datetime +from datetime import datetime +from datetime import timezone from covjson_pydantic.coverage import Coverage from covjson_pydantic.domain import Domain @@ -10,7 +11,7 @@ axes={ "x": {"dataType": "float", "values": [1.23]}, "y": {"values": [4.56]}, - "t": {"dataType": "datetime", "values": [datetime.datetime.now()]}, + "t": {"dataType": "datetime", "values": [datetime.now(tz=timezone.utc)]}, }, ), ranges={"temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])}, diff --git a/src/covjson_pydantic/domain.py b/src/covjson_pydantic/domain.py index a735f0e..c59a51a 100644 --- a/src/covjson_pydantic/domain.py +++ b/src/covjson_pydantic/domain.py @@ -1,4 +1,3 @@ -from datetime import datetime from enum import Enum from typing import Generic from typing import List @@ -8,6 +7,7 @@ from typing import TypeVar from typing import Union +from pydantic import AwareDatetime from pydantic import ConfigDict from pydantic import model_validator from pydantic import PositiveInt @@ -61,7 +61,7 @@ class Axes(BaseModel): x: Optional[Union[ValuesAxis[float], CompactAxis]] = None y: Optional[Union[ValuesAxis[float], CompactAxis]] = None z: Optional[Union[ValuesAxis[float], CompactAxis]] = None - t: Optional[ValuesAxis[datetime]] = None + t: Optional[ValuesAxis[AwareDatetime]] = None composite: Optional[ValuesAxis[Tuple]] = None @model_validator(mode="after") From 5042ecfe9f2b496195f92d5470819482e0670e3a Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 8 Sep 2023 16:34:05 +0200 Subject: [PATCH 08/17] KDP-1761: Update README --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5911298..fb926a6 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ pip install git+https://github.com/KNMI/covjson-pydantic.git ## Usage ```python -import datetime +from datetime import datetime, timezone from covjson_pydantic.coverage import Coverage from covjson_pydantic.domain import Domain from covjson_pydantic.ndarray import NdArray @@ -46,14 +46,15 @@ c = Coverage( axes={ "x": {"dataType": "float", "values": [1.23]}, "y": {"values": [4.56]}, - "t": {"dataType": "datetime", "values": [datetime.datetime.now()]} + "t": {"dataType": "datetime", "values": [datetime.now(tz=timezone.utc)]}, }, ), ranges={ "temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0]) } ) -print(c.json(exclude_none=True,indent=True)) + +print(c.model_dump_json(exclude_none=True, indent=4)) ``` Will print ```json @@ -77,7 +78,7 @@ Will print "t": { "dataType": "datetime", "values": [ - "2023-01-19T13:14:47.126631Z" + "2023-09-08T14:31:41.311717Z" ] } } From 8038695f1d8cd1dcadd342b3e848b3a819a9b02b Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 8 Sep 2023 16:42:55 +0200 Subject: [PATCH 09/17] KDP-1761: Bump version --- pyproject.toml | 2 +- src/covjson_pydantic/i18n.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5e4e132..8bb9666 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: GIS", "Typing :: Typed", ] -version = "0.1.0" +version = "0.2.0" dependencies = ["pydantic>=2,<3"] [project.optional-dependencies] diff --git a/src/covjson_pydantic/i18n.py b/src/covjson_pydantic/i18n.py index 87035f6..4959b37 100644 --- a/src/covjson_pydantic/i18n.py +++ b/src/covjson_pydantic/i18n.py @@ -12,6 +12,7 @@ class LanguageTag(str, Enum): # TODO: This was throwing warning: # Expected `definition-ref` but got `LanguageTag` - serialized value may not be as expected # This may be a bug in Pydantic: https://github.com/pydantic/pydantic/issues/6467 +# or: https://github.com/pydantic/pydantic/issues/6422 # So, for now, reverted to a less strict type # i18n = Dict[LanguageTag, str] i18n = Dict[str, str] From 724856a2359d4e95f8f05efdbd45abd5def7b45e Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Thu, 14 Sep 2023 13:57:35 +0200 Subject: [PATCH 10/17] KDP-1761: Fix example --- README.md | 17 ++++++++--------- example.py | 13 ++++++++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index fb926a6..e21b373 100644 --- a/README.md +++ b/README.md @@ -36,18 +36,19 @@ pip install git+https://github.com/KNMI/covjson-pydantic.git ```python from datetime import datetime, timezone +from pydantic import AwareDatetime from covjson_pydantic.coverage import Coverage -from covjson_pydantic.domain import Domain +from covjson_pydantic.domain import Domain, Axes, ValuesAxis from covjson_pydantic.ndarray import NdArray c = Coverage( domain=Domain( domainType="PointSeries", - axes={ - "x": {"dataType": "float", "values": [1.23]}, - "y": {"values": [4.56]}, - "t": {"dataType": "datetime", "values": [datetime.now(tz=timezone.utc)]}, - }, + axes=Axes( + x=ValuesAxis[float](values=[1.23]), + y=ValuesAxis[float](values=[4.56]), + t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)]) + ) ), ranges={ "temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0]) @@ -65,7 +66,6 @@ Will print "domainType": "PointSeries", "axes": { "x": { - "dataType": "float", "values": [ 1.23 ] @@ -76,9 +76,8 @@ Will print ] }, "t": { - "dataType": "datetime", "values": [ - "2023-09-08T14:31:41.311717Z" + "2023-09-14T11:54:02.151493Z" ] } } diff --git a/example.py b/example.py index 32140ae..bb4d902 100644 --- a/example.py +++ b/example.py @@ -2,17 +2,20 @@ from datetime import timezone from covjson_pydantic.coverage import Coverage +from covjson_pydantic.domain import Axes from covjson_pydantic.domain import Domain +from covjson_pydantic.domain import ValuesAxis from covjson_pydantic.ndarray import NdArray +from pydantic import AwareDatetime c = Coverage( domain=Domain( domainType="PointSeries", - axes={ - "x": {"dataType": "float", "values": [1.23]}, - "y": {"values": [4.56]}, - "t": {"dataType": "datetime", "values": [datetime.now(tz=timezone.utc)]}, - }, + axes=Axes( + x=ValuesAxis[float](values=[1.23]), + y=ValuesAxis[float](values=[4.56]), + t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)]), + ), ), ranges={"temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])}, ) From e1d0f19019002ecc21e33533e16286db921c1535 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Thu, 21 Sep 2023 08:50:16 +0200 Subject: [PATCH 11/17] KDP-1761: Simplify usage of the base models --- src/covjson_pydantic/base_models.py | 6 +----- src/covjson_pydantic/coverage.py | 4 ++-- src/covjson_pydantic/domain.py | 11 +++-------- src/covjson_pydantic/ndarray.py | 7 +++---- src/covjson_pydantic/observed_property.py | 6 +++--- src/covjson_pydantic/parameter.py | 6 +++--- src/covjson_pydantic/reference_system.py | 8 ++++---- src/covjson_pydantic/unit.py | 6 +++--- 8 files changed, 22 insertions(+), 32 deletions(-) diff --git a/src/covjson_pydantic/base_models.py b/src/covjson_pydantic/base_models.py index 7511afe..b5e68af 100644 --- a/src/covjson_pydantic/base_models.py +++ b/src/covjson_pydantic/base_models.py @@ -2,7 +2,7 @@ from pydantic import ConfigDict -class BaseModel(PydanticBaseModel): +class CovJsonBaseModel(PydanticBaseModel): model_config = ConfigDict( str_strip_whitespace=True, str_min_length=1, @@ -10,7 +10,3 @@ class BaseModel(PydanticBaseModel): validate_default=True, validate_assignment=True, ) - - -class CovJsonBaseModel(BaseModel): - model_config = ConfigDict(extra="allow") diff --git a/src/covjson_pydantic/coverage.py b/src/covjson_pydantic/coverage.py index 2bcb9dc..279240b 100644 --- a/src/covjson_pydantic/coverage.py +++ b/src/covjson_pydantic/coverage.py @@ -16,7 +16,7 @@ from .reference_system import ReferenceSystemConnectionObject -class Coverage(CovJsonBaseModel): +class Coverage(CovJsonBaseModel, extra="allow"): id: Optional[str] = None type: Literal["Coverage"] = "Coverage" domain: Domain @@ -25,7 +25,7 @@ class Coverage(CovJsonBaseModel): ranges: Dict[str, Union[NdArray, TiledNdArray, AnyUrl]] -class CoverageCollection(CovJsonBaseModel): +class CoverageCollection(CovJsonBaseModel, extra="allow"): type: Literal["CoverageCollection"] = "CoverageCollection" domainType: Optional[DomainType] = None # noqa: N815 coverages: List[Coverage] diff --git a/src/covjson_pydantic/domain.py b/src/covjson_pydantic/domain.py index c59a51a..cde31f4 100644 --- a/src/covjson_pydantic/domain.py +++ b/src/covjson_pydantic/domain.py @@ -8,16 +8,14 @@ from typing import Union from pydantic import AwareDatetime -from pydantic import ConfigDict from pydantic import model_validator from pydantic import PositiveInt -from .base_models import BaseModel from .base_models import CovJsonBaseModel from .reference_system import ReferenceSystemConnectionObject -class CompactAxis(BaseModel): +class CompactAxis(CovJsonBaseModel): start: float stop: float num: PositiveInt @@ -32,14 +30,11 @@ def single_value_case(self): ValuesT = TypeVar("ValuesT") -class ValuesAxis(BaseModel, Generic[ValuesT]): +class ValuesAxis(CovJsonBaseModel, Generic[ValuesT], extra="allow"): dataType: Optional[str] = None # noqa: N815 coordinates: Optional[List[str]] = None values: List[ValuesT] bounds: Optional[List[ValuesT]] = None - model_config = ConfigDict( - str_strip_whitespace=True, str_min_length=1, extra="allow", validate_default=True, validate_assignment=True - ) @model_validator(mode="after") def bounds_length(self): @@ -57,7 +52,7 @@ class DomainType(str, Enum): multi_point = "MultiPoint" -class Axes(BaseModel): +class Axes(CovJsonBaseModel): x: Optional[Union[ValuesAxis[float], CompactAxis]] = None y: Optional[Union[ValuesAxis[float], CompactAxis]] = None z: Optional[Union[ValuesAxis[float], CompactAxis]] = None diff --git a/src/covjson_pydantic/ndarray.py b/src/covjson_pydantic/ndarray.py index 274a30f..abda788 100644 --- a/src/covjson_pydantic/ndarray.py +++ b/src/covjson_pydantic/ndarray.py @@ -6,7 +6,6 @@ from pydantic import model_validator -from .base_models import BaseModel from .base_models import CovJsonBaseModel @@ -15,7 +14,7 @@ class DataType(str, Enum): float = "float" -class NdArray(CovJsonBaseModel): +class NdArray(CovJsonBaseModel, extra="allow"): type: Literal["NdArray"] = "NdArray" dataType: DataType = DataType.float # noqa: N815 axisNames: Optional[List[str]] = None # noqa: N815 @@ -44,13 +43,13 @@ def check_field_dependencies(self): return self -class TileSet(BaseModel): +class TileSet(CovJsonBaseModel): tileShape: List[Optional[int]] # noqa: N815 urlTemplate: str # noqa: N815 # TODO: Validation of field dependencies -class TiledNdArray(CovJsonBaseModel): +class TiledNdArray(CovJsonBaseModel, extra="allow"): type: Literal["TiledNdArray"] = "TiledNdArray" dataType: DataType = DataType.float # noqa: N815 axisNames: List[str] # noqa: N815 diff --git a/src/covjson_pydantic/observed_property.py b/src/covjson_pydantic/observed_property.py index 71a44e3..976a067 100644 --- a/src/covjson_pydantic/observed_property.py +++ b/src/covjson_pydantic/observed_property.py @@ -1,17 +1,17 @@ from typing import List from typing import Optional -from .base_models import BaseModel +from .base_models import CovJsonBaseModel from .i18n import i18n -class Category(BaseModel): +class Category(CovJsonBaseModel): id: str label: i18n description: Optional[i18n] = None -class ObservedProperty(BaseModel): +class ObservedProperty(CovJsonBaseModel): id: Optional[str] = None label: i18n description: Optional[i18n] = None diff --git a/src/covjson_pydantic/parameter.py b/src/covjson_pydantic/parameter.py index 25b5645..9f684cc 100644 --- a/src/covjson_pydantic/parameter.py +++ b/src/covjson_pydantic/parameter.py @@ -6,13 +6,13 @@ from pydantic import model_validator -from .base_models import BaseModel +from .base_models import CovJsonBaseModel from .i18n import i18n from .observed_property import ObservedProperty from .unit import Unit -class Parameter(BaseModel, extra="allow"): +class Parameter(CovJsonBaseModel, extra="allow"): type: Literal["Parameter"] = "Parameter" id: Optional[str] = None label: Optional[i18n] = None @@ -32,7 +32,7 @@ def must_not_have_unit_if_observed_property_has_categories(self): return self -class ParameterGroup(BaseModel, extra="allow"): +class ParameterGroup(CovJsonBaseModel, extra="allow"): type: Literal["ParameterGroup"] = "ParameterGroup" id: Optional[str] = None label: Optional[i18n] = None diff --git a/src/covjson_pydantic/reference_system.py b/src/covjson_pydantic/reference_system.py index 570610a..7222330 100644 --- a/src/covjson_pydantic/reference_system.py +++ b/src/covjson_pydantic/reference_system.py @@ -7,17 +7,17 @@ from pydantic import AnyUrl from pydantic import model_validator -from .base_models import BaseModel +from .base_models import CovJsonBaseModel from .i18n import i18n -class TargetConcept(BaseModel): +class TargetConcept(CovJsonBaseModel): id: Optional[str] = None # Not in spec, but needed for example in spec for 'Identifier-based Reference Systems' label: i18n description: Optional[i18n] = None -class ReferenceSystem(BaseModel, extra="allow"): +class ReferenceSystem(CovJsonBaseModel, extra="allow"): type: Literal["GeographicCRS", "ProjectedCRS", "VerticalCRS", "TemporalRS", "IdentifierRS"] id: Optional[str] = None description: Optional[i18n] = None @@ -49,6 +49,6 @@ def check_type_specific_fields(self): return self -class ReferenceSystemConnectionObject(BaseModel): +class ReferenceSystemConnectionObject(CovJsonBaseModel): coordinates: List[str] system: ReferenceSystem diff --git a/src/covjson_pydantic/unit.py b/src/covjson_pydantic/unit.py index 63ac5a7..f1783dd 100644 --- a/src/covjson_pydantic/unit.py +++ b/src/covjson_pydantic/unit.py @@ -3,16 +3,16 @@ from pydantic import model_validator -from .base_models import BaseModel +from .base_models import CovJsonBaseModel from .i18n import i18n -class Symbol(BaseModel): +class Symbol(CovJsonBaseModel): value: str type: str -class Unit(BaseModel): +class Unit(CovJsonBaseModel): id: Optional[str] = None label: Optional[i18n] = None symbol: Optional[Union[str, Symbol]] = None From 4e26ed8a87a4c04e04a58595d5e134090c3d02b7 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Thu, 21 Sep 2023 08:52:18 +0200 Subject: [PATCH 12/17] KDP-1761: Minimum pydantic 2.3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8bb9666..5f94691 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Typing :: Typed", ] version = "0.2.0" -dependencies = ["pydantic>=2,<3"] +dependencies = ["pydantic>=2.3,<3"] [project.optional-dependencies] test = ["pytest", "pytest-cov"] From 971c400cb57efe41b346c37a2aeb66e0e0791f02 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 22 Sep 2023 11:33:39 +0200 Subject: [PATCH 13/17] KDP-1761: Also domain can have extra fields --- src/covjson_pydantic/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/covjson_pydantic/domain.py b/src/covjson_pydantic/domain.py index cde31f4..4a41713 100644 --- a/src/covjson_pydantic/domain.py +++ b/src/covjson_pydantic/domain.py @@ -66,7 +66,7 @@ def at_least_one_axes(self): return self -class Domain(CovJsonBaseModel): +class Domain(CovJsonBaseModel, extra="allow"): type: Literal["Domain"] = "Domain" domainType: Optional[DomainType] = None # noqa: N815 axes: Axes From 8b911ffc9444959d2418569aa2105cd5074737f9 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 22 Sep 2023 11:46:45 +0200 Subject: [PATCH 14/17] KDP-1761: Add note about issue --- src/covjson_pydantic/i18n.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/covjson_pydantic/i18n.py b/src/covjson_pydantic/i18n.py index 4959b37..9a345b6 100644 --- a/src/covjson_pydantic/i18n.py +++ b/src/covjson_pydantic/i18n.py @@ -14,5 +14,6 @@ class LanguageTag(str, Enum): # This may be a bug in Pydantic: https://github.com/pydantic/pydantic/issues/6467 # or: https://github.com/pydantic/pydantic/issues/6422 # So, for now, reverted to a less strict type +# See issue: https://github.com/KNMI/covjson-pydantic/issues/3 # i18n = Dict[LanguageTag, str] i18n = Dict[str, str] From 84b0a7f9f9db4b62c9b495b34a351b103c7b95e0 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Fri, 22 Sep 2023 13:38:07 +0200 Subject: [PATCH 15/17] KDP-1761: Enable strict mode --- src/covjson_pydantic/base_models.py | 1 + src/covjson_pydantic/domain.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/covjson_pydantic/base_models.py b/src/covjson_pydantic/base_models.py index b5e68af..4c1c3eb 100644 --- a/src/covjson_pydantic/base_models.py +++ b/src/covjson_pydantic/base_models.py @@ -9,4 +9,5 @@ class CovJsonBaseModel(PydanticBaseModel): extra="forbid", validate_default=True, validate_assignment=True, + strict=True, ) diff --git a/src/covjson_pydantic/domain.py b/src/covjson_pydantic/domain.py index 4a41713..89da847 100644 --- a/src/covjson_pydantic/domain.py +++ b/src/covjson_pydantic/domain.py @@ -30,7 +30,9 @@ def single_value_case(self): ValuesT = TypeVar("ValuesT") -class ValuesAxis(CovJsonBaseModel, Generic[ValuesT], extra="allow"): +# Combination between Generics (ValuesT) and datetime and strict mode causes issues between JSON <-> Pydantic +# conversions. Strict mode has been disabled. Issue: https://github.com/KNMI/covjson-pydantic/issues/4 +class ValuesAxis(CovJsonBaseModel, Generic[ValuesT], extra="allow", strict=False): dataType: Optional[str] = None # noqa: N815 coordinates: Optional[List[str]] = None values: List[ValuesT] From 22e7f01863d5ca8f383bb8bd7c617a9913561b66 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Mon, 25 Sep 2023 09:25:27 +0200 Subject: [PATCH 16/17] KDP-1761: Fix example --- README.md | 4 ++-- example.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e21b373..ae39aca 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,12 @@ pip install git+https://github.com/KNMI/covjson-pydantic.git from datetime import datetime, timezone from pydantic import AwareDatetime from covjson_pydantic.coverage import Coverage -from covjson_pydantic.domain import Domain, Axes, ValuesAxis +from covjson_pydantic.domain import Domain, Axes, ValuesAxis, DomainType from covjson_pydantic.ndarray import NdArray c = Coverage( domain=Domain( - domainType="PointSeries", + domainType=DomainType.point_series, axes=Axes( x=ValuesAxis[float](values=[1.23]), y=ValuesAxis[float](values=[4.56]), diff --git a/example.py b/example.py index bb4d902..a494f56 100644 --- a/example.py +++ b/example.py @@ -4,13 +4,14 @@ from covjson_pydantic.coverage import Coverage from covjson_pydantic.domain import Axes from covjson_pydantic.domain import Domain +from covjson_pydantic.domain import DomainType from covjson_pydantic.domain import ValuesAxis from covjson_pydantic.ndarray import NdArray from pydantic import AwareDatetime c = Coverage( domain=Domain( - domainType="PointSeries", + domainType=DomainType.point_series, axes=Axes( x=ValuesAxis[float](values=[1.23]), y=ValuesAxis[float](values=[4.56]), From 44c7a28e72d50c5a566baf39958cf21739715e92 Mon Sep 17 00:00:00 2001 From: Paul van Schayck Date: Mon, 25 Sep 2023 13:07:09 +0200 Subject: [PATCH 17/17] KDP-1761: Workaround for FastAPI issue --- src/covjson_pydantic/domain.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/covjson_pydantic/domain.py b/src/covjson_pydantic/domain.py index 89da847..1b37489 100644 --- a/src/covjson_pydantic/domain.py +++ b/src/covjson_pydantic/domain.py @@ -8,6 +8,7 @@ from typing import Union from pydantic import AwareDatetime +from pydantic import field_validator from pydantic import model_validator from pydantic import PositiveInt @@ -74,6 +75,16 @@ class Domain(CovJsonBaseModel, extra="allow"): axes: Axes referencing: Optional[List[ReferenceSystemConnectionObject]] = None + # TODO: This is a workaround to allow domainType to work in strict mode, in combination with FastAPI. + # See: https://github.com/tiangolo/fastapi/discussions/9868 + # And: https://github.com/KNMI/covjson-pydantic/issues/5 + @field_validator("domainType", mode="before") + @classmethod + def value_to_enum(cls, v): + if isinstance(v, str): + return DomainType(v) + return v + @staticmethod def check_axis(domain_type, axes, required_axes, allowed_axes, single_value_axes): # Check required axes