diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 40eda99..ed5aa23 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: additional_dependencies: [ 'pep8-naming==0.12.1' ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.971 + rev: v1.5.1 hooks: - id: mypy language_version: python @@ -64,5 +64,4 @@ repos: # Don't pass it the individual filenames because it is already doing the whole folder. pass_filenames: false additional_dependencies: - - orjson - pydantic diff --git a/README.md b/README.md index 5911298..ae39aca 100644 --- a/README.md +++ b/README.md @@ -35,25 +35,27 @@ pip install git+https://github.com/KNMI/covjson-pydantic.git ## Usage ```python -import datetime +from datetime import datetime, timezone +from pydantic import AwareDatetime from covjson_pydantic.coverage import Coverage -from covjson_pydantic.domain import Domain +from covjson_pydantic.domain import Domain, Axes, ValuesAxis, DomainType from covjson_pydantic.ndarray import NdArray c = Coverage( domain=Domain( - domainType="PointSeries", - axes={ - "x": {"dataType": "float", "values": [1.23]}, - "y": {"values": [4.56]}, - "t": {"dataType": "datetime", "values": [datetime.datetime.now()]} - }, + domainType=DomainType.point_series, + axes=Axes( + x=ValuesAxis[float](values=[1.23]), + y=ValuesAxis[float](values=[4.56]), + t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)]) + ) ), ranges={ "temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0]) } ) -print(c.json(exclude_none=True,indent=True)) + +print(c.model_dump_json(exclude_none=True, indent=4)) ``` Will print ```json @@ -64,7 +66,6 @@ Will print "domainType": "PointSeries", "axes": { "x": { - "dataType": "float", "values": [ 1.23 ] @@ -75,9 +76,8 @@ Will print ] }, "t": { - "dataType": "datetime", "values": [ - "2023-01-19T13:14:47.126631Z" + "2023-09-14T11:54:02.151493Z" ] } } diff --git a/example.py b/example.py index 238b4a2..a494f56 100644 --- a/example.py +++ b/example.py @@ -1,19 +1,24 @@ -import datetime +from datetime import datetime +from datetime import timezone from covjson_pydantic.coverage import Coverage +from covjson_pydantic.domain import Axes from covjson_pydantic.domain import Domain +from covjson_pydantic.domain import DomainType +from covjson_pydantic.domain import ValuesAxis from covjson_pydantic.ndarray import NdArray +from pydantic import AwareDatetime c = Coverage( domain=Domain( - domainType="PointSeries", - axes={ - "x": {"dataType": "float", "values": [1.23]}, - "y": {"values": [4.56]}, - "t": {"dataType": "datetime", "values": [datetime.datetime.now()]}, - }, + domainType=DomainType.point_series, + axes=Axes( + x=ValuesAxis[float](values=[1.23]), + y=ValuesAxis[float](values=[4.56]), + t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)]), + ), ), ranges={"temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])}, ) -print(c.json(exclude_none=True, indent=True)) +print(c.model_dump_json(exclude_none=True, indent=4)) diff --git a/pyproject.toml b/pyproject.toml index 0f2e3eb..5f94691 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,8 @@ classifiers = [ "Topic :: Scientific/Engineering :: GIS", "Typing :: Typed", ] -version = "0.1.0" -dependencies = ["pydantic>=1,<2", "orjson>=3"] +version = "0.2.0" +dependencies = ["pydantic>=2.3,<3"] [project.optional-dependencies] test = ["pytest", "pytest-cov"] diff --git a/src/covjson_pydantic/base_models.py b/src/covjson_pydantic/base_models.py index 3836e7b..4c1c3eb 100644 --- a/src/covjson_pydantic/base_models.py +++ b/src/covjson_pydantic/base_models.py @@ -1,32 +1,13 @@ -import orjson from pydantic import BaseModel as PydanticBaseModel -from pydantic import Extra - - -def orjson_dumps(v, *, default, indent=None, sort_keys=False): - options = orjson.OPT_NON_STR_KEYS | orjson.OPT_UTC_Z | orjson.OPT_NAIVE_UTC - if indent: - options |= orjson.OPT_INDENT_2 - - if sort_keys: - options |= orjson.OPT_SORT_KEYS - - # orjson.dumps returns bytes, to match standard json.dumps we need to decode - return orjson.dumps(v, default=default, option=options).decode() - - -class BaseModel(PydanticBaseModel): - class Config: - anystr_strip_whitespace = True - min_anystr_length = 1 - extra = Extra.forbid - validate_all = True - validate_assignment = True - - json_loads = orjson.loads - json_dumps = orjson_dumps - - -class CovJsonBaseModel(BaseModel): - class Config: - extra = Extra.allow # allow custom members +from pydantic import ConfigDict + + +class CovJsonBaseModel(PydanticBaseModel): + model_config = ConfigDict( + str_strip_whitespace=True, + str_min_length=1, + extra="forbid", + validate_default=True, + validate_assignment=True, + strict=True, + ) diff --git a/src/covjson_pydantic/coverage.py b/src/covjson_pydantic/coverage.py index 11d5fd4..279240b 100644 --- a/src/covjson_pydantic/coverage.py +++ b/src/covjson_pydantic/coverage.py @@ -16,19 +16,19 @@ from .reference_system import ReferenceSystemConnectionObject -class Coverage(CovJsonBaseModel): - id: Optional[str] +class Coverage(CovJsonBaseModel, extra="allow"): + id: Optional[str] = None type: Literal["Coverage"] = "Coverage" domain: Domain - parameters: Optional[Dict[str, Parameter]] - parameterGroups: Optional[List[ParameterGroup]] # noqa: N815 + parameters: Optional[Dict[str, Parameter]] = None + parameterGroups: Optional[List[ParameterGroup]] = None # noqa: N815 ranges: Dict[str, Union[NdArray, TiledNdArray, AnyUrl]] -class CoverageCollection(CovJsonBaseModel): +class CoverageCollection(CovJsonBaseModel, extra="allow"): type: Literal["CoverageCollection"] = "CoverageCollection" - domainType: Optional[DomainType] # noqa: N815 + domainType: Optional[DomainType] = None # noqa: N815 coverages: List[Coverage] - parameters: Optional[Dict[str, Parameter]] - parameterGroups: Optional[List[ParameterGroup]] # noqa: N815 - referencing: Optional[List[ReferenceSystemConnectionObject]] + parameters: Optional[Dict[str, Parameter]] = None + parameterGroups: Optional[List[ParameterGroup]] = None # noqa: N815 + referencing: Optional[List[ReferenceSystemConnectionObject]] = None diff --git a/src/covjson_pydantic/domain.py b/src/covjson_pydantic/domain.py index 8141393..1b37489 100644 --- a/src/covjson_pydantic/domain.py +++ b/src/covjson_pydantic/domain.py @@ -1,4 +1,3 @@ -from datetime import datetime from enum import Enum from typing import Generic from typing import List @@ -8,49 +7,43 @@ from typing import TypeVar from typing import Union -from pydantic import Extra +from pydantic import AwareDatetime +from pydantic import field_validator +from pydantic import model_validator from pydantic import PositiveInt -from pydantic.class_validators import root_validator -from pydantic.generics import GenericModel -from .base_models import BaseModel from .base_models import CovJsonBaseModel from .reference_system import ReferenceSystemConnectionObject -class CompactAxis(BaseModel): +class CompactAxis(CovJsonBaseModel): start: float stop: float num: PositiveInt - @root_validator(skip_on_failure=True) - def single_value_case(cls, values): - if values["num"] == 1 and values["start"] != values["stop"]: + @model_validator(mode="after") + def single_value_case(self): + if self.num == 1 and self.start != self.stop: raise ValueError("If the value of 'num' is 1, then 'start' and 'stop' MUST have identical values.") - return values + return self ValuesT = TypeVar("ValuesT") -class ValuesAxis(GenericModel, Generic[ValuesT]): - dataType: Optional[str] # noqa: N815 - coordinates: Optional[List[str]] +# Combination between Generics (ValuesT) and datetime and strict mode causes issues between JSON <-> Pydantic +# conversions. Strict mode has been disabled. Issue: https://github.com/KNMI/covjson-pydantic/issues/4 +class ValuesAxis(CovJsonBaseModel, Generic[ValuesT], extra="allow", strict=False): + dataType: Optional[str] = None # noqa: N815 + coordinates: Optional[List[str]] = None values: List[ValuesT] - bounds: Optional[List[ValuesT]] - - class Config: - anystr_strip_whitespace = True - min_anystr_length = 1 - extra = Extra.allow # allow custom members - validate_all = True - validate_assignment = True - - @root_validator(skip_on_failure=True) - def bounds_length(cls, values): - if values["bounds"] is not None and len(values["bounds"]) != 2 * len(values["values"]): + bounds: Optional[List[ValuesT]] = None + + @model_validator(mode="after") + def bounds_length(self): + if self.bounds is not None and len(self.bounds) != 2 * len(self.values): raise ValueError("If provided, the length of 'bounds' should be twice that of 'values'.") - return values + return self class DomainType(str, Enum): @@ -62,31 +55,35 @@ class DomainType(str, Enum): multi_point = "MultiPoint" -class Axes(BaseModel): - x: Optional[Union[ValuesAxis[float], CompactAxis]] - y: Optional[Union[ValuesAxis[float], CompactAxis]] - z: Optional[Union[ValuesAxis[float], CompactAxis]] - t: Optional[ValuesAxis[datetime]] - composite: Optional[ValuesAxis[Tuple]] - - @root_validator(skip_on_failure=True) - def at_least_one_axes(cls, values): - if ( - values["x"] is None - and values["y"] is None - and values["z"] is None - and values["t"] is None - and values["composite"] is None - ): +class Axes(CovJsonBaseModel): + x: Optional[Union[ValuesAxis[float], CompactAxis]] = None + y: Optional[Union[ValuesAxis[float], CompactAxis]] = None + z: Optional[Union[ValuesAxis[float], CompactAxis]] = None + t: Optional[ValuesAxis[AwareDatetime]] = None + composite: Optional[ValuesAxis[Tuple]] = None + + @model_validator(mode="after") + def at_least_one_axes(self): + if self.x is None and self.y is None and self.z is None and self.t is None and self.composite is None: raise ValueError("At least one axis of x,y,z,t or composite must be given.") - return values + return self -class Domain(CovJsonBaseModel): +class Domain(CovJsonBaseModel, extra="allow"): type: Literal["Domain"] = "Domain" - domainType: Optional[DomainType] # noqa: N815 + domainType: Optional[DomainType] = None # noqa: N815 axes: Axes - referencing: Optional[List[ReferenceSystemConnectionObject]] + referencing: Optional[List[ReferenceSystemConnectionObject]] = None + + # TODO: This is a workaround to allow domainType to work in strict mode, in combination with FastAPI. + # See: https://github.com/tiangolo/fastapi/discussions/9868 + # And: https://github.com/KNMI/covjson-pydantic/issues/5 + @field_validator("domainType", mode="before") + @classmethod + def value_to_enum(cls, v): + if isinstance(v, str): + return DomainType(v) + return v @staticmethod def check_axis(domain_type, axes, required_axes, allowed_axes, single_value_axes): @@ -119,10 +116,10 @@ def check_axis(domain_type, axes, required_axes, allowed_axes, single_value_axes f"of a '{domain_type.value}' domain must contain a single value." ) - @root_validator(skip_on_failure=True) - def check_domain_consistent(cls, values): - domain_type = values.get("domainType") - axes = values.get("axes") + @model_validator(mode="after") + def check_domain_consistent(self): + domain_type = self.domainType + axes = self.axes if domain_type == DomainType.grid: Domain.check_axis( @@ -158,4 +155,4 @@ def check_domain_consistent(cls, values): domain_type, axes, required_axes={"composite"}, allowed_axes={"t"}, single_value_axes={"t"} ) - return values + return self diff --git a/src/covjson_pydantic/i18n.py b/src/covjson_pydantic/i18n.py index 2cc6e0b..9a345b6 100644 --- a/src/covjson_pydantic/i18n.py +++ b/src/covjson_pydantic/i18n.py @@ -9,4 +9,11 @@ class LanguageTag(str, Enum): undefined = "und" -i18n = Dict[LanguageTag, str] +# TODO: This was throwing warning: +# Expected `definition-ref` but got `LanguageTag` - serialized value may not be as expected +# This may be a bug in Pydantic: https://github.com/pydantic/pydantic/issues/6467 +# or: https://github.com/pydantic/pydantic/issues/6422 +# So, for now, reverted to a less strict type +# See issue: https://github.com/KNMI/covjson-pydantic/issues/3 +# i18n = Dict[LanguageTag, str] +i18n = Dict[str, str] diff --git a/src/covjson_pydantic/ndarray.py b/src/covjson_pydantic/ndarray.py index 068bb39..abda788 100644 --- a/src/covjson_pydantic/ndarray.py +++ b/src/covjson_pydantic/ndarray.py @@ -4,10 +4,8 @@ from typing import Literal from typing import Optional -from pydantic import AnyUrl -from pydantic.class_validators import root_validator +from pydantic import model_validator -from .base_models import BaseModel from .base_models import CovJsonBaseModel @@ -16,46 +14,42 @@ class DataType(str, Enum): float = "float" -class NdArray(CovJsonBaseModel): +class NdArray(CovJsonBaseModel, extra="allow"): type: Literal["NdArray"] = "NdArray" dataType: DataType = DataType.float # noqa: N815 - axisNames: Optional[List[str]] # noqa: N815 - shape: Optional[List[int]] + axisNames: Optional[List[str]] = None # noqa: N815 + shape: Optional[List[int]] = None values: List[Optional[float]] - @root_validator(skip_on_failure=True) - def check_field_dependencies(cls, values): - if len(values["values"]) > 1 and (values.get("axisNames") is None or len(values.get("axisNames")) == 0): + @model_validator(mode="after") + def check_field_dependencies(self): + if len(self.values) > 1 and (self.axisNames is None or len(self.axisNames) == 0): raise ValueError("'axisNames' must to be provided if array is not 0D") - if len(values["values"]) > 1 and (values.get("shape") is None or len(values.get("shape")) == 0): + if len(self.values) > 1 and (self.shape is None or len(self.shape) == 0): raise ValueError("'shape' must to be provided if array is not 0D") - if ( - values.get("axisNames") is not None - and values.get("shape") is not None - and len(values.get("axisNames")) != len(values.get("shape")) - ): + if self.axisNames is not None and self.shape is not None and len(self.axisNames) != len(self.shape): raise ValueError("'axisNames' and 'shape' should have equal length") - if values.get("shape") is not None and len(values.get("shape")) >= 1: - prod = math.prod(values["shape"]) - if len(values["values"]) != prod: + if self.shape is not None and len(self.shape) >= 1: + prod = math.prod(self.shape) + if len(self.values) != prod: raise ValueError( "Where 'shape' is present and non-empty, the product of its values MUST equal " "the number of elements in the 'values' array." ) - return values + return self -class TileSet(BaseModel): +class TileSet(CovJsonBaseModel): tileShape: List[Optional[int]] # noqa: N815 - urlTemplate: AnyUrl # noqa: N815 + urlTemplate: str # noqa: N815 # TODO: Validation of field dependencies -class TiledNdArray(CovJsonBaseModel): +class TiledNdArray(CovJsonBaseModel, extra="allow"): type: Literal["TiledNdArray"] = "TiledNdArray" dataType: DataType = DataType.float # noqa: N815 axisNames: List[str] # noqa: N815 diff --git a/src/covjson_pydantic/observed_property.py b/src/covjson_pydantic/observed_property.py index f2f57f1..976a067 100644 --- a/src/covjson_pydantic/observed_property.py +++ b/src/covjson_pydantic/observed_property.py @@ -1,18 +1,18 @@ from typing import List from typing import Optional -from .base_models import BaseModel +from .base_models import CovJsonBaseModel from .i18n import i18n -class Category(BaseModel): +class Category(CovJsonBaseModel): id: str label: i18n - description: Optional[i18n] + description: Optional[i18n] = None -class ObservedProperty(BaseModel): - id: Optional[str] +class ObservedProperty(CovJsonBaseModel): + id: Optional[str] = None label: i18n - description: Optional[i18n] - categories: Optional[List[Category]] + description: Optional[i18n] = None + categories: Optional[List[Category]] = None diff --git a/src/covjson_pydantic/parameter.py b/src/covjson_pydantic/parameter.py index 63c0769..9f684cc 100644 --- a/src/covjson_pydantic/parameter.py +++ b/src/covjson_pydantic/parameter.py @@ -4,51 +4,46 @@ from typing import Optional from typing import Union -from pydantic import Extra -from pydantic.class_validators import root_validator +from pydantic import model_validator -from .base_models import BaseModel +from .base_models import CovJsonBaseModel from .i18n import i18n from .observed_property import ObservedProperty from .unit import Unit -class Parameter(BaseModel, extra=Extra.allow): +class Parameter(CovJsonBaseModel, extra="allow"): type: Literal["Parameter"] = "Parameter" - id: Optional[str] - label: Optional[i18n] - description: Optional[i18n] + id: Optional[str] = None + label: Optional[i18n] = None + description: Optional[i18n] = None observedProperty: ObservedProperty # noqa: N815 - categoryEncoding: Optional[Dict[str, Union[int, List[int]]]] # noqa: N815 - unit: Optional[Unit] - - @root_validator(skip_on_failure=True) - def must_not_have_unit_if_observed_property_has_categories(cls, values): - if ( - values.get("unit") is not None - and values.get("observedProperty") is not None - and values.get("observedProperty").categories is not None - ): + categoryEncoding: Optional[Dict[str, Union[int, List[int]]]] = None # noqa: N815 + unit: Optional[Unit] = None + + @model_validator(mode="after") + def must_not_have_unit_if_observed_property_has_categories(self): + if self.unit is not None and self.observedProperty is not None and self.observedProperty.categories is not None: raise ValueError( "A parameter object MUST NOT have a 'unit' member " "if the 'observedProperty' member has a 'categories' member." ) - return values + return self -class ParameterGroup(BaseModel, extra=Extra.allow): +class ParameterGroup(CovJsonBaseModel, extra="allow"): type: Literal["ParameterGroup"] = "ParameterGroup" - id: Optional[str] - label: Optional[i18n] - description: Optional[i18n] - observedProperty: Optional[ObservedProperty] # noqa: N815 + id: Optional[str] = None + label: Optional[i18n] = None + description: Optional[i18n] = None + observedProperty: Optional[ObservedProperty] = None # noqa: N815 members: List[str] - @root_validator(skip_on_failure=True) - def must_have_label_and_or_observed_property(cls, values): - if values.get("label") is None and values.get("observedProperty") is None: + @model_validator(mode="after") + def must_have_label_and_or_observed_property(self): + if self.label is None and self.observedProperty is None: raise ValueError( "A parameter group object MUST have either or both the members 'label' or/and 'observedProperty'" ) - return values + return self diff --git a/src/covjson_pydantic/reference_system.py b/src/covjson_pydantic/reference_system.py index ba6c795..7222330 100644 --- a/src/covjson_pydantic/reference_system.py +++ b/src/covjson_pydantic/reference_system.py @@ -5,55 +5,50 @@ from typing import Union from pydantic import AnyUrl -from pydantic import Extra -from pydantic.class_validators import root_validator +from pydantic import model_validator -from .base_models import BaseModel +from .base_models import CovJsonBaseModel from .i18n import i18n -class TargetConcept(BaseModel): - id: Optional[str] # Not in spec, but needed for example in spec for 'Identifier-based Reference Systems' +class TargetConcept(CovJsonBaseModel): + id: Optional[str] = None # Not in spec, but needed for example in spec for 'Identifier-based Reference Systems' label: i18n - description: Optional[i18n] + description: Optional[i18n] = None -class ReferenceSystem(BaseModel, extra=Extra.allow): +class ReferenceSystem(CovJsonBaseModel, extra="allow"): type: Literal["GeographicCRS", "ProjectedCRS", "VerticalCRS", "TemporalRS", "IdentifierRS"] - id: Optional[str] - description: Optional[i18n] + id: Optional[str] = None + description: Optional[i18n] = None # Only for TemporalRS - calendar: Optional[Union[Literal["Gregorian"], AnyUrl]] - timeScale: Optional[AnyUrl] # noqa: N815 + calendar: Optional[Union[Literal["Gregorian"], AnyUrl]] = None + timeScale: Optional[AnyUrl] = None # noqa: N815 # Only for IdentifierRS - label: Optional[i18n] - targetConcept: Optional[TargetConcept] # noqa: N815 - identifiers: Optional[Dict[str, TargetConcept]] - - @root_validator(skip_on_failure=True) - def check_type_specific_fields(cls, values): - if values["type"] != "TemporalRS" and ( - values.get("calendar") is not None or values.get("timeScale") is not None - ): + label: Optional[i18n] = None + targetConcept: Optional[TargetConcept] = None # noqa: N815 + identifiers: Optional[Dict[str, TargetConcept]] = None + + @model_validator(mode="after") + def check_type_specific_fields(self): + if self.type != "TemporalRS" and (self.calendar is not None or self.timeScale is not None): raise ValueError("'calendar' and 'timeScale' fields can only be used for type 'TemporalRS'") - if values["type"] != "IdentifierRS" and ( - values.get("label") is not None - or values.get("targetConcept") is not None - or values.get("identifiers") is not None + if self.type != "IdentifierRS" and ( + self.label is not None or self.targetConcept is not None or self.identifiers is not None ): raise ValueError( "'label', 'targetConcept' and 'identifiers' fields can only be used for type 'IdentifierRS'" ) - if values["type"] == "IdentifierRS" and values.get("targetConcept") is None: + if self.type == "IdentifierRS" and self.targetConcept is None: raise ValueError("An identifier RS object MUST have a member 'targetConcept'") - return values + return self -class ReferenceSystemConnectionObject(BaseModel): +class ReferenceSystemConnectionObject(CovJsonBaseModel): coordinates: List[str] system: ReferenceSystem diff --git a/src/covjson_pydantic/unit.py b/src/covjson_pydantic/unit.py index f4a8a44..f1783dd 100644 --- a/src/covjson_pydantic/unit.py +++ b/src/covjson_pydantic/unit.py @@ -1,25 +1,25 @@ from typing import Optional from typing import Union -from pydantic.class_validators import root_validator +from pydantic import model_validator -from .base_models import BaseModel +from .base_models import CovJsonBaseModel from .i18n import i18n -class Symbol(BaseModel): +class Symbol(CovJsonBaseModel): value: str type: str -class Unit(BaseModel): - id: Optional[str] - label: Optional[i18n] - symbol: Optional[Union[str, Symbol]] +class Unit(CovJsonBaseModel): + id: Optional[str] = None + label: Optional[i18n] = None + symbol: Optional[Union[str, Symbol]] = None - @root_validator(skip_on_failure=True) - def check_either_label_or_symbol(cls, values): - if values.get("label") is None and values.get("symbol") is None: + @model_validator(mode="after") + def check_either_label_or_symbol(self): + if self.label is None and self.symbol is None: raise ValueError("Either 'label' or 'symbol' should be set") - return values + return self diff --git a/tests/test_coverage.py b/tests/test_coverage.py index fdfc296..139872b 100644 --- a/tests/test_coverage.py +++ b/tests/test_coverage.py @@ -47,7 +47,7 @@ def test_happy_cases(file_name, object_type): json_string = json.dumps(data, separators=(",", ":")) # Round-trip - assert object_type.parse_raw(json_string).json(exclude_none=True) == json_string + assert object_type.model_validate_json(json_string).model_dump_json(exclude_none=True) == json_string error_cases = [ @@ -70,4 +70,4 @@ def test_error_cases(file_name, object_type, error_message): json_string = json.dumps(data, separators=(",", ":")) with pytest.raises(ValidationError, match=error_message): - object_type.parse_raw(json_string) + object_type.model_validate_json(json_string)