Merge pull request #2 from KNMI/KDP-1761

KDP-1761: Upgrade to Pydantic v2

.pre-commit-config.yaml

@@ -53,7 +53,7 @@ repos:
         additional_dependencies: [ 'pep8-naming==0.12.1' ]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.971
+    rev: v1.5.1
     hooks:
       - id: mypy
         language_version: python
@@ -64,5 +64,4 @@ repos:
         # Don't pass it the individual filenames because it is already doing the whole folder.
         pass_filenames: false
         additional_dependencies:
-          - orjson
           - pydantic

README.md

@@ -35,25 +35,27 @@ pip install git+https://github.com/KNMI/covjson-pydantic.git
 ## Usage
 
 ```python
-import datetime
+from datetime import datetime, timezone
+from pydantic import AwareDatetime
 from covjson_pydantic.coverage import Coverage
-from covjson_pydantic.domain import Domain
+from covjson_pydantic.domain import Domain, Axes, ValuesAxis, DomainType
 from covjson_pydantic.ndarray import NdArray
 
 c = Coverage(
     domain=Domain(
-        domainType="PointSeries",
-        axes={
-            "x": {"dataType": "float", "values": [1.23]},
-            "y": {"values": [4.56]},
-            "t": {"dataType": "datetime", "values": [datetime.datetime.now()]}
-        },
+        domainType=DomainType.point_series,
+        axes=Axes(
+            x=ValuesAxis[float](values=[1.23]),
+            y=ValuesAxis[float](values=[4.56]),
+            t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)])
+        )
     ),
     ranges={
         "temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])
     }
 )
-print(c.json(exclude_none=True,indent=True))
+
+print(c.model_dump_json(exclude_none=True, indent=4))
 ```
 Will print
 ```json
@@ -64,7 +66,6 @@ Will print
         "domainType": "PointSeries",
         "axes": {
             "x": {
-                "dataType": "float",
                 "values": [
                     1.23
                 ]
@@ -75,9 +76,8 @@ Will print
                 ]
             },
             "t": {
-                "dataType": "datetime",
                 "values": [
-                    "2023-01-19T13:14:47.126631Z"
+                    "2023-09-14T11:54:02.151493Z"
                 ]
             }
         }

example.py

@@ -1,19 +1,24 @@
-import datetime
+from datetime import datetime
+from datetime import timezone
 
 from covjson_pydantic.coverage import Coverage
+from covjson_pydantic.domain import Axes
 from covjson_pydantic.domain import Domain
+from covjson_pydantic.domain import DomainType
+from covjson_pydantic.domain import ValuesAxis
 from covjson_pydantic.ndarray import NdArray
+from pydantic import AwareDatetime
 
 c = Coverage(
     domain=Domain(
-        domainType="PointSeries",
-        axes={
-            "x": {"dataType": "float", "values": [1.23]},
-            "y": {"values": [4.56]},
-            "t": {"dataType": "datetime", "values": [datetime.datetime.now()]},
-        },
+        domainType=DomainType.point_series,
+        axes=Axes(
+            x=ValuesAxis[float](values=[1.23]),
+            y=ValuesAxis[float](values=[4.56]),
+            t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)]),
+        ),
     ),
     ranges={"temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])},
 )
 
-print(c.json(exclude_none=True, indent=True))
+print(c.model_dump_json(exclude_none=True, indent=4))

pyproject.toml

@@ -19,8 +19,8 @@ classifiers = [
     "Topic :: Scientific/Engineering :: GIS",
     "Typing :: Typed",
 ]
-version = "0.1.0"
-dependencies = ["pydantic>=1,<2", "orjson>=3"]
+version = "0.2.0"
+dependencies = ["pydantic>=2.3,<3"]
 
 [project.optional-dependencies]
 test = ["pytest", "pytest-cov"]

src/covjson_pydantic/base_models.py

@@ -1,32 +1,13 @@
-import orjson
 from pydantic import BaseModel as PydanticBaseModel
-from pydantic import Extra
-
-
-def orjson_dumps(v, *, default, indent=None, sort_keys=False):
-    options = orjson.OPT_NON_STR_KEYS | orjson.OPT_UTC_Z | orjson.OPT_NAIVE_UTC
-    if indent:
-        options |= orjson.OPT_INDENT_2
-
-    if sort_keys:
-        options |= orjson.OPT_SORT_KEYS
-
-    # orjson.dumps returns bytes, to match standard json.dumps we need to decode
-    return orjson.dumps(v, default=default, option=options).decode()
-
-
-class BaseModel(PydanticBaseModel):
-    class Config:
-        anystr_strip_whitespace = True
-        min_anystr_length = 1
-        extra = Extra.forbid
-        validate_all = True
-        validate_assignment = True
-
-        json_loads = orjson.loads
-        json_dumps = orjson_dumps
-
-
-class CovJsonBaseModel(BaseModel):
-    class Config:
-        extra = Extra.allow  # allow custom members
+from pydantic import ConfigDict
+
+
+class CovJsonBaseModel(PydanticBaseModel):
+    model_config = ConfigDict(
+        str_strip_whitespace=True,
+        str_min_length=1,
+        extra="forbid",
+        validate_default=True,
+        validate_assignment=True,
+        strict=True,
+    )

src/covjson_pydantic/coverage.py

@@ -16,19 +16,19 @@
 from .reference_system import ReferenceSystemConnectionObject
 
 
-class Coverage(CovJsonBaseModel):
-    id: Optional[str]
+class Coverage(CovJsonBaseModel, extra="allow"):
+    id: Optional[str] = None
     type: Literal["Coverage"] = "Coverage"
     domain: Domain
-    parameters: Optional[Dict[str, Parameter]]
-    parameterGroups: Optional[List[ParameterGroup]]  # noqa: N815
+    parameters: Optional[Dict[str, Parameter]] = None
+    parameterGroups: Optional[List[ParameterGroup]] = None  # noqa: N815
     ranges: Dict[str, Union[NdArray, TiledNdArray, AnyUrl]]
 
 
-class CoverageCollection(CovJsonBaseModel):
+class CoverageCollection(CovJsonBaseModel, extra="allow"):
     type: Literal["CoverageCollection"] = "CoverageCollection"
-    domainType: Optional[DomainType]  # noqa: N815
+    domainType: Optional[DomainType] = None  # noqa: N815
     coverages: List[Coverage]
-    parameters: Optional[Dict[str, Parameter]]
-    parameterGroups: Optional[List[ParameterGroup]]  # noqa: N815
-    referencing: Optional[List[ReferenceSystemConnectionObject]]
+    parameters: Optional[Dict[str, Parameter]] = None
+    parameterGroups: Optional[List[ParameterGroup]] = None  # noqa: N815
+    referencing: Optional[List[ReferenceSystemConnectionObject]] = None

src/covjson_pydantic/domain.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from enum import Enum
 from typing import Generic
 from typing import List
@@ -8,49 +7,43 @@
 from typing import TypeVar
 from typing import Union
 
-from pydantic import Extra
+from pydantic import AwareDatetime
+from pydantic import field_validator
+from pydantic import model_validator
 from pydantic import PositiveInt
-from pydantic.class_validators import root_validator
-from pydantic.generics import GenericModel
 
-from .base_models import BaseModel
 from .base_models import CovJsonBaseModel
 from .reference_system import ReferenceSystemConnectionObject
 
 
-class CompactAxis(BaseModel):
+class CompactAxis(CovJsonBaseModel):
     start: float
     stop: float
     num: PositiveInt
 
-    @root_validator(skip_on_failure=True)
-    def single_value_case(cls, values):
-        if values["num"] == 1 and values["start"] != values["stop"]:
+    @model_validator(mode="after")
+    def single_value_case(self):
+        if self.num == 1 and self.start != self.stop:
             raise ValueError("If the value of 'num' is 1, then 'start' and 'stop' MUST have identical values.")
-        return values
+        return self
 
 
 ValuesT = TypeVar("ValuesT")
 
 
-class ValuesAxis(GenericModel, Generic[ValuesT]):
-    dataType: Optional[str]  # noqa: N815
-    coordinates: Optional[List[str]]
+# Combination between Generics (ValuesT) and datetime and strict mode causes issues between JSON <-> Pydantic
+# conversions. Strict mode has been disabled. Issue: https://github.com/KNMI/covjson-pydantic/issues/4
+class ValuesAxis(CovJsonBaseModel, Generic[ValuesT], extra="allow", strict=False):
+    dataType: Optional[str] = None  # noqa: N815
+    coordinates: Optional[List[str]] = None
     values: List[ValuesT]
-    bounds: Optional[List[ValuesT]]
-
-    class Config:
-        anystr_strip_whitespace = True
-        min_anystr_length = 1
-        extra = Extra.allow  # allow custom members
-        validate_all = True
-        validate_assignment = True
-
-    @root_validator(skip_on_failure=True)
-    def bounds_length(cls, values):
-        if values["bounds"] is not None and len(values["bounds"]) != 2 * len(values["values"]):
+    bounds: Optional[List[ValuesT]] = None
+
+    @model_validator(mode="after")
+    def bounds_length(self):
+        if self.bounds is not None and len(self.bounds) != 2 * len(self.values):
             raise ValueError("If provided, the length of 'bounds' should be twice that of 'values'.")
-        return values
+        return self
 
 
 class DomainType(str, Enum):
@@ -62,31 +55,35 @@ class DomainType(str, Enum):
     multi_point = "MultiPoint"
 
 
-class Axes(BaseModel):
-    x: Optional[Union[ValuesAxis[float], CompactAxis]]
-    y: Optional[Union[ValuesAxis[float], CompactAxis]]
-    z: Optional[Union[ValuesAxis[float], CompactAxis]]
-    t: Optional[ValuesAxis[datetime]]
-    composite: Optional[ValuesAxis[Tuple]]
-
-    @root_validator(skip_on_failure=True)
-    def at_least_one_axes(cls, values):
-        if (
-            values["x"] is None
-            and values["y"] is None
-            and values["z"] is None
-            and values["t"] is None
-            and values["composite"] is None
-        ):
+class Axes(CovJsonBaseModel):
+    x: Optional[Union[ValuesAxis[float], CompactAxis]] = None
+    y: Optional[Union[ValuesAxis[float], CompactAxis]] = None
+    z: Optional[Union[ValuesAxis[float], CompactAxis]] = None
+    t: Optional[ValuesAxis[AwareDatetime]] = None
+    composite: Optional[ValuesAxis[Tuple]] = None
+
+    @model_validator(mode="after")
+    def at_least_one_axes(self):
+        if self.x is None and self.y is None and self.z is None and self.t is None and self.composite is None:
             raise ValueError("At least one axis of x,y,z,t or composite must be given.")
-        return values
+        return self
 
 
-class Domain(CovJsonBaseModel):
+class Domain(CovJsonBaseModel, extra="allow"):
     type: Literal["Domain"] = "Domain"
-    domainType: Optional[DomainType]  # noqa: N815
+    domainType: Optional[DomainType] = None  # noqa: N815
     axes: Axes
-    referencing: Optional[List[ReferenceSystemConnectionObject]]
+    referencing: Optional[List[ReferenceSystemConnectionObject]] = None
+
+    # TODO: This is a workaround to allow domainType to work in strict mode, in combination with FastAPI.
+    # See: https://github.com/tiangolo/fastapi/discussions/9868
+    # And: https://github.com/KNMI/covjson-pydantic/issues/5
+    @field_validator("domainType", mode="before")
+    @classmethod
+    def value_to_enum(cls, v):
+        if isinstance(v, str):
+            return DomainType(v)
+        return v
 
     @staticmethod
     def check_axis(domain_type, axes, required_axes, allowed_axes, single_value_axes):
@@ -119,10 +116,10 @@ def check_axis(domain_type, axes, required_axes, allowed_axes, single_value_axes
                         f"of a '{domain_type.value}' domain must contain a single value."
                     )
 
-    @root_validator(skip_on_failure=True)
-    def check_domain_consistent(cls, values):
-        domain_type = values.get("domainType")
-        axes = values.get("axes")
+    @model_validator(mode="after")
+    def check_domain_consistent(self):
+        domain_type = self.domainType
+        axes = self.axes
 
         if domain_type == DomainType.grid:
             Domain.check_axis(
@@ -158,4 +155,4 @@ def check_domain_consistent(cls, values):
                 domain_type, axes, required_axes={"composite"}, allowed_axes={"t"}, single_value_axes={"t"}
             )
 
-        return values
+        return self

src/covjson_pydantic/i18n.py

@@ -9,4 +9,11 @@ class LanguageTag(str, Enum):
     undefined = "und"
 
 
-i18n = Dict[LanguageTag, str]
+# TODO: This was throwing warning:
+#   Expected `definition-ref` but got `LanguageTag` - serialized value may not be as expected
+# This may be a bug in Pydantic: https://github.com/pydantic/pydantic/issues/6467
+# or: https://github.com/pydantic/pydantic/issues/6422
+# So, for now, reverted to a less strict type
+# See issue: https://github.com/KNMI/covjson-pydantic/issues/3
+# i18n = Dict[LanguageTag, str]
+i18n = Dict[str, str]