Improve Item validation rules and add support for py3.12

.github/workflows/cicd.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v4
@@ -27,11 +27,14 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install tox pre-commit
+          python -m pip install '.[lint]'
           pre-commit install
 
+      - name: Lint
+        run: pre-commit run --all
+
       # Run tox using the version of Python in `PATH`
-      - name: Run Tox
+      - name: Test
         run: tox -e py
 
       - name: Upload Results

CHANGELOG.txt

@@ -1,3 +1,14 @@
+- Require `type` property to be set for Catalog and Collections
+- Fix validator for Item `datetime` and Common MetaData `start_datetime` and `end_datetime`
+- Make sure all `datetime` fields are correctly parsed
+- Include `datetime` and `license` to Common MetaData
+- Be more permissive about date format
+- Make sure default values for required but unset fields are correctly parsed
+- Add support from Python 3.12
+- Lint all files
+- Increase test coverage
+
+
 3.0.0 (2024-01-25)
 ------------------
 - Support pydantic>2.0 (@huard)

pyproject.toml

@@ -3,34 +3,40 @@ requires = ["setuptools>=61.0"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name="stac-pydantic"
-description="Pydantic data models for the STAC spec"
-classifiers=[
+name = "stac-pydantic"
+description = "Pydantic data models for the STAC spec"
+classifiers = [
         "Intended Audience :: Developers",
         "Intended Audience :: Information Technology",
         "Intended Audience :: Science/Research",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
-        "License :: OSI Approved :: MIT License",
-    ]
-keywords=["stac", "pydantic", "validation"]
-authors=[{ name = "Arturo Engineering", email = "[email protected]"}]
-license= { text = "MIT" }
-requires-python=">=3.8"
-dependencies = ["click>=8.1.7", "pydantic>=2.4.1", "geojson-pydantic>=1.0.0", "ciso8601~=2.3"]
+        "Programming Language :: Python :: 3.12",
+        "License :: OSI Approved :: MIT License"]
+keywords = ["stac", "pydantic", "validation"]
+authors = [{ name = "Arturo Engineering", email = "[email protected]" }]
+license = { text = "MIT" }
+requires-python = ">=3.8"
+dependencies = [
+        "click>=8.1.7",
+        "pydantic>=2.4.1",
+        "geojson-pydantic>=1.0.0",
+        "ciso8601~=2.3",
+        "python-dateutil>=2.7.0",]
 dynamic = ["version", "readme"]
 
 [project.scripts]
 stac-pydantic = "stac_pydantic.scripts.cli:app"
 
 [project.urls]
 homepage = "https://github.com/stac-utils/stac-pydantic"
-repository ="https://github.com/stac-utils/stac-pydantic.git"
+repository = "https://github.com/stac-utils/stac-pydantic.git"
 
 [project.optional-dependencies]
-dev = ["arrow>=1.2.3",
+dev = [
+        "arrow>=1.2.3",
         "pytest>=7.4.2",
         "pytest-cov>=4.1.0",
         "pytest-icdiff>=0.8",
@@ -39,23 +45,25 @@ dev = ["arrow>=1.2.3",
         "dictdiffer>=0.9.0",
         "jsonschema>=4.19.1",
         "pyyaml>=6.0.1"]
-lint = ["types-requests>=2.31.0.5",
+lint = [
+        "types-requests>=2.31.0.5",
         "types-jsonschema>=4.19.0.3",
         "types-PyYAML>=6.0.12.12",
+        "types-python-dateutil>=2.7.0",
         "black>=23.9.1",
         "isort>=5.12.0",
         "flake8>=6.1.0",
         "Flake8-pyproject>=1.2.3",
-        "mypy>=1.5.1",
+        "mypy==1.4.1",
         "pre-commit>=3.4.0",
         "tox>=4.11.3"]
 
 [tool.setuptools.dynamic]
 version = { attr = "stac_pydantic.version.__version__" }
-readme = {file = ["README.md"], content-type = "text/markdown"}
+readme = { file = ["README.md"], content-type = "text/markdown" }
 
 [tool.setuptools.package-data]
-stac_pydantic= ["*.typed"]
+stac_pydantic = ["*.typed"]
 
 [tool.setuptools.packages.find]
 include = ["stac_pydantic*"]
@@ -81,6 +89,6 @@ exclude = ["tests", ".venv"]
 
 [tool.flake8]
 ignore = ["E203", "E501"]
-select = ["C","E","F","W","B","B950"]
+select = ["C", "E", "F", "W", "B", "B950"]
 exclude = ["tests", ".venv"]
 max-line-length = 88

stac_pydantic/api/search.py

@@ -117,7 +117,9 @@ def validate_datetime(cls, v: str) -> str:
             dates.append(parse_rfc3339(value))
 
         if len(values) > 2:
-            raise ValueError("Invalid datetime range, must match format (begin_date, end_date)")
+            raise ValueError(
+                "Invalid datetime range, must match format (begin_date, end_date)"
+            )
 
         if not {"..", ""}.intersection(set(values)):
             if dates[0] > dates[1]:

stac_pydantic/catalog.py

@@ -1,6 +1,6 @@
-from typing import List, Literal, Optional
+from typing import Any, List, Literal, Optional
 
-from pydantic import AnyUrl, ConfigDict, Field
+from pydantic import AnyUrl, ConfigDict, Field, model_validator
 
 from stac_pydantic.links import Links
 from stac_pydantic.shared import SEMVER_REGEX, StacBaseModel
@@ -14,13 +14,23 @@ class _Catalog(StacBaseModel):
 
     id: str = Field(..., alias="id", min_length=1)
     description: str = Field(..., alias="description", min_length=1)
-    stac_version: str = Field(STAC_VERSION, pattern=SEMVER_REGEX)
+    stac_version: str = Field(..., pattern=SEMVER_REGEX)
     links: Links
-    stac_extensions: Optional[List[AnyUrl]] = []
+    stac_extensions: Optional[List[AnyUrl]] = None
     title: Optional[str] = None
     type: str
     model_config = ConfigDict(use_enum_values=True, extra="allow")
 
+    @model_validator(mode="before")
+    @classmethod
+    def set_default_links(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            if data.get("links") is None:
+                data["links"] = []
+            if data.get("stac_version") is None:
+                data["stac_version"] = STAC_VERSION
+        return data
+
 
 class Catalog(_Catalog):
-    type: Literal["Catalog"] = "Catalog"
+    type: Literal["Catalog"]

stac_pydantic/collection.py

@@ -52,4 +52,4 @@ class Collection(_Catalog):
     keywords: Optional[List[str]] = None
     providers: Optional[List[Provider]] = None
     summaries: Optional[Dict[str, Union[Range, List[Any], Dict[str, Any]]]] = None
-    type: Literal["Collection"] = "Collection"
+    type: Literal["Collection"]

stac_pydantic/item.py

@@ -1,87 +1,67 @@
-from datetime import datetime as dt
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
-from ciso8601 import parse_rfc3339
 from geojson_pydantic import Feature
-from pydantic import (
-    AnyUrl,
-    ConfigDict,
-    Field,
-    field_serializer,
-    model_serializer,
-    model_validator,
-)
+from pydantic import AnyUrl, ConfigDict, Field, model_serializer, model_validator
 
 from stac_pydantic.links import Links
-from stac_pydantic.shared import (
-    DATETIME_RFC339,
-    SEMVER_REGEX,
-    Asset,
-    StacBaseModel,
-    StacCommonMetadata,
-)
+from stac_pydantic.shared import SEMVER_REGEX, Asset, StacBaseModel, StacCommonMetadata
 from stac_pydantic.version import STAC_VERSION
 
 
 class ItemProperties(StacCommonMetadata):
     """
     https://github.com/radiantearth/stac-spec/blob/v1.0.0/item-spec/item-spec.md#properties-object
+    https://github.com/radiantearth/stac-spec/blob/v1.0.0/item-spec/item-spec.md#datetime
     """
 
-    datetime: Union[dt, str] = Field(..., alias="datetime")
-
     # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information.
     model_config = ConfigDict(extra="allow")
 
     @model_validator(mode="before")
     @classmethod
-    def validate_datetime(cls, data: Dict[str, Any]) -> Dict[str, Any]:
-        datetime = data.get("datetime")
-        start_datetime = data.get("start_datetime")
-        end_datetime = data.get("end_datetime")
-
-        if not datetime or datetime == "null":
-            if not start_datetime and not end_datetime:
-                raise ValueError(
-                    "start_datetime and end_datetime must be specified when datetime is null"
-                )
-
-        if isinstance(datetime, str):
-            data["datetime"] = parse_rfc3339(datetime)
+    def validate_datetime(cls, data: Any) -> Any:
+        if isinstance(data, dict):
 
-        if isinstance(start_datetime, str):
-            data["start_datetime"] = parse_rfc3339(start_datetime)
+            datetime = data.get("datetime")
+            start_datetime = data.get("start_datetime")
+            end_datetime = data.get("end_datetime")
 
-        if isinstance(end_datetime, str):
-            data["end_datetime"] = parse_rfc3339(end_datetime)
+            if datetime is None or datetime == "null":
+                if not start_datetime and not end_datetime:
+                    raise ValueError(
+                        "start_datetime and end_datetime must be specified when datetime is null"
+                    )
+                data["datetime"] = None
 
         return data
 
-    @field_serializer("datetime")
-    def serialize_datetime(self, v: dt, _info: Any) -> str:
-        return v.strftime(DATETIME_RFC339)
-
 
 class Item(Feature, StacBaseModel):
     """
     https://github.com/radiantearth/stac-spec/blob/v1.0.0/item-spec/item-spec.md
     """
 
     id: str = Field(..., alias="id", min_length=1)
-    stac_version: str = Field(STAC_VERSION, pattern=SEMVER_REGEX)
+    stac_version: str = Field(..., pattern=SEMVER_REGEX)
     properties: ItemProperties
     assets: Dict[str, Asset]
     links: Links
-    stac_extensions: Optional[List[AnyUrl]] = []
+    stac_extensions: Optional[List[AnyUrl]] = None
     collection: Optional[str] = None
 
     @model_validator(mode="before")
     @classmethod
-    def validate_bbox(cls, values: Dict[str, Any]) -> Dict[str, Any]:
-        if isinstance(values, dict):
-            if values.get("geometry") and values.get("bbox") is None:
+    def validate_defaults(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            if data.get("stac_version") is None:
+                data["stac_version"] = STAC_VERSION
+            if data.get("geometry") and data.get("bbox") is None:
                 raise ValueError("bbox is required if geometry is not null")
-        return values
+            if data.get("assets") is None:
+                data["assets"] = {}
+            if data.get("links") is None:
+                data["links"] = []
+        return data
 
     # https://github.com/developmentseed/geojson-pydantic/issues/147
     @model_serializer(mode="wrap")

stac_pydantic/shared.py

@@ -1,9 +1,10 @@
-from datetime import datetime
+from datetime import datetime as dt
 from enum import Enum, auto
 from typing import Any, Dict, List, Optional, Tuple, Union
 from warnings import warn
 
-from pydantic import BaseModel, ConfigDict, Field
+import dateutil.parser
+from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator
 
 from stac_pydantic.utils import AutoValueEnum
 
@@ -118,17 +119,63 @@ class StacCommonMetadata(StacBaseModel):
 
     title: Optional[str] = Field(None, alias="title")
     description: Optional[str] = Field(None, alias="description")
-    start_datetime: Optional[datetime] = Field(None, alias="start_datetime")
-    end_datetime: Optional[datetime] = Field(None, alias="end_datetime")
-    created: Optional[datetime] = Field(None, alias="created")
-    updated: Optional[datetime] = Field(None, alias="updated")
+    datetime: Optional[dt] = Field(None, alias="datetime")
+    start_datetime: Optional[dt] = Field(None, alias="start_datetime")
+    end_datetime: Optional[dt] = Field(None, alias="end_datetime")
+    created: Optional[dt] = Field(None, alias="created")
+    updated: Optional[dt] = Field(None, alias="updated")
+    license: Optional[str] = Field(None, alias="license")
     platform: Optional[str] = Field(None, alias="platform")
     instruments: Optional[List[str]] = Field(None, alias="instruments")
     constellation: Optional[str] = Field(None, alias="constellation")
     mission: Optional[str] = Field(None, alias="mission")
     providers: Optional[List[Provider]] = Field(None, alias="providers")
     gsd: Optional[float] = Field(None, alias="gsd", gt=0)
 
+    @model_validator(mode="before")
+    @classmethod
+    def validate_start_end_datetime(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+
+            start_datetime = data.get("start_datetime")
+            end_datetime = data.get("end_datetime")
+            datetime = data.get("datetime")
+            created = data.get("created")
+            updated = data.get("updated")
+
+            if not all([start_datetime, end_datetime]) and any(
+                [start_datetime, end_datetime]
+            ):
+                raise ValueError(
+                    "start_datetime and end_datetime must be specified together"
+                )
+
+            if isinstance(datetime, str):
+                data["datetime"] = dateutil.parser.isoparse(datetime)
+
+            if isinstance(start_datetime, str):
+                data["start_datetime"] = dateutil.parser.isoparse(start_datetime)
+
+            if isinstance(end_datetime, str):
+                data["end_datetime"] = dateutil.parser.isoparse(end_datetime)
+
+            if isinstance(created, str):
+                data["created"] = dateutil.parser.isoparse(created)
+
+            if isinstance(updated, str):
+                data["updated"] = dateutil.parser.isoparse(updated)
+
+        return data
+
+    @field_serializer(
+        "datetime", "start_datetime", "end_datetime", "created", "updated"
+    )
+    def serialize_datetime(self, v: dt, _info: Any) -> str:
+        if v is None:
+            return None
+        else:
+            return v.strftime(DATETIME_RFC339)
+
 
 class Asset(StacCommonMetadata):
     """