From 7103c8f32ec69756ea4f3e94e10b293997fd2953 Mon Sep 17 00:00:00 2001 From: Bradley Reynolds Date: Sat, 23 Nov 2024 22:52:55 -0600 Subject: [PATCH] Rewrite on Pydantic (#104) * PWSH -> nox Signed-off-by: GitHub * Update lints Signed-off-by: GitHub * Add Pydantic Signed-off-by: GitHub * Raw conversion of dataclasses to models Signed-off-by: GitHub * Get tests passing Signed-off-by: GitHub * Remove extraneous dash Signed-off-by: GitHub * Remove another extraneous dash Signed-off-by: GitHub * Add tests for new Python verisons Signed-off-by: GitHub --------- Signed-off-by: GitHub --- .devcontainer/devcontainer-lock.json | 5 - .devcontainer/devcontainer.json | 3 +- .github/workflows/python-ci.yaml | 2 +- .pre-commit-config.yaml | 2 +- docs/source/changelog.rst | 4 +- docs/source/conf.py | 6 +- make.ps1 | 119 --------------- noxfile.py | 52 +++++++ pyproject.toml | 26 ++-- src/letsbuilda/pypi/async_client.py | 8 +- src/letsbuilda/pypi/models/models_json.py | 153 ++----------------- src/letsbuilda/pypi/models/models_package.py | 73 +-------- src/letsbuilda/pypi/models/models_rss.py | 57 +++---- src/letsbuilda/pypi/sync_client.py | 6 +- tests/test_json_api_parsing.py | 2 +- tests/test_rss_feed_parsing.py | 4 +- 16 files changed, 124 insertions(+), 398 deletions(-) delete mode 100644 make.ps1 create mode 100644 noxfile.py diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json index ec3da47..e10235c 100644 --- a/.devcontainer/devcontainer-lock.json +++ b/.devcontainer/devcontainer-lock.json @@ -1,10 +1,5 @@ { "features": { - "ghcr.io/devcontainers/features/powershell:1": { - "version": "1.3.5", - "resolved": "ghcr.io/devcontainers/features/powershell@sha256:0dd4e0352cc77ef586f7cca2414d3e8a7c506ad6df9ecd2221d078a961425bd6", - "integrity": "sha256:0dd4e0352cc77ef586f7cca2414d3e8a7c506ad6df9ecd2221d078a961425bd6" - }, "ghcr.io/devcontainers/features/python:1": { "version": "1.6.1", "resolved": "ghcr.io/devcontainers/features/python@sha256:d449aea663ea23ac4a7968719d5920dd57128f0429cd8e216849d5afe67651fb", diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5ed637e..c52f35d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,6 @@ "ghcr.io/devcontainers/features/python:1": { "version": "3.12", "installTools": false - }, - "ghcr.io/devcontainers/features/powershell:1": {} + } } } diff --git a/.github/workflows/python-ci.yaml b/.github/workflows/python-ci.yaml index 345afa7..e5e340b 100644 --- a/.github/workflows/python-ci.yaml +++ b/.github/workflows/python-ci.yaml @@ -19,7 +19,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest ] - python-version: [ "3.11" ] + python-version: [ "3.11", "3.12", "3.13" ] uses: darbiadev/.github/.github/workflows/python-test.yaml@29197a38ef3741064f47b623ede0c1ad22402c57 # v13.0.3 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 955d1da..af048db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index b272a98..15f209f 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -3,10 +3,10 @@ Changelog - :release:`5.2.1 <14th November 2024>` - :bug:`100` Allow new fields to be dynamic -- + - :release:`5.2.0 <14th November 2024>` - :bug:`100` Add fields for fields for PEP 639, Metadata 2.4 -- + - :release:`5.1.0 <26th February 2024>` - :bug:`78` Add ``dynamic`` and ``provides_extra`` to JSON schema diff --git a/docs/source/conf.py b/docs/source/conf.py index a8f640b..c80d3ef 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -62,9 +62,9 @@ def linkcode_resolve(domain: str, info: dict) -> str: if not info["module"]: return None - import importlib # noqa: PLC0415 - import inspect # noqa: PLC0415 - import types # noqa: PLC0415 + import importlib + import inspect + import types mod = importlib.import_module(info["module"]) diff --git a/make.ps1 b/make.ps1 deleted file mode 100644 index d24c299..0000000 --- a/make.ps1 +++ /dev/null @@ -1,119 +0,0 @@ -<# -.SYNOPSIS -Makefile - -.DESCRIPTION -USAGE - .\make.ps1 - -COMMANDS - init install Python build tools - install-dev install local package in editable mode - update-deps update the dependencies - upgrade-deps upgrade the dependencies - lint run `pre-commit`, `black`, and `ruff` - test run `pytest` - build-dist run `python -m build` - clean delete generated content - help, -? show this help message -#> -param( - [Parameter(Position = 0)] - [ValidateSet("init", "install-dev", "update-deps", "upgrade-deps", "lint", "test", "build-dist", "clean", "help")] - [string]$Command -) - -function Invoke-Help -{ - Get-Help $PSCommandPath -} - -function Invoke-Init -{ - python -m pip install --upgrade pip wheel setuptools build -} - -function Invoke-Install-Dev -{ - python -m pip install --upgrade --editable ".[dev, tests, docs]" -} - -function Invoke-Update-Deps -{ - pip-compile --output-file requirements.txt --resolver=backtracking requirements.in -} - -function Invoke-Upgrade-Deps -{ - pre-commit autoupdate - pip-compile --output-file requirements.txt --resolver=backtracking --upgrade requirements.in -} - -function Invoke-Lint -{ - pre-commit run --all-files - python -m ruff --fix . - python -m ruff format . - python -m mypy --strict src/ -} - -function Invoke-Test -{ - python -m pytest -} - -function Invoke-Build-Dist -{ - python -m pip install --upgrade build - python -m build -} - -function Invoke-Clean -{ - $folders = @("build", "dist") - foreach ($folder in $folders) - { - if (Test-Path $folder) - { - - Write-Verbose "Deleting $folder" - Remove-Item $folder -Recurse -Force - } - } -} - -switch ($Command) -{ - "init" { - Invoke-Init - } - "install-dev" { - Invoke-Install-Dev - } - "lint" { - Invoke-Lint - } - "update-deps" { - Invoke-Update-Deps - } - "upgrade-deps" { - Invoke-Upgrade-Deps - } - "test" { - Invoke-Test - } - "build-dist" { - Invoke-Build-Dist - } - "clean" { - Invoke-Clean - } - "help" { - Invoke-Help - } - default - { - Invoke-Init - Invoke-Install-Dev - } -} diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000..38997e0 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,52 @@ +"""Noxfile.""" + +import shutil +from pathlib import Path + +import nox + +nox.options.default_venv_backend = "none" +nox.options.sessions = ["lints"] + + +CLEANABLE_TARGETS = [ + "./dist", + "./build", + "./.nox", + "./.coverage", + "./.coverage.*", + "./coverage.json", + "./**/.mypy_cache", + "./**/.pytest_cache", + "./**/__pycache__", + "./**/*.pyc", + "./**/*.pyo", +] + + +@nox.session +def tests(session: nox.Session) -> None: + """Run tests.""" + session.run("pytest") + + +@nox.session +def lints(session: nox.Session) -> None: + """Run lints.""" + session.run("pre-commit", "run", "--all-files") + session.run("ruff", "format", ".") + session.run("ruff", "check", "--fix", ".") + session.run("mypy", "--strict", "src/") + + +@nox.session +def clean(_: nox.Session) -> None: + """Clean cache, .pyc, .pyo, and test/build artifact files from project.""" + count = 0 + for searchpath in CLEANABLE_TARGETS: + for filepath in Path().glob(searchpath): + if filepath.is_dir(): + shutil.rmtree(filepath) + else: + filepath.unlink() + count += 1 diff --git a/pyproject.toml b/pyproject.toml index 373c9a1..ee56b6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,15 +2,14 @@ name = "letsbuilda-pypi" version = "5.2.1" description = "A wrapper for PyPI's API and RSS feed" -authors = [ - { name = "Bradley Reynolds", email = "bradley.reynolds@darbia.dev" }, -] +authors = [{ name = "Bradley Reynolds", email = "bradley.reynolds@darbia.dev" }] license = { text = "MIT" } readme = "README.md" requires-python = ">=3.11" dependencies = [ "httpx", "xmltodict", + "pydantic", ] [project.urls] @@ -20,6 +19,7 @@ documentation = "https://docs.letsbuilda.dev/letsbuilda-pypi/" [project.optional-dependencies] dev = [ "pre-commit", + "nox", "ruff", "mypy", "types-xmltodict", @@ -42,26 +42,24 @@ build-backend = "setuptools.build_meta" "letsbuilda.pypi" = ["py.typed"] [tool.ruff] -preview = true -unsafe-fixes = true -target-version = "py311" +target-version = "py312" line-length = 120 [tool.ruff.lint] select = ["ALL"] ignore = [ - "CPY001", # (Missing copyright notice at top of file) + "CPY001", # (Missing copyright notice at top of file) "PLC0414", # (Import alias does not rename original package) - Re-exporting ] [tool.ruff.lint.extend-per-file-ignores] "docs/*" = [ "INP001", # (File `tests/*.py` is part of an implicit namespace package. Add an `__init__.py`.) - Docs are not modules - "FA102", # (Missing `from __future__ import annotations`, but uses PEP 585 collection) - Docs are actually built on the latest stable release of Python + "FA102", # (Missing `from __future__ import annotations`, but uses PEP 585 collection) - Docs are actually built on the latest stable release of Python ] "tests/*" = [ "INP001", # (File `tests/*.py` is part of an implicit namespace package. Add an `__init__.py`.) - Tests are not modules - "S101", # (Use of `assert` detected) - Yes, that's the point + "S101", # (Use of `assert` detected) - Yes, that's the point ] [tool.ruff.lint.isort] @@ -70,7 +68,11 @@ known-first-party = ["letsbuilda.pypi"] [tool.ruff.lint.pydocstyle] convention = "numpy" +[tool.mypy] +plugins = ["pydantic.mypy"] + [tool.coverage.run] -source = [ - "letsbuilda.pypi", -] +source = ["letsbuilda.pypi"] + +[tool.pytest.ini_options] +addopts = "--strict-markers" diff --git a/src/letsbuilda/pypi/async_client.py b/src/letsbuilda/pypi/async_client.py index 1c64627..d0520fe 100644 --- a/src/letsbuilda/pypi/async_client.py +++ b/src/letsbuilda/pypi/async_client.py @@ -34,7 +34,7 @@ async def get_rss_feed(self: Self, feed_url: str) -> list[RSSPackageMetadata]: """ response = await self.http_client.get(feed_url) rss_data = xmltodict.parse(response.text)["rss"]["channel"]["item"] - return [RSSPackageMetadata.build_from(package_data) for package_data in rss_data] + return [RSSPackageMetadata.model_validate(package_data) for package_data in rss_data] async def get_package_json_metadata( self: Self, @@ -68,7 +68,7 @@ async def get_package_json_metadata( response = await self.http_client.get(url) if response.status_code == HTTPStatus.NOT_FOUND: raise PackageNotFoundError(package_title, package_version) - return JSONPackageMetadata.from_dict(response.json()) + return JSONPackageMetadata.model_validate(response.json()) async def get_package_metadata( self: Self, @@ -89,4 +89,6 @@ async def get_package_metadata( Package The package object. """ - return Package.from_json_api_data(await self.get_package_json_metadata(package_title, package_version)) + return Package.model_validate( + (await self.get_package_json_metadata(package_title, package_version)).model_dump(), + ) diff --git a/src/letsbuilda/pypi/models/models_json.py b/src/letsbuilda/pypi/models/models_json.py index 0b97765..6cdd939 100644 --- a/src/letsbuilda/pypi/models/models_json.py +++ b/src/letsbuilda/pypi/models/models_json.py @@ -1,98 +1,41 @@ """Models for JSON responses.""" -from dataclasses import dataclass from datetime import datetime -from typing import Literal, Self +from typing import Literal +from pydantic import BaseModel, Field -@dataclass(frozen=True) -class Vulnerability: + +class Vulnerability(BaseModel): """Security vulnerability.""" id: str aliases: list[str] link: str source: str - withdrawn: datetime | None + withdrawn: datetime | None = Field(None) summary: str details: str fixed_in: list[str] - @classmethod - def from_dict(cls: type[Self], data: dict) -> Self: # type: ignore[type-arg] - """ - Build an instance from a dictionary. - - Parameters - ---------- - data - The data for a vulnerability. - - Returns - ------- - Vulnerability - An object storing the details of a security vulnerability. - """ - if data["withdrawn"] is not None: - data["withdrawn"] = datetime.fromisoformat(data["withdrawn"]) - - return cls(**data) - -@dataclass(frozen=True) -class Downloads: +class Downloads(BaseModel): """Release download counts.""" last_day: int last_month: int last_week: int - @classmethod - def from_dict(cls: type[Self], data: dict) -> Self: # type: ignore[type-arg] - """ - Build an instance from a dictionary. - Parameters - ---------- - data - A dictionary containing download statistics. - - Returns - ------- - Downloads - An object storing download statistics. - """ - return cls(**data) - - -@dataclass(frozen=True) -class Digests: +class Digests(BaseModel): """URL file digests.""" - blake2_b_256: str + blake2_b_256: str = Field(validation_alias="blake2b_256") md5: str sha256: str - @classmethod - def from_dict(cls: type[Self], data: dict) -> Self: # type: ignore[type-arg] - """ - Build an instance from a dictionary. - - Parameters - ---------- - data - A dictionary containing checksums of a package release. - Returns - ------- - Digests - An object storing checksums. - """ - return cls(**data) - - -@dataclass(frozen=True) -class URL: +class URL(BaseModel): """Package release URL.""" comment_text: str @@ -111,29 +54,8 @@ class URL: yanked: bool yanked_reason: None - @classmethod - def from_dict(cls: type[Self], data: dict) -> Self: # type: ignore[type-arg] - """ - Build an instance from a dictionary. - - Parameters - ---------- - data - The JSON API metadata for a package release. - - Returns - ------- - URL - An object representing a package release. - """ - data["upload_time"] = datetime.fromisoformat(data["upload_time"]) - data["upload_time_iso_8601"] = datetime.fromisoformat(data["upload_time_iso_8601"]) - - return cls(**data) - -@dataclass(frozen=True) -class Info: +class Info(BaseModel): """Package metadata internal info block.""" author: str @@ -193,59 +115,14 @@ class Info: ] ] | None - ) - provides_extra: list[str] | None - - @classmethod - def from_dict(cls: type[Self], data: dict) -> Self: # type: ignore[type-arg] - """ - Build an instance from a dictionary. - - Parameters - ---------- - data - The JSON API metadata for a package. - - Returns - ------- - Info - An object storing a package's metadata. - """ - if "dynamic" not in data: - data["dynamic"] = None - if "provides_extra" not in data: - data["provides_extra"] = None - return cls(**data) - - -@dataclass(frozen=True) -class JSONPackageMetadata: + ) = Field(None) + provides_extra: list[str] | None = Field(None) + + +class JSONPackageMetadata(BaseModel): """Package metadata.""" info: Info last_serial: int urls: list[URL] vulnerabilities: list[Vulnerability] - - @classmethod - def from_dict(cls: type[Self], data: dict) -> Self: # type: ignore[type-arg] - """ - Build an instance from a dictionary. - - Parameters - ---------- - data - Package metadata from the JSON API. - - Returns - ------- - JSONPackageMetadata - An object storing package metadata. - """ - info = Info.from_dict(data["info"]) - return cls( - info=info, - last_serial=data["last_serial"], - urls=[URL.from_dict(url_data) for url_data in data["urls"]], - vulnerabilities=[Vulnerability.from_dict(vuln_data) for vuln_data in data["vulnerabilities"]], - ) diff --git a/src/letsbuilda/pypi/models/models_package.py b/src/letsbuilda/pypi/models/models_package.py index 9570d0e..58dcddc 100644 --- a/src/letsbuilda/pypi/models/models_package.py +++ b/src/letsbuilda/pypi/models/models_package.py @@ -1,89 +1,24 @@ """Models for package metadata.""" -from dataclasses import dataclass -from typing import Self +from pydantic import BaseModel -from .models_json import URL, JSONPackageMetadata - -@dataclass(frozen=True) -class Distribution: +class Distribution(BaseModel): """Metadata for a distribution.""" filename: str url: str - @classmethod - def from_json_api_data(cls: type[Self], data: URL) -> Self: - """Build an instance from the JSON API data. - - Parameters - ---------- - data - The URL of the file hosting the distribution. - - Returns - ------- - Distribution - An object representing a distribution. - """ - return cls( - filename=data.filename, - url=data.url, - ) - -@dataclass(frozen=True) -class Release: +class Release(BaseModel): """Metadata for a release.""" version: str distributions: list[Distribution] - @classmethod - def from_json_api_data(cls: type[Self], data: JSONPackageMetadata) -> Self: - """ - Build an instance from the JSON API data. - - Parameters - ---------- - data - The JSON API metadata for a package release. - Returns - ------- - Release - An object representing a package release. - """ - return cls( - version=data.info.version, - distributions=[Distribution.from_json_api_data(json_api_data) for json_api_data in data.urls], - ) - - -@dataclass(frozen=True) -class Package: +class Package(BaseModel): """Metadata for a package.""" title: str releases: list[Release] - - @classmethod - def from_json_api_data(cls: type[Self], data: JSONPackageMetadata) -> Self: - """ - Build an instance from the JSON API data. - - Parameters - ---------- - data - The JSON API metadata for a package. - - Returns - ------- - Package - An object representing a package. - """ - return cls( - title=data.info.name, - releases=[Release.from_json_api_data(data)], - ) diff --git a/src/letsbuilda/pypi/models/models_rss.py b/src/letsbuilda/pypi/models/models_rss.py index edd9c1b..24dcc2b 100644 --- a/src/letsbuilda/pypi/models/models_rss.py +++ b/src/letsbuilda/pypi/models/models_rss.py @@ -1,48 +1,31 @@ """Models for RSS responses.""" -from dataclasses import dataclass from datetime import datetime from email.utils import parsedate_to_datetime -from typing import Self +from typing import Annotated +from pydantic import BaseModel, Field, model_validator +from pydantic.functional_validators import BeforeValidator -@dataclass(frozen=True) -class RSSPackageMetadata: +ISODateTime = Annotated[datetime, BeforeValidator(parsedate_to_datetime)] + + +class RSSPackageMetadata(BaseModel): """RSS Package metadata.""" title: str - version: str | None - package_link: str - guid: str | None - description: str | None - author: str | None - publication_date: datetime - + version: str | None = Field(None) + package_link: str = Field(validation_alias="link") + guid: str | None = Field(None) + description: str | None = Field(None) + author: str | None = Field(None) + publication_date: ISODateTime = Field(validation_alias="pubDate") + + @model_validator(mode="before") @classmethod - def build_from(cls: type[Self], data: dict[str, str]) -> Self: - """ - Build an instance from raw data. - - Parameters - ---------- - data - Parsed RSS data from PyPI's RSS API. - - Returns - ------- - RSSPackageMetadata - The pacckage metadata from the RSS API. - """ + def try_split_title(cls, data: dict) -> dict: # type: ignore[type-arg] + """Attempt to split title into package name and version.""" split_title = data["title"].removesuffix(" added to PyPI").split() - title = split_title[0] - version = split_title[1] if len(split_title) == 2 else None # noqa: PLR2004 - is not magic - - return cls( - title=title, - version=version, - package_link=data["link"], - guid=data.get("guid"), - description=data.get("description"), - author=data.get("author"), - publication_date=parsedate_to_datetime(data["pubDate"]), - ) + data["title"] = split_title[0] + data["version"] = split_title[1] if len(split_title) == 2 else None # noqa: PLR2004 - is not magic + return data diff --git a/src/letsbuilda/pypi/sync_client.py b/src/letsbuilda/pypi/sync_client.py index 82691cd..31c2da5 100644 --- a/src/letsbuilda/pypi/sync_client.py +++ b/src/letsbuilda/pypi/sync_client.py @@ -34,7 +34,7 @@ def get_rss_feed(self: Self, feed_url: str) -> list[RSSPackageMetadata]: """ response_text = self.http_client.get(feed_url).text rss_data = xmltodict.parse(response_text)["rss"]["channel"]["item"] - return [RSSPackageMetadata.build_from(package_data) for package_data in rss_data] + return [RSSPackageMetadata.model_validate(package_data) for package_data in rss_data] def get_package_json_metadata( self: Self, @@ -68,7 +68,7 @@ def get_package_json_metadata( response = self.http_client.get(url) if response.status_code == HTTPStatus.NOT_FOUND: raise PackageNotFoundError(package_title, package_version) - return JSONPackageMetadata.from_dict(response.json()) + return JSONPackageMetadata.model_validate(response.json()) def get_package_metadata( self: Self, @@ -89,4 +89,4 @@ def get_package_metadata( Package The package object. """ - return Package.from_json_api_data(self.get_package_json_metadata(package_title, package_version)) + return Package.model_validate(self.get_package_json_metadata(package_title, package_version).model_dump()) diff --git a/tests/test_json_api_parsing.py b/tests/test_json_api_parsing.py index eaa572b..771fc60 100644 --- a/tests/test_json_api_parsing.py +++ b/tests/test_json_api_parsing.py @@ -102,7 +102,7 @@ def test_json_api_data_parsing() -> None: """Confirm sample JSON API data gets parsed correctly.""" - model = JSONPackageMetadata.from_dict(JSON_API_DATA) + model = JSONPackageMetadata.model_validate(JSON_API_DATA) assert model.info.name == "letsbuilda-pypi" assert model.info.version == "4.0.0" diff --git a/tests/test_rss_feed_parsing.py b/tests/test_rss_feed_parsing.py index c085800..317bdf9 100644 --- a/tests/test_rss_feed_parsing.py +++ b/tests/test_rss_feed_parsing.py @@ -22,7 +22,7 @@ def test_parsing_new_package_data() -> None: """Confirm sample new package data gets parsed correctly.""" - parsed_data = RSSPackageMetadata.build_from(NEW_PACKAGE_DATA) + parsed_data = RSSPackageMetadata.model_validate(NEW_PACKAGE_DATA) assert parsed_data.publication_date == datetime(2023, 3, 29, 21, 30, 5, tzinfo=UTC) assert parsed_data.author is None assert parsed_data.description is None @@ -31,5 +31,5 @@ def test_parsing_new_package_data() -> None: def test_parsing_updated_package_data() -> None: """Confirm sample updated package data gets parsed correctly.""" - parsed_data = RSSPackageMetadata.build_from(UPDATED_PACKAGE_DATA) + parsed_data = RSSPackageMetadata.model_validate(UPDATED_PACKAGE_DATA) assert parsed_data.version == "1.0.0"