From 08b19e13b3ece2f74319e4eb95a14787cb35eda1 Mon Sep 17 00:00:00 2001 From: Livio Ribeiro Date: Mon, 19 Feb 2024 11:03:34 -0300 Subject: [PATCH] add redis extension (#39) * add redis extension * add redis documentation --- .github/workflows/docs.yml | 2 +- .github/workflows/test.yml | 24 +- docs/extensions/jinja.md | 2 +- docs/extensions/redis.md | 214 ++++++++++++++++++ docs/extensions/sqlalchemy.md | 5 +- examples/redis/application/__init__.py | 0 examples/redis/application/controller.py | 17 ++ examples/redis/application/service.py | 17 ++ examples/redis/configuration/settings.yaml | 16 ++ mkdocs.yml | 1 + pyproject.toml | 4 +- src/selva/configuration/defaults.py | 5 +- src/selva/ext/data/redis/__init__.py | 16 ++ src/selva/ext/data/redis/service.py | 70 ++++++ src/selva/ext/data/redis/settings.py | 111 +++++++++ src/selva/ext/data/sqlalchemy/__init__.py | 6 +- src/selva/ext/data/sqlalchemy/settings.py | 100 ++++---- src/selva/ext/templates/jinja/settings.py | 48 ++-- src/selva/web/application.py | 8 +- tests/ext/data/redis/__init__.py | 0 tests/ext/data/redis/application.py | 20 ++ tests/ext/data/redis/application_named.py | 20 ++ tests/ext/data/redis/test_application.py | 40 ++++ .../data/redis/test_environment_variables.py | 76 +++++++ tests/ext/data/redis/test_service.py | 138 +++++++++++ tests/ext/data/redis/test_settings.py | 54 +++++ 26 files changed, 923 insertions(+), 91 deletions(-) create mode 100644 docs/extensions/redis.md create mode 100644 examples/redis/application/__init__.py create mode 100644 examples/redis/application/controller.py create mode 100644 examples/redis/application/service.py create mode 100644 examples/redis/configuration/settings.yaml create mode 100644 src/selva/ext/data/redis/__init__.py create mode 100644 src/selva/ext/data/redis/service.py create mode 100644 src/selva/ext/data/redis/settings.py create mode 100644 tests/ext/data/redis/__init__.py create mode 100644 tests/ext/data/redis/application.py create mode 100644 tests/ext/data/redis/application_named.py create mode 100644 tests/ext/data/redis/test_application.py create mode 100644 tests/ext/data/redis/test_environment_variables.py create mode 100644 tests/ext/data/redis/test_service.py create mode 100644 tests/ext/data/redis/test_settings.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a96d242..7b36b0f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,5 +16,5 @@ jobs: - uses: actions/setup-python@v4 with: python-version: 3.x - - run: pip install "mkdocs-material>=9.4,<9.5" + - run: pip install "mkdocs-material>=9.5,<9.6" - run: mkdocs gh-deploy --force diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 516e12b..39892f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,9 +13,6 @@ on: branches: - main -env: - POSTGRES_DB: test_db - jobs: test: runs-on: ubuntu-latest @@ -28,16 +25,31 @@ jobs: image: postgres:16.1-alpine env: POSTGRES_PASSWORD: postgres - POSTGRES_DB: ${{ env.POSTGRES_DB }} + POSTGRES_DB: test options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: bitnami/redis:7.2.4 + env: + REDIS_PASSWORD: password + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v4 - run: pip install poetry - - run: poetry install --with test --extras jinja --extras sqlalchemy --no-interaction + - run: poetry install + --with test + --extras jinja + --extras sqlalchemy + --extras redis + --no-interaction - env: - POSTGRES_URL: postgresql+asyncpg://postgres:postgres@postgres:5432/${{ env.POSTGRES_DB }} + POSTGRES_URL: postgresql+asyncpg://postgres:postgres@postgres:5432/test + REDIS_URL: redis://default:password@redis:6379/0 run: poetry run pytest diff --git a/docs/extensions/jinja.md b/docs/extensions/jinja.md index 86b3758..c2097c1 100644 --- a/docs/extensions/jinja.md +++ b/docs/extensions/jinja.md @@ -1,6 +1,6 @@ # Jinja -This extension offers support for Jinja templates. +This extension provides support for Jinja templates. ## Usage diff --git a/docs/extensions/redis.md b/docs/extensions/redis.md new file mode 100644 index 0000000..39692e0 --- /dev/null +++ b/docs/extensions/redis.md @@ -0,0 +1,214 @@ +# Redis + +This extension provides support for connecting to Redis servers. It registers the +`redis.asyncio.Redis` service. + +## Usage + +First install the `redis` extra: + +```shell +pip install selva[redis] +``` + +Define the configuration properties: + +=== "configuration/settings.yaml" + + ```yaml + extensions: + - selva.ext.data.redis # (1) + + data: + redis: + default: # (2) + url: redis://localhost:6379/0 + other: # (3) + url: redis://localhost:6379/1 + ``` + + 1. Activate the sqlalchemy extension + 2. "default" connection will be registered without a name + 3. Connection registered with name "other" + +Inject the `Redis` service: + +```python +from typing import Annotated +from redis.asyncio import Redis +from selva.di import service, Inject + + +@service +class MyService: + # default service + redis: Annotated[Redis, Inject] + + # named service + other_redis: Annotated[Redis, Inject(name="other")] +``` + +Redis connections can also be defined with username and password separated from +the url, or even with individual components: + +=== "configuration/settings.yaml" + + ```yaml + data: + redis: + url_username_password: # (1) + url: redis://localhost:6379/0 + username: user + password: pass + + individual_components: # (2) + host: localhost + port: 6379 + db: 0 + username: user + password: pass + ``` + + 1. Username and password separated from the redis url + 2. Each component defined individually + +## Using environment variables + +=== "configuration/settings.yaml" + + ```yaml + data: + redis: + default: + url: "${REDIS_URL}" # (1) + + other: # (2) + url: "${REDIS_URL}" + username: "${REDIS_USERNAME}" + password: "${REDIS_PASSWORD}" + + another: # (3) + host: "${REDIS_HOST}" + port: ${REDIS_PORT} + db: "${REDIS_DB}" + username: "${REDIS_USERNAME}" + password: "${REDIS_PASSWORD}" + ``` + + 1. Can be define with just the environment variable `SELVA__DATA__REDIS__DEFAULT__URL` + 2. Can be defined with just the environment variables: + - `SELVA__DATA__REDIS__OTHER__URL` + - `SELVA__DATA__REDIS__OTHER__USERNAME` + - `SELVA__DATA__REDIS__OTHER__PASSWORD` + 3. Can be defined with just the environment variables: + - `SELVA__DATA__REDIS__ANOTHER__HOST` + - `SELVA__DATA__REDIS__ANOTHER__PORT` + - `SELVA__DATA__REDIS__ANOTHER__DB` + - `SELVA__DATA__REDIS__ANOTHER__USERNAME` + - `SELVA__DATA__REDIS__ANOTHER__PASSWORD` + +## Example + +=== "application/controller.py" + + ```python + from typing import Annotated + + from redis.asyncio import Redis + + from asgikit.responses import respond_json + + from selva.di import Inject + from selva.web import controller, get + + + @controller + class Controller: + redis: Annotated[Redis, Inject] + + async def initialize(self): + await self.redis.set("number", 0, nx=True, ex=60) + + @get + async def index(self, request): + number = await self.redis.incr("number") + await respond_json(request.response, {"number": number}) + ``` + +=== "configuration/settings.yaml" + + ```yaml + data: + redis: + default: + url: "redis://localhost:6379/0" + ``` + +## Configuration options + +Selva offers several options to configure Redis. If you need more control over +the SQLAlchemy services, you can create your own `redis.asyncio.Redis` outside +of the DI context. + +The available options are shown below: + +```yaml +data: + redis: + default: + url: "" + host: "" + port: 6379 + db: 0 + username: "" + password: "" + options: # (1) + socket_timeout: 1.0 + socket_connect_timeout: 1.0 + socket_keepalive: false + socket_keepalive_options: {} + unix_socket_path: "" + encoding: "" + encoding_errors: "strict" # or "ignore", "replace" + decode_responses: false + retry_on_timeout: false + retry_on_error: [] + ssl: false + ssl_keyfile: "" + ssl_certfile: "" + ssl_cert_reqs: "" + ssl_ca_certs: "" + ssl_ca_data: "" + ssl_check_hostname: false + max_connections: 1 + single_connection_client: false + health_check_interval: 1 + client_name: "" + lib_name: "" + lib_version: "" + auto_close_connection_pool: false + protocol: 3 + retry: + retries: 1 + supported_errors: [] # (2) + backoff: # (3) + no_backoff: + constant: + backoff: 1 + exponential: + cap: 1 + base: 1 + full_jitter: + cap: 1 + base: 1 + equal_jitter: + cap: 1 + base: 1 + decorrelated_jitter: + cap: 1 + base: 1 +``` + +1. `options` values are described in [`redis.asyncio.Redis`](https://redis.readthedocs.io/en/stable/connections.html#async-client). +2. Dotted path to python classes. +3. Only one option in `backoff` should be set. \ No newline at end of file diff --git a/docs/extensions/sqlalchemy.md b/docs/extensions/sqlalchemy.md index 699d47c..2a4993c 100644 --- a/docs/extensions/sqlalchemy.md +++ b/docs/extensions/sqlalchemy.md @@ -6,7 +6,7 @@ injection context. ## Usage -Install SQLAlchemy python package and the database driver: +Install SQLAlchemy extra and a database driver that supports async: ```shell pip install selva[sqlalchemy] aiosqlite asyncpg aiomysql oracledb @@ -53,12 +53,13 @@ class MyService: # default service sessionmaker: Annotated[async_sessionmaker, Inject] + # named services sessionmaker_postgres: Annotated[async_sessionmaker, Inject(name="postgres")] sessionmaker_mysql: Annotated[async_sessionmaker, Inject(name="mysql")] sessionmaker_oracle: Annotated[async_sessionmaker, Inject(name="oracle")] ``` -Database connection can also be defined with username and password separated from +Database connections can also be defined with username and password separated from the url, or even with individual components: ```yaml diff --git a/examples/redis/application/__init__.py b/examples/redis/application/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/redis/application/controller.py b/examples/redis/application/controller.py new file mode 100644 index 0000000..7ffa68c --- /dev/null +++ b/examples/redis/application/controller.py @@ -0,0 +1,17 @@ +from typing import Annotated + +from asgikit.responses import respond_json +from selva.di import Inject +from selva.web import controller, get + +from .service import RedisService + + +@controller +class Controller: + redis_service: Annotated[RedisService, Inject] + + @get + async def index(self, request): + number = await self.redis_service.get_incr() + await respond_json(request.response, {"number": number}) diff --git a/examples/redis/application/service.py b/examples/redis/application/service.py new file mode 100644 index 0000000..1d3a0be --- /dev/null +++ b/examples/redis/application/service.py @@ -0,0 +1,17 @@ +from typing import Annotated + +from redis.asyncio import Redis + +from selva.di import service, Inject + + +@service +class RedisService: + redis: Annotated[Redis, Inject] + + async def initialize(self): + await self.redis.set("number", 0, ex=60) + + async def get_incr(self) -> int: + return await self.redis.incr("number") + diff --git a/examples/redis/configuration/settings.yaml b/examples/redis/configuration/settings.yaml new file mode 100644 index 0000000..9a1c335 --- /dev/null +++ b/examples/redis/configuration/settings.yaml @@ -0,0 +1,16 @@ +extensions: + - selva.ext.data.redis + +data: + redis: + default: + host: "localhost" + options: + retry: + retries: 1 + backoff: + constant: + backoff: 1 + + other: + url: "redis://localhost:6379/1" \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 0fe5b1c..bcea0b7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -52,4 +52,5 @@ nav: - Extensions: - extensions/overview.md - extensions/sqlalchemy.md + - extensions/redis.md - extensions/jinja.md diff --git a/pyproject.toml b/pyproject.toml index 4859084..b936719 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,10 +35,12 @@ python-dotenv = "^1.0" "ruamel.yaml" = "^0.18" jinja2 = { version = "^3.1", optional = true } SQLAlchemy = { version = "^2.0", optional = true } +redis = { version = "^5.0", optional = true } [tool.poetry.extras] jinja = ["jinja2"] sqlalchemy = ["SQLAlchemy"] +redis = ["redis"] [tool.poetry.group.dev] optional = true @@ -74,7 +76,7 @@ ruff = "^0.2" optional = true [tool.poetry.group.docs.dependencies] -mkdocs-material = "^9.4" +mkdocs-material = "^9.5" [tool.pytest.ini_options] asyncio_mode = "auto" diff --git a/src/selva/configuration/defaults.py b/src/selva/configuration/defaults.py index 047405e..7a56c7c 100644 --- a/src/selva/configuration/defaults.py +++ b/src/selva/configuration/defaults.py @@ -14,5 +14,8 @@ "paths": ["resources/templates"], "jinja": {}, }, - "data": {"sqlalchemy": {}}, + "data": { + "redis": {}, + "sqlalchemy": {}, + }, } diff --git a/src/selva/ext/data/redis/__init__.py b/src/selva/ext/data/redis/__init__.py new file mode 100644 index 0000000..5c50811 --- /dev/null +++ b/src/selva/ext/data/redis/__init__.py @@ -0,0 +1,16 @@ +from importlib.util import find_spec + +from selva.configuration.settings import Settings +from selva.di.container import Container + +from .service import make_service + + +def selva_extension(container: Container, settings: Settings): + if find_spec("redis") is None: + return + + for name in settings.data.redis: + service_name = name if name != "default" else None + + container.register(make_service(name), name=service_name) diff --git a/src/selva/ext/data/redis/service.py b/src/selva/ext/data/redis/service.py new file mode 100644 index 0000000..c5826d1 --- /dev/null +++ b/src/selva/ext/data/redis/service.py @@ -0,0 +1,70 @@ +from redis.asyncio import Redis +from redis.backoff import ( + AbstractBackoff, + ConstantBackoff, + DecorrelatedJitterBackoff, + EqualJitterBackoff, + ExponentialBackoff, + FullJitterBackoff, + NoBackoff, +) +from redis.retry import Retry + +from selva.configuration.settings import Settings +from selva.ext.data.redis.settings import RedisSettings + +from .settings import BackoffSchema, RetrySchema + + +def build_backoff(data: BackoffSchema) -> AbstractBackoff: + if "no_backoff" in data.model_fields_set: + return NoBackoff() + + if value := data.constant: + return ConstantBackoff(**value.model_dump(exclude_unset=True)) + + if value := data.exponential: + return ExponentialBackoff(**value.model_dump(exclude_unset=True)) + + if value := data.full_jitter: + return FullJitterBackoff(**value.model_dump(exclude_unset=True)) + + if value := data.equal_jitter: + return EqualJitterBackoff(**value.model_dump(exclude_unset=True)) + + if value := data.decorrelated_jitter: + return DecorrelatedJitterBackoff(**value.model_dump(exclude_unset=True)) + + +def build_retry(data: RetrySchema): + kwargs = { + "backoff": build_backoff(data.backoff), + "retries": data.retries, + } + + if supported_errors := data.supported_errors: + kwargs["supported_errors"] = supported_errors + + return Retry(**kwargs) + + +def make_service(name: str): + async def redis_service( + settings: Settings, + ) -> Redis: + redis_settings = RedisSettings.model_validate(dict(settings.data.redis[name])) + + kwargs = redis_settings.model_dump(exclude_unset=True) + if (options := redis_settings.options) and (retry := options.retry): + kwargs["retry"] = build_retry(retry) + + if url := kwargs.pop("url", ""): + redis = Redis.from_url(url, **kwargs) + else: + redis = Redis(**kwargs) + + await redis.initialize() + yield redis + await redis.aclose() + + return redis_service diff --git a/src/selva/ext/data/redis/settings.py b/src/selva/ext/data/redis/settings.py new file mode 100644 index 0000000..5f62339 --- /dev/null +++ b/src/selva/ext/data/redis/settings.py @@ -0,0 +1,111 @@ +from types import NoneType +from typing import Self, Type, Literal + +from pydantic import BaseModel, ConfigDict, model_serializer, model_validator +from redis import RedisError + +from selva._util.pydantic import DottedPath + + +class ConstantBackoffSchema(BaseModel): + model_config = ConfigDict(extra="forbid") + + backoff: int + + +class ExponentialBackoffSchema(BaseModel): + model_config = ConfigDict(extra="forbid") + + cap: int = None + base: int = None + + +class BackoffSchema(BaseModel): + model_config = ConfigDict(extra="forbid") + + no_backoff: NoneType = None + constant: ConstantBackoffSchema = None + exponential: ExponentialBackoffSchema = None + full_jitter: ExponentialBackoffSchema = None + equal_jitter: ExponentialBackoffSchema = None + decorrelated_jitter: ExponentialBackoffSchema = None + + @model_validator(mode="before") + def validator(cls, data): + if len(data) != 1: + raise ValueError("Only one backoff value can be set") + + return data + + +class RetrySchema(BaseModel): + model_config = ConfigDict(extra="forbid") + + backoff: BackoffSchema + retries: int + supported_errors: tuple[DottedPath[Type[RedisError]], ...] = None + + +class RedisOptions(BaseModel): + model_config = ConfigDict(extra="forbid") + + socket_timeout: float = None + socket_connect_timeout: float = None + socket_keepalive: bool = None + socket_keepalive_options: dict[int, int | bytes] = None + unix_socket_path: str = None + encoding: str = None + encoding_errors: Literal["strict", "ignore", "replace"] = None + decode_responses: bool = None + retry_on_timeout: bool = None + retry_on_error: list = None + ssl: bool = None + ssl_keyfile: str = None + ssl_certfile: str = None + ssl_cert_reqs: str = None + ssl_ca_certs: str = None + ssl_ca_data: str = None + ssl_check_hostname: bool = None + max_connections: int = None + single_connection_client: bool = None + health_check_interval: int = None + client_name: str = None + lib_name: str = None + lib_version: str = None + retry: RetrySchema = None + auto_close_connection_pool: bool = None + protocol: int = None + + +class RedisSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + url: str = None + host: str = None + port: int = None + db: int = None + username: str = None + password: str = None + options: RedisOptions = None + + @model_validator(mode="after") + def validator(self) -> Self: + if self.url and (self.host or self.port or (self.db is not None)): + raise ValueError( + "Either 'url' should be provided, or 'host', 'port' and 'db'" + ) + + return self + + @model_serializer(when_used="unless-none") + def serializer(self): + data = { + field: getattr(self, field) + for field in self.model_fields_set + if field != "options" + } + + if self.options: + data |= self.options.model_dump(exclude_unset=True) + + return data diff --git a/src/selva/ext/data/sqlalchemy/__init__.py b/src/selva/ext/data/sqlalchemy/__init__.py index f6dd427..aba6add 100644 --- a/src/selva/ext/data/sqlalchemy/__init__.py +++ b/src/selva/ext/data/sqlalchemy/__init__.py @@ -2,10 +2,8 @@ from selva.configuration.settings import Settings from selva.di.container import Container -from selva.ext.data.sqlalchemy.service import ( - make_engine_service, - make_sessionmaker_service, -) + +from .service import make_engine_service, make_sessionmaker_service def selva_extension(container: Container, settings: Settings): diff --git a/src/selva/ext/data/sqlalchemy/settings.py b/src/selva/ext/data/sqlalchemy/settings.py index 4e1a2be..3f93486 100644 --- a/src/selva/ext/data/sqlalchemy/settings.py +++ b/src/selva/ext/data/sqlalchemy/settings.py @@ -1,8 +1,8 @@ from collections.abc import Callable from types import ModuleType -from typing import Annotated, Any, Literal, Self +from typing import Any, Literal, Self -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, ConfigDict, model_validator from sqlalchemy import URL, make_url from selva._util.pydantic import DottedPath @@ -14,14 +14,16 @@ class SqlAlchemyExecutionOptions(BaseModel): Defined in https://docs.sqlalchemy.org/en/20/core/connections.html#sqlalchemy.engine.Connection.execution_options """ - logging_token: Annotated[str, Field(default=None)] - isolation_level: Annotated[str, Field(default=None)] - no_parameters: Annotated[bool, Field(default=None)] - stream_results: Annotated[bool, Field(default=None)] - max_row_buffer: Annotated[int, Field(default=None)] - yield_per: Annotated[int, Field(default=None)] - insertmanyvalues_page_size: Annotated[int, Field(default=None)] - schema_translate_map: Annotated[dict[str | None, str], Field(default=None)] + model_config = ConfigDict(extra="forbid") + + logging_token: str = None + isolation_level: str = None + no_parameters: bool = None + stream_results: bool = None + max_row_buffer: int = None + yield_per: int = None + insertmanyvalues_page_size: int = None + schema_translate_map: dict[str | None, str] = None class SqlAlchemyOptions(BaseModel): @@ -30,49 +32,51 @@ class SqlAlchemyOptions(BaseModel): Defined in https://docs.sqlalchemy.org/en/20/core/engines.html#sqlalchemy.create_engine """ - connect_args: Annotated[dict[str, Any], Field(default=None)] - echo: Annotated[bool, Field(default=None)] - echo_pool: Annotated[bool, Field(default=None)] - enable_from_linting: Annotated[bool, Field(default=None)] - execution_options: Annotated[SqlAlchemyExecutionOptions, Field(default=None)] - hide_parameters: Annotated[bool, Field(default=None)] - insertmanyvalues_page_size: Annotated[int, Field(default=None)] - isolation_level: Annotated[str, Field(default=None)] - json_deserializer: Annotated[DottedPath[Callable], Field(default=None)] - json_serializer: Annotated[DottedPath[Callable], Field(default=None)] - label_length: Annotated[int, Field(default=None)] - logging_name: Annotated[str, Field(default=None)] - max_identifier_length: Annotated[int, Field(default=None)] - max_overflow: Annotated[int, Field(default=None)] - module: Annotated[DottedPath[ModuleType], Field(default=None)] - paramstyle: Annotated[ - Literal["qmark", "numeric", "named", "format", "pyformat"], Field(default=None) - ] - poolclass: Annotated[DottedPath, Field(default=None)] - pool_logging_name: Annotated[str, Field(default=None)] - pool_pre_ping: Annotated[bool, Field(default=None)] - pool_size: Annotated[int, Field(default=None)] - pool_recycle: Annotated[int, Field(default=None)] - pool_reset_on_return: Annotated[Literal["rollback", "commit"], Field(default=None)] - pool_timeout: Annotated[int, Field(default=None)] - pool_use_lifo: Annotated[bool, Field(default=None)] - plugins: Annotated[list[str], Field(default=None)] - query_cache_size: Annotated[int, Field(default=None)] - use_insertmanyvalues: Annotated[bool, Field(default=None)] + model_config = ConfigDict(extra="forbid") + + connect_args: dict[str, Any] = None + echo: bool = None + echo_pool: bool = None + enable_from_linting: bool = None + execution_options: SqlAlchemyExecutionOptions = None + hide_parameters: bool = None + insertmanyvalues_page_size: int = None + isolation_level: str = None + json_deserializer: DottedPath[Callable] = None + json_serializer: DottedPath[Callable] = None + label_length: int = None + logging_name: str = None + max_identifier_length: int = None + max_overflow: int = None + module: DottedPath[ModuleType] = None + paramstyle: Literal["qmark", "numeric", "named", "format", "pyformat"] = None + poolclass: DottedPath = None + pool_logging_name: str = None + pool_pre_ping: bool = None + pool_size: int = None + pool_recycle: int = None + pool_reset_on_return: Literal["rollback", "commit"] = None + pool_timeout: int = None + pool_use_lifo: bool = None + plugins: list[str] = None + query_cache_size: int = None + use_insertmanyvalues: bool = None class SqlAlchemySettings(BaseModel): """Settings for a SQLAlchemy connection defined in a settings file.""" - url: Annotated[str, Field(default=None)] - username: Annotated[str, Field(default=None)] - password: Annotated[str, Field(default=None)] - drivername: Annotated[str, Field(default=None)] - host: Annotated[str, Field(default=None)] - port: Annotated[int, Field(default=None)] - database: Annotated[str, Field(default=None)] - query: Annotated[dict[str, str], Field(default=None)] - options: Annotated[SqlAlchemyOptions, Field(default=None)] + model_config = ConfigDict(extra="forbid") + + url: str = None + username: str = None + password: str = None + drivername: str = None + host: str = None + port: int = None + database: str = None + query: dict[str, str] = None + options: SqlAlchemyOptions = None @model_validator(mode="after") def validate(self) -> Self: diff --git a/src/selva/ext/templates/jinja/settings.py b/src/selva/ext/templates/jinja/settings.py index dba1ba6..301b4b6 100644 --- a/src/selva/ext/templates/jinja/settings.py +++ b/src/selva/ext/templates/jinja/settings.py @@ -1,33 +1,33 @@ from collections.abc import Callable -from typing import Annotated, Literal, Type +from typing import Literal, Type from jinja2 import BaseLoader, BytecodeCache, Undefined -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict from selva._util.pydantic import DottedPath class JinjaTemplateSettings(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) + model_config = ConfigDict(extra="forbid") - block_start_string: Annotated[str, Field(default=None)] - block_end_string: Annotated[str, Field(default=None)] - variable_start_string: Annotated[str, Field(default=None)] - variable_end_string: Annotated[str, Field(default=None)] - comment_start_string: Annotated[str, Field(default=None)] - comment_end_string: Annotated[str, Field(default=None)] - line_statement_prefix: Annotated[str, Field(default=None)] - line_comment_prefix: Annotated[str, Field(default=None)] - trim_blocks: Annotated[bool, Field(default=None)] - lstrip_blocks: Annotated[bool, Field(default=None)] - newline_sequence: Annotated[Literal["\n", "\r\n", "\r"], Field(default=None)] - keep_trailing_newline: Annotated[bool, Field(default=None)] - extensions: Annotated[list[str], Field(default=None)] - optimized: Annotated[bool, Field(default=None)] - undefined: Annotated[DottedPath[Type[Undefined]], Field(default=None)] - finalize: Annotated[DottedPath[Callable[..., None]], Field(default=None)] - autoescape: Annotated[bool | DottedPath[Callable[[str], bool]], Field(default=None)] - loader: Annotated[DottedPath[BaseLoader], Field(default=None)] - cache_size: Annotated[int, Field(default=None)] - auto_reload: Annotated[bool, Field(default=None)] - bytecode_cache: Annotated[DottedPath[BytecodeCache], Field(default=None)] + block_start_string: str = None + block_end_string: str = None + variable_start_string: str = None + variable_end_string: str = None + comment_start_string: str = None + comment_end_string: str = None + line_statement_prefix: str = None + line_comment_prefix: str = None + trim_blocks: bool = None + lstrip_blocks: bool = None + newline_sequence: Literal["\n", "\r\n", "\r"] = None + keep_trailing_newline: bool = None + extensions: list[str] = None + optimized: bool = None + undefined: DottedPath[Type[Undefined]] = None + finalize: DottedPath[Callable[..., None]] = None + autoescape: bool | DottedPath[Callable[[str], bool]] = None + loader: DottedPath[BaseLoader] = None + cache_size: int = None + auto_reload: bool = None + bytecode_cache: DottedPath[BytecodeCache] = None diff --git a/src/selva/web/application.py b/src/selva/web/application.py index c5cde4c..8fd5714 100644 --- a/src/selva/web/application.py +++ b/src/selva/web/application.py @@ -7,7 +7,7 @@ from asgikit.errors.websocket import WebSocketDisconnectError, WebSocketError from asgikit.requests import Request -from asgikit.responses import respond_status +from asgikit.responses import respond_status, respond_text from asgikit.websockets import WebSocket from loguru import logger @@ -211,9 +211,11 @@ async def _handle_request(self, scope, receive, send): return await respond_status(response, err.status) - except Exception: + except Exception as err: logger.exception("Error processing request") - await respond_status(request.response, HTTPStatus.INTERNAL_SERVER_ERROR) + await respond_text( + request.response, str(err), status=HTTPStatus.INTERNAL_SERVER_ERROR + ) async def _process_request(self, request: Request): method = request.method diff --git a/tests/ext/data/redis/__init__.py b/tests/ext/data/redis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/ext/data/redis/application.py b/tests/ext/data/redis/application.py new file mode 100644 index 0000000..3557487 --- /dev/null +++ b/tests/ext/data/redis/application.py @@ -0,0 +1,20 @@ +from typing import Annotated + +from asgikit.responses import respond_text +from redis.asyncio import Redis + +from selva.di import Inject +from selva.web import controller, get + + +@controller +class Controller: + redis: Annotated[Redis, Inject] + + @get + async def index(self, request): + await self.redis.set("key", "value") + result = (await self.redis.get("key")).decode("utf-8") + + await respond_text(request.response, result) + await self.redis.delete("key") diff --git a/tests/ext/data/redis/application_named.py b/tests/ext/data/redis/application_named.py new file mode 100644 index 0000000..47107f4 --- /dev/null +++ b/tests/ext/data/redis/application_named.py @@ -0,0 +1,20 @@ +from typing import Annotated + +from asgikit.responses import respond_text +from redis.asyncio import Redis + +from selva.di import Inject +from selva.web import controller, get + + +@controller +class Controller: + redis: Annotated[Redis, Inject(name="other")] + + @get + async def index(self, request): + await self.redis.set("key", "value") + result = (await self.redis.get("key")).decode("utf-8") + + await respond_text(request.response, result) + await self.redis.delete("key") diff --git a/tests/ext/data/redis/test_application.py b/tests/ext/data/redis/test_application.py new file mode 100644 index 0000000..aa1a631 --- /dev/null +++ b/tests/ext/data/redis/test_application.py @@ -0,0 +1,40 @@ +import os +from http import HTTPStatus + +import pytest +from httpx import AsyncClient + +from selva.configuration.defaults import default_settings +from selva.configuration.settings import Settings +from selva.web.application import Selva + +REDIS_URL = os.environ.get("REDIS_URL") + + +@pytest.mark.skipif(REDIS_URL is None, reason="REDIS_URL not set") +@pytest.mark.parametrize( + "application,database", + [ + ("application", "default"), + ("application_named", "other"), + ], + ids=["default", "named"], +) +async def test_application(application: str, database: str): + settings = Settings( + default_settings + | { + "application": f"{__package__}.{application}", + "extensions": ["selva.ext.data.redis"], + "data": {"redis": {database: {"url": REDIS_URL}}}, + } + ) + + app = Selva(settings) + + await app._lifespan_startup() + + client = AsyncClient(app=app) + response = await client.get("http://localhost:8000/") + + assert response.status_code == HTTPStatus.OK diff --git a/tests/ext/data/redis/test_environment_variables.py b/tests/ext/data/redis/test_environment_variables.py new file mode 100644 index 0000000..793606c --- /dev/null +++ b/tests/ext/data/redis/test_environment_variables.py @@ -0,0 +1,76 @@ +import os +from importlib.util import find_spec +from urllib.parse import urlparse + +import pytest + +from selva.configuration.settings import _get_settings_nocache +from selva.ext.data.redis.service import make_service + +REDIS_URL = os.getenv("REDIS_URL") +PARSED_URL = urlparse(REDIS_URL) if REDIS_URL else None + + +pytestmark = [ + pytest.mark.skipif(REDIS_URL is None, reason="REDIS_URL not defined"), + pytest.mark.skipif(find_spec("redis") is None, reason="redis not present"), +] + + +async def test_redis_url_from_environment_variables(monkeypatch): + monkeypatch.setenv("SELVA__DATA__REDIS__DEFAULT__URL", REDIS_URL) + settings = _get_settings_nocache() + + service = make_service("default")(settings) + async for redis in service: + connection_kwargs = redis.connection_pool.connection_kwargs + assert connection_kwargs["host"] == PARSED_URL.hostname + assert connection_kwargs["port"] == PARSED_URL.port + assert connection_kwargs["db"] == int(PARSED_URL.path.strip("/")) + assert connection_kwargs.get("username") == PARSED_URL.username + assert connection_kwargs.get("password") == PARSED_URL.password + + +async def test_redis_url_username_password_from_environment_variables(monkeypatch): + url = f"redis://{PARSED_URL.hostname}:{PARSED_URL.port}/{PARSED_URL.path}" + username = PARSED_URL.username + password = PARSED_URL.password + + monkeypatch.setenv("SELVA__DATA__REDIS__DEFAULT__URL", url) + monkeypatch.setenv("SELVA__DATA__REDIS__DEFAULT__USERNAME", username) + monkeypatch.setenv("SELVA__DATA__REDIS__DEFAULT__PASSWORD", password) + settings = _get_settings_nocache() + + service = make_service("default")(settings) + async for redis in service: + connection_kwargs = redis.connection_pool.connection_kwargs + assert connection_kwargs["host"] == PARSED_URL.hostname + assert connection_kwargs["port"] == PARSED_URL.port + assert connection_kwargs["db"] == int(PARSED_URL.path.strip("/")) + assert connection_kwargs.get("username") == PARSED_URL.username + assert connection_kwargs.get("password") == PARSED_URL.password + + +async def test_redis_url_components_from_environment_variables(monkeypatch): + host = PARSED_URL.hostname + port = str(PARSED_URL.port) + database = PARSED_URL.path.strip("/") + username = PARSED_URL.username + password = PARSED_URL.password + + monkeypatch.setenv("SELVA__DATA__REDIS__DEFAULT__HOST", host) + monkeypatch.setenv("SELVA__DATA__REDIS__DEFAULT__PORT", port) + monkeypatch.setenv("SELVA__DATA__REDIS__DEFAULT__DB", database) + monkeypatch.setenv("SELVA__DATA__REDIS__DEFAULT__USERNAME", username) + monkeypatch.setenv("SELVA__DATA__REDIS__DEFAULT__PASSWORD", password) + + settings = _get_settings_nocache() + + service = make_service("default")(settings) + async for redis in service: + connection_kwargs = redis.connection_pool.connection_kwargs + assert connection_kwargs["host"] == PARSED_URL.hostname + assert connection_kwargs["port"] == PARSED_URL.port + assert connection_kwargs["db"] == int(PARSED_URL.path.strip("/")) + assert connection_kwargs.get("username") == PARSED_URL.username + assert connection_kwargs.get("password") == PARSED_URL.password diff --git a/tests/ext/data/redis/test_service.py b/tests/ext/data/redis/test_service.py new file mode 100644 index 0000000..03ff675 --- /dev/null +++ b/tests/ext/data/redis/test_service.py @@ -0,0 +1,138 @@ +import os +from importlib.util import find_spec +from urllib.parse import urlparse + +import pytest + +from selva.configuration.defaults import default_settings +from selva.configuration.settings import Settings +from selva.ext.data.redis.service import make_service + +REDIS_URL = os.getenv("REDIS_URL") + +pytestmark = [ + pytest.mark.skipif(REDIS_URL is None, reason="REDIS_URL not defined"), + pytest.mark.skipif(find_spec("redis") is None, reason="redis not present"), +] + +PARSED_URL = urlparse(REDIS_URL) if REDIS_URL else None + + +async def _test_engine_service(settings: Settings): + service = make_service("default")(settings) + async for redis in service: + result = await redis.ping() + assert result + + +async def test_make_service_with_url(): + settings = Settings( + default_settings + | { + "data": { + "redis": { + "default": { + "url": REDIS_URL, + }, + }, + }, + } + ) + + await _test_engine_service(settings) + + +@pytest.mark.skipif(PARSED_URL.password is None, reason="url without password") +async def test_make_service_with_url_username_password(): + settings = Settings( + default_settings + | { + "data": { + "redis": { + "default": { + "url": f"redis://{PARSED_URL.hostname}:{PARSED_URL.port}{PARSED_URL.path}", + "username": PARSED_URL.username, + "password": PARSED_URL.password, + }, + }, + }, + } + ) + + await _test_engine_service(settings) + + +async def test_make_service_with_url_components(): + params = { + "host": PARSED_URL.hostname, + "port": PARSED_URL.port, + "db": int(PARSED_URL.path.strip("/")), + } + + if (username := PARSED_URL.username) and (password := PARSED_URL.password): + params |= { + "username": username, + "password": password, + } + + settings = Settings( + default_settings + | { + "data": { + "redis": { + "default": params, + }, + }, + } + ) + + await _test_engine_service(settings) + + +async def test_make_service_with_options(): + settings = Settings( + default_settings + | { + "data": { + "redis": { + "default": { + "url": REDIS_URL, + "options": { + "client_name": "selva", + }, + }, + }, + }, + } + ) + + service = make_service("default")(settings) + async for redis in service: + connection = await redis.connection_pool.get_connection("") + assert connection.client_name == "selva" + + +async def test_make_service_with_retry(): + settings = Settings( + default_settings + | { + "data": { + "redis": { + "default": { + "url": REDIS_URL, + "options": { + "retry": { + "retries": 1, + "backoff": {"constant": {"backoff": 1}}, + }, + }, + }, + }, + }, + } + ) + + service = make_service("default")(settings) + async for redis in service: + connection = await redis.connection_pool.get_connection("") + assert connection.retry is not None diff --git a/tests/ext/data/redis/test_settings.py b/tests/ext/data/redis/test_settings.py new file mode 100644 index 0000000..034a6c5 --- /dev/null +++ b/tests/ext/data/redis/test_settings.py @@ -0,0 +1,54 @@ +import itertools + +import pytest + +from selva.ext.data.redis.settings import BackoffSchema, RedisOptions, RedisSettings + + +@pytest.mark.parametrize( + "values", + [ + {"host": "host"}, + {"port": 5432}, + {"db": 0}, + {"host": "host", "port": 5423}, + {"host": "host", "port": 5432, "db": 0}, + ], + ids=[ + "host", + "port", + "db", + "host_port", + "host_port_db", + ], +) +def test_mutually_exclusive_connection_properties(values: dict): + with pytest.raises(ValueError): + RedisSettings.model_validate({"url": "url"} | values) + + +@pytest.mark.parametrize( + "values", + [ + left | right + for left, right in itertools.combinations( + [ + {"no_backoff": None}, + {"constant": {"backoff": 1}}, + {"exponential": {"cap": 1, "base": 1}}, + {"full_jitter": {"cap": 1, "base": 1}}, + {"equal_jitter": {"cap": 1, "base": 1}}, + {"decorrelated_jitter": {"cap": 1, "base": 1}}, + ], + 2, + ) + ], +) +def test_mutually_exclusive_retry_properties(values: dict): + with pytest.raises(ValueError): + BackoffSchema.model_validate(values) + + +def test_invalid_encode_errors_property(): + with pytest.raises(ValueError): + RedisOptions.model_validate({"encoding_errors": "invalid"})