From c6ecaf89058c3c774d0ab03075ab039777b6a781 Mon Sep 17 00:00:00 2001 From: Livio Ribeiro Date: Fri, 27 Oct 2023 11:21:14 -0300 Subject: [PATCH] Feature/strictyaml (#32) * change settings to yaml with strictyaml * add handling of env vars in settings and add tests * change settings to settings.yaml * restore info logs when settings files are not found * fix logging setup and add more settings tests * fix erros on tutorial * minor fixes * bump version to 0.7.0 * improve settings processing * improve settings from environment variables * add docs for logging * bump version to 0.7.1 --- docs/configuration.md | 125 ++++++ docs/logging.md | 61 +++ docs/middleware.md | 11 +- docs/tutorial.md | 7 +- examples/database/application/service.py | 2 +- examples/database/configuration/settings.py | 5 - examples/database/configuration/settings.yaml | 6 + examples/hello_world/application.py | 2 +- examples/middleware/application/middleware.py | 7 +- examples/middleware/configuration/settings.py | 12 - .../middleware/configuration/settings.yaml | 10 + examples/settings/application.py | 2 +- examples/settings/configuration/settings.py | 1 - examples/settings/configuration/settings.yaml | 7 + .../settings/configuration/settings_dev.py | 1 - .../settings/configuration/settings_dev.yaml | 2 + examples/websocket/configuration/settings.py | 3 - .../websocket/configuration/settings.yaml | 3 + mkdocs.yml | 2 + pyproject.toml | 5 +- src/selva/__init__.py | 9 - src/selva/_util/import_item.py | 15 + src/selva/configuration/defaults.py | 41 +- src/selva/configuration/environment.py | 151 +++---- src/selva/configuration/settings.py | 136 +++--- src/selva/logging/setup.py | 56 +++ src/selva/web/application.py | 14 +- src/selva/web/exception_handler.py | 1 - src/selva/web/middleware.py | 3 +- .../alternate/configuration/application.yaml | 9 + .../configuration/application_prd.yaml | 1 + .../base/configuration/settings.py | 8 - .../base/configuration/settings.yaml | 9 + .../env_var/configuration/settings.yaml | 1 + .../configuration/settings_profile.yaml | 1 + .../envs/configuration/settings.py | 1 - .../envs/configuration/settings_dev.py | 1 - .../envs/configuration/settings_hlg.py | 1 - .../envs/configuration/settings_prd.py | 1 - .../invalid_yaml/configuration/settings.yaml | 1 + .../configuration/settings.yaml | 1 + .../configuration/settings_bool.py | 3 - .../configuration/settings_dict.py | 3 - .../configuration/settings_float.py | 3 - .../configuration/settings_int.py | 3 - .../configuration/settings_json.py | 3 - .../non_existent/configuration/settings.py | 3 - .../configuration/settings.py | 1 - .../override/configuration/settings.py | 1 - .../override/configuration/settings.yaml | 1 + .../override/configuration/settings_dev.py | 1 - .../override/configuration/settings_dev.yaml | 1 + .../override/configuration/settings_hlg.py | 1 - .../override/configuration/settings_prd.py | 1 - .../override/configuration/settings_prd.yaml | 1 + .../override/configuration/settings_stg.yaml | 1 + .../profiles/configuration/settings.yaml | 1 + .../profiles/configuration/settings_dev.yaml | 1 + .../profiles/configuration/settings_prd.yaml | 1 + .../profiles/configuration/settings_stg.yaml | 1 + tests/configuration/test_environment.py | 315 +++----------- tests/configuration/test_settings.py | 404 +++++++++++------- tests/conftest.py | 2 + tests/web/test_middleware.py | 5 +- 64 files changed, 794 insertions(+), 698 deletions(-) create mode 100644 docs/configuration.md create mode 100644 docs/logging.md delete mode 100644 examples/database/configuration/settings.py create mode 100644 examples/database/configuration/settings.yaml delete mode 100644 examples/middleware/configuration/settings.py create mode 100644 examples/middleware/configuration/settings.yaml delete mode 100644 examples/settings/configuration/settings.py create mode 100644 examples/settings/configuration/settings.yaml delete mode 100644 examples/settings/configuration/settings_dev.py create mode 100644 examples/settings/configuration/settings_dev.yaml delete mode 100644 examples/websocket/configuration/settings.py create mode 100644 examples/websocket/configuration/settings.yaml create mode 100644 src/selva/_util/import_item.py create mode 100644 src/selva/logging/setup.py create mode 100644 tests/configuration/alternate/configuration/application.yaml create mode 100644 tests/configuration/alternate/configuration/application_prd.yaml delete mode 100644 tests/configuration/base/configuration/settings.py create mode 100644 tests/configuration/base/configuration/settings.yaml create mode 100644 tests/configuration/env_var/configuration/settings.yaml create mode 100644 tests/configuration/env_var/configuration/settings_profile.yaml delete mode 100644 tests/configuration/envs/configuration/settings.py delete mode 100644 tests/configuration/envs/configuration/settings_dev.py delete mode 100644 tests/configuration/envs/configuration/settings_hlg.py delete mode 100644 tests/configuration/envs/configuration/settings_prd.py create mode 100644 tests/configuration/invalid_configuration/invalid_yaml/configuration/settings.yaml create mode 100644 tests/configuration/invalid_configuration/non_existent_env_var/configuration/settings.yaml delete mode 100644 tests/configuration/invalid_environment/invalid_value/configuration/settings_bool.py delete mode 100644 tests/configuration/invalid_environment/invalid_value/configuration/settings_dict.py delete mode 100644 tests/configuration/invalid_environment/invalid_value/configuration/settings_float.py delete mode 100644 tests/configuration/invalid_environment/invalid_value/configuration/settings_int.py delete mode 100644 tests/configuration/invalid_environment/invalid_value/configuration/settings_json.py delete mode 100644 tests/configuration/invalid_environment/non_existent/configuration/settings.py delete mode 100644 tests/configuration/invalid_settings/configuration/settings.py delete mode 100644 tests/configuration/override/configuration/settings.py create mode 100644 tests/configuration/override/configuration/settings.yaml delete mode 100644 tests/configuration/override/configuration/settings_dev.py create mode 100644 tests/configuration/override/configuration/settings_dev.yaml delete mode 100644 tests/configuration/override/configuration/settings_hlg.py delete mode 100644 tests/configuration/override/configuration/settings_prd.py create mode 100644 tests/configuration/override/configuration/settings_prd.yaml create mode 100644 tests/configuration/override/configuration/settings_stg.yaml create mode 100644 tests/configuration/profiles/configuration/settings.yaml create mode 100644 tests/configuration/profiles/configuration/settings_dev.yaml create mode 100644 tests/configuration/profiles/configuration/settings_prd.yaml create mode 100644 tests/configuration/profiles/configuration/settings_stg.yaml diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..94f2bf9 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,125 @@ +# Configuration + +Settings in Selva are handled through YAML files. +Internally it uses [strictyaml](https://pypi.org/project/strictyaml/) to parse the +yaml files in order to do the parsing in a safe and predictable way. + +Settings files are located by default in the `configuration` directory with the +base name `settings.yaml`: + +``` +project/ +├── application/ +│ └── ... +└── configuration/ + ├── settings.yaml + ├── settings_dev.yaml + └── settings_prod.yaml +``` + +## Accessing the configuration + +The configuration values can be accessed by injecting `selva.configuration.Settings`. + + +```python +from typing import Annotated +from selva.configuration import Settings +from selva.di import Inject, service + + +@service +class MyService: + settings: Annotated[Settings, Inject] +``` + +The `selva.configuration.Settings` is a dict like object that can also be accessed +using property syntax: + +```python +from selva.configuration import Settings + +settings = Settings({"config": "value"}) +assert settings["config"] == "value" +assert settings.config == "value" +``` + +### Typed settings + +Since `strictyaml` is used to parse the yaml files, all values `str`s. However, we +can use `pydantic` and Selva dependency injection system to provide access to the +settings in a typed manner: + +=== "application.py" + + ```python + from pydantic import BaseModel + from selva.configuration import Settings + from selva.di import service + + + class MySettings(BaseModel): + int_property: int + bool_property: bool + + + @service + def my_settings(settings: Settings) -> MySettings: + return MySettings.model_validate(settings.my_settings) + ``` + +=== "configuration/settings.yaml" + + ```yaml + my_settings: + int_property: 1 + bool_property: true + ``` + +## Environment substitution + +The settings files can include references to environment variables that takes the +format `${ENV_VAR:default_value}`. The default value is optional and an error will +be raised if neither the environment variable nor the default value are defined. + +```yaml +required: ${ENV_VAR} # required environment variable +optional: ${OPT_VAR:default} # optional environment variable +``` + +## Profiles + +Optional profiles can be activated by settings the environment variable `SELVA_PROFILE`. +The framework will look for a file named `settings_${SELVA_PROFILE}.yaml` and merge +the values with the main `settings.yaml`. Values from the profile settings take +precedence over the values from the main settings. + +As an example, if we define `SELVA_PROFILE=dev`, the file `settings_dev.yaml` will +be loaded. If instead we define `SELVA_PROFILE=prod`, then the file `settings_prod.yaml` +will be loaded. + +## Environment variables + +Settings can also be defined with environment variables whose names start with `SELVA__`, +where subsequent double undercores (`__`) indicates nesting (variable is a mapping). +Also, variable names will be lowercased. + +For example, consider the following environment variables: + +```dotenv +SELVA__PROPERTY=1 +SELVA__MAPPING__PROPERTY=2 +SELVA__MAPPING__ANOTHER_PROPERTY=3 +``` + +Those variables will be collected as the following: + +```python +{ + "property": "1", + "mapping": { + "property": "2", + "another_property": "3", + }, +} +``` diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 0000000..98f2ad8 --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,61 @@ +# Logging + +Selva uses [loguru](https://pypi.org/project/loguru/) for logging, but provides +some facilities on top of it to make its usage a bit closer to other frameworks +like Spring Boot. + +First, an interceptor to the standard `logging` module is configured by default, +as suggested in . + +Second, a custom logging filter is provided in order to set the logging level for +each package independently. + +## Configuring logging + +Logging is configured in the Selva configuration: + +```yaml +logging: + root: WARNING + level: + application: INFO + application.service: TRACE + sqlalchemy: DEBUG + enable: + - packages_to_activate_logging + disabled: + - packages_to_deactivate_logging +``` + +The `root` property is the *root* level. It is used if no other level is set for the +package where the log comes from. + +The `level` property defines the logging level for each package independently. + +The `enable` and `disable` properties lists the packages to enable or disable logging. +This comes from loguru, as can be seen in . + +## Manual logger setup + +If you want full control of how loguru is configured, you can provide a logger setup +function and reference it in the configuration file: + +=== "application/logging.py" + + ```python + from loguru import logger + + + def setup(settings): + logger.configure(...) + ``` + +=== "configuration/settings.yaml" + + ```yaml + logging: + setup: application.logging.setup + ``` + +The setup function receives a parameter of type `selva.configuration.Settings`, +so you can have access to the whole settings. diff --git a/docs/middleware.md b/docs/middleware.md index 967de54..596675c 100644 --- a/docs/middleware.md +++ b/docs/middleware.md @@ -46,14 +46,11 @@ in the processing of the request: 1. Invoke the middleware chain to process the request -=== "configuration/settings.py" +=== "configuration/settings.yaml" - ```python - from application.middleware import TimingMiddleware - - MIDDLEWARE = [ - TimingMiddleware, - ] + ```yaml + middleware: + - application.middleware.TimingMiddleware ``` ## Middleware dependencies diff --git a/docs/tutorial.md b/docs/tutorial.md index 9d5b4ba..9a92f68 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -24,7 +24,7 @@ project/ │ ├── repository.py │ └── service.py ├── configuration/ -│ └── settings.py +│ └── settings.yaml └── resources/ ``` @@ -222,9 +222,9 @@ with `@service`, so in this case we need to create a factory function for it: await respond_json(request.response, {"greeting": greeting}) ``` -## Deferred actions +## Execute actions after response -The greetings are being save to the database, but now we have a problem: the +The greetings are being saved to the database, but now we have a problem: the user has to wait until the greeting is saved before receiving it. To solve this problem and improve the user experience, we can use save the greeging @@ -239,7 +239,6 @@ after the request is completed: from asgikit.responses improt respond_json from selva.di import Inject from selva.web import controller, get, FromPath - from selva.web.response import JSONResponse, BackgroundTask from .repository import GreetingRepository from .service import Greeter diff --git a/examples/database/application/service.py b/examples/database/application/service.py index 30cff20..0d65654 100644 --- a/examples/database/application/service.py +++ b/examples/database/application/service.py @@ -10,7 +10,7 @@ @service def database_factory(settings: Settings) -> Database: - database = Database(settings.DATABASE_URL) + database = Database(settings.database.url) logger.info("Sqlite database created") yield database diff --git a/examples/database/configuration/settings.py b/examples/database/configuration/settings.py deleted file mode 100644 index 8d677ef..0000000 --- a/examples/database/configuration/settings.py +++ /dev/null @@ -1,5 +0,0 @@ -DATABASE_URL = "sqlite:///database.sqlite3" - -LOGGING_LEVEL = { - "application": "INFO", -} diff --git a/examples/database/configuration/settings.yaml b/examples/database/configuration/settings.yaml new file mode 100644 index 0000000..55d07ad --- /dev/null +++ b/examples/database/configuration/settings.yaml @@ -0,0 +1,6 @@ +database: + url: sqlite:///database.sqlite3 + +logging: + level: + application: INFO diff --git a/examples/hello_world/application.py b/examples/hello_world/application.py index a72ce19..3bdfbf2 100644 --- a/examples/hello_world/application.py +++ b/examples/hello_world/application.py @@ -31,8 +31,8 @@ async def greet_query( name: Annotated[str, FromQuery("name")] = "World", number: Annotated[int, FromQuery] = 1, ): - logger.info("message") greeting = self.greeter.greet(name) + logger.info(greeting) await respond_json(request.response, {"greeting": greeting, "number": number}) @get("/:name") diff --git a/examples/middleware/application/middleware.py b/examples/middleware/application/middleware.py index fcc8794..5c09637 100644 --- a/examples/middleware/application/middleware.py +++ b/examples/middleware/application/middleware.py @@ -1,7 +1,6 @@ import base64 from datetime import datetime from http import HTTPStatus - from asgikit.requests import Request from asgikit.responses import respond_status from loguru import logger @@ -20,7 +19,7 @@ async def __call__(self, chain, request: Request): request_end = datetime.now() delta = request_end - request_start - logger.warning("Request time: {}", delta) + logger.info("Request time: {}", delta) class LoggingMiddleware(Middleware): @@ -35,7 +34,7 @@ async def __call__(self, chain, request: Request): request_line = f"{request.method} {request.path} HTTP/{request.http_version}" status = request.response.status - logger.warning( + logger.info( '{} "{}" {} {}', client, request_line, status.value, status.phrase ) @@ -57,4 +56,4 @@ async def __call__(self, chain, request: Request): logger.info("User '{}' with password '{}'", user, password) request["user"] = user - await chain(request, response) + await chain(request) diff --git a/examples/middleware/configuration/settings.py b/examples/middleware/configuration/settings.py deleted file mode 100644 index ff9f7cd..0000000 --- a/examples/middleware/configuration/settings.py +++ /dev/null @@ -1,12 +0,0 @@ -from application.middleware import AuthMiddleware, LoggingMiddleware, TimingMiddleware - -MIDDLEWARE = [ - TimingMiddleware, - LoggingMiddleware, - AuthMiddleware, -] - -LOGGING_LEVEL = { - "application": "INFO", - "selva": "INFO", -} diff --git a/examples/middleware/configuration/settings.yaml b/examples/middleware/configuration/settings.yaml new file mode 100644 index 0000000..de24246 --- /dev/null +++ b/examples/middleware/configuration/settings.yaml @@ -0,0 +1,10 @@ +middleware: + - application.middleware.TimingMiddleware + - application.middleware.LoggingMiddleware + - application.middleware.AuthMiddleware + +logging: + root: WARNING + level: + application: INFO + selva: INFO diff --git a/examples/settings/application.py b/examples/settings/application.py index 1e40bb7..1aea226 100644 --- a/examples/settings/application.py +++ b/examples/settings/application.py @@ -14,4 +14,4 @@ class Controller: @get async def index(self, request: Request): - await respond_text(request.response, self.settings.MESSAGE) + await respond_text(request.response, self.settings.application.message) diff --git a/examples/settings/configuration/settings.py b/examples/settings/configuration/settings.py deleted file mode 100644 index 026eb84..0000000 --- a/examples/settings/configuration/settings.py +++ /dev/null @@ -1 +0,0 @@ -MESSAGE = "Hello, World!" diff --git a/examples/settings/configuration/settings.yaml b/examples/settings/configuration/settings.yaml new file mode 100644 index 0000000..7f71292 --- /dev/null +++ b/examples/settings/configuration/settings.yaml @@ -0,0 +1,7 @@ +message: Hello, World! +application: + message: ${MESSAGE:Hello, World!} +logging: + root: INFO + level: + application: WARNING \ No newline at end of file diff --git a/examples/settings/configuration/settings_dev.py b/examples/settings/configuration/settings_dev.py deleted file mode 100644 index 267c415..0000000 --- a/examples/settings/configuration/settings_dev.py +++ /dev/null @@ -1 +0,0 @@ -MESSAGE = "Hello, dev World!" diff --git a/examples/settings/configuration/settings_dev.yaml b/examples/settings/configuration/settings_dev.yaml new file mode 100644 index 0000000..fe293c1 --- /dev/null +++ b/examples/settings/configuration/settings_dev.yaml @@ -0,0 +1,2 @@ +application: + message: ${MESSAGE:Hello, dev World!} diff --git a/examples/websocket/configuration/settings.py b/examples/websocket/configuration/settings.py deleted file mode 100644 index 7023d8d..0000000 --- a/examples/websocket/configuration/settings.py +++ /dev/null @@ -1,3 +0,0 @@ -LOGGING_LEVEL = { - "application": "INFO", -} diff --git a/examples/websocket/configuration/settings.yaml b/examples/websocket/configuration/settings.yaml new file mode 100644 index 0000000..d9db3b7 --- /dev/null +++ b/examples/websocket/configuration/settings.yaml @@ -0,0 +1,3 @@ +logging: + level: + application: INFO diff --git a/mkdocs.yml b/mkdocs.yml index 4abea36..786ebbd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,4 +27,6 @@ nav: - tutorial.md - controllers.md - routing.md + - configuration.md - middleware.md + - logging.md diff --git a/pyproject.toml b/pyproject.toml index 90d05ba..634306d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "selva" -version = "0.6.7" +version = "0.7.1" description = "ASGI Web Framework with Dependency Injection" authors = ["Livio Ribeiro "] license = "MIT" @@ -15,7 +15,7 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Internet :: WWW/HTTP", "Topic :: Internet :: WWW/HTTP :: Dynamic Content", @@ -31,6 +31,7 @@ python = "^3.12" asgikit = "^0.5" pydantic = "^2.4" loguru = "^0.7" +strictyaml = "^1.7" [tool.poetry.group.dev] optional = true diff --git a/src/selva/__init__.py b/src/selva/__init__.py index 7796d34..e69de29 100644 --- a/src/selva/__init__.py +++ b/src/selva/__init__.py @@ -1,9 +0,0 @@ -from loguru import logger - -from selva.logging.stdlib import setup_loguru_std_logging_interceptor - -# disable asgikit and selva loggers by default -logger.disable("asgikit") -logger.disable("selva") - -setup_loguru_std_logging_interceptor() diff --git a/src/selva/_util/import_item.py b/src/selva/_util/import_item.py new file mode 100644 index 0000000..69d7987 --- /dev/null +++ b/src/selva/_util/import_item.py @@ -0,0 +1,15 @@ +from importlib import import_module + + +__all__ = ("import_item",) + + +def import_item(name: str): + """Import an item from a module""" + + match name.rsplit(".", 1): + case [module_name, item_name]: + module = import_module(module_name) + return getattr(module, item_name) + case _: + raise ValueError("name must be in 'module.item' format") diff --git a/src/selva/configuration/defaults.py b/src/selva/configuration/defaults.py index 2d9915e..6470a63 100644 --- a/src/selva/configuration/defaults.py +++ b/src/selva/configuration/defaults.py @@ -1,34 +1,11 @@ -from selva.configuration.environment import get_dict - -COMPONENTS = [] - -MIDDLEWARE = [] - -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "default": { - "format": "{asctime} {levelname:<8} {name}[{threadName}] {message}", - "style": "{", - }, - }, - "handlers": { - "console": { - "class": "logging.StreamHandler", - "formatter": "default", - }, - }, - "loggers": { - "application": { - "level": "WARNING", - "handlers": ["console"], - }, - "selva": { - "level": "WARNING", - "handlers": ["console"], - }, +default_settings = { + "components": [], + "middleware": [], + "logging": { + "setup": "selva.logging.setup.setup_logger", + "root": "WARNING", + "level": {}, + "enable": [], + "disable": [], }, } - -LOGGING_LEVEL = get_dict("LOG_LEVEL", {}) diff --git a/src/selva/configuration/environment.py b/src/selva/configuration/environment.py index 48130b1..5281108 100644 --- a/src/selva/configuration/environment.py +++ b/src/selva/configuration/environment.py @@ -1,91 +1,82 @@ -import json -import os -from collections.abc import Callable, Mapping, Sequence -from typing import Any, Optional - -_UNSET = () +import copy +import re + +RE_VARIABLE = re.compile( + r""" + \$\{ # Expression start with '${' + \ ? # Blank space before variable name + (?P\w+?) # Environment variable name + \ ? # Blank space after variable name + (?: # Default value non-capture group + : # Collon separating variable name and default value + \ ? # Blank space before default value + (?P.*?) # Default value + \ ? # Blank space after default value + )? # End of default value non-capture group + } # Expression end with '}' + """, + re.VERBOSE, +) + + +SELVA_PREFIX = "SELVA__" + + +def parse_settings_from_env(source: dict[str, str]) -> dict: + result = {} + + for name, value in source.items(): + if not name.startswith(SELVA_PREFIX): + continue + + current_ref = result + keys = name.removeprefix(SELVA_PREFIX).split("__") + + match keys: + case [key] if key.strip() != "": + current_key = key.lower() + case [first_key, *keys, last_key]: + for key in [first_key] + keys: + key = key.lower() + if key not in current_ref: + current_ref[key] = {} + current_ref = current_ref[key] + current_key = last_key.lower() + case _: + continue + current_ref[current_key] = value -def get_str(key: str, default: Optional[str] = _UNSET) -> str: - if value := os.getenv(key): - return value + return result - if default is not _UNSET: - return default - raise KeyError(f"Environment variable '{key}' is not defined") +def replace_variables_with_env(settings: str, environ: dict[str, str]): + for match in RE_VARIABLE.finditer(settings): + name = match.group("name") + if value := environ.get(name): + settings = settings.replace(match.group(), value) + continue -def _get_env_and_convert[T](key: str, default: T | None, converter: Callable[[str], T], type_name: str) -> T: - value = get_str(key, None) - if value is None: - return default + if value := match.group("default"): + settings = settings.replace(match.group(), value) + continue - try: - return converter(value) - except ValueError as err: - message = ( - f"Environment variable '{key}'" - f" is not compatible with type '{type_name}': '{value}'" + raise ValueError( + f"{name} environment variable is not defined and does not contain a default value" ) - raise ValueError(message) from err - - -def get_int(key: str, default: Optional[int] = _UNSET) -> int: - return _get_env_and_convert(key, default, int, "int") - - -def get_float(key: str, default: Optional[float] = _UNSET) -> float: - return _get_env_and_convert(key, default, float, "float") - - -def _bool_converter(value: str) -> bool: - if value in ("1", "true", "True"): - return True - - if value in ("0", "false", "False"): - return False - - raise ValueError( - "bool value must be one of ['1', 'true', 'True', '0', 'false', 'False']" - ) - - -def get_bool(key: str, default: Optional[bool] = _UNSET) -> bool: - return _get_env_and_convert(key, default, _bool_converter, "bool") - - -def _list_converter(value: str) -> list[str]: - return [v.strip().strip("'\"") for v in value.split(",")] - - -def get_list(key: str, default: Optional[Sequence[str]] = _UNSET) -> Sequence[str]: - return _get_env_and_convert(key, default, _list_converter, "list") - - -def _dict_converter(value: str) -> dict[str, str]: - result: dict[str, str] = {} - - for item in value.split(","): - match item.split("=", maxsplit=1): - case [key, val]: - key = key.strip().strip("'\"") - val = val.strip().strip("'\"") - result[key] = val - case _: - message = f"{item} is not a valid dict item" - raise ValueError(message) - - return result - - -def get_dict(key, default: Optional[Mapping[str, str]] = _UNSET) -> Mapping[str, str]: - return _get_env_and_convert(key, default, _dict_converter, "dict") - -def _json_converter(value: str) -> dict[str, Any]: - return json.loads(value) + return settings -def get_json(key, default: Optional[Mapping[str, Any]] = _UNSET) -> Mapping[str, Any]: - return _get_env_and_convert(key, default, _json_converter, "json") +def replace_variables_recursive(settings: dict | list | str, environ: dict): + if isinstance(settings, dict): + for key, value in settings.items(): + settings[key] = replace_variables_recursive(value, environ) + return settings + elif isinstance(settings, list): + return [replace_variables_recursive(value, environ) for value in settings] + elif isinstance(settings, str): + return replace_variables_with_env(settings, environ) + else: + raise TypeError("settings should contain only str, list or dict") diff --git a/src/selva/configuration/settings.py b/src/selva/configuration/settings.py index 5e26b40..1c52954 100644 --- a/src/selva/configuration/settings.py +++ b/src/selva/configuration/settings.py @@ -1,113 +1,95 @@ -import importlib -import importlib.util -import inspect import os +from collections import UserDict +from copy import deepcopy from pathlib import Path -from types import ModuleType, SimpleNamespace from typing import Any +import strictyaml from loguru import logger -from selva.configuration import defaults +from selva.configuration.defaults import default_settings +from selva.configuration.environment import parse_settings_from_env, replace_variables_recursive -__all__ = ("Settings", "SettingsModuleError", "get_settings") +__all__ = ("Settings", "SettingsError", "get_settings") -SELVA_SETTINGS_MODULE = "SELVA_SETTINGS_MODULE" -DEFAULT_SELVA_SETTINGS_MODULE = str(Path("configuration") / "settings.py") +SETTINGS_DIR_ENV = "SELVA_SETTINGS_DIR" +SETTINGS_FILE_ENV = "SELVA_SETTINGS_FILE" -SELVA_ENV = "SELVA_ENV" +DEFAULT_SETTINGS_DIR = str(Path("configuration")) +DEFAULT_SETTINGS_FILE = "settings.yaml" +SELVA_PROFILE = "SELVA_PROFILE" -class SettingsModuleError(Exception): - def __init__(self, path: Path): - super().__init__(f"cannot load settings module: {path}") - self.path = path +class Settings(UserDict): + def __init__(self, data: dict): + for key, value in data.items(): + if isinstance(value, dict): + data[key] = Settings(value) + + super().__init__(data) -def is_valid_conf(conf: str) -> bool: - """Checks if the config item can be collected into settings + def __getattr__(self, item: str): + try: + return self.data[item] + except KeyError: + raise AttributeError(item) - Config settings that are exported must start with an uppercase letter - followed by other uppercase letters, numbers or underscores - """ - if not (conf[0].isalpha() and conf[0].isupper()): - return False +class SettingsError(Exception): + def __init__(self, path: Path): + super().__init__(f"cannot load settings from {path}") + self.path = path - return all((i.isalpha() and i.isupper()) or i.isnumeric() or i == "_" for i in conf) +def get_settings() -> Settings: + # get default settings + settings = deepcopy(default_settings) -def extract_valid_keys(settings: ModuleType) -> dict[str, Any]: - """Collect settings from module into dict""" - return { - name: value - for name, value in inspect.getmembers(settings) - if is_valid_conf(name) - } + # merge with main settings file (settings.yaml) + merge_recursive(settings, get_settings_for_profile()) + # merge with environment settings file (settings_$SELVA_ENV.yaml) + if active_env := os.getenv(SELVA_PROFILE): + merge_recursive(settings, get_settings_for_profile(active_env)) -def get_default_settings(): - return extract_valid_keys(defaults) + # merge with environment variables (SELVA_*) + from_env_vars = parse_settings_from_env(os.environ) + merge_recursive(settings, from_env_vars) + settings = replace_variables_recursive(settings, os.environ) + return Settings(settings) -def get_settings_for_env(env: str = None) -> dict[str, Any]: - settings_module_name = "selva_settings" - settings_module_path = Path( - os.getenv(SELVA_SETTINGS_MODULE, DEFAULT_SELVA_SETTINGS_MODULE) - ) - settings_module_path = settings_module_path.with_suffix(".py") +def get_settings_for_profile(env: str = None) -> dict[str, Any]: + settings_file = os.getenv(SETTINGS_FILE_ENV, DEFAULT_SETTINGS_FILE) + settings_dir_path = Path(os.getenv(SETTINGS_DIR_ENV, DEFAULT_SETTINGS_DIR)) + settings_file_path = settings_dir_path / settings_file if env is not None: - settings_module_name += f"_{env}" - settings_module_path = settings_module_path.with_stem( - f"{settings_module_path.stem}_{env}" + settings_file_path = settings_file_path.with_stem( + f"{settings_file_path.stem}_{env}" ) - settings_module_path = settings_module_path.absolute() + settings_file_path = settings_file_path.absolute() try: - spec = importlib.util.spec_from_file_location( - settings_module_name, settings_module_path - ) - settings_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(settings_module) + settings_yaml = settings_file_path.read_text("utf-8") + return strictyaml.load(settings_yaml).data except FileNotFoundError: - logger.info("settings module not found: {}", settings_module_path) + logger.info("settings file not found: {}", settings_file_path) return {} except (KeyError, ValueError): raise except Exception as err: - raise SettingsModuleError(settings_module_path) from err - - return extract_valid_keys(settings_module) - + raise SettingsError(settings_file_path) from err -class Settings(SimpleNamespace): - def __init__(self, settings: dict[str, Any]): - super().__init__(**settings) - def __getitem__(self, item): - if (value := self.get(item)) is not None: - return value - - raise KeyError(item) - - def __setattr__(self, key, value): - raise AttributeError("can't set attribute") - - def __delattr__(self, item): - raise AttributeError("can't del attribute") - - def get(self, name: str, default=None) -> Any | None: - return getattr(self, name, default) - - -def get_settings() -> Settings: - settings = get_default_settings() - settings |= get_settings_for_env() - - if active_env := os.getenv(SELVA_ENV): - settings |= get_settings_for_env(active_env) - - return Settings(settings) +def merge_recursive(destination: dict, source: dict): + for key in source: + if key in destination and all( + isinstance(arg[key], dict) for arg in (destination, source) + ): + merge_recursive(destination[key], source[key]) + else: + destination[key] = deepcopy(source[key]) diff --git a/src/selva/logging/setup.py b/src/selva/logging/setup.py new file mode 100644 index 0000000..5604384 --- /dev/null +++ b/src/selva/logging/setup.py @@ -0,0 +1,56 @@ +import sys +from functools import cache + +from loguru import logger + +from selva.configuration.settings import Settings +from selva.logging.stdlib import setup_loguru_std_logging_interceptor + + +def setup_logger(settings: Settings): + enable = [(name, True) for name in settings.logging.enable] + disable = [(name, False) for name in settings.logging.disable] + + # enabling has precedence over disabling + # therefore the "enable" list comes after the "disable" list + activation = disable + enable + + log_config = settings.get("logging", {}) + root_level = logger.level(log_config.get("root", "WARNING")) + log_level = { + name: logger.level(value) + for name, value in log_config.get("level", {}).items() + } + + filter_func = filter_func_factory(root_level, log_level) + handler = {"sink": sys.stderr, "filter": filter_func} + + logger.configure( + handlers=[handler], + activation=activation + ) + + setup_loguru_std_logging_interceptor() + + +def filter_func_factory(root_level, log_level: dict): + @cache + def has_level(name: str, record_level): + level = log_level.get(name) + + while not level: + match name.rsplit(".", 1): + case [first, _last]: + name = first + level = log_level.get(name) + case _: + level = root_level + + return record_level.no >= level.no + + def filter_func(record): + name = record["name"] + record_level = record["level"] + return has_level(name, record_level) + + return filter_func diff --git a/src/selva/web/application.py b/src/selva/web/application.py index 7474d9a..62b8e53 100644 --- a/src/selva/web/application.py +++ b/src/selva/web/application.py @@ -15,6 +15,7 @@ from selva._util.base_types import get_base_types from selva._util.maybe_async import maybe_async +from selva._util.import_item import import_item from selva.configuration.settings import Settings, get_settings from selva.di.container import Container from selva.di.decorator import DI_SERVICE_ATTRIBUTE @@ -72,11 +73,14 @@ def __init__(self): param_extractor_impl, param_converter_impl, ) - self.di.scan(self.settings.COMPONENTS) - components = self.settings.COMPONENTS + components = self.settings.components + self.di.scan(components) self._register_components(components) + setup_logger = import_item(self.settings.logging.setup) + setup_logger(self.settings) + async def __call__(self, scope, receive, send): match scope["type"]: case "http" | "websocket": @@ -120,10 +124,12 @@ def _register_components( self.router.route(impl) async def _initialize_middleware(self): - middleware = self.settings.MIDDLEWARE + middleware = self.settings.middleware if len(middleware) == 0: return + middleware = [import_item(name) for name in middleware] + if middleware_errors := [ m for m in middleware if not issubclass(m, Middleware) ]: @@ -135,7 +141,7 @@ async def _initialize_middleware(self): f"Middleware classes must inherit from '{mid_class_name}': {mid_classes}" ) - for cls in reversed(self.settings.MIDDLEWARE): + for cls in reversed(middleware): mid = await self.di.create(cls) chain = functools.partial(mid, self.handler) self.handler = chain diff --git a/src/selva/web/exception_handler.py b/src/selva/web/exception_handler.py index 24e7eda..65f9631 100644 --- a/src/selva/web/exception_handler.py +++ b/src/selva/web/exception_handler.py @@ -1,7 +1,6 @@ from typing import Protocol, Type, runtime_checkable from asgikit.requests import Request -from asgikit.responses import Response from selva.di.decorator import service diff --git a/src/selva/web/middleware.py b/src/selva/web/middleware.py index 78bf0d5..963b30f 100644 --- a/src/selva/web/middleware.py +++ b/src/selva/web/middleware.py @@ -2,7 +2,6 @@ from collections.abc import Awaitable, Callable from asgikit.requests import Request -from asgikit.responses import Response __all__ = ("Middleware",) @@ -11,7 +10,7 @@ class Middleware(ABC): @abstractmethod async def __call__( self, - call: Callable[[Request, Response], Awaitable], + call: Callable[[Request], Awaitable], request: Request, ): raise NotImplementedError() diff --git a/tests/configuration/alternate/configuration/application.yaml b/tests/configuration/alternate/configuration/application.yaml new file mode 100644 index 0000000..fedfb5c --- /dev/null +++ b/tests/configuration/alternate/configuration/application.yaml @@ -0,0 +1,9 @@ +prop: value +list: + - 1 + - 2 + - 3 +dict: + a: 1 + b: 2 + c: 3 diff --git a/tests/configuration/alternate/configuration/application_prd.yaml b/tests/configuration/alternate/configuration/application_prd.yaml new file mode 100644 index 0000000..ee56c26 --- /dev/null +++ b/tests/configuration/alternate/configuration/application_prd.yaml @@ -0,0 +1 @@ +environment: prd \ No newline at end of file diff --git a/tests/configuration/base/configuration/settings.py b/tests/configuration/base/configuration/settings.py deleted file mode 100644 index 067a2c9..0000000 --- a/tests/configuration/base/configuration/settings.py +++ /dev/null @@ -1,8 +0,0 @@ -CONF_STR = "str" -CONF_INT = 1 -CONF_LIST = [1, 2, 3] -CONF_DICT = { - "a": 1, - "b": 2, - "c": 3, -} diff --git a/tests/configuration/base/configuration/settings.yaml b/tests/configuration/base/configuration/settings.yaml new file mode 100644 index 0000000..fedfb5c --- /dev/null +++ b/tests/configuration/base/configuration/settings.yaml @@ -0,0 +1,9 @@ +prop: value +list: + - 1 + - 2 + - 3 +dict: + a: 1 + b: 2 + c: 3 diff --git a/tests/configuration/env_var/configuration/settings.yaml b/tests/configuration/env_var/configuration/settings.yaml new file mode 100644 index 0000000..f838336 --- /dev/null +++ b/tests/configuration/env_var/configuration/settings.yaml @@ -0,0 +1 @@ +name: ${VAR_NAME} \ No newline at end of file diff --git a/tests/configuration/env_var/configuration/settings_profile.yaml b/tests/configuration/env_var/configuration/settings_profile.yaml new file mode 100644 index 0000000..c6c40b9 --- /dev/null +++ b/tests/configuration/env_var/configuration/settings_profile.yaml @@ -0,0 +1 @@ +name: profile \ No newline at end of file diff --git a/tests/configuration/envs/configuration/settings.py b/tests/configuration/envs/configuration/settings.py deleted file mode 100644 index 50a0095..0000000 --- a/tests/configuration/envs/configuration/settings.py +++ /dev/null @@ -1 +0,0 @@ -NAME = "application" diff --git a/tests/configuration/envs/configuration/settings_dev.py b/tests/configuration/envs/configuration/settings_dev.py deleted file mode 100644 index f3511de..0000000 --- a/tests/configuration/envs/configuration/settings_dev.py +++ /dev/null @@ -1 +0,0 @@ -ENVIRONMENT = "dev" diff --git a/tests/configuration/envs/configuration/settings_hlg.py b/tests/configuration/envs/configuration/settings_hlg.py deleted file mode 100644 index 4d6f882..0000000 --- a/tests/configuration/envs/configuration/settings_hlg.py +++ /dev/null @@ -1 +0,0 @@ -ENVIRONMENT = "hlg" diff --git a/tests/configuration/envs/configuration/settings_prd.py b/tests/configuration/envs/configuration/settings_prd.py deleted file mode 100644 index c237f0f..0000000 --- a/tests/configuration/envs/configuration/settings_prd.py +++ /dev/null @@ -1 +0,0 @@ -ENVIRONMENT = "prd" diff --git a/tests/configuration/invalid_configuration/invalid_yaml/configuration/settings.yaml b/tests/configuration/invalid_configuration/invalid_yaml/configuration/settings.yaml new file mode 100644 index 0000000..a6bea6f --- /dev/null +++ b/tests/configuration/invalid_configuration/invalid_yaml/configuration/settings.yaml @@ -0,0 +1 @@ +invalid: yaml: \ No newline at end of file diff --git a/tests/configuration/invalid_configuration/non_existent_env_var/configuration/settings.yaml b/tests/configuration/invalid_configuration/non_existent_env_var/configuration/settings.yaml new file mode 100644 index 0000000..583331d --- /dev/null +++ b/tests/configuration/invalid_configuration/non_existent_env_var/configuration/settings.yaml @@ -0,0 +1 @@ +does_not_exist: ${DOES_NOT_EXIST} diff --git a/tests/configuration/invalid_environment/invalid_value/configuration/settings_bool.py b/tests/configuration/invalid_environment/invalid_value/configuration/settings_bool.py deleted file mode 100644 index 406deb7..0000000 --- a/tests/configuration/invalid_environment/invalid_value/configuration/settings_bool.py +++ /dev/null @@ -1,3 +0,0 @@ -from selva.configuration import environment as env - -env.get_bool("INVALID") diff --git a/tests/configuration/invalid_environment/invalid_value/configuration/settings_dict.py b/tests/configuration/invalid_environment/invalid_value/configuration/settings_dict.py deleted file mode 100644 index cc6961e..0000000 --- a/tests/configuration/invalid_environment/invalid_value/configuration/settings_dict.py +++ /dev/null @@ -1,3 +0,0 @@ -from selva.configuration import environment as env - -env.get_dict("INVALID") diff --git a/tests/configuration/invalid_environment/invalid_value/configuration/settings_float.py b/tests/configuration/invalid_environment/invalid_value/configuration/settings_float.py deleted file mode 100644 index 1e84ac5..0000000 --- a/tests/configuration/invalid_environment/invalid_value/configuration/settings_float.py +++ /dev/null @@ -1,3 +0,0 @@ -from selva.configuration import environment as env - -env.get_float("INVALID") diff --git a/tests/configuration/invalid_environment/invalid_value/configuration/settings_int.py b/tests/configuration/invalid_environment/invalid_value/configuration/settings_int.py deleted file mode 100644 index 08090ad..0000000 --- a/tests/configuration/invalid_environment/invalid_value/configuration/settings_int.py +++ /dev/null @@ -1,3 +0,0 @@ -from selva.configuration import environment as env - -env.get_int("INVALID") diff --git a/tests/configuration/invalid_environment/invalid_value/configuration/settings_json.py b/tests/configuration/invalid_environment/invalid_value/configuration/settings_json.py deleted file mode 100644 index 0174abd..0000000 --- a/tests/configuration/invalid_environment/invalid_value/configuration/settings_json.py +++ /dev/null @@ -1,3 +0,0 @@ -from selva.configuration import environment as env - -env.get_json("INVALID") diff --git a/tests/configuration/invalid_environment/non_existent/configuration/settings.py b/tests/configuration/invalid_environment/non_existent/configuration/settings.py deleted file mode 100644 index 621c887..0000000 --- a/tests/configuration/invalid_environment/non_existent/configuration/settings.py +++ /dev/null @@ -1,3 +0,0 @@ -from selva.configuration import environment as env - -env.get_str("DOES_NOT_EXIST") diff --git a/tests/configuration/invalid_settings/configuration/settings.py b/tests/configuration/invalid_settings/configuration/settings.py deleted file mode 100644 index fdad2f1..0000000 --- a/tests/configuration/invalid_settings/configuration/settings.py +++ /dev/null @@ -1 +0,0 @@ -VALUE = \ No newline at end of file diff --git a/tests/configuration/override/configuration/settings.py b/tests/configuration/override/configuration/settings.py deleted file mode 100644 index 6a48b8e..0000000 --- a/tests/configuration/override/configuration/settings.py +++ /dev/null @@ -1 +0,0 @@ -VALUE = "base" diff --git a/tests/configuration/override/configuration/settings.yaml b/tests/configuration/override/configuration/settings.yaml new file mode 100644 index 0000000..1f48b0e --- /dev/null +++ b/tests/configuration/override/configuration/settings.yaml @@ -0,0 +1 @@ +value: base diff --git a/tests/configuration/override/configuration/settings_dev.py b/tests/configuration/override/configuration/settings_dev.py deleted file mode 100644 index ff48678..0000000 --- a/tests/configuration/override/configuration/settings_dev.py +++ /dev/null @@ -1 +0,0 @@ -VALUE = "dev" diff --git a/tests/configuration/override/configuration/settings_dev.yaml b/tests/configuration/override/configuration/settings_dev.yaml new file mode 100644 index 0000000..b6a0d24 --- /dev/null +++ b/tests/configuration/override/configuration/settings_dev.yaml @@ -0,0 +1 @@ +value: dev diff --git a/tests/configuration/override/configuration/settings_hlg.py b/tests/configuration/override/configuration/settings_hlg.py deleted file mode 100644 index fe23a06..0000000 --- a/tests/configuration/override/configuration/settings_hlg.py +++ /dev/null @@ -1 +0,0 @@ -VALUE = "hlg" diff --git a/tests/configuration/override/configuration/settings_prd.py b/tests/configuration/override/configuration/settings_prd.py deleted file mode 100644 index a527665..0000000 --- a/tests/configuration/override/configuration/settings_prd.py +++ /dev/null @@ -1 +0,0 @@ -VALUE = "prd" diff --git a/tests/configuration/override/configuration/settings_prd.yaml b/tests/configuration/override/configuration/settings_prd.yaml new file mode 100644 index 0000000..89ba112 --- /dev/null +++ b/tests/configuration/override/configuration/settings_prd.yaml @@ -0,0 +1 @@ +value: prd diff --git a/tests/configuration/override/configuration/settings_stg.yaml b/tests/configuration/override/configuration/settings_stg.yaml new file mode 100644 index 0000000..53c2bbd --- /dev/null +++ b/tests/configuration/override/configuration/settings_stg.yaml @@ -0,0 +1 @@ +value: stg diff --git a/tests/configuration/profiles/configuration/settings.yaml b/tests/configuration/profiles/configuration/settings.yaml new file mode 100644 index 0000000..87deafc --- /dev/null +++ b/tests/configuration/profiles/configuration/settings.yaml @@ -0,0 +1 @@ +name: application diff --git a/tests/configuration/profiles/configuration/settings_dev.yaml b/tests/configuration/profiles/configuration/settings_dev.yaml new file mode 100644 index 0000000..2713f19 --- /dev/null +++ b/tests/configuration/profiles/configuration/settings_dev.yaml @@ -0,0 +1 @@ +environment: dev diff --git a/tests/configuration/profiles/configuration/settings_prd.yaml b/tests/configuration/profiles/configuration/settings_prd.yaml new file mode 100644 index 0000000..4c0f335 --- /dev/null +++ b/tests/configuration/profiles/configuration/settings_prd.yaml @@ -0,0 +1 @@ +environment: prd diff --git a/tests/configuration/profiles/configuration/settings_stg.yaml b/tests/configuration/profiles/configuration/settings_stg.yaml new file mode 100644 index 0000000..2daa011 --- /dev/null +++ b/tests/configuration/profiles/configuration/settings_stg.yaml @@ -0,0 +1 @@ +environment: stg diff --git a/tests/configuration/test_environment.py b/tests/configuration/test_environment.py index de523ed..f187391 100644 --- a/tests/configuration/test_environment.py +++ b/tests/configuration/test_environment.py @@ -1,273 +1,92 @@ import pytest -from selva.configuration import environment as env - - -def test_get_str(monkeypatch): - monkeypatch.setenv("VALUE", "str") - result = env.get_str("VALUE") - assert result == "str" - - -def test_get_str_with_default(): - result = env.get_str("VALUE", "default") - assert result == "default" - - -def test_get_str_with_value_and_default(monkeypatch): - monkeypatch.setenv("VALUE", "str") - result = env.get_str("VALUE", "default") - assert result == "str" - - -def test_undefined_environment_variable_should_fail(): - with pytest.raises( - KeyError, match=f"Environment variable 'DOES_NOT_EXIST' is not defined" - ): - env.get_str("DOES_NOT_EXIST") - - -def test_get_int(monkeypatch): - monkeypatch.setenv("VALUE", "123") - result = env.get_int("VALUE") - assert result == 123 - - -def test_get_int_with_default(): - result = env.get_int("VALUE", 123) - assert result == 123 - - -def test_get_int_with_value_and_default(monkeypatch): - monkeypatch.setenv("VALUE", "123") - result = env.get_int("VALUE", 987) - assert result == 123 - - -def test_get_int_with_invalid_value_should_fail(monkeypatch): - monkeypatch.setenv("VALUE", "abc") - with pytest.raises(ValueError): - env.get_int("VALUE") - - -def test_get_float(monkeypatch): - monkeypatch.setenv("VALUE", "1.23") - result = env.get_float("VALUE") - assert result == 1.23 - - -def test_get_float_with_default(): - result = env.get_float("VALUE", 1.23) - assert result == 1.23 - - -def test_get_float_with_value_and_default(monkeypatch): - monkeypatch.setenv("VALUE", "1.23") - result = env.get_float("VALUE", 9.87) - assert result == 1.23 - - -def test_get_float_with_invalid_should_fail(monkeypatch): - monkeypatch.setenv("VALUE", "abc") - with pytest.raises(ValueError): - env.get_float("VALUE") - - -@pytest.mark.parametrize( - "raw,expected", - [ - ("true", True), - ("True", True), - ("1", True), - ("false", False), - ("False", False), - ("0", False), - ], - ids=["true", "True", "1", "false", "False", "0"], +from selva.configuration.environment import ( + parse_settings_from_env, + replace_variables_with_env, + replace_variables_recursive, ) -def test_get_bool(raw, expected, monkeypatch): - monkeypatch.setenv("VALUE", raw) - result = env.get_bool("VALUE") - assert result == expected - - -def test_get_bool_with_default(): - result = env.get_bool("VALUE", True) - assert result is True -@pytest.mark.parametrize( - "value", - [ - "true", - "True", - "1", - ], -) -def test_get_bool_with_value_and_default(monkeypatch, value): - monkeypatch.setenv("VALUE", value) - result = env.get_bool("VALUE", False) - assert result is True - - -def test_get_bool_with_invalid_should_fail(monkeypatch): - monkeypatch.setenv("VALUE", "abc") - with pytest.raises(ValueError): - env.get_bool("VALUE") - - -@pytest.mark.parametrize( - "raw,expected", - [ - ("abc", ["abc"]), - ("abc,", ["abc", ""]), - ("abc,,", ["abc", "", ""]), - (",abc", ["", "abc"]), - (",,abc", ["", "", "abc"]), - ("abc,def,ghi", ["abc", "def", "ghi"]), - ('"abc","def","ghi"', ["abc", "def", "ghi"]), - ("'abc','def','ghi'", ["abc", "def", "ghi"]), - ('"abc",def,ghi', ["abc", "def", "ghi"]), - ("'abc',def,ghi", ["abc", "def", "ghi"]), - (" abc,def , ghi ", ["abc", "def", "ghi"]), - ("'abc ',' def',' ghi '", ["abc ", " def", " ghi "]), - (" 'abc ',' def', ' ghi ' ", ["abc ", " def", " ghi "]), - ], - ids=[ - "abc", - "abc,", - "abc,,", - ",abc", - ",,abc", - "abc,def,ghi", - '"abc","def","ghi"', - "'abc','def','ghi'", - '"abc",def,ghi', - "'abc',def,ghi", - " abc,def , ghi ", - "'abc ',' def',' ghi '", - " 'abc ',' def', ' ghi ' ", - ], -) -def test_get_list(raw, expected, monkeypatch): - monkeypatch.setenv("VALUE", raw) - result = env.get_list("VALUE") +def test_parse_settings(): + source = { + "SELVA__ENV1": "1", + "SELVA__PATH1__ENV2": "2", + "SELVA__PATH1__PATH2__SUB_ENV3": "3", + "SELVA__PATH1__PATH2__SUB_ENV4": "4", + "SELVA__": "None", + "NOT_SELVA_ENV": "None", + } + + result = parse_settings_from_env(source) + + expected = { + "env1": "1", + "path1": { + "env2": "2", + "path2": { + "sub_env3": "3", + "sub_env4": "4", + }, + }, + } assert result == expected -def test_get_list_with_default(): - result = env.get_list("VALUE", [1, 2, 3]) - assert result == [1, 2, 3] - +def test_replace_variable_from_environ(): + yaml = "${VAR}" + result = replace_variables_with_env(yaml, {"VAR": "value"}) -def test_get_list_with_value_and_default(monkeypatch): - monkeypatch.setenv("VALUE", "1, 2, 3") - result = env.get_list("VALUE", ["9", "8", "7"]) - assert result == ["1", "2", "3"] + assert result == "value" -@pytest.mark.parametrize( - "raw,expected", - [ - ("a=1", {"a": "1"}), - ("a=1,b=2", {"a": "1", "b": "2"}), - ("'a'=1,b='2','c'='3'", {"a": "1", "b": "2", "c": "3"}), - ('"a"=1,b="2","c"="3"', {"a": "1", "b": "2", "c": "3"}), - ( - "'a'=\"1\",\"b\"='2','c'='3',\"d\"=\"4\"", - {"a": "1", "b": "2", "c": "3", "d": "4"}, - ), - ("a =1 , b= 2, c = 3 ", {"a": "1", "b": "2", "c": "3"}), - ("'a '='1 ',' b'=' 2',' c '=' 3 '", {"a ": "1 ", " b": " 2", " c ": " 3 "}), - ], - ids=[ - "a=1", - "a=1,b=2", - "'a'=1,b='2','c'='3'", - '"a"=1,b="2","c"="3"', - "'a'=\"1\",\"b\"='2','c'='3',\"d\"=\"4\"", - "a =1 , b= 2, c = 3 ", - "'a '='1 ',' b'=' 2',' c '=' 3 '", - ], -) -def test_get_dict(raw, expected, monkeypatch): - monkeypatch.setenv("VALUE", raw) - result = env.get_dict("VALUE") - assert result == expected +def test_replace_variable_from_default(): + yaml = "${VAR:default}" + result = replace_variables_with_env(yaml, {}) + assert result == "default" -def test_get_dict_with_default(): - result = env.get_dict("VALUE", {"a": 1, "b": 2, "c": 3}) - assert result == {"a": 1, "b": 2, "c": 3} +def test_replace_variable_environ_precedence(): + yaml = "${VAR:default}" + result = replace_variables_with_env(yaml, {"VAR": "value"}) -def test_get_dict_with_value_and_default(monkeypatch): - monkeypatch.setenv("VALUE", "a=1, b=2, c=3") - result = env.get_dict("VALUE", {"z": 9, "y": 8, "x": 7}) - assert result == {"a": "1", "b": "2", "c": "3"} + assert result == "value" -@pytest.mark.parametrize("value", ["a", "a=", "a=1,", ",a=1"]) -def test_get_dict_with_invalid_should_fail(value, monkeypatch): - monkeypatch.setenv("VALUE", value) - with pytest.raises(ValueError): - env.get_bool("VALUE") +def test_replace_multiple_variables(): + yaml = "${VAR1} ${VAR2}" + result = replace_variables_with_env(yaml, {"VAR1": "1", "VAR2": "2"}) + assert result == "1 2" -@pytest.mark.parametrize( - "raw,expected", - [ - ("1", 1), - ("1.2", 1.2), - ("true", True), - ("false", False), - ("null", None), - ('"1"', "1"), - ('"1.2"', "1.2"), - ('"true"', "true"), - ('"false"', "false"), - ('"null"', "null"), - ("[1, 2, 3]", [1, 2, 3]), - ('["a", "b", "c"]', ["a", "b", "c"]), - ( - '{"a": 1, "b": 1.2, "c": "3", "d": true, "e": null}', - {"a": 1, "b": 1.2, "c": "3", "d": True, "e": None}, - ), - ], - ids=[ - "1", - "1.2", - "true", - "false", - "null", - '"1"', - '"1.2"', - '"true"', - '"false"', - '"null"', - "[1, 2, 3]", - '["a", "b", "c"]', - '{"a": 1, "b": 1.2, "c": "3", "d": true, "e": null}', - ], -) -def test_get_json(raw, expected, monkeypatch): - monkeypatch.setenv("VALUE", raw) - result = env.get_json("VALUE") - assert result == expected +def test_replace_variables_recursive(): + settings = { + "prop": "${VAR1}", + "list": ["${VAR2}"], + "dict": { + "var": "${VAR3}", + "subdict": {"var": "${VAR4}"}, + }, + } -def test_get_json_with_default(): - result = env.get_json("VALUE", {"a": 1, "b": 2, "c": 3}) - assert result == {"a": 1, "b": 2, "c": 3} + environ = {"VAR1": "1", "VAR2": "2", "VAR3": "3", "VAR4": "4"} + result = replace_variables_recursive(settings, environ) + assert result == { + "prop": "1", + "list": ["2"], + "dict": { + "var": "3", + "subdict": {"var": "4"}, + } + } -def test_get_json_with_value_and_default(monkeypatch): - monkeypatch.setenv("VALUE", '{"a": 1, "b": 2, "c": 3}') - result = env.get_json("VALUE", {"z": 9, "y": 8, "x": 7}) - assert result == {"a": 1, "b": 2, "c": 3} +def test_replace_variables_recursive_with_invalid_value_should_fail(): + settings = { + "prop": 1 + } -def test_get_json_with_invalid_should_fail(monkeypatch): - monkeypatch.setenv("VALUE", "abc") - with pytest.raises(ValueError): - env.get_bool("VALUE") + with pytest.raises(TypeError, match="settings should contain only str, list or dict"): + replace_variables_recursive(settings, {}) \ No newline at end of file diff --git a/tests/configuration/test_settings.py b/tests/configuration/test_settings.py index 2836972..9620f73 100644 --- a/tests/configuration/test_settings.py +++ b/tests/configuration/test_settings.py @@ -1,296 +1,370 @@ import logging from pathlib import Path -from types import SimpleNamespace import pytest +from selva.configuration.defaults import default_settings from selva.configuration.settings import ( Settings, - SettingsModuleError, - extract_valid_keys, - get_default_settings, + SettingsError, get_settings, - get_settings_for_env, - is_valid_conf, + get_settings_for_profile, + merge_recursive, ) -@pytest.mark.parametrize( - "name", - [ - "ALL_UPPERCASE", - "WITH_NUMBER_1", - "MULTI__UNDERSCORE", - "END_WITH_UNDERLINE_", - ], -) -def test_valid_config_names(name: str): - result = is_valid_conf(name) - assert result +def test_get_settings(monkeypatch): + monkeypatch.chdir(Path(__file__).parent / "base") + + result = get_settings() + assert result.data == default_settings | { + "prop": "value", + "list": ["1", "2", "3"], + "dict": Settings( + { + "a": "1", + "b": "2", + "c": "3", + } + ), + } @pytest.mark.parametrize( - "name", - [ - "all_undercase", - "ONE_UNDERCASe", - "_STARTS_WITH_UNDERSCORE", - ], + "profile", + ["dev", "stg", "prd"], ) -def test_invalid_config_names(name: str): - result = is_valid_conf(name) - assert not result - - -def test_extract_valid_settings(): - module = SimpleNamespace() - module.ALL_UPPERCASE = "" - module.WITH_NUMBER_1 = "" - module.MULTI__UNDERSCORE = "" - module.END_WITH_UNDERLINE_ = "" - module.all_undercase = "" - module.ONE_UNDERCASe = "" - module._STARTS_WITH_UNDERSCORE = "" - - result = extract_valid_keys(module) - assert result == { - "ALL_UPPERCASE": "", - "WITH_NUMBER_1": "", - "MULTI__UNDERSCORE": "", - "END_WITH_UNDERLINE_": "", +def test_get_settings_with_profile(monkeypatch, profile): + monkeypatch.chdir(Path(__file__).parent / "profiles") + monkeypatch.setenv("SELVA_PROFILE", profile) + + result = get_settings() + assert result.data == default_settings | { + "name": "application", + "environment": profile, } @pytest.mark.parametrize( - "env,expected", + "profile,expected", [ - (None, {"NAME": "application"}), - ("dev", {"ENVIRONMENT": "dev"}), - ("hlg", {"ENVIRONMENT": "hlg"}), - ("prd", {"ENVIRONMENT": "prd"}), + (None, {"name": "application"}), + ("dev", {"environment": "dev"}), + ("stg", {"environment": "stg"}), + ("prd", {"environment": "prd"}), ], - ids=["None", "dev", "hlg", "prd"], + ids=["None", "dev", "stg", "prd"], ) -def test_get_settings_for_env(monkeypatch, env, expected): - monkeypatch.chdir(Path(__file__).parent / "envs") +def test_get_settings_for_profile(monkeypatch, profile, expected): + monkeypatch.chdir(Path(__file__).parent / "profiles") - result = get_settings_for_env(env) + result = get_settings_for_profile(profile) assert result == expected -def test_get_settings(monkeypatch): - monkeypatch.chdir(Path(__file__).parent / "base") +def test_configure_settings_dir(monkeypatch): + monkeypatch.setenv( + "SELVA_SETTINGS_DIR", + str(Path(__file__).parent / "base" / "configuration"), + ) result = get_settings() - assert result.__dict__ == get_default_settings() | { - "CONF_STR": "str", - "CONF_INT": 1, - "CONF_LIST": [1, 2, 3], - "CONF_DICT": { - "a": 1, - "b": 2, - "c": 3, - }, + assert result.data == default_settings | { + "prop": "value", + "list": ["1", "2", "3"], + "dict": Settings( + { + "a": "1", + "b": "2", + "c": "3", + } + ), } -def test_configure_settings_module(monkeypatch): +def test_configure_settings_file(monkeypatch): + monkeypatch.chdir(str(Path(__file__).parent / "alternate")) monkeypatch.setenv( - "SELVA_SETTINGS_MODULE", - str(Path(__file__).parent / "base/configuration/settings"), + "SELVA_SETTINGS_FILE", + "application.yaml", ) result = get_settings() - assert result.__dict__ == get_default_settings() | { - "CONF_STR": "str", - "CONF_INT": 1, - "CONF_LIST": [1, 2, 3], - "CONF_DICT": { - "a": 1, - "b": 2, - "c": 3, - }, + assert result.data == default_settings | { + "prop": "value", + "list": ["1", "2", "3"], + "dict": Settings( + { + "a": "1", + "b": "2", + "c": "3", + } + ), } -@pytest.mark.parametrize( - "env", - ["dev", "hlg", "prd"], -) -def test_get_env_setttings(monkeypatch, env): - monkeypatch.chdir(Path(__file__).parent / "envs") - monkeypatch.setenv("SELVA_ENV", env) +def test_configure_settings_dir_and_file(monkeypatch): + monkeypatch.setenv( + "SELVA_SETTINGS_DIR", + str(Path(__file__).parent / "alternate" / "configuration"), + ) + monkeypatch.setenv( + "SELVA_SETTINGS_FILE", + "application.yaml", + ) result = get_settings() - assert result.__dict__ == get_default_settings() | { - "NAME": "application", - "ENVIRONMENT": env, + assert result.data == default_settings | { + "prop": "value", + "list": ["1", "2", "3"], + "dict": Settings( + { + "a": "1", + "b": "2", + "c": "3", + } + ), + } + + +def test_configure_settings_file_with_profile(monkeypatch): + monkeypatch.chdir(str(Path(__file__).parent / "alternate")) + monkeypatch.setenv( + "SELVA_SETTINGS_FILE", + "application.yaml", + ) + + monkeypatch.setenv("SELVA_PROFILE", "prd") + + result = get_settings() + assert result.data == default_settings | { + "environment": "prd", + "prop": "value", + "list": ["1", "2", "3"], + "dict": { + "a": "1", + "b": "2", + "c": "3", + }, } @pytest.mark.parametrize( "env", - ["dev", "hlg", "prd"], + ["dev", "stg", "prd"], ) -def test_configure_env_setttings_module(monkeypatch, env): +def test_configure_env_setttings(monkeypatch, env): monkeypatch.setenv( - "SELVA_SETTINGS_MODULE", - str(Path(__file__).parent / "envs/configuration/settings"), + "SELVA_SETTINGS_DIR", + str(Path(__file__).parent / "profiles/configuration"), ) - monkeypatch.setenv("SELVA_ENV", env) + monkeypatch.setenv("SELVA_PROFILE", env) result = get_settings() - assert result.__dict__ == get_default_settings() | { - "NAME": "application", - "ENVIRONMENT": env, + assert result.data == default_settings | { + "name": "application", + "environment": env, } @pytest.mark.parametrize( "env", - ["dev", "hlg", "prd"], + ["dev", "stg", "prd"], ) def test_override_settings(monkeypatch, env): - default_settings = get_default_settings() monkeypatch.chdir(Path(__file__).parent / "override") result = get_settings() - assert result.__dict__ == default_settings | {"VALUE": "base"} + assert result.data == default_settings | {"value": "base"} - monkeypatch.setenv("SELVA_ENV", env) + monkeypatch.setenv("SELVA_PROFILE", env) result = get_settings() - assert result.__dict__ == default_settings | {"VALUE": env} + assert result.data == default_settings | {"value": env} def test_settings_class(monkeypatch): monkeypatch.chdir(Path(__file__).parent / "base") settings = get_settings() - assert settings.CONF_STR == "str" - assert settings.CONF_INT == 1 - assert settings.CONF_LIST == [1, 2, 3] - assert settings.CONF_DICT == { - "a": 1, - "b": 2, - "c": 3, + assert settings["prop"] == "value" + assert settings["list"] == ["1", "2", "3"] + assert settings["dict"] == { + "a": "1", + "b": "2", + "c": "3", } @pytest.mark.parametrize( "env", - ["dev", "hlg", "prd"], + ["dev", "stg", "prd"], ) def test_setttings_class_env(monkeypatch, env): - monkeypatch.chdir(Path(__file__).parent / "envs") - monkeypatch.setenv("SELVA_ENV", env) + monkeypatch.chdir(Path(__file__).parent / "profiles") + monkeypatch.setenv("SELVA_PROFILE", env) settings = get_settings() - assert settings.NAME == "application" - assert settings.ENVIRONMENT == env + assert settings["name"] == "application" + assert settings["environment"] == env def test_no_settings_file_should_log_info(monkeypatch, caplog): - monkeypatch.setenv("SELVA_SETTINGS_MODULE", "does_not_exist.py") + monkeypatch.setenv("SELVA_SETTINGS_FILE", "does_not_exist.yaml") - settings_path = Path.cwd() / "does_not_exist.py" + settings_path = Path.cwd() / "configuration" / "does_not_exist.yaml" with caplog.at_level(logging.INFO, logger="selva"): get_settings() - assert f"settings module not found: {settings_path}" in caplog.text + assert f"settings file not found: {settings_path}" in caplog.text def test_no_env_settings_file_should_log_info(monkeypatch, caplog): - monkeypatch.chdir(Path(__file__).parent / "envs") - monkeypatch.setenv("SELVA_ENV", "does_not_exist") + monkeypatch.chdir(Path(__file__).parent / "profiles") + monkeypatch.setenv("SELVA_PROFILE", "does_not_exist") - settings_path = Path.cwd() / "configuration" / "settings_does_not_exist.py" + settings_path = Path.cwd() / "configuration" / "settings_does_not_exist.yaml" with caplog.at_level(logging.INFO, logger="selva"): get_settings() - assert f"settings module not found: {settings_path}" in caplog.text + assert f"settings file not found: {settings_path}" in caplog.text -def test_invalid_settings_module_should_fail(monkeypatch): - monkeypatch.chdir(Path(__file__).parent / "invalid_settings") - with pytest.raises(SettingsModuleError): +def test_override_settings_with_env_var(monkeypatch): + monkeypatch.chdir(Path(__file__).parent / "base") + monkeypatch.setenv("SELVA__PROP", "override") + + settings = get_settings() + + assert settings.prop == "override" + + +def test_override_nested_settings_with_env_var(monkeypatch): + monkeypatch.chdir(Path(__file__).parent / "base") + monkeypatch.setenv("SELVA__DICT__A", "override") + + settings = get_settings() + + assert settings.dict.a == "override" + + +def test_non_existent_env_var_should_fail(monkeypatch): + monkeypatch.chdir( + Path(__file__).parent / "invalid_configuration" / "non_existent_env_var" + ) + + with pytest.raises( + ValueError, + match=f"DOES_NOT_EXIST environment variable is not defined and does not contain a default value", + ): get_settings() -def test_non_existent_environment_variable_should_fail(monkeypatch): - monkeypatch.chdir(Path(__file__).parent / "invalid_environment" / "non_existent") +def test_invalid_yaml_should_fail(monkeypatch): + settings_path = Path(__file__).parent / "invalid_configuration" / "invalid_yaml" + monkeypatch.chdir(Path(__file__).parent / "invalid_configuration" / "invalid_yaml") + with pytest.raises( - KeyError, match=f"Environment variable 'DOES_NOT_EXIST' is not defined" + SettingsError, match=f"cannot load settings from {settings_path}" ): - get_settings_for_env() + get_settings_for_profile() -@pytest.mark.parametrize("value_type", ["int", "float", "bool", "dict", "json"]) -def test_invalid_value_should_fail(monkeypatch, value_type): - monkeypatch.chdir(Path(__file__).parent / "invalid_environment" / "invalid_value") - monkeypatch.setenv("INVALID", "abc") +@pytest.mark.parametrize( + "settings,extra,expected", + [ + ({"a": 1}, {"b": 2}, {"a": 1, "b": 2}), + ({"a": 1}, {"a": 2}, {"a": 2}), + ({"a": {"a": 1}}, {"a": {"b": 2}}, {"a": {"a": 1, "b": 2}}), + ({"a": {"a": 1}}, {"a": {"a": 2}}, {"a": {"a": 2}}), + ({"a": {"a": 1}}, {"a": 1}, {"a": 1}), + ({"a": 1}, {"a": {"a": 1}}, {"a": {"a": 1}}), + ], + ids=[ + "add value", + "replace value", + "add nested value", + "replace nested value", + "replace dict with value", + "replace value with dict", + ] +) +def test_merge_recursive(settings, extra, expected): + merge_recursive(settings, extra) + + assert settings == expected - message = ( - "Environment variable 'INVALID'" - f" is not compatible with type '{value_type}': 'abc'" - ) - with pytest.raises(ValueError, match=message): - get_settings_for_env(value_type) +def test_settings_item_access(): + settings = Settings({"a": 1}) + + assert settings["a"] == 1 + + +def test_settings_nested_item_access(): + settings = Settings({"a": {"b": 1}}) + + assert settings["a"]["b"] == 1 def test_settings_attribute_access(): - settings = Settings({"A": 1}) + settings = Settings({"a": 1}) - assert settings.A == 1 + assert settings.a == 1 -def test_settings_method_access(): - settings = Settings({"A": 1}) +def test_settings_nested_attribute_access(): + settings = Settings({"a": {"b": 1}}) - assert settings.get("A") == 1 + assert settings.a.b == 1 -def test_settings_method_access_default_value(): +def test_settings_non_existent_attribute_should_fail(): settings = Settings({}) - assert settings.get("A") is None - assert settings.get("A", "B") == "B" + with pytest.raises(AttributeError, match="a"): + settings.a -def test_settings_item_access(): - settings = Settings({"A": 1}) +def test_settings_mixed_item_attribute_access(): + settings = Settings({"a": {"b": 1}}) - assert settings["A"] == 1 + assert settings["a"].b == 1 + assert settings.a["b"] == 1 -def test_settings_set_attribute_should_fail(): +def test_settings_method_access(): + settings = Settings({"a": 1}) + + assert settings.get("a") == 1 + + +def test_settings_method_access_default_value(): settings = Settings({}) - with pytest.raises(AttributeError, match="can't set attribute"): - settings.A = 1 + assert settings.get("a") is None + assert settings.get("a", "b") == "b" -def test_settings_setattr_should_fail(): + +def test_settings_get_nonexistent_item_should_fail(): settings = Settings({}) - with pytest.raises(AttributeError, match="can't set attribute"): - setattr(settings, "A", 1) + with pytest.raises(KeyError, match="a"): + settings["a"] -def test_settings_del_attribute_should_fail(): - settings = Settings({"A": 1}) - with pytest.raises(AttributeError, match="can't del attribute"): - del settings.A +def test_settings_with_env_var(monkeypatch): + monkeypatch.chdir(Path(__file__).parent / "env_var") + monkeypatch.setenv("VAR_NAME", "test") + settings = get_settings() + assert settings.name == "test" -def test_settings_delattr_should_fail(): - settings = Settings({"A": 1}) - with pytest.raises(AttributeError, match="can't del attribute"): - delattr(settings, "A") +def test_settings_with_env_var_replaced_in_profile(monkeypatch): + monkeypatch.chdir(Path(__file__).parent / "env_var") + monkeypatch.setenv("SELVA_PROFILE", "profile") -def test_settings_get_nonexistent_item_should_fail(): - settings = Settings({}) - with pytest.raises(KeyError, match="A"): - settings["A"] + settings = get_settings() + assert settings.name == "profile" diff --git a/tests/conftest.py b/tests/conftest.py index 94169a4..79eee1b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,8 @@ from _pytest.logging import LogCaptureFixture from loguru import logger +logger.remove() + @pytest.fixture def caplog(caplog: LogCaptureFixture): diff --git a/tests/web/test_middleware.py b/tests/web/test_middleware.py index 6034497..37da391 100644 --- a/tests/web/test_middleware.py +++ b/tests/web/test_middleware.py @@ -1,13 +1,12 @@ import pytest from asgikit.requests import Request -from asgikit.responses import Response from selva.web.middleware import Middleware class Mid(Middleware): - async def __call__(self, chain, request: Request, response: Response): - await chain(request, response) + async def __call__(self, chain, request: Request): + await chain(request) async def test_non_async_chain_should_fail():