diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 330fbc09..4ab6cc8a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.0 + rev: v0.8.1 hooks: - id: ruff name: ruff unused imports diff --git a/.ruff.toml b/.ruff.toml index 83f9d0bc..ecfee7bf 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -111,3 +111,8 @@ lines-after-imports = 2 # https://docs.astral.sh/ruff/settings/#lint_isort_l "interface_*.py" = [ "F401" # F401 [*] {name} imported but unused ] + + +"run/*" = [ + "S101" # Use of assert detected +] diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst index 9837b4ec..7fc7c963 100644 --- a/docs/advanced_usage.rst +++ b/docs/advanced_usage.rst @@ -47,7 +47,9 @@ File properties ------------------------------ For every HABApp file it is possible to specify some properties. The properties are specified as a comment (prefixed with ``#``) somewhere at the beginning of the file -and are in the yml format. They keyword ``HABApp`` can be arbitrarily intended. +and are in the yml format. +Make sure **no empty line** is before the property definition in the file. +They keyword ``HABApp`` can be arbitrarily intended. .. hint:: File names are not absolute but relative with a folder specific prefix. diff --git a/docs/requirements.txt b/docs/requirements.txt index 9b5b4892..04d17b0e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,6 +2,6 @@ sphinx == 8.1.3 sphinx-autodoc-typehints == 2.5.0 sphinx_rtd_theme == 3.0.2 -sphinx-exec-code == 0.14 +sphinx-exec-code == 0.15 autodoc_pydantic == 2.2.0 sphinx-copybutton == 0.5.2 diff --git a/readme.md b/readme.md index d622771c..eea19155 100644 --- a/readme.md +++ b/readme.md @@ -127,6 +127,10 @@ MyOpenhabRule() ``` # Changelog + +#### 24.11.1-DEV-1 (2024-XX-XX) +- Updated thread pool and asyncio handling + #### 24.11.1 (2024-11-25) Fixed an issue with the logging Queue diff --git a/requirements.txt b/requirements.txt index 81cc2573..b886fac0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ # Packages for source formatting # ----------------------------------------------------------------------------- pre-commit == 4.0.1 -ruff == 0.8.0 +ruff == 0.8.2 autotyping == 24.9.0 # ----------------------------------------------------------------------------- # Packages for other developement tasks diff --git a/requirements_setup.txt b/requirements_setup.txt index 68e7b688..a3c083cf 100644 --- a/requirements_setup.txt +++ b/requirements_setup.txt @@ -1,14 +1,14 @@ -aiohttp == 3.11.7 -pydantic == 2.10.1 +aiohttp == 3.11.10 +pydantic == 2.10.3 bidict == 0.23.1 -watchdog == 6.0.0 +watchfiles == 1.0.3 ujson == 5.10.0 aiomqtt == 2.3.0 eascheduler == 0.2.1 immutables == 0.21 -easyconfig == 0.3.2 +easyconfig == 0.4.0 stack_data == 0.6.3 colorama == 0.4.6 fastnumbers == 5.1.0 @@ -20,4 +20,4 @@ typing-extensions == 4.12.2 aiohttp-sse-client == 0.2.1 -javaproperties == 0.8.1 +javaproperties == 0.8.2 diff --git a/requirements_tests.txt b/requirements_tests.txt index b0c32dfb..df847e49 100644 --- a/requirements_tests.txt +++ b/requirements_tests.txt @@ -7,5 +7,5 @@ # Packages to run source tests # ----------------------------------------------------------------------------- packaging == 24.2 -pytest == 8.3.3 +pytest == 8.3.4 pytest-asyncio == 0.24.0 diff --git a/run/conf_testing/lib/HABAppTests/__init__.py b/run/conf_testing/lib/HABAppTests/__init__.py index c6138f1e..e70b073f 100644 --- a/run/conf_testing/lib/HABAppTests/__init__.py +++ b/run/conf_testing/lib/HABAppTests/__init__.py @@ -1,8 +1,10 @@ -from .utils import get_random_name, run_coro, find_astro_sun_thing, get_bytes_text +from .test_data import get_openhab_test_events, get_openhab_test_states, get_openhab_test_types +from .utils import find_astro_sun_thing, get_random_name + + +# isort: split -from .test_rule import TestBaseRule, TestResult from .event_waiter import EventWaiter from .item_waiter import ItemWaiter -from .openhab_tmp_item import OpenhabTmpItem - -from .test_data import get_openhab_test_events, get_openhab_test_states, get_openhab_test_types +from .openhab_tmp_item import AsyncOpenhabTmpItem, OpenhabTmpItem +from .test_rule import TestBaseRule, TestResult, TestRunnerRule diff --git a/run/conf_testing/lib/HABAppTests/compare_values.py b/run/conf_testing/lib/HABAppTests/compare_values.py index dc14543d..06cf1f1d 100644 --- a/run/conf_testing/lib/HABAppTests/compare_values.py +++ b/run/conf_testing/lib/HABAppTests/compare_values.py @@ -1,9 +1,16 @@ -from .utils import get_bytes_text +from binascii import b2a_hex -def get_equal_text(value1, value2) -> str: - return f'{get_value_text(value1)} {"==" if value1 == value2 else "!="} {get_value_text(value2)}' +def get_bytes_text(value: object) -> object: + if isinstance(value, bytes) and len(value) > 300: + return b2a_hex(value[:40]).decode() + ' ... ' + b2a_hex(value[-40:]).decode() + return value -def get_value_text(value) -> str: +def get_equal_text(value1: object, value2: object) -> str: + equal = value1 == value2 and isinstance(value1, value2.__class__) + return f'{get_value_text(value1):s} {"==" if equal else "!="} {get_value_text(value2):s}' + + +def get_value_text(value: object) -> str: return f'{get_bytes_text(value)} ({str(type(value))[8:-2]})' diff --git a/run/conf_testing/lib/HABAppTests/errors.py b/run/conf_testing/lib/HABAppTests/errors.py index 9528c5fa..f73e47c9 100644 --- a/run/conf_testing/lib/HABAppTests/errors.py +++ b/run/conf_testing/lib/HABAppTests/errors.py @@ -1,8 +1,3 @@ -class TestCaseFailed(Exception): - def __init__(self, msg: str) -> None: - self.msg = msg - - -class TestCaseWarning(Exception): +class TestCaseFailed(Exception): # noqa: N818 def __init__(self, msg: str) -> None: self.msg = msg diff --git a/run/conf_testing/lib/HABAppTests/event_waiter.py b/run/conf_testing/lib/HABAppTests/event_waiter.py index ee527912..6b381de6 100644 --- a/run/conf_testing/lib/HABAppTests/event_waiter.py +++ b/run/conf_testing/lib/HABAppTests/event_waiter.py @@ -1,7 +1,10 @@ +import asyncio import logging import time +from collections.abc import Generator +from time import monotonic from types import TracebackType -from typing import Any, TypeVar +from typing import Any from HABApp.core.events.filter import EventFilter from HABApp.core.internals import ( @@ -18,8 +21,6 @@ log = logging.getLogger('HABApp.Tests') -EVENT_TYPE = TypeVar('EVENT_TYPE') - class EventWaiter: def __init__(self, name: BaseValueItem | str, @@ -29,61 +30,85 @@ def __init__(self, name: BaseValueItem | str, assert isinstance(name, str) assert isinstance(event_filter, EventFilterBase) - self.name = name - self.event_filter = event_filter - self.timeout = timeout + self._name = name + self._event_filter = event_filter + self._timeout = timeout - self.event_listener = EventBusListener( - self.name, + self._event_listener = EventBusListener( + self._name, wrap_func(self.__process_event), - self.event_filter + self._event_filter ) self._received_events = [] def __process_event(self, event) -> None: - if isinstance(self.event_filter, EventFilter): - assert isinstance(event, self.event_filter.event_class) + if isinstance(self._event_filter, EventFilter): + assert isinstance(event, self._event_filter.event_class) self._received_events.append(event) def clear(self) -> None: self._received_events.clear() - def wait_for_event(self, **kwargs) -> EVENT_TYPE: - - start = time.time() - - while True: - time.sleep(0.02) + def _check_wait_event(self, attribs: dict[str, Any]) -> Generator[float, Any, Any]: + start = monotonic() + end = start + self._timeout - if time.time() > start + self.timeout: - expected_values = 'with ' + ', '.join([f'{__k}={__v}' for __k, __v in kwargs.items()]) if kwargs else '' - msg = f'Timeout while waiting for {self.event_filter.describe()} for {self.name} {expected_values}' - raise TestCaseFailed(msg) + while monotonic() < end: + yield 0.01 if not self._received_events: continue event = self._received_events.pop() - if kwargs: - if self.compare_event_value(event, kwargs): + if attribs: + if self.compare_event_value(event, attribs): return event continue return event - raise ValueError() + expected_values = 'with ' + ', '.join([f'{__k}={__v}' for __k, __v in attribs.items()]) if attribs else '' + msg = f'Timeout while waiting for {self._event_filter.describe()} for {self._name} {expected_values}' + raise TestCaseFailed(msg) + + def wait_for_event(self, **kwargs: Any) -> Any: + gen = self._check_wait_event(kwargs) + try: + while True: + delay = next(gen) + time.sleep(delay) + except StopIteration as e: + event = e.value + + if event is None: + raise ValueError() + return event + + async def async_wait_for_event(self, **kwargs: Any) -> Any: + gen = self._check_wait_event(kwargs) + try: + while True: + delay = next(gen) + await asyncio.sleep(delay) + except StopIteration as e: + event = e.value + + if event is None: + raise ValueError() + return event def __enter__(self) -> 'EventWaiter': - get_current_context().add_event_listener(self.event_listener) + get_current_context().add_event_listener(self._event_listener) return self - def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None) -> bool: - get_current_context().remove_event_listener(self.event_listener) + def __exit__(self, exc_type: type[BaseException] | None, + exc_val: BaseException | None, exc_tb: TracebackType | None) -> bool: + get_current_context().remove_event_listener(self._event_listener) @staticmethod - def compare_event_value(event, kwargs: dict[str, Any]): + def compare_event_value(event: Any, kwargs: dict[str, Any]) -> bool: only_value = 'value' in kwargs and len(kwargs) == 1 val_msg = [] diff --git a/run/conf_testing/lib/HABAppTests/item_waiter.py b/run/conf_testing/lib/HABAppTests/item_waiter.py index 4905c6ee..36671a33 100644 --- a/run/conf_testing/lib/HABAppTests/item_waiter.py +++ b/run/conf_testing/lib/HABAppTests/item_waiter.py @@ -1,6 +1,10 @@ +import asyncio import logging import time +from collections.abc import Generator +from time import monotonic from types import TracebackType +from typing import Any, Final from HABApp.core.items import BaseValueItem from HABAppTests.compare_values import get_equal_text @@ -11,38 +15,48 @@ class ItemWaiter: - def __init__(self, item, timeout=1) -> None: - self.item = item - assert isinstance(item, BaseValueItem), f'{item} is not an Item' + def __init__(self, item: str | BaseValueItem, timeout: float = 1) -> None: + self._item: Final = item if not isinstance(item, str) else BaseValueItem.get_item(item) + assert isinstance(self._item, BaseValueItem), f'{self._item} is not an Item' - self.timeout = timeout + self._timeout: Final = timeout - def wait_for_attribs(self, **kwargs) -> bool: - start = time.time() - end = start + self.timeout + def _check_attribs(self, attribs: dict[str, Any]) -> Generator[float, Any, None]: + start = monotonic() + end = start + self._timeout - while True: - time.sleep(0.01) + while monotonic() < end: + yield 0.01 - for name, target in kwargs.items(): - if getattr(self.item, name) != target: + for name, target in attribs.items(): + if getattr(self._item, name) != target: break else: - return True - - if time.time() > end: - indent = max(map(len, kwargs)) - failed = [ - f'{name:>{indent:d}s}: {get_equal_text(getattr(self.item, name), target)}' - for name, target in kwargs.items() - ] - failed_msg = '\n'.join(failed) - msg = f'Timeout waiting for {self.item.name}!\n{failed_msg}' - raise TestCaseFailed(msg) - - def wait_for_state(self, state=None): + return None + + indent = max(map(len, attribs)) + failed = [ + f'{name:>{indent:d}s}: {get_equal_text(getattr(self._item, name), target)}' + for name, target in attribs.items() + ] + failed_msg = '\n'.join(failed) + msg = f'Timeout waiting for {self._item.name}!\n{failed_msg}' + raise TestCaseFailed(msg) + + def wait_for_attribs(self, **kwargs) -> None: + for delay in self._check_attribs(kwargs): + time.sleep(delay) + + async def async_wait_for_attribs(self, **kwargs) -> None: + for delay in self._check_attribs(kwargs): + await asyncio.sleep(delay) + + def wait_for_state(self, state=None) -> None: return self.wait_for_attribs(value=state) + async def async_wait_for_state(self, state=None) -> None: + return await self.async_wait_for_attribs(value=state) + def __enter__(self) -> 'ItemWaiter': return self diff --git a/run/conf_testing/lib/HABAppTests/openhab_tmp_item.py b/run/conf_testing/lib/HABAppTests/openhab_tmp_item.py index f8de4515..5fbfb9ad 100644 --- a/run/conf_testing/lib/HABAppTests/openhab_tmp_item.py +++ b/run/conf_testing/lib/HABAppTests/openhab_tmp_item.py @@ -1,45 +1,87 @@ +import asyncio import time +from collections.abc import Generator from functools import wraps +from time import monotonic from types import TracebackType +from typing import Any, NotRequired, Self, TypedDict, Unpack import HABApp +from HABApp.core.asyncio import AsyncContextError, thread_context from HABApp.openhab.definitions.topics import TOPIC_ITEMS +from HABApp.openhab.items import OpenhabItem from . import EventWaiter, get_random_name -class OpenhabTmpItem: +class ItemApiKwargs(TypedDict): + label: NotRequired[str] + category: NotRequired[str] + tags: NotRequired[list[str]] + groups: NotRequired[list[str]] + group_type: NotRequired[str] + group_function: NotRequired[str] + group_function_params: NotRequired[list[str]] + + +class OpenhabTmpItemBase: + @classmethod + def _insert_kwargs(cls, item_type: str, name: str | None, kwargs: dict[str, Any], arg_name: str | None) -> Self: + item = cls(item_type, name) + if arg_name is not None: + if arg_name in kwargs: + msg = f'Arg {arg_name} already set!' + raise ValueError(msg) + kwargs[arg_name] = item + return item + + def __init__(self, item_type: str, item_name: str | None = None) -> None: + self._type: str = item_type + self._name = get_random_name(item_type) if item_name is None else item_name + + @property + def name(self) -> str: + return self._name + + def _wait_until_item_exists(self) -> Generator[float, Any, Any]: + # wait max 1 sec for the item to be created + stop = monotonic() + 1.5 + while not HABApp.core.Items.item_exists(self.name): + if monotonic() > stop: + msg = f'Item {self.name} was not found!' + raise TimeoutError(msg) + + yield 0.01 + + +class OpenhabTmpItem(OpenhabTmpItemBase): @staticmethod - def use(item_type: str, name: str | None = None, arg_name: str = 'item'): + def use(item_type: str, name: str | None = None, *, arg_name: str = 'item'): def decorator(func): @wraps(func) def new_func(*args, **kwargs): - assert arg_name not in kwargs, f'arg {arg_name} already set' - item = OpenhabTmpItem(item_type, name) + item = OpenhabTmpItem._insert_kwargs(item_type, name, kwargs, arg_name) try: - kwargs[arg_name] = item return func(*args, **kwargs) finally: item.remove() + return new_func + return decorator @staticmethod - def create(item_type: str, name: str | None = None, arg_name: str | None = None): + def create(item_type: str, name: str | None = None, *, arg_name: str | None = None): def decorator(func): @wraps(func) def new_func(*args, **kwargs): - with OpenhabTmpItem(item_type, name) as f: - if arg_name is not None: - assert arg_name not in kwargs, f'arg {arg_name} already set' - kwargs[arg_name] = f + + with OpenhabTmpItem._insert_kwargs(item_type, name, kwargs, arg_name): return func(*args, **kwargs) + return new_func - return decorator - def __init__(self, item_type: str, item_name: str | None = None) -> None: - self.type: str = item_type - self.name = get_random_name(item_type) if item_name is None else item_name + return decorator def __enter__(self) -> HABApp.openhab.items.OpenhabItem: return self.create_item() @@ -49,39 +91,82 @@ def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException return False def remove(self) -> None: + if thread_context.get(None) is None: + raise AsyncContextError(self.remove) HABApp.openhab.interface_sync.remove_item(self.name) - def _create(self, label='', category='', tags: list[str] = [], groups: list[str] = [], - group_type: str = '', group_function: str = '', - group_function_params: list[str] = []) -> None: + def _create(self, **kwargs: Unpack[ItemApiKwargs]) -> None: + if thread_context.get(None) is None: + raise AsyncContextError(self._create) + interface = HABApp.openhab.interface_sync - interface.create_item(self.type, self.name, label=label, category=category, - tags=tags, groups=groups, group_type=group_type, - group_function=group_function, group_function_params=group_function_params) + interface.create_item(self._type, self._name, **kwargs) - def create_item(self, label='', category='', tags: list[str] = [], groups: list[str] = [], - group_type: str = '', group_function: str = '', - group_function_params: list[str] = []) -> HABApp.openhab.items.OpenhabItem: + def create_item(self, **kwargs: Unpack[ItemApiKwargs]) -> OpenhabItem: - self._create(label=label, category=category, tags=tags, groups=groups, group_type=group_type, - group_function=group_function, group_function_params=group_function_params) + self._create(**kwargs) + for delay in self._wait_until_item_exists(): + time.sleep(delay) + return OpenhabItem.get_item(self.name) - # wait max 1 sec for the item to be created - stop = time.time() + 1 - while not HABApp.core.Items.item_exists(self.name): - time.sleep(0.01) - if time.time() > stop: - msg = f'Item {self.name} was not found!' - raise TimeoutError(msg) + def modify(self, **kwargs: Unpack[ItemApiKwargs]) -> None: + with EventWaiter(TOPIC_ITEMS, HABApp.core.events.EventFilter(HABApp.openhab.events.ItemUpdatedEvent)) as w: + self._create(**kwargs) + w.wait_for_event() - return HABApp.openhab.items.OpenhabItem.get_item(self.name) - def modify(self, label='', category='', tags: list[str] = [], groups: list[str] = [], - group_type: str = '', group_function: str = '', group_function_params: list[str] = []) -> None: +class AsyncOpenhabTmpItem(OpenhabTmpItemBase): + @staticmethod + def use(item_type: str, name: str | None = None, *, arg_name: str = 'item'): + def decorator(func): + @wraps(func) + async def new_func(*args, **kwargs): + item = AsyncOpenhabTmpItem._insert_kwargs(item_type, name, kwargs, arg_name) + try: + return await func(*args, **kwargs) + finally: + await item.remove() - with EventWaiter(TOPIC_ITEMS, HABApp.core.events.EventFilter(HABApp.openhab.events.ItemUpdatedEvent)) as w: + return new_func - self._create(label=label, category=category, tags=tags, groups=groups, group_type=group_type, - group_function=group_function, group_function_params=group_function_params) + return decorator - w.wait_for_event() + @staticmethod + def create(item_type: str, name: str | None = None, *, arg_name: str | None = None): + + def decorator(func): + @wraps(func) + async def new_func(*args, **kwargs): + + async with AsyncOpenhabTmpItem._insert_kwargs(item_type, name, kwargs, arg_name): + return await func(*args, **kwargs) + + return new_func + + return decorator + + async def __aenter__(self) -> HABApp.openhab.items.OpenhabItem: + return await self.create_item() + + async def __aexit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None) -> bool: + await self.remove() + return False + + async def remove(self) -> None: + await HABApp.openhab.interface_async.async_remove_item(self.name) + + async def _create(self, **kwargs: Unpack[ItemApiKwargs]) -> None: + interface = HABApp.openhab.interface_async + await interface.async_create_item(self._type, self._name, **kwargs) + + async def create_item(self, **kwargs: Unpack[ItemApiKwargs]) -> OpenhabItem: + + await self._create(**kwargs) + for delay in self._wait_until_item_exists(): + await asyncio.sleep(delay) + return OpenhabItem.get_item(self.name) + + async def modify(self, **kwargs: Unpack[ItemApiKwargs]) -> None: + with EventWaiter(TOPIC_ITEMS, HABApp.core.events.EventFilter(HABApp.openhab.events.ItemUpdatedEvent)) as w: + await self._create(**kwargs) + await w.async_wait_for_event() diff --git a/run/conf_testing/lib/HABAppTests/test_rule/__init__.py b/run/conf_testing/lib/HABAppTests/test_rule/__init__.py index 233adc7b..fa10d3f0 100644 --- a/run/conf_testing/lib/HABAppTests/test_rule/__init__.py +++ b/run/conf_testing/lib/HABAppTests/test_rule/__init__.py @@ -1 +1,2 @@ -from .test_rule import TestBaseRule, TestResult +from .test_rule import TestBaseRule, TestResult, run_test_cases +from .test_runner_rule import TestRunnerRule diff --git a/run/conf_testing/lib/HABAppTests/test_rule/_com_patcher.py b/run/conf_testing/lib/HABAppTests/test_rule/_com_patcher.py new file mode 100644 index 00000000..a8caf150 --- /dev/null +++ b/run/conf_testing/lib/HABAppTests/test_rule/_com_patcher.py @@ -0,0 +1,156 @@ +import json +import logging +import pprint +from collections.abc import Callable +from types import ModuleType, TracebackType +from typing import Any, Final + +from pytest import MonkeyPatch + +import HABApp.mqtt.connection.publish +import HABApp.mqtt.connection.subscribe +import HABApp.openhab.connection.handler +import HABApp.openhab.connection.handler.func_async +import HABApp.openhab.process_events +from HABApp.config import CONFIG + + +class PatcherName: + def __init__(self, header: str) -> None: + self.header = header + self.logged = False + + +class BasePatcher: + @staticmethod + def create_name(header: str) -> PatcherName: + return PatcherName(header) + + def __init__(self, name: PatcherName, logger_name: str) -> None: + self._log: Final = logging.getLogger('Com').getChild(logger_name) + self.name: Final = name + self.monkeypatch: Final = MonkeyPatch() + + def log(self, msg: str) -> None: + if not self.name.logged: + self._log.debug('') + self._log.debug(self.name.header) + self.name.logged = True + self._log.debug(msg) + + def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, + exc_tb: TracebackType | None) -> bool: + + self.monkeypatch.undo() + return False + + +def shorten_url(url: str) -> str: + url = str(url) + cfg = CONFIG.openhab.connection.url + if url.startswith(cfg): + return url[len(cfg):] + return url + + +class RestPatcher(BasePatcher): + + def __init__(self, name: str) -> None: + super().__init__(name, 'Rest') + + def wrap_http(self, to_call): + async def resp_wrap(*args, **kwargs): + + resp = await to_call(*args, **kwargs) + + out = '' + if kwargs.get('json') is not None: + out = f' {kwargs["json"]}' + if kwargs.get('data') is not None: + out = f' "{kwargs["data"]}"' + + self.log( + f'{resp.request_info.method:^6s} {shorten_url(resp.request_info.url)} ({resp.status}){out}' + ) + + if resp.status >= 300 and kwargs.get('log_404', True): + self.log(f'{"":6s} Header request : {resp.request_info.headers}') + self.log(f'{"":6s} Header response: {resp.headers}') + + def wrap_content(content_func): + async def content_func_wrap(*cargs, **ckwargs): + t = await content_func(*cargs, **ckwargs) + + if isinstance(t, (dict, list)): + txt = json.dumps(t, indent=2) + else: + txt = pprint.pformat(t, indent=2) + + lines = txt.splitlines() + for i, l in enumerate(lines): + self.log(f'{"->" if not i else "":^6s} {l}') + + return t + return content_func_wrap + + resp.text = wrap_content(resp.text) + resp.json = wrap_content(resp.json) + return resp + return resp_wrap + + def __enter__(self) -> None: + m = self.monkeypatch + + # http functions + to_patch: Final[tuple[tuple[ModuleType, tuple[str, ...]]], ...] = ( + (HABApp.openhab.connection.handler, ('get', 'put', 'post', 'delete')), + (HABApp.openhab.connection.handler.func_async, ('get', 'put', 'post', 'delete')), + (HABApp.openhab.connection.plugins.out, ('put', 'post')), + ) + + for module, methods in to_patch: + for name in methods: + m.setattr(module, name, self.wrap_http(getattr(module, name))) + + +class SsePatcher(BasePatcher): + + def __init__(self, name: str) -> None: + super().__init__(name, 'SSE') + + def wrap_sse(self, to_wrap: Callable[[dict], Any]) -> Callable[[dict], Any]: + def new_call(_dict: dict) -> Any: + self.log(f'{"SSE":^6s} {_dict}') + return to_wrap(_dict) + return new_call + + def __enter__(self) -> None: + module = HABApp.openhab.process_events + self.monkeypatch.setattr(module, 'get_event', self.wrap_sse(module.get_event)) + + +class MqttPatcher(BasePatcher): + + def __init__(self, name: str) -> None: + super().__init__(name, 'Mqtt') + + def wrap_msg(self, func: Callable[[str, Any, bool], Any]) -> Callable[[str, Any, bool], Any]: + def new_call(topic: str, payload: Any, retain: bool) -> Any: + self.log(f'{"MSG":^6s} {"R" if retain else " "} {topic} {payload}') + return func(topic, payload, retain) + return new_call + + def pub_msg(self, func: Callable[[str, Any, int, bool], Any]) -> Callable[[str, Any, int, bool], Any]: + async def wrapped_publish(topic: str, payload: Any, qos: int = 0, retain: bool = False) -> Any: + self.log(f'{"PUB":^6s} {"R" if retain else " "}{qos:d} {topic} {payload}') + return await func(topic, payload, qos, retain) + return wrapped_publish + + def __enter__(self) -> None: + m = self.monkeypatch + + module = HABApp.mqtt.connection.subscribe + m.setattr(module, 'msg_to_event', self.wrap_msg(module.msg_to_event)) + + obj = HABApp.mqtt.connection.publish.PUBLISH_HANDLER.plugin_connection.context + m.setattr(obj, 'publish', self.pub_msg(obj.publish)) diff --git a/run/conf_testing/lib/HABAppTests/test_rule/_rest_patcher.py b/run/conf_testing/lib/HABAppTests/test_rule/_rest_patcher.py deleted file mode 100644 index 06077def..00000000 --- a/run/conf_testing/lib/HABAppTests/test_rule/_rest_patcher.py +++ /dev/null @@ -1,110 +0,0 @@ -import json -import logging -import pprint -from types import TracebackType - -from pytest import MonkeyPatch # noqa: PT013 - -import HABApp.openhab.connection.handler -import HABApp.openhab.connection.handler.func_async -import HABApp.openhab.process_events -from HABApp.config import CONFIG - - -def shorten_url(url: str) -> str: - url = str(url) - cfg = CONFIG.openhab.connection.url - if url.startswith(cfg): - return url[len(cfg):] - return url - - -class RestPatcher: - def __init__(self, name: str) -> None: - self.name = name - self.logged_name = False - self._log = logging.getLogger('HABApp.Rest') - - self.monkeypatch = MonkeyPatch() - - def log(self, msg: str) -> None: - # Log name when we log the first message - if not self.logged_name: - self.logged_name = True - self._log.debug('') - self._log.debug(f'{self.name}:') - - self._log.debug(msg) - - def wrap(self, to_call): - async def resp_wrap(*args, **kwargs): - - resp = await to_call(*args, **kwargs) - - out = '' - if kwargs.get('json') is not None: - out = f' {kwargs["json"]}' - if kwargs.get('data') is not None: - out = f' "{kwargs["data"]}"' - - # Log name when we log the first message - if not self.logged_name: - self.logged_name = True - self.log('') - self.log(f'{self.name}:') - - self.log( - f'{resp.request_info.method:^6s} {shorten_url(resp.request_info.url)} ({resp.status}){out}' - ) - - if resp.status >= 300 and kwargs.get('log_404', True): - self.log(f'{"":6s} Header request : {resp.request_info.headers}') - self.log(f'{"":6s} Header response: {resp.headers}') - - def wrap_content(content_func): - async def content_func_wrap(*cargs, **ckwargs): - t = await content_func(*cargs, **ckwargs) - - if isinstance(t, (dict, list)): - txt = json.dumps(t, indent=2) - else: - txt = pprint.pformat(t, indent=2) - - lines = txt.splitlines() - for i, l in enumerate(lines): - self.log(f'{"->" if not i else "":^6s} {l}') - - return t - return content_func_wrap - - resp.text = wrap_content(resp.text) - resp.json = wrap_content(resp.json) - return resp - return resp_wrap - - def wrap_sse(self, to_wrap): - def new_call(_dict): - self.log(f'{"SSE":^6s} {_dict}') - return to_wrap(_dict) - return new_call - - def __enter__(self) -> None: - m = self.monkeypatch - - # event handler - module = HABApp.openhab.process_events - m.setattr(module, 'get_event', self.wrap_sse(module.get_event)) - - # http functions - for module in (HABApp.openhab.connection.handler, HABApp.openhab.connection.handler.func_async,): - for name in ('get', 'put', 'post', 'delete'): - m.setattr(module, name, self.wrap(getattr(module, name))) - - # additional communication - module = HABApp.openhab.connection.plugins.out - m.setattr(module, 'put', self.wrap(getattr(module, 'put'))) - m.setattr(module, 'post', self.wrap(getattr(module, 'post'))) - - def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None) -> bool: - self.monkeypatch.undo() - return False diff --git a/run/conf_testing/lib/HABAppTests/test_rule/_rule_ids.py b/run/conf_testing/lib/HABAppTests/test_rule/_rule_ids.py deleted file mode 100644 index 85800fae..00000000 --- a/run/conf_testing/lib/HABAppTests/test_rule/_rule_ids.py +++ /dev/null @@ -1,62 +0,0 @@ -import threading -import typing - -import HABAppTests - -from ._rule_status import TestRuleStatus - - -LOCK = threading.Lock() - - -RULE_CTR = 0 -TESTS_RULES: typing.Dict[int, 'HABAppTests.TestBaseRule'] = {} - - -class RuleID: - def __init__(self, id: int) -> None: - self.__id = id - - def is_newest(self) -> bool: - with LOCK: - if self.__id != RULE_CTR: - return False - return True - - def remove(self) -> None: - pop_test_rule(self.__id) - - -def get_next_id(rule) -> RuleID: - global RULE_CTR - with LOCK: - RULE_CTR += 1 - TESTS_RULES[RULE_CTR] = rule - - obj = RuleID(RULE_CTR) - return obj - - -def pop_test_rule(id: int) -> None: - with LOCK: - rule = TESTS_RULES.pop(id) - rule._rule_status = TestRuleStatus.FINISHED - - -def get_test_rules() -> typing.Iterable['HABAppTests.TestBaseRule']: - ret = [] - for k, rule in sorted(TESTS_RULES.items()): - assert isinstance(rule, HABAppTests.TestBaseRule) - if rule._rule_status is not TestRuleStatus.CREATED: - continue - ret.append(rule) - - return tuple(ret) - - -def test_rules_running() -> bool: - for rule in TESTS_RULES.values(): - status = rule._rule_status - if status is not TestRuleStatus.CREATED and status is not TestRuleStatus.FINISHED: - return True - return False diff --git a/run/conf_testing/lib/HABAppTests/test_rule/_rule_status.py b/run/conf_testing/lib/HABAppTests/test_rule/_rule_status.py deleted file mode 100644 index 27df05eb..00000000 --- a/run/conf_testing/lib/HABAppTests/test_rule/_rule_status.py +++ /dev/null @@ -1,8 +0,0 @@ -from enum import Enum, auto - - -class TestRuleStatus(Enum): - CREATED = auto() - PENDING = auto() - RUNNING = auto() - FINISHED = auto() diff --git a/run/conf_testing/lib/HABAppTests/test_rule/test_case.py b/run/conf_testing/lib/HABAppTests/test_rule/test_case.py new file mode 100644 index 00000000..eb2edba8 --- /dev/null +++ b/run/conf_testing/lib/HABAppTests/test_rule/test_case.py @@ -0,0 +1,179 @@ +import functools +import logging +from asyncio import sleep +from collections.abc import Callable, Coroutine, Sequence +from inspect import iscoroutinefunction +from types import TracebackType +from typing import Any, Final + +import HABApp +from HABApp.core.events import NoEventFilter +from HABApp.core.internals import EventBusListener, WrappedFunctionBase, wrap_func +from HABAppTests.test_rule._com_patcher import BasePatcher, MqttPatcher, RestPatcher, SsePatcher +from HABAppTests.test_rule.test_result import TestResult, TestResultStatus +from HABAppTests.utils import get_file_path_of_obj + + +class TmpLogLevel: + def __init__(self, name: str) -> None: + self.log = logging.getLogger(name) + self.old = self.log.level + + def __enter__(self) -> None: + self.old = self.log.level + self.log.setLevel(logging.INFO) + + def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None) -> bool: + self.log.setLevel(self.old) + return False + + +class ExecutionEventCatcher: + def __init__(self, event_name: str, event_bus_name: str) -> None: + self._event_name: Final = event_name + self._event_bus_name: Final = event_bus_name + + self._listener: EventBusListener | None = None + self._events = [] + + async def _event(self, event: Any) -> None: + self._events.append(event) + + def get_message(self) -> str: + if not self._events: + return '' + return f'{(ct := len(self._events))} {self._event_name}{"s" if ct != 1 else ""} in worker' + + async def __aenter__(self): + if self._listener is None: + ebl = EventBusListener(self._event_bus_name, wrap_func(self._event), NoEventFilter()) + self._listener = ebl + + with TmpLogLevel('HABApp'): + HABApp.core.EventBus.add_listener(ebl) + + return self + + async def __aexit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None) -> bool: + if (ebl := self._listener) is not None: + self._listener = None + with TmpLogLevel('HABApp'): + ebl.cancel() + return False + + +def tc_wrap_func(func: Callable | Callable[[...], Coroutine], res: TestResult) -> WrappedFunctionBase: + if iscoroutinefunction(func): + @functools.wraps(func) + async def tc_async_wrapped(*args, **kwargs): + try: + return await func(*args, **kwargs) + except Exception as e: + res.exception(e) + return None + + return wrap_func(tc_async_wrapped) + + @functools.wraps(func) + def tc_wrapped(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + res.exception(e) + return None + + return wrap_func(tc_wrapped) + + +class TestCase: + def __init__(self, name: str, func: Callable | Callable[[...], Coroutine], + args: Sequence[Any] = (), kwargs: dict[str, Any] | None = None, + setup_up: Callable | Callable[[...], Coroutine] | None = None, + tear_down: Callable | Callable[[...], Coroutine] | None = None) -> None: + self.name: Final = name + self.func: Final = func + self.args: Final = args + self.kwargs: Final = kwargs if kwargs is not None else {} + + self.set_up: Final = setup_up if setup_up is not None else None + self.tear_down: Final = tear_down if tear_down is not None else None + + async def run(self, res: TestResult) -> TestResult: + + async with ExecutionEventCatcher('warning', HABApp.core.const.topics.TOPIC_WARNINGS) as worker_warnings, \ + ExecutionEventCatcher('error', HABApp.core.const.topics.TOPIC_ERRORS) as worker_errors: + + try: + suffix = f' (from "{get_file_path_of_obj(self.func)}")' + except ValueError: + suffix = '' + + name = RestPatcher.create_name(f'{res.group_name:s}.{res.test_name:s}{suffix:s}') + b = BasePatcher(name, 'TC') + + try: + with RestPatcher(name), SsePatcher(name), MqttPatcher(name): + if s := self.set_up: + await tc_wrap_func(s, res).async_run() + await sleep(0.1) + b.log('Setup done') + + ret = await tc_wrap_func(self.func, res).async_run(*self.args, **self.kwargs) + if ret: + res.set_state(TestResultStatus.FAILED) + res.add_msg(f'{ret}') + else: + res.set_state(TestResultStatus.PASSED) + + await sleep(0.1) + except Exception as e: + res.exception(e) + + try: + if t := self.tear_down: + b.log('Tear down') + await tc_wrap_func(t, res).async_run() + await sleep(0.1) + except Exception as e: + res.exception(e) + + await sleep(0.1) + + if msg := worker_warnings.get_message(): + res.set_state(TestResultStatus.WARNING) + res.add_msg(msg) + + if msg := worker_errors.get_message(): + res.set_state(TestResultStatus.ERROR) + res.add_msg(msg) + + return res + + +log = logging.getLogger('HABApp.Tests') + + +async def run_test_cases(test_cases: Sequence[TestCase], group_name: str, source: str | object, *, + skip_on_failure: bool = False) -> list[TestResult]: + source_text = source if isinstance(source, str) else get_file_path_of_obj(source) + + count = len(test_cases) + width = len(str(count)) + + results = [TestResult(group_name, tc.name, f'{i:{width}d}/{count}') for i, tc in enumerate(test_cases, 1)] + + log.info('') + log.info( + f'Running {count:d} test{"s" if count != 1 else ""} for {group_name:s} (from "{source_text:s}")' + ) + + for res, tc in zip(results, test_cases, strict=True): + if skip_on_failure and max(r.state for r in results) >= TestResultStatus.FAILED: + res.set_state(TestResultStatus.SKIPPED) + res.log() + continue + + await tc.run(res) + res.log() + + return results diff --git a/run/conf_testing/lib/HABAppTests/test_rule/test_case/__init__.py b/run/conf_testing/lib/HABAppTests/test_rule/test_case/__init__.py deleted file mode 100644 index a77584fd..00000000 --- a/run/conf_testing/lib/HABAppTests/test_rule/test_case/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .test_result import TestResultStatus, TestResult -# isort: split -from .test_case import TestCase diff --git a/run/conf_testing/lib/HABAppTests/test_rule/test_case/test_case.py b/run/conf_testing/lib/HABAppTests/test_rule/test_case/test_case.py deleted file mode 100644 index 02035508..00000000 --- a/run/conf_testing/lib/HABAppTests/test_rule/test_case/test_case.py +++ /dev/null @@ -1,30 +0,0 @@ -import time -from collections.abc import Callable - -from HABAppTests.test_rule._rest_patcher import RestPatcher -from HABAppTests.test_rule.test_case import TestResult, TestResultStatus - - -class TestCase: - def __init__(self, name: str, func: Callable, args=[], kwargs={}) -> None: - self.name = name - self.func = func - self.args = args - self.kwargs = kwargs - - def run(self, res: TestResult) -> TestResult: - time.sleep(0.05) - - try: - with RestPatcher(f'{res.cls_name}.{res.test_name}'): - ret = self.func(*self.args, **self.kwargs) - if ret: - res.set_state(TestResultStatus.FAILED) - res.add_msg(f'{ret}') - else: - res.set_state(TestResultStatus.PASSED) - except Exception as e: - res.exception(e) - - time.sleep(0.05) - return res diff --git a/run/conf_testing/lib/HABAppTests/test_rule/test_case/test_result.py b/run/conf_testing/lib/HABAppTests/test_rule/test_result.py similarity index 75% rename from run/conf_testing/lib/HABAppTests/test_rule/test_case/test_result.py rename to run/conf_testing/lib/HABAppTests/test_rule/test_result.py index 4dbb1b3a..a2b84db4 100644 --- a/run/conf_testing/lib/HABAppTests/test_rule/test_case/test_result.py +++ b/run/conf_testing/lib/HABAppTests/test_rule/test_result.py @@ -1,8 +1,9 @@ import logging from enum import IntEnum, auto +from typing import Final import HABApp -from HABAppTests.errors import TestCaseFailed, TestCaseWarning +from HABAppTests.errors import TestCaseFailed log = logging.getLogger('HABApp.Tests') @@ -29,29 +30,28 @@ class TestResultStatus(IntEnum): class TestResult: def __init__(self, cls_name: str, test_name: str, test_nr: str = '') -> None: - self.cls_name = cls_name - self.test_name = test_name - self.test_nr = test_nr + self.group_name: Final = cls_name + self.test_name: Final = test_name + self.test_nr: Final = test_nr self.state = TestResultStatus.NOT_SET - self.msgs: list[str] = [] - - def is_set(self): - return self.state != TestResultStatus.NOT_SET + self.msgs: Final[list[str]] = [] def set_state(self, new_state: TestResultStatus) -> None: if self.state <= new_state: self.state = new_state - def exception(self, e: Exception): + def exception(self, e: Exception) -> None: if isinstance(e, TestCaseFailed): self.set_state(TestResultStatus.FAILED) self.add_msg(e.msg) return None - if isinstance(e, TestCaseWarning): - self.set_state(TestResultStatus.WARNING) - self.add_msg(e.msg) - return None + + # if isinstance(e, AssertionError): + # self.set_state(TestResultStatus.FAILED) + # self.add_msg(f'{e}') + # self.add_msg(f' {e.args}') + # return None self.add_msg(f'Exception: {e}') self.state = TestResultStatus.ERROR @@ -63,9 +63,9 @@ def add_msg(self, msg: str) -> None: for line in msg.splitlines(): self.msgs.append(line) - def log(self, name: str | None = None): + def log(self, name: str | None = None) -> None: if name is None: - name = f'{self.cls_name}.{self.test_name}' + name = f'{self.group_name}.{self.test_name}' nr = f' {self.test_nr} ' if self.test_nr else ' ' prefix = f'{nr}"{name}"' @@ -83,3 +83,4 @@ def log(self, name: str | None = None): log_func(f'{prefix} {self.state.name.lower()}: {first_msg}') for msg in self.msgs[1:]: log_func(f'{"":8s}{msg}') + return None diff --git a/run/conf_testing/lib/HABAppTests/test_rule/test_rule.py b/run/conf_testing/lib/HABAppTests/test_rule/test_rule.py index 9250ab0f..f7a5558d 100644 --- a/run/conf_testing/lib/HABAppTests/test_rule/test_rule.py +++ b/run/conf_testing/lib/HABAppTests/test_rule/test_rule.py @@ -1,12 +1,17 @@ import logging -from collections.abc import Callable -from pathlib import Path +from collections.abc import Callable, Coroutine +from enum import Enum, auto +from typing import Any, Self, overload import HABApp -from HABAppTests.test_rule.test_case import TestCase, TestResult, TestResultStatus +from HABAppTests.test_rule.test_case import TestCase, TestResult, run_test_cases -from ._rule_ids import get_next_id, get_test_rules, test_rules_running -from ._rule_status import TestRuleStatus + +class TestRuleStatus(Enum): + CREATED = auto() + PENDING = auto() + RUNNING = auto() + FINISHED = auto() log = logging.getLogger('HABApp.Tests') @@ -19,181 +24,66 @@ def __init__(self) -> None: class TestBaseRule(HABApp.Rule): - """This rule is testing the OpenHAB data types by posting values and checking the events""" - def __init__(self) -> None: super().__init__() - self._rule_status = TestRuleStatus.CREATED - self._rule_id = get_next_id(self) - self._tests: dict[str, TestCase] = {} - - self.__warnings = [] - self.__errors = [] - self.__sub_warning = None - self.__sub_errors = None self.config = TestConfig() + self._rule_status = TestRuleStatus.CREATED + self._test_cases: dict[str, TestCase] = {} - self.__worst_result = TestResultStatus.PASSED - - # we have to chain the rules later, because we register the rules only once we loaded successfully. - self.run.at(2, self.__execute_run) + @overload + def set_up(self) -> None: ... - def on_rule_unload(self) -> None: - self._rule_id.remove() + @overload + async def set_up(self) -> None: ... - # ------------------------------------------------------------------------------------------------------------------ - # Overrides and test def set_up(self) -> None: pass + @overload + def tear_down(self) -> None: ... + + @overload + async def tear_down(self) -> None: ... + def tear_down(self) -> None: pass - def add_test(self, name, func: Callable, *args, **kwargs) -> None: - tc = TestCase(name, func, args, kwargs) - assert tc.name not in self._tests - self._tests[tc.name] = tc - - # ------------------------------------------------------------------------------------------------------------------ - # Rule execution - def __execute_run(self): - if not self._rule_id.is_newest(): - return None - - # If we currently run a test wait until it is complete - if test_rules_running(): - self.run.at(2, self.__execute_run) - return None - - ergs = [] - rules = get_test_rules() - for rule in rules: - # mark rules for execution - rule._rule_status = TestRuleStatus.PENDING - for rule in rules: - # It's possible that we unload a rule before it was run - if rule._rule_status is not TestRuleStatus.PENDING: - continue - ergs.extend(rule._run_tests()) - - skipped = tuple(filter(lambda x: x.state is TestResultStatus.SKIPPED, ergs)) - passed = tuple(filter(lambda x: x.state is TestResultStatus.PASSED, ergs)) - warning = tuple(filter(lambda x: x.state is TestResultStatus.WARNING, ergs)) - failed = tuple(filter(lambda x: x.state is TestResultStatus.FAILED, ergs)) - error = tuple(filter(lambda x: x.state is TestResultStatus.ERROR, ergs)) - - def plog(msg: str) -> None: - print(msg) - log.info(msg) - - parts = [f'{len(ergs)} executed', f'{len(passed)} passed'] - if skipped: - parts.append(f'{len(skipped)} skipped') - if warning: - parts.append(f'{len(warning)} warning{"" if len(warning) == 1 else "s"}') - parts.append(f'{len(failed)} failed') - if error: - parts.append(f'{len(error)} error{"" if len(error) == 1 else "s"}') - - plog('') - plog('-' * 120) - plog(', '.join(parts)) - - # ------------------------------------------------------------------------------------------------------------------ - # Event from the worker - def __event_warning(self, event) -> None: - self.__warnings.append(event) - - def __event_error(self, event) -> None: - self.__errors.append(event) - - def _worker_events_sub(self) -> None: - assert self.__sub_warning is None - assert self.__sub_errors is None - self.__sub_warning = self.listen_event(HABApp.core.const.topics.TOPIC_WARNINGS, self.__event_warning) - self.__sub_errors = self.listen_event(HABApp.core.const.topics.TOPIC_ERRORS, self.__event_error) - - def _worker_events_cancel(self) -> None: - if self.__sub_warning is not None: - self.__sub_warning.cancel() - if self.__sub_errors is not None: - self.__sub_errors.cancel() - - # ------------------------------------------------------------------------------------------------------------------ - # Test execution - def __exec_tc(self, res: TestResult, tc: TestCase) -> None: - self.__warnings.clear() - self.__errors.clear() - - tc.run(res) - - if self.__warnings: - res.set_state(TestResultStatus.WARNING) - ct = len(self.__warnings) - msg = f'{ct} warning{"s" if ct != 1 else ""} in worker' - res.add_msg(msg) - self.__warnings.clear() - - if self.__errors: - res.set_state(TestResultStatus.ERROR) - ct = len(self.__errors) - msg = f'{ct} error{"s" if ct != 1 else ""} in worker' - res.add_msg(msg) - self.__errors.clear() - - self.__worst_result = max(self.__worst_result, res.state) - - def _run_tests(self) -> list[TestResult]: + def add_test(self, name: str, func: Callable | Callable[[...], Coroutine], *args: Any, + setup_up: Callable | Callable[[...], Coroutine] | None = None, + tear_down: Callable | Callable[[...], Coroutine] | None = None, + **kwargs: Any) -> Self: + + tc = TestCase(name, func, args, kwargs, setup_up, tear_down) + assert tc.name not in self._test_cases + self._test_cases[tc.name] = tc + return self + + async def run_test_cases(self) -> list[TestResult]: self._rule_status = TestRuleStatus.RUNNING - self._worker_events_sub() - results = [] + results: list[TestResult] = [] # setup tc = TestCase('set_up', self.set_up) tr = TestResult(self.__class__.__name__, tc.name) - self.__exec_tc(tr, tc) + await tc.run(tr) if tr.state is not tr.state.PASSED: results.append(tr) - results.extend(self.__run_tests()) + results.extend( + await run_test_cases( + tuple(self._test_cases.values()), self.__class__.__name__, self, + skip_on_failure=self.config.skip_on_failure + ) + ) # tear down - tc = TestCase('tear_down', self.set_up) + tc = TestCase('tear_down', self.tear_down) tr = TestResult(self.__class__.__name__, tc.name) - self.__exec_tc(tr, tc) + await tc.run(tr) if tr.state is not tr.state.PASSED: results.append(tr) - self._worker_events_cancel() self._rule_status = TestRuleStatus.FINISHED return results - - def __run_tests(self) -> list[TestResult]: - count = len(self._tests) - width = 1 - while count >= 10 ** width: - width += 1 - - c_name = self.__class__.__name__ - results = [ - TestResult(c_name, tc.name, f'{i + 1:{width}d}/{count}') for i, tc in enumerate(self._tests.values()) - ] - - module_of_class = Path(self.__class__.__module__) - relative_path = module_of_class.relative_to(HABApp.CONFIG.directories.rules) - - log.info('') - log.info(f'Running {count} tests for {c_name} (from "{relative_path}")') - - for res, tc in zip(results, self._tests.values()): - if self.config.skip_on_failure and self.__worst_result >= TestResultStatus.FAILED: - res.set_state(TestResultStatus.SKIPPED) - res.log() - continue - - self.__exec_tc(res, tc) - res.log() - - return results diff --git a/run/conf_testing/lib/HABAppTests/test_rule/test_runner_rule.py b/run/conf_testing/lib/HABAppTests/test_rule/test_runner_rule.py new file mode 100644 index 00000000..e8d6d711 --- /dev/null +++ b/run/conf_testing/lib/HABAppTests/test_rule/test_runner_rule.py @@ -0,0 +1,76 @@ +import logging + +import HABApp +from HABApp.core import shutdown +from HABApp.core.const.topics import TOPIC_FILES +from HABApp.core.events import EventFilter +from HABApp.core.events.habapp_events import RequestFileLoadEvent +from HABApp.core.lib import SingleTask +from HABApp.core.wrapper import ignore_exception +from HABAppTests.test_rule.test_case import TestResult, TestResultStatus + +from .test_rule import TestBaseRule, TestRuleStatus + + +log = logging.getLogger('HABApp.Tests') + + +class TestRunnerRule(HABApp.Rule): + def __init__(self) -> None: + super().__init__() + + self.listen_event(TOPIC_FILES, self._file_event, EventFilter(event_class=RequestFileLoadEvent)) + self.countdown = self.run.countdown(3, self._files_const) + self.countdown.reset() + + self.task = SingleTask(self._run_tests) + + async def _file_event(self, event) -> None: + self.countdown.reset() + + async def _files_const(self) -> None: + self.task.start_if_not_running() + + def _get_next_rule(self) -> TestBaseRule | None: + all_rules = [r for r in self.get_rule(None) if isinstance(r, TestBaseRule)] + for rule in all_rules: + if rule._rule_status is TestRuleStatus.CREATED: + rule._rule_status = TestRuleStatus.PENDING + if rule._rule_status is TestRuleStatus.PENDING: + return rule + return None + + @ignore_exception + async def _run_tests(self) -> None: + results: list[TestResult] = [] + + while (rule := self._get_next_rule()) is not None and not shutdown.is_requested(): + results.extend(await rule.run_test_cases()) + + await self.tests_done(results) + + skipped = tuple(x for x in results if x.state is TestResultStatus.SKIPPED) + passed = tuple(x for x in results if x.state is TestResultStatus.PASSED) + warning = tuple(x for x in results if x.state is TestResultStatus.WARNING) + failed = tuple(x for x in results if x.state is TestResultStatus.FAILED) + error = tuple(x for x in results if x.state is TestResultStatus.ERROR) + + def plog(msg: str) -> None: + print(msg) + log.info(msg) + + parts = [f'{len(results)} executed', f'{len(passed)} passed'] + if skipped: + parts.append(f'{len(skipped)} skipped') + if warning: + parts.append(f'{len(warning)} warning{"" if len(warning) == 1 else "s"}') + parts.append(f'{len(failed)} failed') + if error: + parts.append(f'{len(error)} error{"" if len(error) == 1 else "s"}') + + plog('') + plog('-' * 120) + plog(', '.join(parts)) + + async def tests_done(self, results: list[TestResult]) -> None: + raise NotImplementedError() diff --git a/run/conf_testing/lib/HABAppTests/utils.py b/run/conf_testing/lib/HABAppTests/utils.py index f810366e..807000f4 100644 --- a/run/conf_testing/lib/HABAppTests/utils.py +++ b/run/conf_testing/lib/HABAppTests/utils.py @@ -1,8 +1,7 @@ -import asyncio import random import string import typing -from binascii import b2a_hex +from pathlib import Path import HABApp from HABApp.openhab.items import Thing @@ -16,18 +15,19 @@ def __get_fill_char(skip: str, upper=False) -> str: - skip += 'il' + skip += 'ilo' skip = skip.upper() if upper else skip.lower() - rnd = random.choice(string.ascii_uppercase if upper else string.ascii_lowercase) - while rnd in skip: - rnd = random.choice(string.ascii_uppercase if upper else string.ascii_lowercase) + + letters = string.ascii_uppercase if upper else string.ascii_lowercase + while (rnd := random.choice(letters)) in skip: + pass return rnd def get_random_name(item_type: str) -> str: name = name_prev = __RAND_PREFIX[item_type.split(':')[0]] - for c in range(3): + for _ in range(3): name += __get_fill_char(name_prev, upper=True) while len(name) < 10: @@ -35,21 +35,21 @@ def get_random_name(item_type: str) -> str: return name -def run_coro(coro: typing.Coroutine): - fut = asyncio.run_coroutine_threadsafe(coro, HABApp.core.const.loop) - return fut.result() - - def find_astro_sun_thing() -> str: items = HABApp.core.Items.get_items() for item in items: if isinstance(item, Thing) and item.name.startswith('astro:sun'): return item.name - raise ValueError('No astro thing found!') + msg = 'No astro thing found!' + raise ValueError(msg) + +def get_file_path_of_obj(obj: typing.Any) -> str: + try: + module = obj.__module__ + except AttributeError: + module = obj.__class__.__module__ -def get_bytes_text(value): - if isinstance(value, bytes) and len(value) > 300: - return b2a_hex(value[:40]).decode() + ' ... ' + b2a_hex(value[-40:]).decode() - return value + module_of_class = Path(module) + return str(module_of_class.relative_to(HABApp.CONFIG.directories.rules)) diff --git a/run/conf_testing/logging.yml b/run/conf_testing/logging.yml index 079dcc1e..58e43328 100644 --- a/run/conf_testing/logging.yml +++ b/run/conf_testing/logging.yml @@ -1,7 +1,7 @@ formatters: HABApp_format: format: '[%(asctime)s] [%(name)25s] %(levelname)8s | %(message)s' - HABApp_REST: + HABApp_COM: format: '[%(asctime)s] [%(name)11s] %(levelname)8s | %(message)s' @@ -33,13 +33,13 @@ handlers: formatter: HABApp_format level: DEBUG - HABApp_rest_file: + HABApp_com_file: class: logging.handlers.RotatingFileHandler - filename: 'test_rest.log' + filename: 'test_com.log' maxBytes: 10_485_760 backupCount: 3 - formatter: HABApp_REST + formatter: HABApp_COM level: DEBUG @@ -74,8 +74,8 @@ loggers: - HABApp_test_file propagate: False - HABApp.Rest: + Com: level: DEBUG handlers: - - HABApp_rest_file + - HABApp_com_file propagate: False diff --git a/run/conf_testing/rules/habapp/test_event_listener.py b/run/conf_testing/rules/habapp/test_event_listener.py index 77e3bf3a..480a7e19 100644 --- a/run/conf_testing/rules/habapp/test_event_listener.py +++ b/run/conf_testing/rules/habapp/test_event_listener.py @@ -1,5 +1,3 @@ -import logging - from HABAppTests import TestBaseRule, get_random_name from HABApp.core.events import ValueChangeEventFilter @@ -8,9 +6,6 @@ from HABApp.util import EventListenerGroup -log = logging.getLogger('HABApp.Tests.MultiMode') - - class TestNoWarningOnRuleUnload(TestBaseRule): """This rule tests that multiple listen/cancel commands don't create warnings on unload""" @@ -19,7 +14,7 @@ def __init__(self) -> None: self.add_test('CheckWarning', self.test_unload) - def test_unload(self) -> None: + async def test_unload(self) -> None: item = Item.get_create_item(get_random_name('HABApp')) grp = EventListenerGroup().add_listener(item, self.cb, ValueChangeEventFilter()) @@ -28,13 +23,7 @@ def test_unload(self) -> None: grp.listen() grp.cancel() - self._habapp_ctx.unload_rule() - - # workaround so we don't get Errors - for k in ['_TestBaseRule__sub_warning', '_TestBaseRule__sub_errors']: - obj = self.__dict__[k] - self.__dict__[k] = None - assert obj._parent_ctx is None + await self._habapp_ctx.unload_rule() # Workaround to so we don't crash self.on_rule_unload = lambda: None diff --git a/run/conf_testing/rules/habapp/test_internals.py b/run/conf_testing/rules/habapp/test_internals.py new file mode 100644 index 00000000..a86111d5 --- /dev/null +++ b/run/conf_testing/rules/habapp/test_internals.py @@ -0,0 +1,104 @@ +from HABApp import Rule +from HABApp.core.asyncio import ( + create_task, + create_task_from_async, + run_coro_from_thread, + run_func_from_async, + thread_context, +) +from HABApp.core.items import Item + + +def check_in_thread() -> None: + if thread_context.get() is None: + msg = 'Thread context not set!' + raise ValueError(msg) + + +def check_in_task() -> None: + if thread_context.get(None) is not None: + msg = 'Thread context set!' + raise ValueError(msg) + + +class TestThreadPool(Rule): + """This rule is testing the Scheduler implementation""" + + def __init__(self) -> None: + super().__init__() + + self.run.soon(self.sync_func_task) + self.run.soon(self.async_func) + + self.run.soon(self.test_sync_calls) + self.run.soon(self.test_async_calls) + + def sync_func_task(self) -> int: + check_in_thread() + return 7 + + def sync_func_async(self) -> int: + check_in_task() + return 7 + + async def async_func(self) -> int: + check_in_task() + return 7 + + def test_sync_calls(self) -> None: + check_in_thread() + + f = create_task(self.async_func()) + assert f.result() == 7 + + f = run_func_from_async(self.sync_func_async) + assert f == 7 + + f = run_coro_from_thread(self.async_func(), self.async_func) + assert f == 7 + + async def test_async_calls(self) -> None: + check_in_task() + + f = create_task(self.async_func()) + assert await f == 7 + + f = run_func_from_async(self.sync_func_async) + assert f == 7 + + f = await create_task_from_async(self.async_func()) + assert f == 7 + + +TestThreadPool() + + +item = Item.get_create_item('RuleLifecycleMethods', initial_value=[]) + + +class TestRuleLifecycleThread(Rule): + + def on_rule_loaded(self) -> None: + check_in_thread() + item.value.append('on_rule_loaded_thread') + + def on_rule_removed(self) -> None: + check_in_thread() + item.value.append('on_rule_removed_thread') + + +TestRuleLifecycleThread() + + +class TestRuleLifecycleTask(Rule): + + async def on_rule_loaded(self) -> None: + check_in_task() + item.value.append('on_rule_loaded_task') + + async def on_rule_removed(self) -> None: + check_in_task() + item.value.append('on_rule_removed_task') + + +TestRuleLifecycleTask() diff --git a/run/conf_testing/rules/openhab/test_habapp_internals.py b/run/conf_testing/rules/openhab/test_habapp_internals.py index 6492829e..af1bb9ed 100644 --- a/run/conf_testing/rules/openhab/test_habapp_internals.py +++ b/run/conf_testing/rules/openhab/test_habapp_internals.py @@ -1,4 +1,4 @@ -from HABAppTests import OpenhabTmpItem, TestBaseRule, run_coro +from HABAppTests import AsyncOpenhabTmpItem, TestBaseRule from HABApp.openhab.connection.handler.func_async import ( async_get_item_with_habapp_meta, @@ -14,31 +14,31 @@ def __init__(self) -> None: super().__init__() self.add_test('async', self.create_meta) - def create_meta(self) -> None: - with OpenhabTmpItem('String') as tmpitem: - d = run_coro(async_get_item_with_habapp_meta(tmpitem.name)) + @AsyncOpenhabTmpItem.create('String', arg_name='tmpitem') + async def create_meta(self, tmpitem: AsyncOpenhabTmpItem) -> None: + d = await async_get_item_with_habapp_meta(tmpitem.name) assert d.metadata['HABApp'] is None # create empty set - run_coro(async_set_habapp_metadata(tmpitem.name, HABAppThingPluginData())) + await async_set_habapp_metadata(tmpitem.name, HABAppThingPluginData()) - d = run_coro(async_get_item_with_habapp_meta(tmpitem.name)) + d = await async_get_item_with_habapp_meta(tmpitem.name) assert isinstance(d.metadata['HABApp'], HABAppThingPluginData) # create valid data - run_coro(async_set_habapp_metadata( - tmpitem.name, HABAppThingPluginData(created_link='asdf', created_ns=['a', 'b'])) + await async_set_habapp_metadata( + tmpitem.name, HABAppThingPluginData(created_link='asdf', created_ns=['a', 'b']) ) - d = run_coro(async_get_item_with_habapp_meta(tmpitem.name)) + d = await async_get_item_with_habapp_meta(tmpitem.name) d = d.metadata['HABApp'] assert isinstance(d, HABAppThingPluginData) assert d.created_link == 'asdf' assert d.created_ns == ['a', 'b'] # remove metadata again - run_coro(async_remove_habapp_metadata(tmpitem.name)) - d = run_coro(async_get_item_with_habapp_meta(tmpitem.name)) + await async_remove_habapp_metadata(tmpitem.name) + d = await async_get_item_with_habapp_meta(tmpitem.name) assert d.metadata['HABApp'] is None diff --git a/run/conf_testing/rules/openhab/test_items.py b/run/conf_testing/rules/openhab/test_items.py index b6a2e834..ff3a55ba 100644 --- a/run/conf_testing/rules/openhab/test_items.py +++ b/run/conf_testing/rules/openhab/test_items.py @@ -1,23 +1,27 @@ -import asyncio -from datetime import datetime + +from typing import TYPE_CHECKING from HABAppTests import EventWaiter, ItemWaiter, OpenhabTmpItem, TestBaseRule from immutables import Map from whenever import Instant, OffsetDateTime, SystemDateTime -from HABApp.core.const import loop from HABApp.core.events import ValueUpdateEventFilter from HABApp.core.types import HSB, RGB from HABApp.openhab.interface_async import async_get_items from HABApp.openhab.items import ColorItem, DatetimeItem, GroupItem, NumberItem, StringItem +if TYPE_CHECKING: + from datetime import datetime + + class OpenhabItems(TestBaseRule): def __init__(self) -> None: super().__init__() - self.add_test('ApiDoc', self.test_api) + self.add_test('Api', self.test_api) + self.add_test('AsyncApi', self.test_api_async) self.add_test('MemberTags', self.test_tags) self.add_test('MemberGroups', self.test_groups) self.add_test('TestExisting', self.test_existing) @@ -68,7 +72,9 @@ def test_api(self) -> None: self.openhab.get_item(self.item_switch.name) self.openhab.get_item(self.item_group.name) - asyncio.run_coroutine_threadsafe(async_get_items(), loop).result() + + async def test_api_async(self) -> None: + await async_get_items() @OpenhabTmpItem.create('Number', arg_name='tmp_item') def test_small_float_values(self, tmp_item: OpenhabTmpItem) -> None: diff --git a/run/conf_testing/rules/openhab/test_links.py b/run/conf_testing/rules/openhab/test_links.py index 50be3052..b958d0f0 100644 --- a/run/conf_testing/rules/openhab/test_links.py +++ b/run/conf_testing/rules/openhab/test_links.py @@ -1,5 +1,4 @@ from HABAppTests import TestBaseRule -from HABAppTests.utils import find_astro_sun_thing, run_coro from HABApp.openhab.connection.handler.func_async import async_get_link, async_get_links @@ -8,14 +7,7 @@ class OpenhabLinkApi(TestBaseRule): def __init__(self) -> None: super().__init__() - self.add_test('AllLinks', self.wrap_async, self.api_get_links) - - def wrap_async(self, coro, *args, **kwargs) -> None: - # create valid data - run_coro(coro(*args, **kwargs)) - - def set_up(self) -> None: - self.thing = self.openhab.get_thing(find_astro_sun_thing()) + self.add_test('AllLinks', self.api_get_links) async def api_get_links(self) -> None: objs = await async_get_links() diff --git a/run/conf_testing/rules/test_mqtt.py b/run/conf_testing/rules/test_mqtt.py index db22735d..9ba9a526 100644 --- a/run/conf_testing/rules/test_mqtt.py +++ b/run/conf_testing/rules/test_mqtt.py @@ -1,7 +1,7 @@ +import asyncio import logging -import time -from HABAppTests import EventWaiter, ItemWaiter, TestBaseRule, run_coro +from HABAppTests import EventWaiter, ItemWaiter, TestBaseRule import HABApp from HABApp.core.connections import Connections, ConnectionStatus @@ -63,23 +63,24 @@ def test_mqtt_state(self) -> None: my_item.publish(data) waiter.wait_for_state(data) - def test_mqtt_item_creation(self) -> None: + async def test_mqtt_item_creation(self) -> None: topic = 'mqtt/item/creation' assert HABApp.core.Items.item_exists(topic) is False self.mqtt.publish(topic, 'asdf') - time.sleep(0.1) + await asyncio.sleep(0.1) assert HABApp.core.Items.item_exists(topic) is False # We create the item only on retain self.mqtt.publish(topic, 'asdf', retain=True) - time.sleep(0.2) + await asyncio.sleep(0.2) - run_coro(self.trigger_reconnect()) + await self.trigger_reconnect() + await asyncio.sleep(0.2) connection = Connections.get('mqtt') while not connection.is_online: - time.sleep(0.2) + await asyncio.sleep(0.2) assert HABApp.core.Items.item_exists(topic) is True diff --git a/run/conf_testing/rules/test_runner.py b/run/conf_testing/rules/test_runner.py new file mode 100644 index 00000000..eb2f3f18 --- /dev/null +++ b/run/conf_testing/rules/test_runner.py @@ -0,0 +1,64 @@ +import asyncio + +from HABAppTests import TestResult, TestRunnerRule +from HABAppTests.test_rule.test_case import TestCase, run_test_cases + +from HABApp import CONFIG +from HABApp.core.const.topics import TOPIC_FILES +from HABApp.core.events.habapp_events import RequestFileUnloadEvent +from HABApp.core.items import Item + + +class TestRunnerImpl(TestRunnerRule): + + def __init__(self) -> None: + super().__init__() + self._file_pos = 0 + + async def tests_done(self, results: list[TestResult]) -> None: + results.extend( + await run_test_cases([TestCase('RuleLifeCycle', self.test_lifecycle_methods)], 'RuleLifeCycle', self) + ) + + # show errors of HABApp.log + log_file = CONFIG.directories.logging / 'HABApp.log' + + seek = self._file_pos + self._file_pos = log_file.stat().st_size + + with log_file.open('r') as f: + f.seek(seek) + text = f.read() + + show = [] + for line in text.splitlines(): + if ' ERROR ' in line: + show.append(line) + continue + + if ' WARNING ' in line: + if 'DeprecationWarning' in line or 'is a UoM item but "unit" is not found in item metadata' in line: + continue + if 'Item type changed from' in line: + continue + show.append(line) + + if show: + print('-' * 120) + for line in show: + print(line) + print('-' * 120) + print() + + async def test_lifecycle_methods(self) -> None: + item = Item.get_item('RuleLifecycleMethods') + + assert item.value[0:2] == ['on_rule_loaded_thread', 'on_rule_loaded_task'] + + self.post_event(TOPIC_FILES, RequestFileUnloadEvent('rules/habapp/test_internals.py')) + await asyncio.sleep(1) + + assert item.value[2:4] == ['on_rule_removed_thread', 'on_rule_removed_task'] + + +TestRunnerImpl() diff --git a/src/HABApp/__check_dependency_packages__.py b/src/HABApp/__check_dependency_packages__.py index 2aff19cc..51882269 100644 --- a/src/HABApp/__check_dependency_packages__.py +++ b/src/HABApp/__check_dependency_packages__.py @@ -19,7 +19,7 @@ def get_dependencies() -> list[str]: 'pydantic', 'stack_data', 'voluptuous', - 'watchdog', + 'watchfiles', 'ujson', 'immutables', 'javaproperties', diff --git a/src/HABApp/__version__.py b/src/HABApp/__version__.py index 6d00aa28..c38b509d 100644 --- a/src/HABApp/__version__.py +++ b/src/HABApp/__version__.py @@ -10,4 +10,4 @@ # Development versions contain the DEV-COUNTER postfix: # - 24.01.0.DEV-1 -__version__ = '24.11.1' +__version__ = '24.11.1-DEV-1' diff --git a/src/HABApp/config/__init__.py b/src/HABApp/config/__init__.py index 60ff7186..5621c0c1 100644 --- a/src/HABApp/config/__init__.py +++ b/src/HABApp/config/__init__.py @@ -6,4 +6,4 @@ # isort: split -from .loader import load_config +from .loader import setup_habapp_configuration diff --git a/src/HABApp/config/loader.py b/src/HABApp/config/loader.py index 55b3fbf8..9b04c51f 100644 --- a/src/HABApp/config/loader.py +++ b/src/HABApp/config/loader.py @@ -5,10 +5,11 @@ import eascheduler import pydantic -import HABApp from HABApp import __version__ from HABApp.config.config import CONFIG from HABApp.config.logging import HABAppQueueHandler, load_logging_file +from HABApp.core import shutdown +from HABApp.core.internals.proxy.proxies import uses_file_manager from .debug import setup_debug from .errors import AbsolutePathExpected, InvalidConfigError @@ -19,7 +20,10 @@ log = logging.getLogger('HABApp.Config') -def load_config(config_folder: Path) -> None: +file_manager = uses_file_manager() + + +def setup_habapp_configuration(config_folder: Path) -> None: CONFIG.set_file_path(config_folder / 'config.yml') @@ -40,16 +44,14 @@ def load_config(config_folder: Path) -> None: if not loaded_logging: load_logging_cfg(logging_cfg_path) + shutdown.register(stop_queue_handlers, msg='Stop logging queue handlers', last=True) + setup_debug() - # Watch folders, so we can reload the config on the fly - filter = HABApp.core.files.watcher.FileEndingFilter('.yml') - watcher = HABApp.core.files.watcher.AggregatingAsyncEventHandler( - config_folder, config_files_changed, filter, watch_subfolders=False - ) - HABApp.core.files.watcher.add_folder_watch(watcher) + watcher = file_manager.get_file_watcher() + watcher.watch_file('config.log_file', config_file_changed, config_folder / 'logging.yml', habapp_internal=True) + watcher.watch_file('config.cfg_file', config_file_changed, config_folder / 'config.yml', habapp_internal=True) - HABApp.core.shutdown.register(stop_queue_handlers, last=True, msg='Stopping logging threads') CONFIG.habapp.logging.subscribe_for_changes(set_flush_delay) @@ -57,12 +59,12 @@ def set_flush_delay() -> None: HABAppQueueHandler.FLUSH_DELAY = CONFIG.habapp.logging.flush_every -async def config_files_changed(paths: list[Path]) -> None: - for path in paths: - if path.name == 'config.yml': - load_habapp_cfg() - if path.name == 'logging.yml': - load_logging_cfg(path) +async def config_file_changed(path: str) -> None: + file = Path(path) + if file.name == 'config.yml': + load_habapp_cfg() + if file.name == 'logging.yml': + load_logging_cfg(file) def load_habapp_cfg(do_print=False) -> None: diff --git a/src/HABApp/core/asyncio.py b/src/HABApp/core/asyncio.py index 91a28783..4df83739 100644 --- a/src/HABApp/core/asyncio.py +++ b/src/HABApp/core/asyncio.py @@ -4,8 +4,7 @@ from asyncio import Task as _Task from asyncio import run_coroutine_threadsafe as _run_coroutine_threadsafe from contextvars import ContextVar as _ContextVar -from contextvars import Token as _Token -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING from typing import Any as _Any from typing import ParamSpec as _ParamSpec from typing import TypeVar as _TypeVar @@ -14,34 +13,12 @@ if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Awaitable as _Awaitable from collections.abc import Callable as _Callable from collections.abc import Coroutine as _Coroutine - from types import TracebackType -async_context = _ContextVar('async_ctx') - - -class AsyncContext: - def __init__(self, value: str) -> None: - self.value: Final = value - self.token: _Token[str] | None = None - self.parent: AsyncContext | None = None - - def __enter__(self) -> None: - assert self.token is None, self - self.parent = async_context.get(None) - self.token = async_context.set(self.value) - - def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None) -> None: - async_context.reset(self.token) - - def __repr__(self) -> str: - parent: str = '' - if self.parent: - parent = f'{self.parent} -> ' - return f'<{self.__class__.__name__} {parent:s}{self.value:s}>' +thread_context = _ContextVar('thread_ctx') class AsyncContextError(Exception): @@ -61,8 +38,8 @@ def __str__(self) -> str: def create_task(coro: _Coroutine[_Any, _Any, _T], name: str | None = None) -> _Future[_T]: # https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task - if async_context.get(None) is None: - f = _run_coroutine_threadsafe(coro, loop) + if thread_context.get(None) is not None: + f = _run_coroutine_threadsafe(_async_execute_awaitable(coro), loop) _tasks.add(f) f.add_done_callback(_tasks.discard) return f @@ -82,29 +59,39 @@ def create_task_from_async(coro: _Coroutine[_Any, _Any, _T], name: str | None = def run_coro_from_thread(coro: _Coroutine[_Any, _Any, _T], calling: _Callable) -> _T: # This function call is blocking, so it can't be called in the async context - if async_context.get(None) is not None: + if thread_context.get(None) is None: raise AsyncContextError(calling) - fut = _run_coroutine_threadsafe(coro, loop) + fut = _run_coroutine_threadsafe(_async_execute_awaitable(coro), loop) return fut.result() _P = _ParamSpec('_P') -def run_func_from_async(func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> _T: +def run_func_from_async(func: _Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> _T: # we already have an async context - if async_context.get(None) is not None: + if thread_context.get(None) is None: return func(*args, **kwargs) # we are in a thread, that's why we can wait (and block) for the future - future = _run_coroutine_threadsafe(_run_func_from_async_helper(func, *args, **kwargs), loop) + future = _run_coroutine_threadsafe(_async_execute_func(func, *args, **kwargs), loop) return future.result() -async def _run_func_from_async_helper(func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> _T: - token = async_context.set('run_func_from_async') +async def _async_execute_func(func: _Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs) -> _T: + ctx = thread_context.set(None) + try: return func(*args, **kwargs) finally: - async_context.reset(token) + thread_context.reset(ctx) + + +async def _async_execute_awaitable(awaitable: _Awaitable[_T]) -> _T: + ctx = thread_context.set(None) + + try: + return await awaitable + finally: + thread_context.reset(ctx) diff --git a/src/HABApp/core/connections/manager.py b/src/HABApp/core/connections/manager.py index 2cb523da..f1fd3e1f 100644 --- a/src/HABApp/core/connections/manager.py +++ b/src/HABApp/core/connections/manager.py @@ -1,13 +1,17 @@ from __future__ import annotations import asyncio -from typing import Final, TypeVar +from typing import TYPE_CHECKING, Final, TypeVar import HABApp from HABApp.core.connections import BaseConnection from HABApp.core.connections._definitions import connection_log +if TYPE_CHECKING: + from collections.abc import Generator + + T = TypeVar('T', bound=BaseConnection) @@ -16,16 +20,21 @@ def __init__(self) -> None: self.connections: dict[str, BaseConnection] = {} def add(self, connection: T) -> T: - assert connection.name not in self.connections + if connection.name in self.connections: + msg = f'Connection {connection.name:s} already exists!' + raise ValueError(msg) + self.connections[connection.name] = connection connection_log.debug(f'Added {connection.name:s}') - return connection def get(self, name: str) -> BaseConnection: return self.connections[name] - def remove(self, name): + def get_names(self) -> Generator[str, None, None]: + yield from self.connections.keys() + + def remove(self, name: str) -> None: con = self.get(name) if not con.is_shutdown: raise ValueError() diff --git a/src/HABApp/core/connections/plugin_callback.py b/src/HABApp/core/connections/plugin_callback.py index a83e2f60..fac9eef3 100644 --- a/src/HABApp/core/connections/plugin_callback.py +++ b/src/HABApp/core/connections/plugin_callback.py @@ -6,6 +6,8 @@ from inspect import getmembers, iscoroutinefunction, signature from typing import TYPE_CHECKING, Any +from typing_extensions import Self + from ._definitions import ConnectionStatus @@ -54,9 +56,10 @@ async def run(self, connection: BaseConnection, context: Any): return await self.coro(**kwargs) @staticmethod - def _get_coro_kwargs(plugin: BaseConnectionPlugin, coro: Callable[[...], Awaitable]): + def _get_coro_kwargs(plugin: BaseConnectionPlugin, coro: Callable[[...], Awaitable]) -> tuple[str, ...]: if not iscoroutinefunction(coro): - raise ValueError(f'Coroutine function expected for {plugin.plugin_name}.{coro.__name__}') + msg = f'Coroutine function expected for {plugin.plugin_name}.{coro.__name__}' + raise ValueError(msg) sig = signature(coro) @@ -65,9 +68,10 @@ def _get_coro_kwargs(plugin: BaseConnectionPlugin, coro: Callable[[...], Awaitab if name in ('connection', 'context'): kwargs.append(name) else: - raise ValueError(f'Invalid parameter name "{name:s}" for {plugin.plugin_name}.{coro.__name__}') + msg = f'Invalid parameter name "{name:s}" for {plugin.plugin_name}.{coro.__name__}' + raise ValueError(msg) return tuple(kwargs) @classmethod - def create(cls, plugin: BaseConnectionPlugin, coro: Callable[[...], Awaitable]): + def create(cls, plugin: BaseConnectionPlugin, coro: Callable[[...], Awaitable]) -> Self: return cls(plugin, coro, cls._get_coro_kwargs(plugin, coro)) diff --git a/src/HABApp/core/connections/status_transitions.py b/src/HABApp/core/connections/status_transitions.py index 7d1848ac..da0df692 100644 --- a/src/HABApp/core/connections/status_transitions.py +++ b/src/HABApp/core/connections/status_transitions.py @@ -77,11 +77,11 @@ def _next_step(self) -> ConnectionStatus: return transitions.get(status) def __repr__(self) -> str: - return f'<{self.__class__.__name__} {self.status} ' \ - f'[{"x" if self.error else " "}] Error, ' \ - f'[{"x" if self.setup else " "}] Setup>' + return (f'<{self.__class__.__name__} {self.status} ' + f'[{"x" if self.error else " "}] Error, ' + f'[{"x" if self.setup else " "}] Setup>') - def __eq__(self, other: ConnectionStatus): + def __eq__(self, other: ConnectionStatus) -> bool: if not isinstance(other, ConnectionStatus): return NotImplemented return self.status == other diff --git a/src/HABApp/core/events/habapp_events.py b/src/HABApp/core/events/habapp_events.py index f971561d..34eb8a39 100644 --- a/src/HABApp/core/events/habapp_events.py +++ b/src/HABApp/core/events/habapp_events.py @@ -8,6 +8,9 @@ def __init__(self, name: str) -> None: def __repr__(self) -> str: return f'<{self.__class__.__name__} filename: {self.name}>' + def __eq__(self, other: object) -> bool: + return isinstance(other, self.__class__) and self.name == other.name + class RequestFileLoadEvent(__FileEventBase): """Request (re-) loading of the specified file diff --git a/src/HABApp/core/files/__init__.py b/src/HABApp/core/files/__init__.py index 020ab40c..43782f5b 100644 --- a/src/HABApp/core/files/__init__.py +++ b/src/HABApp/core/files/__init__.py @@ -1,8 +1,2 @@ -from . import errors - -from . import watcher -from . import file -from . import folders -from . import manager - -from .setup import setup +from .watcher import HABAppFileWatcher, FolderDispatcher, FileDispatcher +from .manager import FileManager diff --git a/src/HABApp/core/files/errors.py b/src/HABApp/core/files/errors.py index 7b9c9a20..4766579e 100644 --- a/src/HABApp/core/files/errors.py +++ b/src/HABApp/core/files/errors.py @@ -1,8 +1,5 @@ -from collections.abc import Iterable as _Iterable - - class CircularReferenceError(Exception): - def __init__(self, stack: _Iterable[str]) -> None: + def __init__(self, stack: tuple[str, ...]) -> None: self.stack = stack def __repr__(self) -> str: diff --git a/src/HABApp/core/files/file.py b/src/HABApp/core/files/file.py new file mode 100644 index 00000000..a0056408 --- /dev/null +++ b/src/HABApp/core/files/file.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from enum import Enum, auto +from hashlib import blake2b +from typing import TYPE_CHECKING, Final + +from HABApp.core.files.errors import AlreadyHandledFileError, CircularReferenceError, DependencyDoesNotExistError +from HABApp.core.files.file_properties import FileProperties +from HABApp.core.wrapper import process_exception + + +if TYPE_CHECKING: + import logging + from pathlib import Path + + from HABApp.core.files.manager import FileManager, FileTypeHandler + + +class FileState(Enum): + LOADED = auto() + FAILED = auto() + + DEPENDENCIES_OK = auto() + DEPENDENCIES_MISSING = auto() + DEPENDENCIES_ERROR = auto() + + PROPERTIES_INVALID = auto() # Properties could not be parsed + + # initial and last state + PENDING = auto() + REMOVED = auto() + + def __str__(self) -> str: + return str(self.name) + + +class HABAppFile: + + @staticmethod + def create_checksum(text: str) -> bytes: + b = blake2b() + b.update(text.encode()) + return b.digest() + + def __init__(self, name: str, path: Path, checksum: bytes, properties: FileProperties | None) -> None: + self.name: Final = name + self.path: Final = path + self.checksum: Final = checksum + self.properties: Final = properties if properties is not None else FileProperties() + self._state: FileState = FileState.PENDING if properties is not None else FileState.PROPERTIES_INVALID + + def __repr__(self) -> str: + return f'<{self.__class__.__name__} {self.name} state: {self._state}>' + + def set_state(self, new_state: FileState, manager: FileManager) -> None: + if self._state is new_state: + return None + + self._state = new_state + manager.file_state_changed(self, str(new_state)) + + def _check_circ_refs(self, stack: tuple[str, ...], prop: str, manager: FileManager) -> None: + c: list[str] = getattr(self.properties, prop) + for f in c: + _stack = stack + (f, ) + if f in stack: + raise CircularReferenceError(_stack) + + next_file = manager.get_file(f) + if next_file is not None: + next_file._check_circ_refs(_stack, prop, manager) + + def _check_properties(self, manager: FileManager, log: logging.Logger) -> None: + # check dependencies + missing = {name for name in self.properties.depends_on if manager.get_file(name) is None} + if missing: + one = len(missing) == 1 + msg = (f'File {self.path} depends on file{"" if one else "s"} that ' + f'do{"es" if one else ""}n\'t exist: {", ".join(sorted(missing))}') + raise DependencyDoesNotExistError(msg) + + # check reload + missing = {name for name in self.properties.reloads_on if manager.get_file(name) is None} + if missing: + one = len(missing) == 1 + log.warning(f'File {self.path} reloads on file{"" if one else "s"} that ' + f'do{"es" if one else ""}n\'t exist: {", ".join(sorted(missing))}') + + def check_properties(self, manager: FileManager, log: logging.Logger, *, log_msg: bool = False) -> None: + if self._state is not FileState.PENDING and self._state is not FileState.DEPENDENCIES_ERROR: + return None + + try: + self._check_properties(manager, log) + except DependencyDoesNotExistError as e: + if log_msg: + log.error(e.msg) + return self.set_state(FileState.DEPENDENCIES_ERROR, manager) + + try: + # check for circular references + self._check_circ_refs((self.name, ), 'depends_on', manager) + self._check_circ_refs((self.name, ), 'reloads_on', manager) + except CircularReferenceError as e: + log.error(f'Circular reference: {" -> ".join(e.stack)}') + return self.set_state(FileState.DEPENDENCIES_ERROR, manager) + + # Check if we can already load it + new_state = FileState.DEPENDENCIES_OK if not self.properties.depends_on else FileState.DEPENDENCIES_MISSING + self.set_state(new_state, manager) + return None + + def check_dependencies(self, manager: FileManager) -> None: + if self._state is not FileState.DEPENDENCIES_MISSING: + return None + + for name in self.properties.depends_on: + if (file := manager.get_file(name)) is None: + return None + if file._state is not FileState.LOADED: + return None + + self.set_state(FileState.DEPENDENCIES_OK, manager) + return None + + def can_be_loaded(self) -> bool: + return self._state is FileState.DEPENDENCIES_OK + + def can_be_removed(self) -> bool: + return self._state is FileState.REMOVED + + async def load(self, handler: FileTypeHandler, manager: FileManager) -> None: + if not self.can_be_loaded(): + msg = f'File {self.name} can not be loaded because current state is {self._state}!' + raise ValueError(msg) + + try: + await handler.on_load(self.name, self.path) + except Exception as e: + if not isinstance(e, AlreadyHandledFileError): + process_exception(handler.on_load, e, logger=handler.logger) + self.set_state(FileState.FAILED, manager) + return None + + self.set_state(FileState.LOADED, manager) + return None + + async def unload(self, handler: FileTypeHandler, manager: FileManager) -> None: + try: + await handler.on_unload(self.name, self.path) + except Exception as e: + if not isinstance(e, AlreadyHandledFileError): + process_exception(handler.on_unload, e, logger=handler.logger) + self.set_state(FileState.FAILED, manager) + return None + + self.set_state(FileState.REMOVED, manager) + return None + + def file_state_changed(self, file: HABAppFile, manager: FileManager) -> None: + name = file.name + if name in self.properties.reloads_on: + self.set_state(FileState.PENDING, manager) diff --git a/src/HABApp/core/files/file/__init__.py b/src/HABApp/core/files/file/__init__.py deleted file mode 100644 index e4229245..00000000 --- a/src/HABApp/core/files/file/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .file_state import FileState -from .file import HABAppFile -from .file_types import create_file, register_file_type diff --git a/src/HABApp/core/files/file/file.py b/src/HABApp/core/files/file/file.py deleted file mode 100644 index d2f187e8..00000000 --- a/src/HABApp/core/files/file/file.py +++ /dev/null @@ -1,136 +0,0 @@ -from __future__ import annotations - -import logging -from collections.abc import Awaitable, Callable -from pathlib import Path -from typing import Any - -from HABApp.core.files.errors import AlreadyHandledFileError, CircularReferenceError, DependencyDoesNotExistError -from HABApp.core.files.file.properties import FileProperties -from HABApp.core.files.manager.files import FILES, file_state_changed -from HABApp.core.wrapper import process_exception - -from . import FileState - - -log = logging.getLogger('HABApp.files') - - -class HABAppFile: - LOGGER: logging.Logger - LOAD_FUNC: Callable[[str, Path], Awaitable[Any]] - UNLOAD_FUNC: Callable[[str, Path], Awaitable[Any]] - - def __init__(self, name: str, path: Path, properties: FileProperties) -> None: - self.name: str = name - self.path: Path = path - - self.state: FileState = FileState.PENDING - self.properties: FileProperties = properties - log.debug(f'{self.name} added') - - def __repr__(self) -> str: - return f'<{self.__class__.__name__} {self.name} state: {self.state}>' - - def set_state(self, new_state: FileState): - if self.state is new_state: - return None - - self.state = new_state - log.debug(f'{self.name} changed to {self.state}') - file_state_changed(self) - - def _check_circ_refs(self, stack, prop: str): - c: list[str] = getattr(self.properties, prop) - for f in c: - _stack = stack + (f, ) - if f in stack: - raise CircularReferenceError(_stack) - - next_file = FILES.get(f) - if next_file is not None: - next_file._check_circ_refs(_stack, prop) - - def _check_properties(self): - # check dependencies - mis = set(filter(lambda x: x not in FILES, self.properties.depends_on)) - if mis: - one = len(mis) == 1 - msg = f'File {self.path} depends on file{"" if one else "s"} that ' \ - f'do{"es" if one else ""}n\'t exist: {", ".join(sorted(mis))}' - - raise DependencyDoesNotExistError(msg) - - # check reload - mis = set(filter(lambda x: x not in FILES, self.properties.reloads_on)) - if mis: - one = len(mis) == 1 - log.warning(f'File {self.path} reloads on file{"" if one else "s"} that ' - f'do{"es" if one else ""}n\'t exist: {", ".join(sorted(mis))}') - - def check_properties(self, log_msg: bool = False): - if self.state is not FileState.PENDING and self.state is not FileState.DEPENDENCIES_ERROR: - return None - - try: - self._check_properties() - except DependencyDoesNotExistError as e: - if log_msg: - log.error(e.msg) - return self.set_state(FileState.DEPENDENCIES_ERROR) - - try: - # check for circular references - self._check_circ_refs((self.name, ), 'depends_on') - self._check_circ_refs((self.name, ), 'reloads_on') - except CircularReferenceError as e: - log.error(f'Circular reference: {" -> ".join(e.stack)}') - return self.set_state(FileState.DEPENDENCIES_ERROR) - - # Check if we can already load it - self.set_state(FileState.DEPENDENCIES_OK if not self.properties.depends_on else FileState.DEPENDENCIES_MISSING) - - def check_dependencies(self): - if self.state is not FileState.DEPENDENCIES_MISSING: - return None - - for name in self.properties.depends_on: - f = FILES.get(name, None) - if f is None: - return None - if f.state is not FileState.LOADED: - return None - - self.set_state(FileState.DEPENDENCIES_OK) - return None - - async def load(self): - assert self.state is FileState.DEPENDENCIES_OK, self.state - - try: - await self.__class__.LOAD_FUNC(self.name, self.path) - except Exception as e: - if not isinstance(e, AlreadyHandledFileError): - process_exception(self.__class__.LOAD_FUNC, e, logger=self.LOGGER) - self.set_state(FileState.FAILED) - return None - - self.set_state(FileState.LOADED) - return None - - async def unload(self): - try: - await self.__class__.UNLOAD_FUNC(self.name, self.path) - except Exception as e: - if not isinstance(e, AlreadyHandledFileError): - process_exception(self.__class__.UNLOAD_FUNC, e, logger=self.LOGGER) - self.set_state(FileState.FAILED) - return None - - self.set_state(FileState.REMOVED) - return None - - def file_changed(self, file: HABAppFile) -> None: - name = file.name - if name in self.properties.reloads_on: - self.set_state(FileState.PENDING) diff --git a/src/HABApp/core/files/file/file_state.py b/src/HABApp/core/files/file/file_state.py deleted file mode 100644 index 5d725a9a..00000000 --- a/src/HABApp/core/files/file/file_state.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -from enum import Enum, auto - - -class FileState(Enum): - LOADED = auto() - FAILED = auto() - - DEPENDENCIES_OK = auto() - DEPENDENCIES_MISSING = auto() - DEPENDENCIES_ERROR = auto() - - PROPERTIES_INVALID = auto() # Properties could not be parsed - - # initial and last state - PENDING = auto() - REMOVED = auto() - - def __str__(self) -> str: - return str(self.name) diff --git a/src/HABApp/core/files/file/file_types.py b/src/HABApp/core/files/file/file_types.py deleted file mode 100644 index bf206a2c..00000000 --- a/src/HABApp/core/files/file/file_types.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -import logging -from pathlib import Path - -from pydantic import ValidationError - -from HABApp.core.files.file import FileState, HABAppFile -from HABApp.core.files.file.properties import FileProperties, get_properties -from HABApp.core.logger import HABAppError - - -FILE_TYPES: dict[str, type[HABAppFile]] = {} - - -log = logging.getLogger('HABApp.files') - - -def register_file_type(prefix: str, cls: type[HABAppFile]) -> None: - assert prefix not in FILE_TYPES - - assert cls.LOGGER - assert cls.LOAD_FUNC - assert cls.UNLOAD_FUNC - - FILE_TYPES[prefix] = cls - - -def create_file(name: str, path: Path) -> HABAppFile: - for prefix, cls in FILE_TYPES.items(): - if name.startswith(prefix): - break - else: - raise ValueError(f'Unknown file type for "{name}"!') - - with path.open('r', encoding='utf-8') as f: - txt = f.read(10 * 1024) - - validation_error = True - - try: - properties = get_properties(txt) - validation_error = False - except ValidationError as e: - logger = HABAppError(log) - logger.add(f'Error while parsing properties for {name:s}:') - for line in str(e).splitlines()[1:]: - logger.add(f' {line:s}') - logger.dump() - - properties = FileProperties() - - obj = cls(name, path, properties) - if validation_error: - obj.set_state(FileState.PROPERTIES_INVALID) - - return obj diff --git a/src/HABApp/core/files/file/properties.py b/src/HABApp/core/files/file_properties.py similarity index 60% rename from src/HABApp/core/files/file/properties.py rename to src/HABApp/core/files/file_properties.py index 4ace7ccb..fb3c288e 100644 --- a/src/HABApp/core/files/file/properties.py +++ b/src/HABApp/core/files/file_properties.py @@ -12,45 +12,37 @@ class FileProperties(BaseModel): model_config = ConfigDict(extra='forbid', populate_by_name=True) -RE_START = re.compile(r'^#(\s*)HABApp\s*:', re.IGNORECASE) +RE_START = re.compile(r'^(\s*#\s*)HABApp\s*:', re.IGNORECASE) -def get_properties(_str: str) -> FileProperties: +def get_file_properties(_str: str) -> FileProperties: cfg = [] cut = 0 # extract the property string for line in _str.splitlines(): - line = line.strip() - if cut and not line: + line_strip = line.strip() + if cut and not line_strip: break - if not line: + if not line_strip: continue # break on first non-empty line that is not a comment - if line and not line.startswith('#'): + if line_strip and not line_strip.startswith('#'): break if not cut: # find out how much from the start we have to cut - m = RE_START.search(line) - if m: - cut = len(m.group(1)) + 1 + if m := RE_START.search(line): + cut = m.end(1) cfg.append(line[cut:].lower()) else: - do_break = False - for i, c in enumerate(line): - if i > cut: - break - - if c not in ('#', ' ', '\t'): - do_break = True - break - if do_break: + # If we would cut away characters it's not the yaml definition any more + # Here it's cut + 1 because it must be indented + if line[:cut + 1].strip() not in ('', '#'): break - cfg.append(line[cut:]) data = yml.load('\n'.join(cfg)) diff --git a/src/HABApp/core/files/folders/__init__.py b/src/HABApp/core/files/folders/__init__.py deleted file mode 100644 index 65c4a101..00000000 --- a/src/HABApp/core/files/folders/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .folders import add_folder, get_name, get_path, get_prefixes diff --git a/src/HABApp/core/files/folders/folders.py b/src/HABApp/core/files/folders/folders.py deleted file mode 100644 index 606be30e..00000000 --- a/src/HABApp/core/files/folders/folders.py +++ /dev/null @@ -1,71 +0,0 @@ -from pathlib import Path - -import HABApp -from HABApp.core.const.topics import TOPIC_FILES as T_FILES -from HABApp.core.events.habapp_events import RequestFileLoadEvent, RequestFileUnloadEvent -from HABApp.core.files.watcher import AggregatingAsyncEventHandler -from HABApp.core.internals import uses_post_event - - -FOLDERS: dict[str, 'ConfiguredFolder'] = {} - -post_event = uses_post_event() - - -async def _generate_file_events(files: list[Path]) -> None: - for file in files: - name = get_name(file) - post_event(T_FILES, RequestFileLoadEvent(name) if file.is_file() else RequestFileUnloadEvent(name)) - - -class ConfiguredFolder: - def __init__(self, prefix: str, folder: Path, priority: int) -> None: - self.prefix = prefix - self.folder = folder - self.priority: int = priority # priority determines the order how the files will be loaded - - def add_watch(self, file_ending: str, watch_subfolders: bool = True) -> AggregatingAsyncEventHandler: - filter = HABApp.core.files.watcher.FileEndingFilter(file_ending) - handler = AggregatingAsyncEventHandler(self.folder, _generate_file_events, filter, watch_subfolders) - HABApp.core.files.watcher.add_folder_watch(handler) - return handler - - def add_file_type(self, cls: type['HABApp.core.files.file.HABAppFile']) -> None: - HABApp.core.files.file.register_file_type(self.prefix, cls) - - -def get_prefixes() -> list[str]: - return list(map(lambda x: x.prefix, sorted(FOLDERS.values(), key=lambda x: x.priority, reverse=True))) - - -def add_folder(prefix: str, folder: Path, priority: int) -> ConfiguredFolder: - """Make a folder known - - :param prefix: HABApp file name prefix - :param folder: Folder path - :param priority: Priority (used to determine the load order) - :return: ConfiguredFolder - """ - assert prefix and prefix.endswith('/') - for obj in FOLDERS.values(): - assert obj.priority != priority - FOLDERS[prefix] = c = ConfiguredFolder(prefix, folder, priority) - return c - - -def get_name(path: Path) -> str: - path_str = path.as_posix() - for prefix, cfg in sorted(FOLDERS.items(), key=lambda x: len(x[0]), reverse=True): - folder = cfg.folder.as_posix() - if path_str.startswith(folder): - return prefix + path_str[len(folder) + 1:] - - raise ValueError(f'Path "{path_str}" is not part of the configured folders!') - - -def get_path(name: str) -> Path: - for prefix, obj in FOLDERS.items(): - if name.startswith(prefix): - return obj.folder / name[len(prefix):] - - raise ValueError(f'Prefix not found for "{name}"!') diff --git a/src/HABApp/core/files/manager.py b/src/HABApp/core/files/manager.py new file mode 100644 index 00000000..dc51b83d --- /dev/null +++ b/src/HABApp/core/files/manager.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import asyncio +import logging +from asyncio import sleep +from pathlib import Path +from time import monotonic +from typing import TYPE_CHECKING, Final + +from pydantic import ValidationError + +import HABApp +from HABApp.core.const.topics import TOPIC_FILES +from HABApp.core.files.file import HABAppFile +from HABApp.core.files.file_properties import get_file_properties +from HABApp.core.files.name_builder import FileNameBuilder +from HABApp.core.lib import SingleTask, ValueChange + + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + from re import Pattern + + from HABApp.core.events.habapp_events import RequestFileLoadEvent, RequestFileUnloadEvent + from HABApp.core.files.watcher import HABAppFileWatcher + + +log = logging.getLogger('HABApp.files') + + +class FileTypeHandler: + def __init__(self, name: str, logger: logging.Logger, *, + prefix: str, pattern: Pattern | None = None, + on_load: Callable[[str, Path], Awaitable[None]], + on_unload: Callable[[str, Path], Awaitable[None]]) -> None: + self.name: Final = name + self.logger: Final = logger + + self.prefix: Final = prefix + self.pattern: Final = pattern + + self.on_load: Final = on_load + self.on_unload: Final = on_unload + + def matches(self, name: str) -> bool: + if not name.startswith(self.prefix): + return False + + if (p := self.pattern) is not None and not p.search(name): # noqa: SIM103 + return False + + return True + + def __repr__(self) -> str: + return f'<{self.__class__.__name__} {self.name:s}>' + + +class FileManager: + def __init__(self, watcher: HABAppFileWatcher | None) -> None: + self._lock = asyncio.Lock() + self._files: Final[dict[str, HABAppFile]] = {} + self._file_names: Final = FileNameBuilder() + self._file_handlers: tuple[FileTypeHandler, ...] = () + self._task: Final = SingleTask(self._load_file_task, name='file load worker') + self._watcher: Final = watcher + + def add_folder(self, prefix: str, folder: Path, *, name: str, priority: int, pattern: Pattern | None = None) -> None: + self._file_names.add_folder(prefix, folder, priority=priority, pattern=pattern) + if self._watcher is not None: + self._watcher.watch_folder(name, self.file_watcher_event, folder) + + def get_file_watcher(self) -> HABAppFileWatcher: + if self._watcher is None: + raise ValueError() + return self._watcher + + def get_folders(self): # noqa: ANN201 + return self._file_names.get_folders() + + def add_handler(self, name: str, logger: logging.Logger, *, + prefix: str, pattern: Pattern | None = None, + on_load: Callable[[str, Path], Awaitable[None]], + on_unload: Callable[[str, Path], Awaitable[None]]) -> None: + + for h in self._file_handlers: + if h.name == name: + msg = f'Handler {name:s} already exists!' + raise ValueError(msg) + if h.prefix == prefix and h.pattern == pattern: + msg = f'Handler with prefix {prefix:s} and pattern {pattern} already exists!' + raise ValueError(msg) + + new = FileTypeHandler(name, logger, prefix=prefix, pattern=pattern, on_load=on_load, on_unload=on_unload) + self._file_handlers += (new, ) + log.debug(f'Added handler {new.name}') + + def get_file(self, name: str) -> HABAppFile | None: + return self._files.get(name) + + def file_state_changed(self, file: HABAppFile, new_state: str) -> None: + log.debug(f'{file.name} changed to {new_state:s}') + for f in self._files.values(): + f.file_state_changed(file, self) + + def _get_file_handler(self, name: str) -> FileTypeHandler: + handlers = [h for h in self._file_handlers if h.matches(name)] + if not handlers: + msg = f'No handler matched for {name:s}' + raise ValueError(msg) + + if len(handlers) > 1: + msg = f'Multiple handlers matches for {name:s}: {", ".join(str(h) for h in handlers)}' + raise ValueError(msg) + + return handlers[0] + + async def _do_file_load(self, name: str, *, aquire_lock: bool = True) -> None: + if aquire_lock: + await self._lock.acquire() + + try: + if not (file := self.get_file(name)): + return None + + await file.load(self._get_file_handler(name), manager=self) + finally: + if aquire_lock: + self._lock.release() + + async def _do_file_unload(self, name: str, *, aquire_lock: bool = True) -> None: + if aquire_lock: + await self._lock.acquire() + + try: + if not (file := self.get_file(name)): + return None + + await file.unload(self._get_file_handler(name), manager=self) + + if file.can_be_removed(): + self._files.pop(name) + finally: + if aquire_lock: + self._lock.release() + + async def _load_file_task(self) -> None: + try: + task_sleep = 0.3 + task_alive = 15 + + task_shutdown = False + last_process = monotonic() + + files_count = ValueChange[int]() + + while True: + await sleep(0) + + # wait until we have all files + while files_count.set_value(len(self._files)).changed: # noqa: ASYNC110 + await sleep(task_sleep) + + async with self._lock: + # check files for dependencies etc. + for file in self._files.values(): + file.check_properties(self, log, log_msg=task_shutdown) + file.check_dependencies(self) + + if can_be_loaded := [f.name for f in self._files.values() if f.can_be_loaded()]: + name = next(self._file_names.get_names(can_be_loaded)) + await self._do_file_load(name, aquire_lock=False) + last_process = monotonic() + + if task_shutdown: + break + task_shutdown = monotonic() - last_process > task_alive + + except Exception as e: + HABApp.core.wrapper.process_exception(self._task.name, e, logger=log) + log.debug('Worker done!') + + def __accept_event(self, event: RequestFileLoadEvent | RequestFileUnloadEvent) -> bool: + if not self._file_names.is_accepted_name(event.name): + HABApp.core.logger.log_error(log, f'Ignoring {event.__class__.__name__} for invalid name "{event.name}"') + return False + return True + + def __create_file(self, name: str) -> HABAppFile: + path = self._file_names.create_path(name) + text = path.read_text() + checksum = HABAppFile.create_checksum(text) + try: + properties = get_file_properties(text) + except ValidationError as e: + properties = None + HABApp.core.logger.log_error(log, str(e)) + + return HABAppFile(name, path, checksum, properties) + + async def event_load(self, event: RequestFileLoadEvent) -> None: + if not self.__accept_event(event): + return None + + self._task.start_if_not_running() + + name = event.name + file = self.__create_file(name) + + async with self._lock: + # file already exists -> unload first + if name in self._files: + await self._do_file_unload(name, aquire_lock=False) + + self._files[name] = file + + async def event_unload(self, event: RequestFileUnloadEvent) -> None: + if not self.__accept_event(event): + return None + + self._task.start_if_not_running() + await self._do_file_unload(event.name) + return None + + async def file_watcher_event(self, path: str) -> None: + if not self._file_names.is_accepted_path(path): + return None + + obj = Path(path) + name = self._file_names.create_name(path) + + if obj.is_dir(): + return None + + if not obj.is_file(): + HABApp.core.EventBus.post_event(TOPIC_FILES, HABApp.core.events.habapp_events.RequestFileUnloadEvent(name)) + return None + + if existing := self.get_file(name): + checksum = HABAppFile.create_checksum(obj.read_text()) + if existing.checksum == checksum: + log.debug(f'Skip file system event because file {name:s} did not change') + return None + + HABApp.core.EventBus.post_event(TOPIC_FILES, HABApp.core.events.habapp_events.RequestFileLoadEvent(name)) + return None + + def setup(self) -> None: + HABApp.core.EventBus.add_listener( + HABApp.core.internals.EventBusListener( + TOPIC_FILES, HABApp.core.internals.wrap_func(self.event_load), + HABApp.core.events.EventFilter(HABApp.core.events.habapp_events.RequestFileLoadEvent) + ) + ) + + HABApp.core.EventBus.add_listener( + HABApp.core.internals.EventBusListener( + TOPIC_FILES, HABApp.core.internals.wrap_func(self.event_unload), + HABApp.core.events.EventFilter(HABApp.core.events.habapp_events.RequestFileUnloadEvent) + ) + ) diff --git a/src/HABApp/core/files/manager/__init__.py b/src/HABApp/core/files/manager/__init__.py deleted file mode 100644 index 95f70830..00000000 --- a/src/HABApp/core/files/manager/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .files import FILES, file_state_changed -from .listen_events import setup_file_manager -from .worker import process_file diff --git a/src/HABApp/core/files/manager/files.py b/src/HABApp/core/files/manager/files.py deleted file mode 100644 index 6e5a52ce..00000000 --- a/src/HABApp/core/files/manager/files.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import TYPE_CHECKING - - -if TYPE_CHECKING: - import HABApp - -FILES: dict[str, 'HABApp.core.files.file.HABAppFile'] = {} - - -def file_state_changed(file: 'HABApp.core.files.file.HABAppFile') -> None: - for f in FILES.values(): - if f is not file: - f.file_changed(file) diff --git a/src/HABApp/core/files/manager/listen_events.py b/src/HABApp/core/files/manager/listen_events.py deleted file mode 100644 index 4c99a5c4..00000000 --- a/src/HABApp/core/files/manager/listen_events.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging - -import HABApp -from HABApp.core.const.topics import TOPIC_FILES as T_FILES -from HABApp.core.events import EventFilter -from HABApp.core.events.habapp_events import RequestFileLoadEvent, RequestFileUnloadEvent -from HABApp.core.internals import EventBusListener, uses_event_bus, wrap_func - - -log = logging.getLogger('HABApp.Files') -event_bus = uses_event_bus() - - -async def _process_event(event: RequestFileUnloadEvent | RequestFileLoadEvent) -> None: - name = event.name - await HABApp.core.files.manager.process_file(name, HABApp.core.files.folders.get_path(name)) - - -async def setup_file_manager() -> None: - # Setup events so we can process load/unload - event_bus.add_listener( - EventBusListener( - T_FILES, wrap_func(_process_event), EventFilter(RequestFileUnloadEvent) - ) - ) - event_bus.add_listener( - EventBusListener( - T_FILES, wrap_func(_process_event), EventFilter(RequestFileLoadEvent) - ) - ) diff --git a/src/HABApp/core/files/manager/worker.py b/src/HABApp/core/files/manager/worker.py deleted file mode 100644 index ee5398f7..00000000 --- a/src/HABApp/core/files/manager/worker.py +++ /dev/null @@ -1,89 +0,0 @@ -import logging -import time -from asyncio import Future, create_task, sleep -from pathlib import Path - -import HABApp -from HABApp.core.files.file import FileState -from HABApp.core.files.folders import get_prefixes - -from . import FILES - - -log = logging.getLogger('HABApp.files') - - -TASK: Future | None = None -TASK_SLEEP: float = 0.3 -TASK_DURATION: float = 15 - - -async def process_file(name: str, file: Path): - global TASK - - # unload file - if not file.is_file(): - existing = FILES.pop(name, None) - if existing is None: - return None - - await existing.unload() - log.debug(f'Removed {existing.name}') - return None - - # add file - FILES[name] = HABApp.core.files.file.create_file(name, file) - if TASK is None: - TASK = create_task(_process()) - - -async def _process() -> None: - global TASK - - prefixes = get_prefixes() - - ct = -1 - log_msg = False - last_process = time.time() - - try: - while True: - - # wait until files are stable - while ct != len(FILES): - ct = len(FILES) - await sleep(TASK_SLEEP) - - # check files for dependencies etc. - for file in FILES.values(): - file.check_properties(log_msg) - - # Load order - for prefix in prefixes: - file_loaded = False - for name in filter(lambda x: x.startswith(prefix), sorted(FILES.keys())): - file = FILES[name] - file.check_dependencies() - - if file.state is FileState.DEPENDENCIES_OK: - await file.load() - last_process = time.time() - file_loaded = True - break - if file_loaded: - break - - # if we don't have any files left to load we sleep! - if not any(map(lambda x: x.state is FileState.DEPENDENCIES_OK, FILES.values())): - await sleep(TASK_SLEEP) - - # Emit an error message during the last run - if log_msg: - break - log_msg = time.time() - last_process > TASK_DURATION - - except Exception as e: - HABApp.core.wrapper.process_exception('file load worker', e, logger=log) - finally: - TASK = None - log.debug('Worker done!') diff --git a/src/HABApp/core/files/name_builder.py b/src/HABApp/core/files/name_builder.py new file mode 100644 index 00000000..b3fe4377 --- /dev/null +++ b/src/HABApp/core/files/name_builder.py @@ -0,0 +1,99 @@ +from collections.abc import Generator, Iterable +from pathlib import Path +from re import Pattern +from typing import Final + + +class FileNameBuilderRule: + def __init__(self, prefix: str, folder: str, *, + priority: int, pattern: Pattern | None = None) -> None: + self.prefix: Final = prefix + self.folder: Final = folder + self.priority: Final = priority + self.pattern: Final = pattern + + def create_name(self, path: str) -> str | None: + if not path.startswith(folder := self.folder): + return None + + if (p := self.pattern) is not None and not p.search(path): + return None + return self.prefix + path.removeprefix(folder) + + def create_path(self, name: str) -> Path | None: + if not name.startswith(prefix := self.prefix): + return None + + if (p := self.pattern) is not None and not p.search(name): + return None + + return Path(self.folder + name.removeprefix(prefix)) + + def matches_name(self, name: str) -> bool: + return name.startswith(self.prefix) and (self.pattern is None or self.pattern.search(name)) + + +class FileNameBuilder: + def __init__(self) -> None: + self._builders: tuple[FileNameBuilderRule, ...] = () + + def add_folder(self, prefix: str, folder: Path, *, + priority: int, pattern: Pattern | None = None) -> None: + for b in self._builders: + if b.priority == priority: + msg = f'Priority {priority} already exists for {b.prefix}!' + raise ValueError(msg) + + new = FileNameBuilderRule(prefix, folder.as_posix() + '/', priority=priority, pattern=pattern) + self._builders = tuple(sorted(self._builders + (new,), key=lambda x: x.priority, reverse=True)) + + def create_name(self, path: str) -> str: + paths = [n for b in self._builders if (n := b.create_name(path)) is not None] + if not paths: + msg = f'Nothing matched for path {path:s}' + raise ValueError(msg) + + if len(paths) > 1: + msg = f'Multiple matches for path {path:s}: {", ".join(paths)}' + raise ValueError(msg) + + return paths[0] + + def create_path(self, name: str) -> Path: + paths = [p for b in self._builders if (p := b.create_path(name)) is not None] + if not paths: + msg = f'Nothing matched for name {name:s}' + raise ValueError(msg) + + if len(paths) > 1: + msg = f'Multiple matches for name {name:s}: {", ".join(p.as_posix() for p in paths)}' + raise ValueError(msg) + + return paths[0] + + def is_accepted_path(self, path: str) -> bool: + return any(b.create_name(path) is not None for b in self._builders) + + def is_accepted_name(self, path: str) -> bool: + return any(b.matches_name(path) for b in self._builders) + + def get_folders(self) -> list[str]: + ret: list[str] = [] + for b in self._builders: + if b.folder not in ret: + ret.append(b.folder) + return ret + + def get_names_with_path(self, paths: list[str]) -> list[tuple[str, Path]]: + ret = [] + for b in self._builders: + ret.extend((n, Path(p)) for p in paths if (n := b.create_name(p)) is not None) + return ret + + def get_names(self, names: Iterable[str]) -> Generator[str, None, None]: + """Get sorted names""" + names = sorted(names) + for b in self._builders: + for name in names: + if b.matches_name(name): + yield name diff --git a/src/HABApp/core/files/setup.py b/src/HABApp/core/files/setup.py deleted file mode 100644 index d361a5df..00000000 --- a/src/HABApp/core/files/setup.py +++ /dev/null @@ -1,5 +0,0 @@ -from .manager import setup_file_manager - - -async def setup() -> None: - await setup_file_manager() diff --git a/src/HABApp/core/files/watcher.py b/src/HABApp/core/files/watcher.py new file mode 100644 index 00000000..57b580d9 --- /dev/null +++ b/src/HABApp/core/files/watcher.py @@ -0,0 +1,258 @@ +import asyncio +import contextlib +import logging +import re +from asyncio import Event, Task +from collections.abc import Awaitable, Callable +from pathlib import Path +from re import Pattern +from typing import Any, Final, override + +from watchfiles import Change, DefaultFilter, awatch + +from HABApp.core.asyncio import create_task_from_async +from HABApp.core.wrapper import process_exception + + +log = logging.getLogger('HABApp.file.events') +log.setLevel(logging.INFO) + + +DEFAULT_FILTER = DefaultFilter() + +HABAPP_DISPATCHER_PREFIX: Final = 'HABAppInternal-' + + +class FileWatcherDispatcherBase: + + def __init__(self, name: str, coro: Callable[[str], Awaitable[Any]],) -> None: + self._name: Final = name + self._coro: Final = coro + + def __repr__(self) -> str: + return f'<{self.__class__.__name__} {self._name:s}>' + + def __eq__(self, other: object) -> bool: + raise NotImplementedError() + + @property + def name(self) -> str: + return self._name + + def allow(self, change: Change | None, path: str) -> bool: + raise NotImplementedError() + + async def dispatch(self, path: str) -> None: + if not self.allow(None, path): + return None + + try: + await self._coro(path) + except Exception as e: + process_exception(self._coro, e, logger=log) + + +class FolderDispatcher(FileWatcherDispatcherBase): + def __init__(self, name: str, coro: Callable[[str], Awaitable[Any]], folder: str) -> None: + super().__init__(name, coro) + self._folder: Final = folder + + @override + def __eq__(self, other: object) -> bool: + if not isinstance(other, FolderDispatcher): + return False + return self._name == other._name and self._coro is other._coro and self._folder == other._folder + + @override + def allow(self, change: Change, path: str) -> bool: + return path.startswith(self._folder) + + +class FileDispatcher(FileWatcherDispatcherBase): + def __init__(self, name: str, coro: Callable[[str], Awaitable[Any]], file: str) -> None: + super().__init__(name, coro) + self._file: Final = file + + @override + def __eq__(self, other: object) -> bool: + if not isinstance(other, FileDispatcher): + return False + return self._name == other._name and self._coro is other._coro and self._file == other._file + + @override + def allow(self, change: Change, path: str) -> bool: + return path == self._file + + +class HABAppFileWatcher: + def __init__(self) -> None: + self._dispatchers: tuple[FileWatcherDispatcherBase, ...] = () + self._paths: tuple[str, ...] = () + self._files_task: Task | None = None + self._stop_event: Final = Event() + + def __repr__(self) -> str: + return f'<{self.__class__.__name__:s}>' + + def cancel(self, dispatcher: FileWatcherDispatcherBase | str) -> None: + if isinstance(dispatcher, str): + for d in self._dispatchers: + if d.name == dispatcher: + dispatcher = d + break + else: + msg = f'No dispatcher with name "{dispatcher:s}" found' + raise ValueError(msg) + + self._dispatchers = tuple(d for d in self._dispatchers if d is not dispatcher) + + def __notify_task(self) -> None: + if self._files_task is None: + self._files_task = create_task_from_async(self._watcher_task()) + else: + self._stop_event.set() + + def watch_folder(self, name: str, coro: Callable[[str], Awaitable[Any]], folder: Path, *, + habapp_internal: bool = False) -> FolderDispatcher: + d = FolderDispatcher( + name if not habapp_internal else f'{HABAPP_DISPATCHER_PREFIX}{name}', coro, folder.as_posix() + ) + self.add_dispatcher(d) + self.add_path(folder) + return d + + def watch_file(self, name: str, coro: Callable[[str], Awaitable[Any]], file: Path, *, + habapp_internal: bool = False) -> FileDispatcher: + d = FileDispatcher( + name if not habapp_internal else f'{HABAPP_DISPATCHER_PREFIX}{name}', coro, file.as_posix() + ) + self.add_dispatcher(d) + self.add_path(file) + return d + + def add_dispatcher(self, dispatcher: FileWatcherDispatcherBase) -> None: + name = dispatcher.name.lower() + for d in self._dispatchers: + if d.name.lower() != name or d == dispatcher: + continue + msg = f'Dispatcher with name "{dispatcher.name:s}" already exists' + raise ValueError(msg) + + self._dispatchers += (dispatcher, ) + log.debug(f'Added dispatcher {dispatcher.name:s}') + self.__notify_task() + + def add_path(self, path: Path) -> None: + if path.as_posix() in self._paths: + return None + + if not path.is_dir() and not path.is_file(): + msg = f'Path {path} does not exist' + raise FileNotFoundError(msg) + + self._paths += (path.as_posix(), ) + log.debug(f'Watching {path}') + + self.__notify_task() + + def _watch_filter(self, change: Change | None, path: str, *, + dispatchers: list[FileWatcherDispatcherBase] | None = None) -> bool: + if not DEFAULT_FILTER(change, path): + return False + + if dispatchers is not None: + return any(dispatcher.allow(change, path) for dispatcher in dispatchers) + + log.debug(f'{change.name:s} {path:s}') + return any(dispatcher.allow(change, path) for dispatcher in self._dispatchers) + + async def _watcher_task(self) -> None: + delay = 1 + while self._dispatchers: + await asyncio.sleep(1) + + try: + self._stop_event.clear() + log.debug('Starting file watcher') + async for changes in awatch(*self._paths, watch_filter=self._watch_filter, stop_event=self._stop_event): + file_names = [Path(p).as_posix() for _, p in changes] + for dispatcher in self._dispatchers: + for path in file_names: + await dispatcher.dispatch(path) + + log.debug('File watcher stopped') + except Exception as e: # noqa: PERF203 + process_exception(self._watcher_task, e, logger=log) + delay *= 2 + await asyncio.sleep(delay) + + log.debug('File watcher shutdown') + + async def shutdown(self) -> None: + self._dispatchers = () + self._stop_event.set() + if self._files_task is None: + return None + + task = self._files_task + self._files_task = None + + try: + return await asyncio.wait_for(task, 2) + except asyncio.TimeoutError: + pass + + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + return None + + async def load_files(self, name_include: Pattern | str | None = None, name_exclude: Pattern | str | None = None, *, + exclude_habapp_config: bool = True) -> None: + + if isinstance(name_include, str): + name_include = re.compile(f'^{name_include}$') + if isinstance(name_exclude, str): + name_exclude = re.compile(f'^{name_exclude}$') + + dispatchers = [] + for d in self._dispatchers: + name = d.name + + if name.startswith(HABAPP_DISPATCHER_PREFIX): + if exclude_habapp_config: + continue + name = name.removeprefix(HABAPP_DISPATCHER_PREFIX) + + if exclude_habapp_config and d.name.lower().startswith(HABAPP_DISPATCHER_PREFIX): + continue + if name_include is not None and not name_include.search(name): + continue + if name_exclude is not None and name_exclude.search(name): + continue + dispatchers.append(d) + + if not dispatchers: + msg = 'No dispatchers selected!' + raise ValueError(msg) + + files: list[str] = [] + for path_str in self._paths: + if not self._watch_filter(None, path_str, dispatchers=dispatchers): + continue + path = Path(path_str) + + if path.is_file(): # noqa: PTH113 + files.append(path.as_posix()) + continue + + if path.is_dir(): + for obj in path.glob('**/*'): + obj_str = obj.as_posix() + if self._watch_filter(None, obj_str, dispatchers=dispatchers): + files.append(obj_str) + + for file in sorted(files): + for dispatcher in self._dispatchers: + if dispatcher.allow(None, file): + await dispatcher.dispatch(file) diff --git a/src/HABApp/core/files/watcher/__init__.py b/src/HABApp/core/files/watcher/__init__.py deleted file mode 100644 index bbb27003..00000000 --- a/src/HABApp/core/files/watcher/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .folder_watcher import start, remove_folder_watch, add_folder_watch -from .file_watcher import AggregatingAsyncEventHandler -from .base_watcher import FileEndingFilter diff --git a/src/HABApp/core/files/watcher/base_watcher.py b/src/HABApp/core/files/watcher/base_watcher.py deleted file mode 100644 index bc449b4f..00000000 --- a/src/HABApp/core/files/watcher/base_watcher.py +++ /dev/null @@ -1,63 +0,0 @@ -import logging -from pathlib import Path - -from watchdog.events import EVENT_TYPE_CLOSED as WD_EVENT_TYPE_CLOSED -from watchdog.events import EVENT_TYPE_CLOSED_NO_WRITE as WD_EVENT_TYPE_CLOSED_NO_WRITE -from watchdog.events import EVENT_TYPE_OPENED as WD_EVENT_TYPE_OPENED -from watchdog.events import FileSystemEvent - - -log = logging.getLogger('HABApp.file.events') -log.setLevel(logging.INFO) - - -class EventFilterBase: - def notify(self, path: str) -> bool: - raise NotImplementedError() - - -class FileEndingFilter(EventFilterBase): - def __init__(self, ending: str) -> None: - self.ending: str = ending - - def notify(self, path: str) -> bool: - return path.endswith(self.ending) - - def __repr__(self) -> str: - return f'<{self.__class__.__name__} ending: {self.ending}>' - - -class FileSystemEventHandler: - def __init__(self, folder: Path, filter: EventFilterBase, watch_subfolders: bool = False) -> None: - assert isinstance(folder, Path), type(folder) - assert watch_subfolders is True or watch_subfolders is False - - self.folder: Path = folder - self.watch_subfolders: bool = watch_subfolders - - self.filter: EventFilterBase = filter - - def dispatch(self, event: FileSystemEvent): - log.debug(event) - - # we don't process directory events - if event.is_directory: - return None - - # we don't process open and close events - if event.event_type in (WD_EVENT_TYPE_OPENED, WD_EVENT_TYPE_CLOSED, WD_EVENT_TYPE_CLOSED_NO_WRITE): - return None - - src = event.src_path - if self.filter.notify(src): - self.file_changed(src) - - # moved events have a dst, so we process it, too - if hasattr(event, 'dest_path'): - dst = event.dest_path - if self.filter.notify(dst): - self.file_changed(dst) - return None - - def file_changed(self, dst: str): - raise NotImplementedError() diff --git a/src/HABApp/core/files/watcher/file_watcher.py b/src/HABApp/core/files/watcher/file_watcher.py deleted file mode 100644 index d43fa4c2..00000000 --- a/src/HABApp/core/files/watcher/file_watcher.py +++ /dev/null @@ -1,55 +0,0 @@ -from asyncio import run_coroutine_threadsafe, sleep -from collections.abc import Awaitable, Callable -from pathlib import Path -from time import monotonic -from typing import Any - -import HABApp -from HABApp.core.asyncio import AsyncContext -from HABApp.core.wrapper import ignore_exception - -from .base_watcher import EventFilterBase, FileSystemEventHandler - - -DEBOUNCE_TIME: float = 0.6 - - -class AggregatingAsyncEventHandler(FileSystemEventHandler): - def __init__(self, folder: Path, func: Callable[[list[Path]], Awaitable[Any]], filter: EventFilterBase, - watch_subfolders: bool = False) -> None: - super().__init__(folder, filter, watch_subfolders=watch_subfolders) - - self.func = func - - self._files: set[Path] = set() - self.last_event: float = 0 - - @ignore_exception - def file_changed(self, dst: str) -> None: - # Map from thread to async - run_coroutine_threadsafe(self._event_waiter(Path(dst)), loop=HABApp.core.const.loop) - - @ignore_exception - async def _event_waiter(self, dst: Path): - self.last_event = ts = monotonic() - self._files.add(dst) - - # debounce time - await sleep(DEBOUNCE_TIME) - - # check if a new event came - if self.last_event > ts: - return None - - # Copy Path so we're done here - files = list(self._files) - self._files.clear() - - # process - with AsyncContext('FileWatcherEvent'): - await self.func(HABApp.core.lib.sort_files(files)) - - async def trigger_all(self) -> None: - files = HABApp.core.lib.list_files(self.folder, self.filter, self.watch_subfolders) - with AsyncContext('FileWatcherAll'): - await self.func(files) diff --git a/src/HABApp/core/files/watcher/folder_watcher.py b/src/HABApp/core/files/watcher/folder_watcher.py deleted file mode 100644 index 8df80547..00000000 --- a/src/HABApp/core/files/watcher/folder_watcher.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging -from pathlib import Path -from threading import Lock - -from watchdog.observers import Observer -from watchdog.observers.api import ObservedWatch - -from HABApp.core import shutdown - -from .base_watcher import FileSystemEventHandler - - -log = logging.getLogger('HABApp.files.watcher') - -LOCK = Lock() - -OBSERVER: Observer | None = None -WATCHES: dict[str, ObservedWatch] = {} - - -def start(): - global OBSERVER - - # start only once! - assert OBSERVER is None - - OBSERVER = Observer() - OBSERVER.start() - - # register for proper shutdown - shutdown.register(OBSERVER.stop, msg='Stopping folder observer') - shutdown.register(OBSERVER.join, last=True, msg='Joining folder observer threads') - return None - - -def add_folder_watch(handler: FileSystemEventHandler) -> None: - assert OBSERVER is not None - assert isinstance(handler, FileSystemEventHandler), type(handler) - assert isinstance(handler.folder, Path) and handler.folder.is_dir() - - log.debug( - f'Adding {"recursive " if handler.watch_subfolders else ""}watcher for {handler.folder} with {handler.filter}' - ) - - with LOCK: - _folder = str(handler.folder) - assert _folder not in WATCHES - - WATCHES[_folder] = OBSERVER.schedule(handler, _folder, recursive=handler.watch_subfolders) - - -def remove_folder_watch(folder: Path) -> None: - assert OBSERVER is not None - assert isinstance(folder, Path) - - with LOCK: - OBSERVER.unschedule(WATCHES.pop(str(folder))) diff --git a/src/HABApp/core/internals/event_bus/event_bus.py b/src/HABApp/core/internals/event_bus/event_bus.py index a9513b7a..68de1976 100644 --- a/src/HABApp/core/internals/event_bus/event_bus.py +++ b/src/HABApp/core/internals/event_bus/event_bus.py @@ -13,7 +13,7 @@ class EventBus: - __slots__ = ('_lock', '_listeners') + __slots__ = ('_listeners', '_lock') def __init__(self) -> None: self._lock = threading.Lock() diff --git a/src/HABApp/core/internals/item_registry/item_registry.py b/src/HABApp/core/internals/item_registry/item_registry.py index 48c90965..1e7dbc0d 100644 --- a/src/HABApp/core/internals/item_registry/item_registry.py +++ b/src/HABApp/core/internals/item_registry/item_registry.py @@ -2,7 +2,7 @@ import logging import threading -from typing import Final, TypeVar +from typing import Final, TypeVar, overload from HABApp.core.errors import ItemAlreadyExistsError, ItemNotFoundException from HABApp.core.internals.item_registry import ItemRegistryItem @@ -59,7 +59,15 @@ def add_item(self, item: ITEM_TYPE) -> ITEM_TYPE: item._on_item_added() return item - def pop_item(self, name: str | ITEM_TYPE) -> ITEM_TYPE: + @overload + def pop_item(self, name: ITEM_TYPE) -> ITEM_TYPE: + ... + + @overload + def pop_item(self, name: str) -> ItemRegistryItem: + ... + + def pop_item(self, name: str | ItemRegistryItem) -> ItemRegistryItem: if not isinstance(name, str): name = name.name @@ -72,3 +80,9 @@ def pop_item(self, name: str | ITEM_TYPE) -> ITEM_TYPE: log.debug(f'Removed {name} ({item.__class__.__name__})') item._on_item_removed() return item + + def __bool__(self) -> bool: + return bool(self._items) + + def __len__(self) -> int: + return len(self._items) diff --git a/src/HABApp/core/internals/proxy/__init__.py b/src/HABApp/core/internals/proxy/__init__.py index 348ccf3d..5e4e9868 100644 --- a/src/HABApp/core/internals/proxy/__init__.py +++ b/src/HABApp/core/internals/proxy/__init__.py @@ -2,4 +2,4 @@ # isort: split -from .proxies import uses_get_item, uses_item_registry, uses_post_event, uses_event_bus, setup_internals +from .proxies import uses_get_item, uses_item_registry, uses_post_event, uses_event_bus, setup_internals, uses_file_manager diff --git a/src/HABApp/core/internals/proxy/proxies.py b/src/HABApp/core/internals/proxy/proxies.py index c49255be..9b064b7e 100644 --- a/src/HABApp/core/internals/proxy/proxies.py +++ b/src/HABApp/core/internals/proxy/proxies.py @@ -24,11 +24,17 @@ def uses_item_registry() -> 'HABApp.core.internals.ItemRegistry': return create_proxy(uses_item_registry) +def uses_file_manager() -> 'HABApp.core.files.FileManager': + return create_proxy(uses_file_manager) + + def setup_internals(ir: 'HABApp.core.internals.ItemRegistry', - eb: 'HABApp.core.internals.EventBus', final=True): + eb: 'HABApp.core.internals.EventBus', + file_manager: 'HABApp.core.files.FileManager', final=True): """Replace the proxy objects with the real thing""" replacements = { uses_item_registry: ir, uses_get_item: ir.get_item, uses_event_bus: eb, uses_post_event: eb.post_event, + uses_file_manager: file_manager, } return replace_proxies(replacements, final=final) diff --git a/src/HABApp/core/internals/wrapped_function/__init__.py b/src/HABApp/core/internals/wrapped_function/__init__.py index 14690eab..0eb431ef 100644 --- a/src/HABApp/core/internals/wrapped_function/__init__.py +++ b/src/HABApp/core/internals/wrapped_function/__init__.py @@ -1,5 +1,6 @@ from HABApp.core.internals.wrapped_function.base import WrappedFunctionBase + # isort: split from HABApp.core.internals.wrapped_function.wrapper import wrap_func diff --git a/src/HABApp/core/internals/wrapped_function/wrapped_async.py b/src/HABApp/core/internals/wrapped_function/wrapped_async.py index 589e41ff..a57bf142 100644 --- a/src/HABApp/core/internals/wrapped_function/wrapped_async.py +++ b/src/HABApp/core/internals/wrapped_function/wrapped_async.py @@ -4,7 +4,7 @@ from typing_extensions import override -from HABApp.core.asyncio import async_context, create_task +from HABApp.core.asyncio import create_task from HABApp.core.internals import Context from HABApp.core.internals.wrapped_function.base import P, R, WrappedFunctionBase @@ -27,12 +27,8 @@ def run(self, *args: P.args, **kwargs: P.kwargs) -> None: @override async def async_run(self, *args: P.args, **kwargs: P.kwargs) -> R | None: - token = async_context.set('WrappedAsyncFunction') - try: return await self.coro(*args, **kwargs) except Exception as e: self.process_exception(e, *args, **kwargs) return None - finally: - async_context.reset(token) diff --git a/src/HABApp/core/internals/wrapped_function/wrapped_sync.py b/src/HABApp/core/internals/wrapped_function/wrapped_sync.py index 7d15a77b..bf01c3fc 100644 --- a/src/HABApp/core/internals/wrapped_function/wrapped_sync.py +++ b/src/HABApp/core/internals/wrapped_function/wrapped_sync.py @@ -5,7 +5,7 @@ from typing_extensions import override -from HABApp.core.asyncio import async_context, create_task +from HABApp.core.asyncio import create_task from HABApp.core.internals import Context from HABApp.core.internals.wrapped_function.base import P, R, WrappedFunctionBase @@ -31,11 +31,8 @@ def run(self, *args: P.args, **kwargs: P.kwargs) -> None: @override async def async_run(self, *args: P.args, **kwargs: P.kwargs) -> R | None: - token = async_context.set('WrappedSyncFunction') - try: return self.func(*args, **kwargs) except Exception as e: self.process_exception(e, *args, **kwargs) - finally: - async_context.reset(token) + return None diff --git a/src/HABApp/core/internals/wrapped_function/wrapped_thread.py b/src/HABApp/core/internals/wrapped_function/wrapped_thread.py index 4e113aaa..0591dbad 100644 --- a/src/HABApp/core/internals/wrapped_function/wrapped_thread.py +++ b/src/HABApp/core/internals/wrapped_function/wrapped_thread.py @@ -7,6 +7,7 @@ from typing_extensions import override +from HABApp.core.asyncio import thread_context from HABApp.core.const import loop from HABApp.core.internals import Context, ContextProvidingObj from HABApp.core.internals.wrapped_function.base import P, R, WrappedFunctionBase, default_logger @@ -21,6 +22,10 @@ POOL_THREADS: int = 0 +def _initialize_thread() -> None: + thread_context.set('HABAppWorker') + + def create_thread_pool(count: int) -> None: global POOL, POOL_THREADS @@ -32,7 +37,7 @@ def create_thread_pool(count: int) -> None: stop_thread_pool() POOL_THREADS = count - POOL = ThreadPoolExecutor(count, 'HabAppWorker') + POOL = ThreadPoolExecutor(count, 'HabAppWorker', initializer=_initialize_thread) def stop_thread_pool() -> None: diff --git a/src/HABApp/core/lib/__init__.py b/src/HABApp/core/lib/__init__.py index 216c938b..d17c6eff 100644 --- a/src/HABApp/core/lib/__init__.py +++ b/src/HABApp/core/lib/__init__.py @@ -1,5 +1,4 @@ from .exceptions import HINT_EXCEPTION, format_exception -from .funcs import list_files, sort_files from .instant_view import InstantView from .pending_future import PendingFuture from .priority_list import PriorityList diff --git a/src/HABApp/core/lib/exceptions/format_frame.py b/src/HABApp/core/lib/exceptions/format_frame.py index d3fa661f..93f7cbef 100644 --- a/src/HABApp/core/lib/exceptions/format_frame.py +++ b/src/HABApp/core/lib/exceptions/format_frame.py @@ -1,4 +1,7 @@ import re +import sys +from pathlib import Path +from typing import Final from stack_data import LINE_GAP, FrameInfo @@ -25,12 +28,33 @@ re.compile(r'[/\\]HABApp[/\\]core[/\\]connections[/\\]'), ) -SUPPRESSED_PATHS = ( - # Libraries of base installation - re.compile(r'[/\\](?:python\d\.\d+|python\d{2,3})[/\\](?:lib[/\\]|site-packages[/\\]|\w+\.py.*$)', re.IGNORECASE), - # Libraries in venv - re.compile(r'[/\\]lib[/\\]site-packages[/\\]', re.IGNORECASE), -) + +def __get_library_paths() -> tuple[str, ...]: + ret = [] + exec_folter = Path(sys.executable).parent + ret.append(str(exec_folter)) + + # detect virtual environment + if exec_folter.with_name('pyvenv.cfg').is_file(): + folder_names = ('bin', 'include', 'lib', 'lib64', 'scripts') + for p in exec_folter.parent.iterdir(): + if p.name.lower() in folder_names and (value := str(p)) not in ret and p.is_dir(): + ret.append(value) + + return tuple(ret) + + +def _get_habapp_module_path() -> str: + this = Path(__file__) + while this.name != 'HABApp' or not this.is_dir(): + this = this.parent + return str(this) + + +SUPPRESSED_PATHS: Final = __get_library_paths() +HABAPP_MODULE_PATH: Final = _get_habapp_module_path() +del __get_library_paths +del _get_habapp_module_path def is_suppressed_habapp_file(name: str) -> bool: @@ -41,12 +65,11 @@ def is_suppressed_habapp_file(name: str) -> bool: def is_lib_file(name: str) -> bool: - for r in SUPPRESSED_PATHS: - if r.search(name): - if '/HABApp/' in name or '\\HABApp\\' in name: - continue - return True - return False + if name.startswith(HABAPP_MODULE_PATH): + return False + + return bool(name.startswith(SUPPRESSED_PATHS)) + def format_frame_info(tb: list[str], frame_info: FrameInfo, is_last=False) -> bool: diff --git a/src/HABApp/core/lib/exceptions/format_frame_vars.py b/src/HABApp/core/lib/exceptions/format_frame_vars.py index 0a579e13..7acc925a 100644 --- a/src/HABApp/core/lib/exceptions/format_frame_vars.py +++ b/src/HABApp/core/lib/exceptions/format_frame_vars.py @@ -55,7 +55,7 @@ def _filter_expressions(name: str, value: Any) -> bool: return False -SKIPPED_OBJS: Final = ( +SKIPPED_OBJS: Final[tuple[str, ...]] = ( 'HABApp.core.Items', ) @@ -143,6 +143,18 @@ def format_frame_variables(tb: list[str], stack_variables: list[Variable]): tb.append(SEPARATOR_VARIABLES) for name, value in variables.items(): - tb.append(f'{" " * (PRE_INDENT + 1)}{name} = {value!r}') + # both name and value can be a multiline string + # -> try to format it nicely + + name_lines = name.splitlines() + for line in name_lines[:-1]: + tb.append(f'{" " * (PRE_INDENT + 1):s}{line:s}') + + last_name_line = name_lines[-1] + for nr, line in enumerate(repr(value).splitlines()): + if not nr: + tb.append(f'{" " * (PRE_INDENT + 1):s}{last_name_line:s} = {line}') + else: + tb.append(f'{" " * (PRE_INDENT + 1):s}{" " * len(last_name_line):s} {line}') tb.append(SEPARATOR_VARIABLES) diff --git a/src/HABApp/core/lib/funcs.py b/src/HABApp/core/lib/funcs.py index f3869dc2..f32c17d5 100644 --- a/src/HABApp/core/lib/funcs.py +++ b/src/HABApp/core/lib/funcs.py @@ -1,27 +1,11 @@ import operator as _operator -from collections.abc import Iterable -from pathlib import Path -from typing import TYPE_CHECKING +from collections.abc import Callable +from typing import Any, Final from HABApp.core.const import MISSING -if TYPE_CHECKING: - import HABApp - - -def list_files(folder: Path, file_filter: 'HABApp.core.files.watcher.file_watcher.EventFilterBase', - recursive: bool = False) -> list[Path]: - # glob is much quicker than iter_dir() - files = folder.glob('**/*' if recursive else '*') - return sorted(filter(lambda x: file_filter.notify(str(x)), files), key=lambda x: x.relative_to(folder)) - - -def sort_files(files: Iterable[Path]) -> list[Path]: - return sorted(files) - - -CMP_OPS = { +CMP_OPS: Final[dict[str, Callable[[Any, Any], bool]]] = { 'lt': _operator.lt, 'lower_than': _operator.lt, 'le': _operator.le, 'lower_equal': _operator.le, 'eq': _operator.eq, 'equal': _operator.eq, @@ -34,7 +18,7 @@ def sort_files(files: Iterable[Path]) -> list[Path]: } -def compare(value, **kwargs) -> bool: +def compare(value: Any, **kwargs) -> bool: for name, cmp_value in kwargs.items(): if cmp_value is MISSING: diff --git a/src/HABApp/core/logger.py b/src/HABApp/core/logger.py index 5b3d9e27..8ddb016b 100644 --- a/src/HABApp/core/logger.py +++ b/src/HABApp/core/logger.py @@ -19,6 +19,7 @@ def log_error(logger: logging.Logger, text: str) -> None: logger.error(line) else: logger.error(text) + post_event( _T_ERRORS, text ) diff --git a/src/HABApp/core/shutdown.py b/src/HABApp/core/shutdown.py index 7bf89678..d86612f2 100644 --- a/src/HABApp/core/shutdown.py +++ b/src/HABApp/core/shutdown.py @@ -8,16 +8,20 @@ from dataclasses import dataclass from types import BuiltinMethodType, FunctionType, MethodType from typing import TYPE_CHECKING +from collections.abc import Awaitable -from HABApp.core.asyncio import async_context, create_task +from HABApp.core.asyncio import create_task from HABApp.core.const import loop if TYPE_CHECKING: - from collections.abc import Callable, Coroutine + from collections.abc import Callable from typing import Any, NoReturn +log = logging.getLogger('HABApp.Shutdown') + + @dataclass(frozen=True) class ShutdownBase: msg: str @@ -37,18 +41,18 @@ async def run(self) -> None: @dataclass(frozen=True) class ShutdownAwaitable(ShutdownBase): - func: Callable[[], Coroutine[Any, Any, Any]] + func: Callable[[], Awaitable[Any]] async def run(self) -> None: await self.func() -_REGISTERED: tuple[ShutdownBase, ...] = () +_REGISTERED: tuple[ShutdownFunction | ShutdownAwaitable, ...] = () _REQUESTED: bool = False -def register(func: Callable[[], Any], *, last: bool = False, msg: str = '') -> None: +def register(func: Callable[[], Any | Awaitable[Any]], *, last: bool = False, msg: str = '') -> None: global _REGISTERED if last is not True and last is not False: @@ -60,6 +64,17 @@ def register(func: Callable[[], Any], *, last: bool = False, msg: str = '') -> N if not msg: msg = f'{func.__module__}.{func.__name__}' + for existing in _REGISTERED: + if existing.func is func: + # If it's the same thing we don't call it multiple times + if existing.msg == msg and existing.last == last: + return None + + log.warning(f'Function {func} is already registered with a different message!') + log.warning(f' - {existing.msg:s}') + log.warning(f' - {msg:s}') + return None + if iscoroutinefunction(func): _REGISTERED += (ShutdownAwaitable(func=func, last=last, msg=msg), ) elif isinstance(func, (FunctionType, MethodType, BuiltinMethodType)): @@ -75,10 +90,6 @@ async def _shutdown() -> None: return None _REQUESTED = True - - async_context.set('Shutdown') - - log = logging.getLogger('HABApp.Shutdown') log.debug('Requested shutdown') objs = ( diff --git a/src/HABApp/core/types/color.py b/src/HABApp/core/types/color.py index b43242c2..b909ded2 100644 --- a/src/HABApp/core/types/color.py +++ b/src/HABApp/core/types/color.py @@ -88,7 +88,7 @@ def replace(self, r: int | None = None, g: int | None = None, b: int | None = No def __str__(self) -> str: return f'{self.__class__.__name__}({self._r:d}, {self._g:d}, {self._b})' - def __eq__(self, other): + def __eq__(self, other: object) -> bool: if isinstance(other, self.__class__): return self._r == other._r and self._g == other._g and self._b == other._b if isinstance(other, HSB): @@ -234,7 +234,7 @@ def replace(self, h: float | None = None, s: float | None = None, b: float | Non def __str__(self) -> str: return f'{self.__class__.__name__}({self._hue:.2f}, {self._saturation:.2f}, {self._brightness:.2f})' - def __eq__(self, other): + def __eq__(self, other: object) -> bool: if isinstance(other, self.__class__): return self._hue == other._hue and \ self._saturation == other._saturation and \ diff --git a/src/HABApp/core/wrapper.py b/src/HABApp/core/wrapper.py index 9ef8971a..28094d1f 100644 --- a/src/HABApp/core/wrapper.py +++ b/src/HABApp/core/wrapper.py @@ -1,13 +1,13 @@ import asyncio import functools import logging -import typing -from collections.abc import Callable +from collections.abc import Awaitable, Callable from logging import Logger # noinspection PyProtectedMember from sys import _getframe as sys_get_frame from types import TracebackType +from typing import ParamSpec, TypeVar, overload from HABApp.core.const.topics import TOPIC_ERRORS, TOPIC_WARNINGS from HABApp.core.events.habapp_events import HABAppException @@ -20,6 +20,10 @@ post_event = uses_post_event() +T = TypeVar('T') # the callable/awaitable return type +P = ParamSpec('P') # the callable parameters + + def process_exception(func: Callable | str, e: Exception, do_print=False, logger: logging.Logger = log) -> None: lines = format_exception(e) @@ -39,9 +43,17 @@ def process_exception(func: Callable | str, e: Exception, post_event(TOPIC_ERRORS, HABAppException(func_name=func_name, exception=e, traceback='\n'.join(lines))) +@overload +def log_exception(func: Callable[P, T]) -> Callable[P, T]: ... + + +@overload +def log_exception(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]: ... + + def log_exception(func): # return async wrapper - if asyncio.iscoroutinefunction(func) or asyncio.iscoroutine(func): + if asyncio.iscoroutinefunction(func): @functools.wraps(func) async def a(*args, **kwargs): try: @@ -65,9 +77,17 @@ def f(*args, **kwargs): return f +@overload +def ignore_exception(func: Callable[P, T]) -> Callable[P, T]: ... + + +@overload +def ignore_exception(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]: ... + + def ignore_exception(func): # return async wrapper - if asyncio.iscoroutinefunction(func) or asyncio.iscoroutine(func): + if asyncio.iscoroutinefunction(func): @functools.wraps(func) async def a(*args, **kwargs): try: @@ -89,7 +109,7 @@ def f(*args, **kwargs): class ExceptionToHABApp: - def __init__(self, logger: Logger | None = None, log_level: int = logging.ERROR, + def __init__(self, logger: Logger | None = None, log_level: int = logging.ERROR, *, ignore_exception: bool = True) -> None: self.log: Logger | None = logger self.log_level = log_level @@ -97,12 +117,14 @@ def __init__(self, logger: Logger | None = None, log_level: int = logging.ERROR, self.raised_exception = False - self.proc_tb: typing.Callable[[list], list] | None = None + self.proc_tb: Callable[[list], list] | None = None def __enter__(self) -> None: self.raised_exception = False - def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None): + def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, + exc_tb: TracebackType | None): + # no exception -> we exit gracefully if exc_type is None and exc_val is None: return True diff --git a/src/HABApp/mqtt/connection/connection.py b/src/HABApp/mqtt/connection/connection.py index 8add0739..ccd1b481 100644 --- a/src/HABApp/mqtt/connection/connection.py +++ b/src/HABApp/mqtt/connection/connection.py @@ -6,7 +6,6 @@ from aiomqtt import Client, MqttError import HABApp -from HABApp.core.asyncio import AsyncContext from HABApp.core.connections import AutoReconnectPlugin, BaseConnection, Connections, ConnectionStateToEventBusPlugin from HABApp.core.connections.base_connection import AlreadyHandledException from HABApp.core.connections.base_plugin import BaseConnectionPluginConnectedTask @@ -65,7 +64,7 @@ async def _mqtt_wrap_task(self) -> None: log = connection.log log.debug(f'{self.task.name} task start') try: - with AsyncContext('MQTT'), connection.handle_exception(self.mqtt_task): + with connection.handle_exception(self.mqtt_task): await self.mqtt_task() except AlreadyHandledException: pass diff --git a/src/HABApp/openhab/connection/plugins/events_sse.py b/src/HABApp/openhab/connection/plugins/events_sse.py index 0dd1b512..b32334cd 100644 --- a/src/HABApp/openhab/connection/plugins/events_sse.py +++ b/src/HABApp/openhab/connection/plugins/events_sse.py @@ -8,7 +8,6 @@ import HABApp import HABApp.core import HABApp.openhab.events -from HABApp.core.asyncio import AsyncContext from HABApp.core.connections import BaseConnectionPlugin from HABApp.core.const.json import load_json from HABApp.core.const.log import TOPIC_EVENTS @@ -35,51 +34,50 @@ async def on_disconnected(self) -> None: async def sse_task(self) -> None: try: - with AsyncContext('SSE'): - # cache so we don't have to look up every event - _load_json = load_json - _see_handler = on_sse_event - context = self.plugin_connection.context - oh3 = context.is_oh3 - - log_events = logging.getLogger(f'{TOPIC_EVENTS}.openhab') - DEBUG = logging.DEBUG - - async with sse_client.EventSource( - url=f'/rest/events?topics={HABApp.CONFIG.openhab.connection.topic_filter}', - session=context.session, **context.session_options) as event_source: - async for event in event_source: - - e_str = event.data - - try: - e_json = _load_json(e_str) - except (ValueError, TypeError): - log_events.warning(f'Invalid json: {e_str}') - continue - - # Alive event from openhab to detect dropped connections - # -> Can be ignored on the HABApp side - e_type = e_json.get('type') - if e_type == 'ALIVE': - continue - - # Log raw sse event - if log_events.isEnabledFor(logging.DEBUG): - log_events._log(DEBUG, e_str, []) - - # With OH4 we have the ItemStateUpdatedEvent, so we can ignore the ItemStateEvent - if not oh3 and e_type == 'ItemStateEvent': - continue - - # https://github.com/spacemanspiff2007/HABApp/issues/437 - # https://github.com/spacemanspiff2007/HABApp/issues/449 - # openHAB will automatically restore the future states of the item - # which means we can safely ignore these events because we will see the ItemStateUpdatedEvent - if e_type in ('ItemTimeSeriesUpdatedEvent', 'ItemTimeSeriesEvent'): - continue - - # process - _see_handler(e_json, oh3) + # cache so we don't have to look up every event + _load_json = load_json + _see_handler = on_sse_event + context = self.plugin_connection.context + oh3 = context.is_oh3 + + log_events = logging.getLogger(f'{TOPIC_EVENTS}.openhab') + DEBUG = logging.DEBUG + + async with sse_client.EventSource( + url=f'/rest/events?topics={HABApp.CONFIG.openhab.connection.topic_filter}', + session=context.session, **context.session_options) as event_source: + async for event in event_source: + + e_str = event.data + + try: + e_json = _load_json(e_str) + except (ValueError, TypeError): + log_events.warning(f'Invalid json: {e_str}') + continue + + # Alive event from openhab to detect dropped connections + # -> Can be ignored on the HABApp side + e_type = e_json.get('type') + if e_type == 'ALIVE': + continue + + # Log raw sse event + if log_events.isEnabledFor(logging.DEBUG): + log_events._log(DEBUG, e_str, []) + + # With OH4 we have the ItemStateUpdatedEvent, so we can ignore the ItemStateEvent + if not oh3 and e_type == 'ItemStateEvent': + continue + + # https://github.com/spacemanspiff2007/HABApp/issues/437 + # https://github.com/spacemanspiff2007/HABApp/issues/449 + # openHAB will automatically restore the future states of the item + # which means we can safely ignore these events because we will see the ItemStateUpdatedEvent + if e_type in ('ItemTimeSeriesUpdatedEvent', 'ItemTimeSeriesEvent'): + continue + + # process + _see_handler(e_json, oh3) except Exception as e: self.plugin_connection.process_exception(e, self.sse_task) diff --git a/src/HABApp/openhab/connection/plugins/plugin_things/plugin_things.py b/src/HABApp/openhab/connection/plugins/plugin_things/plugin_things.py index 5061d3f9..cb6ddd88 100644 --- a/src/HABApp/openhab/connection/plugins/plugin_things/plugin_things.py +++ b/src/HABApp/openhab/connection/plugins/plugin_things/plugin_things.py @@ -8,8 +8,6 @@ import HABApp.openhab.events from HABApp.core.connections import BaseConnectionPlugin from HABApp.core.files.file import HABAppFile -from HABApp.core.files.folders import add_folder as add_habapp_folder -from HABApp.core.files.watcher import AggregatingAsyncEventHandler from HABApp.core.lib import PendingFuture from HABApp.core.logger import HABAppError, log_warning from HABApp.openhab.connection.connection import OpenhabConnection @@ -48,6 +46,9 @@ async def on_setup(self): if path is None: return None + log.warning('TextualThingConfig deactivated') + return None + if self.watcher is not None: return None diff --git a/src/HABApp/openhab/connection/plugins/wait_for_startlevel.py b/src/HABApp/openhab/connection/plugins/wait_for_startlevel.py index 4cb9f811..dad94a95 100644 --- a/src/HABApp/openhab/connection/plugins/wait_for_startlevel.py +++ b/src/HABApp/openhab/connection/plugins/wait_for_startlevel.py @@ -1,10 +1,12 @@ from __future__ import annotations import asyncio +import logging import HABApp import HABApp.core import HABApp.openhab.events +from HABApp.config.models.openhab import General as OpenHABGeneralConfig from HABApp.core import shutdown from HABApp.core.connections import BaseConnectionPlugin from HABApp.core.lib import Timeout, ValueChange @@ -30,6 +32,20 @@ async def __on_connected_new(self, context: OpenhabContext, connection: OpenhabC # If openHAB is already running we have a fast exit path here if system_info.uptime >= oh_general.min_uptime and system_info.start_level >= oh_general.min_start_level: context.waited_for_openhab = False + + # Show a hint in case it's possible to increase the start level + # A higher start level means a more consistent startup and thus is more desirable + if system_info.start_level > oh_general.min_start_level: + _field_name_cfg = 'min_start_level' + if (alias := OpenHABGeneralConfig.model_fields[_field_name_cfg].alias) is not None: + _field_name_cfg = alias + + logging.getLogger('HABApp').info( + f'Openhab reached start level {system_info.start_level:d} but HABApp only waits until ' + f'level {oh_general.min_start_level:d} is reached. ' + f'Consider increasing "{_field_name_cfg:s}" in the HABApp configuration. ' + ) + return None log = connection.log diff --git a/src/HABApp/openhab/items/switch_item.py b/src/HABApp/openhab/items/switch_item.py index 4924dae2..ece44878 100644 --- a/src/HABApp/openhab/items/switch_item.py +++ b/src/HABApp/openhab/items/switch_item.py @@ -1,5 +1,5 @@ from collections.abc import Mapping -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Any, Final from HABApp.core.errors import InvalidItemValue, ItemValueIsNoneError from HABApp.openhab.definitions import OnOffValue @@ -30,11 +30,12 @@ class SwitchItem(OpenhabItem, OnOffCommand): @staticmethod def _state_from_oh_str(state: str): - if state != ON and state != OFF: - raise ValueError(f'Invalid value for SwitchItem: {state}') + if state not in (ON, OFF): + msg = f'Invalid value for SwitchItem: {state}' + raise ValueError(msg) return state - def set_value(self, new_value) -> bool: + def set_value(self, new_value: str | None) -> bool: if isinstance(new_value, OnOffValue): new_value = new_value.value @@ -66,7 +67,7 @@ def toggle(self): def __str__(self) -> str: return str(self.value) - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, SwitchItem): return self.value == other.value if isinstance(other, str): diff --git a/src/HABApp/parameters/parameter_files.py b/src/HABApp/parameters/parameter_files.py index 023280d9..10646af0 100644 --- a/src/HABApp/parameters/parameter_files.py +++ b/src/HABApp/parameters/parameter_files.py @@ -1,10 +1,11 @@ import logging +import re import threading from pathlib import Path import HABApp from HABApp.core.files.file import HABAppFile -from HABApp.core.files.folders import add_folder as add_habapp_folder +from HABApp.core.internals.proxy import uses_file_manager from .parameters import get_parameter_file, remove_parameter_file, set_parameter_file @@ -14,6 +15,8 @@ LOCK = threading.Lock() PARAM_PREFIX = 'params/' +file_manager = uses_file_manager() + async def load_file(name: str, path: Path) -> None: with LOCK: # serialize to get proper error messages @@ -59,10 +62,12 @@ async def setup_param_files() -> bool: if path is None: return False - folder = add_habapp_folder(PARAM_PREFIX, path, 100) - folder.add_file_type(HABAppParameterFile) - watcher = folder.add_watch('.yml') - await watcher.trigger_all() + prefix = 'params/' + file_manager.add_handler('ParamFiles', log, prefix=prefix, on_load=load_file, on_unload=unload_file) + file_manager.add_folder( + prefix, path, priority=100, pattern=re.compile(r'.yml$', re.IGNORECASE), name='rules-parameters' + ) + return True diff --git a/src/HABApp/rule/rule.py b/src/HABApp/rule/rule.py index 464fa723..fea9bf36 100644 --- a/src/HABApp/rule/rule.py +++ b/src/HABApp/rule/rule.py @@ -83,11 +83,13 @@ def __init__(self) -> None: self.openhab: Final = self.oh def on_rule_loaded(self) -> None: - """Override this to implement logic that will be called when the rule and the file has been successfully loaded + """Override this to method to implement logic that will be called when + the rule and the file has been successfully loaded. Can be sync or async. """ def on_rule_removed(self) -> None: - """Override this to implement logic that will be called when the rule has been unloaded. + """Override this method to implement logic that will be called when the rule has been unloaded. + Can be sync or async. """ def __repr__(self) -> str: diff --git a/src/HABApp/rule/scheduler/job_builder.py b/src/HABApp/rule/scheduler/job_builder.py index a1ea7932..70468a87 100644 --- a/src/HABApp/rule/scheduler/job_builder.py +++ b/src/HABApp/rule/scheduler/job_builder.py @@ -14,7 +14,7 @@ from eascheduler.schedulers.async_scheduler import AsyncScheduler from typing_extensions import ParamSpec, Self, override -from HABApp.core.asyncio import async_context, create_task_from_async, run_func_from_async +from HABApp.core.asyncio import create_task_from_async, run_func_from_async from HABApp.core.const import loop from HABApp.core.internals import Context, wrap_func from HABApp.core.internals.wrapped_function.wrapped_async import WrappedAsyncFunction @@ -67,15 +67,7 @@ def wrapped_func_executor(func: Any, args: Iterable = (), kwargs: Mapping[str, A class AsyncHABAppScheduler(AsyncScheduler): @override - def run_jobs(self) -> None: - ctx = async_context.set('Scheduler') - try: - super().run_jobs() - finally: - async_context.reset(ctx) - - @override - def set_enabled(self, enabled: bool) -> Self: # noqa: FBT001 + def set_enabled(self, enabled: bool) -> Self: return run_func_from_async(super().set_enabled, enabled) diff --git a/src/HABApp/rule_ctx/rule_ctx.py b/src/HABApp/rule_ctx/rule_ctx.py index 7741a3ce..6d756346 100644 --- a/src/HABApp/rule_ctx/rule_ctx.py +++ b/src/HABApp/rule_ctx/rule_ctx.py @@ -5,7 +5,7 @@ import HABApp from HABApp.core.const.topics import ALL_TOPICS -from HABApp.core.internals import Context, EventBusListener, uses_event_bus, uses_item_registry +from HABApp.core.internals import Context, EventBusListener, uses_event_bus, uses_item_registry, wrap_func from HABApp.core.internals.event_bus import EventBusBaseListener @@ -40,7 +40,7 @@ def remove_event_listener(self, listener: TB) -> TB: event_bus.remove_listener(listener) return listener - def unload_rule(self) -> None: + async def unload_rule(self) -> None: with HABApp.core.wrapper.ExceptionToHABApp(log): rule = self.rule @@ -61,12 +61,12 @@ def unload_rule(self) -> None: rule._habapp_rule_ctx = None # user implementation - rule.on_rule_removed() + await wrap_func(rule.on_rule_removed).async_run() - def check_rule(self) -> None: + async def check_rule(self) -> None: with HABApp.core.wrapper.ExceptionToHABApp(log): # We need items if we want to run the test - if item_registry.get_items(): + if item_registry: # Check if we have a valid item for all listeners for listener in self.objs: @@ -86,4 +86,4 @@ def check_rule(self) -> None: self.rule.run._scheduler.set_enabled(True) # user implementation - self.rule.on_rule_loaded() + await wrap_func(self.rule.on_rule_loaded).async_run() diff --git a/src/HABApp/rule_manager/rule_file.py b/src/HABApp/rule_manager/rule_file.py index 3cf98c64..58ac2e4d 100644 --- a/src/HABApp/rule_manager/rule_file.py +++ b/src/HABApp/rule_manager/rule_file.py @@ -38,11 +38,11 @@ def suggest_rule_name(self, obj: 'HABApp.Rule') -> str: return f'{name:s}.{found:d}' if found > 1 else f'{name:s}' - def check_all_rules(self) -> None: + async def check_all_rules(self) -> None: for rule in self.rules.values(): # type: HABApp.Rule - get_current_context(rule).check_rule() + await get_current_context(rule).check_rule() - def unload(self): + async def unload(self) -> None: # If we don't have any rules we can not unload if not self.rules: @@ -50,7 +50,7 @@ def unload(self): # unload all registered callbacks for rule in self.rules.values(): # type: HABApp.Rule - get_current_context(rule).unload_rule() + await get_current_context(rule).unload_rule() log.debug(f'File {self.name} successfully unloaded!') return None diff --git a/src/HABApp/rule_manager/rule_manager.py b/src/HABApp/rule_manager/rule_manager.py index 397ad8a7..cedd604f 100644 --- a/src/HABApp/rule_manager/rule_manager.py +++ b/src/HABApp/rule_manager/rule_manager.py @@ -1,4 +1,5 @@ import logging +import re import threading import typing from asyncio import sleep @@ -9,10 +10,8 @@ from HABApp.core import shutdown from HABApp.core.connections import Connections from HABApp.core.files.errors import AlreadyHandledFileError -from HABApp.core.files.file import HABAppFile -from HABApp.core.files.folders import add_folder as add_habapp_folder -from HABApp.core.files.watcher import AggregatingAsyncEventHandler from HABApp.core.internals import uses_item_registry +from HABApp.core.internals.proxy import uses_file_manager from HABApp.core.internals.wrapped_function import wrap_func from HABApp.core.logger import log_warning from HABApp.core.wrapper import log_exception @@ -22,6 +21,7 @@ log = logging.getLogger('HABApp.Rules') item_registry = uses_item_registry() +file_manager = uses_file_manager() class RuleManager: @@ -36,11 +36,6 @@ def __init__(self, parent) -> None: self.__load_lock = threading.Lock() self.__files_lock = threading.Lock() - # Processing - self.__process_last_sec = 60 - - self.watcher: AggregatingAsyncEventHandler | None = None - async def setup(self): # shutdown @@ -54,18 +49,18 @@ async def setup(self): log.error('Failed to load Benchmark!') shutdown.request() return None - file.check_all_rules() - return - - class HABAppRuleFile(HABAppFile): - LOGGER = log - LOAD_FUNC = self.request_file_load - UNLOAD_FUNC = self.request_file_unload + await file.check_all_rules() path = HABApp.CONFIG.directories.rules - folder = add_habapp_folder('rules/', path, 0) - folder.add_file_type(HABAppRuleFile) - self.watcher = folder.add_watch('.py', True) + prefix = 'rules/' + + file_manager.add_handler( + self.__class__.__name__, log, prefix=prefix, + on_load=self.request_file_load, on_unload=self.request_file_unload + ) + file_manager.add_folder( + prefix, path, priority=0, pattern=re.compile(r'.py$', re.IGNORECASE), name='rules-python' + ) # Initial loading of rules HABApp.core.internals.wrap_func(self.load_rules_on_startup, logger=log).run() @@ -84,7 +79,7 @@ async def load_rules_on_startup(self): return None # trigger event for every file - await self.watcher.trigger_all() + await file_manager.get_file_watcher().load_files(name_include=r'^rules.*$') return None @log_exception @@ -125,7 +120,7 @@ async def request_file_unload(self, name: str, path: Path, request_lock=True): with self.__files_lock: rule = self.files.pop(path_str) - await wrap_func(rule.unload).async_run() + await rule.unload() finally: if request_lock: self.__load_lock.release() @@ -163,8 +158,8 @@ async def request_file_load(self, name: str, path: Path): log.debug(f'File {name} successfully loaded!') # Do simple checks which prevent errors - file.check_all_rules() + await file.check_all_rules() - def shutdown(self) -> None: + async def shutdown(self) -> None: for f in self.files.values(): - f.unload() + await f.unload() diff --git a/src/HABApp/runtime/runtime.py b/src/HABApp/runtime/runtime.py index 05675b9e..1b9497a3 100644 --- a/src/HABApp/runtime/runtime.py +++ b/src/HABApp/runtime/runtime.py @@ -12,7 +12,6 @@ import HABApp.rule_manager import HABApp.util from HABApp.core import Connections, shutdown -from HABApp.core.asyncio import async_context from HABApp.core.internals import setup_internals from HABApp.core.internals.proxy import ConstProxyObj from HABApp.core.wrapper import process_exception @@ -22,37 +21,35 @@ class Runtime: def __init__(self) -> None: - self.config: HABApp.config.Config = None - # Rule engine self.rule_manager: HABApp.rule_manager.RuleManager = None async def start(self, config_folder: Path) -> None: try: - token = async_context.set('HABApp startup') - # shutdown setup shutdown.register(Connections.on_application_shutdown, msg='Shutting down connections') # setup exception handler for the scheduler eascheduler.set_exception_handler(lambda x: process_exception('HABApp.scheduler', x)) - # Start Folder watcher! - HABApp.core.files.watcher.start() - - # Load config - HABApp.config.load_config(config_folder) + file_watcher = HABApp.core.files.HABAppFileWatcher() + shutdown.register(file_watcher.shutdown, msg='Shutdown file watcher') # replace proxy objects ir = HABApp.core.internals.ItemRegistry() eb = HABApp.core.internals.EventBus() - setup_internals(ir, eb) + file_manager = HABApp.core.files.FileManager(file_watcher) + + setup_internals(ir, eb, file_manager) assert isinstance(HABApp.core.Items, ConstProxyObj) HABApp.core.Items = ir assert isinstance(HABApp.core.EventBus, ConstProxyObj) HABApp.core.EventBus = eb - await HABApp.core.files.setup() + file_manager.setup() + + # Load config + HABApp.config.setup_habapp_configuration(config_folder) # generic HTTP await HABApp.rule.interfaces._http.create_client() @@ -71,8 +68,6 @@ async def start(self, config_folder: Path) -> None: Connections.application_startup_complete() - async_context.reset(token) - except HABApp.config.InvalidConfigError: shutdown.request() except Exception as e: diff --git a/src/HABApp/util/rate_limiter/limits/base.py b/src/HABApp/util/rate_limiter/limits/base.py index a7c058cd..af5f0858 100644 --- a/src/HABApp/util/rate_limiter/limits/base.py +++ b/src/HABApp/util/rate_limiter/limits/base.py @@ -35,13 +35,13 @@ def __repr__(self) -> str: f'{self.repr_text():s}>' ) - def do_test_allow(self): + def do_test_allow(self) -> None: raise NotImplementedError() - def do_allow(self): + def do_allow(self) -> None: raise NotImplementedError() - def do_deny(self): + def do_deny(self) -> None: raise NotImplementedError() def info(self) -> BaseRateLimitInfo: diff --git a/src/HABApp/util/rate_limiter/limits/fixed_window.py b/src/HABApp/util/rate_limiter/limits/fixed_window.py index 5aa46155..1fa1a646 100644 --- a/src/HABApp/util/rate_limiter/limits/fixed_window.py +++ b/src/HABApp/util/rate_limiter/limits/fixed_window.py @@ -1,6 +1,8 @@ from dataclasses import dataclass from time import monotonic +from typing_extensions import override + from .base import BaseRateLimit, BaseRateLimitInfo @@ -16,14 +18,17 @@ def __init__(self, allowed: int, interval: int, hits: int = 0) -> None: self.start: float = -1.0 self.stop: float = -1.0 + @override def repr_text(self) -> str: return f'window={self.stop - self.start:.0f}s' + @override def do_test_allow(self) -> None: if self.stop <= monotonic(): self.hits = 0 self.skips = 0 + @override def do_allow(self) -> None: now = monotonic() @@ -33,9 +38,11 @@ def do_allow(self) -> None: self.start = now self.stop = now + self.interval + @override def do_deny(self) -> None: self.stop = monotonic() + self.interval + @override def info(self) -> FixedWindowElasticExpiryLimitInfo: self.do_test_allow() diff --git a/src/HABApp/util/rate_limiter/limits/leaky_bucket.py b/src/HABApp/util/rate_limiter/limits/leaky_bucket.py index 650f6ac9..73df7148 100644 --- a/src/HABApp/util/rate_limiter/limits/leaky_bucket.py +++ b/src/HABApp/util/rate_limiter/limits/leaky_bucket.py @@ -2,6 +2,8 @@ from time import monotonic from typing import Final +from typing_extensions import override + from .base import BaseRateLimit, BaseRateLimitInfo @@ -17,9 +19,11 @@ def __init__(self, allowed: int, interval: int, hits: int = 0) -> None: self.drop_interval: Final = interval / allowed self.next_drop: float = monotonic() + self.drop_interval + @override def repr_text(self) -> str: return f'drop_interval={self.drop_interval:.1f}s' + @override def do_test_allow(self) -> None: while self.next_drop <= monotonic(): @@ -38,6 +42,7 @@ def do_test_allow(self) -> None: do_allow = do_test_allow do_deny = None + @override def info(self) -> LeakyBucketLimitInfo: self.do_test_allow() diff --git a/src/HABApp/util/rate_limiter/registry.py b/src/HABApp/util/rate_limiter/registry.py index 048c7e40..ff14c0a4 100644 --- a/src/HABApp/util/rate_limiter/registry.py +++ b/src/HABApp/util/rate_limiter/registry.py @@ -10,10 +10,10 @@ _LIMITERS: dict[str, Limiter] = {} -def RateLimiter(name: str) -> Limiter: +def RateLimiter(name: str) -> Limiter: # noqa: N802 """Create a new rate limiter or return an already existing one with a given name. - :param name: case insensitive name of limiter + :param name: case-insensitive name of limiter :return: Rate limiter object """ diff --git a/tests/conftest.py b/tests/conftest.py index 714afd8f..e260f948 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,7 @@ import pytest import HABApp -from HABApp.core.asyncio import async_context +from HABApp.core.files import FileManager from HABApp.core.internals import EventBus, ItemRegistry, setup_internals from tests.helpers import LogCollector, eb, get_dummy_cfg, params, parent_rule, sync_worker from tests.helpers.log.log_matcher import AsyncDebugWarningMatcher, LogLevelMatcher @@ -52,27 +52,28 @@ def use_dummy_cfg(monkeypatch): @pytest.fixture(autouse=True, scope='session') def event_loop(): - token = async_context.set('pytest') - yield HABApp.core.const.loop - async_context.reset(token) - @pytest.fixture() def ir(): return ItemRegistry() +@pytest.fixture() +def file_manager(): + return FileManager(None) + + @pytest.fixture(autouse=True) -def clean_objs(ir: ItemRegistry, eb: EventBus, request): +def clean_objs(ir: ItemRegistry, eb: EventBus, file_manager: FileManager, request): markers = request.node.own_markers for marker in markers: if marker.name == 'no_internals': yield None return None - restore = setup_internals(ir, eb, final=False) + restore = setup_internals(ir, eb, file_manager, final=False) yield diff --git a/tests/helpers/log/log_collector.py b/tests/helpers/log/log_collector.py index c881a8e0..a9a98331 100644 --- a/tests/helpers/log/log_collector.py +++ b/tests/helpers/log/log_collector.py @@ -87,7 +87,8 @@ def update(self) -> Self: for phase in self.phases: if phase not in ALL_PYTEST_PHASES: - raise ValueError(f'Unknown pytest phase: {phase}') + msg = f'Unknown pytest phase: {phase}' + raise ValueError(msg) prev_rec = None for record in self.caplog.get_records(phase): diff --git a/tests/helpers/traceback.py b/tests/helpers/traceback.py index b3d51dcb..ee40c295 100644 --- a/tests/helpers/traceback.py +++ b/tests/helpers/traceback.py @@ -12,6 +12,10 @@ def remove_dyn_parts_from_traceback(traceback: str) -> str: fname = '/'.join(Path(m.group(1)).parts[-3:]) traceback = traceback.replace(m.group(0), f'File "{fname}"') + # Line nrs + traceback = re.sub(r'line\s+(\d+)', 'line x', traceback) + traceback = re.sub(r'^(-->|\s{3})\s{2}\d+ \|', '\g<1> x |', traceback, flags=re.MULTILINE) + return traceback @@ -22,12 +26,18 @@ def test_remove_dyn_parts_from_traceback() -> None: File "/My/Folder/HABApp/tests/test_core/test_lib/test_format_traceback.py", line 19 in exec_func func = File "C:\\My\\Folder\\HABApp\\tests\\test_core\\test_lib\\test_format_traceback.py", line 19, in exec_func + 16 | try: +--> 17 | func() + 18 | except Exception as e: ''' processed = remove_dyn_parts_from_traceback(traceback) assert processed == ''' -File "test_core/test_lib/test_format_traceback.py", line 19 in exec_func -File "test_core/test_lib/test_format_traceback.py", line 19 in exec_func +File "test_core/test_lib/test_format_traceback.py", line x in exec_func +File "test_core/test_lib/test_format_traceback.py", line x in exec_func func = - File "test_core/test_lib/test_format_traceback.py", line 19, in exec_func + File "test_core/test_lib/test_format_traceback.py", line x, in exec_func + x | try: +--> x | func() + x | except Exception as e: ''' diff --git a/tests/rule_runner/rule_runner.py b/tests/rule_runner/rule_runner.py index 6ef25c65..91dfc74e 100644 --- a/tests/rule_runner/rule_runner.py +++ b/tests/rule_runner/rule_runner.py @@ -1,4 +1,3 @@ -import asyncio from types import TracebackType from astral import Observer @@ -9,7 +8,7 @@ import HABApp.core.lib.exceptions.format import HABApp.rule.rule as rule_module import HABApp.rule.scheduler.job_builder as job_builder_module -from HABApp.core.asyncio import async_context +from HABApp.core.files import FileManager from HABApp.core.internals import EventBus, ItemRegistry, setup_internals from HABApp.core.internals.proxy import ConstProxyObj from HABApp.core.internals.wrapped_function import wrapped_thread, wrapper @@ -19,6 +18,19 @@ from HABApp.runtime import Runtime +# def _get_loop_modules() -> tuple[ModuleType, ...]: +# ret = [] +# for module in habapp_modules(): +# for name, obj in getmembers(module): +# if obj is loop: +# ret.append(module) +# assert name == 'loop' +# return tuple(ret) +# +# +# LOOP_MODULES = _get_loop_modules() + + def suggest_rule_name(obj: object) -> str: return f'TestRule.{obj.__class__.__name__}' @@ -65,7 +77,6 @@ def __init__(self) -> None: self.monkeypatch = MonkeyPatch() self.restore = [] - self.ctx = asyncio.Future() def submit(self, callback, *args, **kwargs) -> None: # This executes the callback so we can not ignore exceptions @@ -76,12 +87,10 @@ def set_up(self) -> None: assert isinstance(HABApp.core.Items, ConstProxyObj) assert isinstance(HABApp.core.EventBus, ConstProxyObj) - # prevent we're calling from asyncio - this works because we don't use threads - self.ctx = async_context.set('Rule Runner') - ir = ItemRegistry() eb = EventBus() - self.restore = setup_internals(ir, eb, final=False) + file_manager = FileManager(None) + self.restore = setup_internals(ir, eb, file_manager, final=False) # Scheduler self.monkeypatch.setattr(prod_sun_module, 'OBSERVER', Observer(52.51870523376821, 13.376072914752532, 10)) @@ -105,18 +114,11 @@ def set_up(self) -> None: self.monkeypatch.setattr(job_builder_module, 'AsyncHABAppScheduler', SyncScheduler) def tear_down(self) -> None: - - for rule in self.loaded_rules: - rule._habapp_ctx.unload_rule() self.loaded_rules.clear() # restore patched self.monkeypatch.undo() - # restore async context - async_context.reset(self.ctx) - self.ctx = None - for r in self.restore: r.restore() @@ -128,7 +130,8 @@ def process_events(self) -> None: def __enter__(self) -> None: self.set_up() - def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None) -> bool: + def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, + exc_tb: TracebackType | None) -> bool: self.tear_down() # do not supress exception return False diff --git a/tests/test_core/test_context.py b/tests/test_core/test_context.py index a799ef53..57ea1fef 100644 --- a/tests/test_core/test_context.py +++ b/tests/test_core/test_context.py @@ -1,15 +1,17 @@ import pytest -from HABApp.core.asyncio import AsyncContextError, async_context +from HABApp.core.asyncio import AsyncContextError, thread_context async def test_error_msg() -> None: def my_sync_func(): - if async_context.get(None) is not None: + if thread_context.get(None) is None: raise AsyncContextError(my_sync_func) - async_context.set('Test') with pytest.raises(AsyncContextError) as e: my_sync_func() assert str(e.value) == 'Function "my_sync_func" may not be called from an async context!' + + thread_context.set('Test') + my_sync_func() diff --git a/tests/test_core/test_files/test_file.py b/tests/test_core/test_files/test_file.py new file mode 100644 index 00000000..1b30ef2e --- /dev/null +++ b/tests/test_core/test_files/test_file.py @@ -0,0 +1,65 @@ +from pathlib import Path + +import pytest + +from HABApp.core.files.file import CircularReferenceError, FileProperties, FileState, HABAppFile +from HABApp.core.files.manager import log as file_manager_logger +from tests.helpers import LogCollector + + +@pytest.fixture +def files(file_manager) -> dict[str, HABAppFile]: + assert not file_manager._files + return file_manager._files + + +def test_depends(test_logs: LogCollector, files, file_manager) -> None: + files['name1'] = f1 = HABAppFile('name1', Path('path1'), b'checksum', FileProperties(depends_on=['name2'])) + files['name2'] = f2 = HABAppFile('name2', Path('path2'), b'checksum', FileProperties()) + + f1.check_properties(file_manager, file_manager_logger, log_msg=True) + f2.check_properties(file_manager, file_manager_logger, log_msg=True) + + assert f1._state is FileState.DEPENDENCIES_MISSING + assert f2._state is FileState.DEPENDENCIES_OK + + f2._state = FileState.LOADED + f1.check_dependencies(file_manager) + assert f1._state is FileState.DEPENDENCIES_OK + + files['name3'] = f3 = HABAppFile('name3', Path('path3'), b'checksum', FileProperties(depends_on=['asdf'])) + f3.check_properties(file_manager, file_manager_logger, log_msg=True) + test_logs.add_expected('HABApp.files', 'ERROR', "File path3 depends on file that doesn't exist: asdf") + + +def test_reloads(test_logs: LogCollector, files, file_manager) -> None: + files['name1'] = f1 = HABAppFile('name1', Path('path1'), b'checksum', FileProperties(reloads_on=['name2', 'asdf'])) + files['name2'] = f2 = HABAppFile('name2', Path('path2'), b'checksum', FileProperties()) + + f1.check_properties(file_manager, file_manager_logger) + assert f1.properties.reloads_on == ['name2', 'asdf'] + assert f2.properties.reloads_on == [] + + test_logs.add_expected('HABApp.files', 'WARNING', "File path1 reloads on file that doesn't exist: asdf") + + +def test_circ(test_logs: LogCollector, files, file_manager) -> None: + files['name1'] = f1 = HABAppFile('name1', Path('path1'), b'checksum', FileProperties(depends_on=['name2'])) + files['name2'] = f2 = HABAppFile('name2', Path('path2'), b'checksum', FileProperties(depends_on=['name3'])) + files['name3'] = f3 = HABAppFile('name3', Path('path3'), b'checksum', FileProperties(depends_on=['name1'])) + + with pytest.raises(CircularReferenceError) as e: + f1._check_circ_refs((f1.name,), 'depends_on', file_manager) + assert e.value.stack == ('name1', 'name2', 'name3', 'name1') + + # Check log output + f1.check_properties(file_manager, file_manager_logger) + test_logs.add_expected('HABApp.files', 'ERROR', 'Circular reference: name1 -> name2 -> name3 -> name1') + + with pytest.raises(CircularReferenceError) as e: + f2._check_circ_refs((f2.name,), 'depends_on', file_manager) + assert e.value.stack == ('name2', 'name3', 'name1', 'name2',) + + with pytest.raises(CircularReferenceError) as e: + f3._check_circ_refs((f3.name,), 'depends_on', file_manager) + assert e.value.stack == ('name3', 'name1', 'name2', 'name3', ) diff --git a/tests/test_core/test_files/test_file_dependencies.py b/tests/test_core/test_files/test_file_dependencies.py deleted file mode 100644 index ea0034f8..00000000 --- a/tests/test_core/test_files/test_file_dependencies.py +++ /dev/null @@ -1,212 +0,0 @@ -import logging -from asyncio import sleep -from pathlib import Path - -import pytest - -import HABApp -from HABApp.core.files.file.file import FileProperties, HABAppFile -from HABApp.core.files.folders import add_folder -from HABApp.core.files.folders.folders import FOLDERS -from HABApp.core.files.manager import process_file -from tests.helpers import LogCollector - - -class MockFile: - def __init__(self, name: str) -> None: - self.name = name.split('/')[1] - - def as_posix(self) -> str: - return f'/my_param/{self.name}' - - def is_file(self) -> bool: - return True - - def __repr__(self) -> str: - return f'' - - -class CfgObj: - def __init__(self) -> None: - self.properties = {} - self.operation: list[tuple[str, str]] = [] - - class TestFile(HABAppFile): - LOGGER = logging.getLogger('test') - LOAD_FUNC = self.load_file - UNLOAD_FUNC = self.unload_file - self.cls = TestFile - - async def load_file(self, name: str, path: Path) -> None: - self.operation.append(('load', name)) - - async def unload_file(self, name: str, path: Path) -> None: - self.operation.append(('unload', name)) - - async def wait_complete(self) -> None: - while HABApp.core.files.manager.worker.TASK is not None: - await sleep(0.05) - - async def process_file(self, name: str) -> None: - await process_file(name, MockFile(name)) - - def create_file(self, name, path) -> HABAppFile: - return self.cls(name, MockFile(name), self.properties[name]) - - -@pytest.fixture() -def cfg(monkeypatch): - obj = CfgObj() - - monkeypatch.setattr(HABApp.core.files.manager.worker, 'TASK_SLEEP', 0.001) - monkeypatch.setattr(HABApp.core.files.manager.worker, 'TASK_DURATION', 0.001) - monkeypatch.setattr(HABApp.core.files.file, 'create_file', obj.create_file) - - FOLDERS.clear() - add_folder('rules/', Path('c:/HABApp/my_rules/'), 0) - add_folder('configs/', Path('c:/HABApp/my_config/'), 10) - add_folder('params/', Path('c:/HABApp/my_param/'), 20) - - yield obj - - FOLDERS.clear() - - -# def test_reload_on(cfg, sync_worker, event_bus: TmpEventBus): -# order = [] -# -# def process_event(event): -# order.append(event.name) -# file_load_ok(event.name) -# -# FILE_PROPS.clear() -# FILE_PROPS['params/param1'] = FileProperties(depends_on=[], reloads_on=['params/param2']) -# FILE_PROPS['params/param2'] = FileProperties() -# -# event_bus.listen_events(HABApp.core.const.topics.TOPIC_FILES, process_event) -# -# process([MockFile('param2'), MockFile('param1')]) -# -# assert order == ['params/param1', 'params/param2', 'params/param1'] -# order.clear() -# -# process([]) -# assert order == [] -# -# process([MockFile('param2')]) -# assert order == ['params/param2', 'params/param1'] -# order.clear() -# -# process([MockFile('param1')]) -# assert order == ['params/param1'] -# order.clear() -# -# process([MockFile('param2')]) -# assert order == ['params/param2', 'params/param1'] -# order.clear() - - -async def test_reload_dep(cfg: CfgObj, caplog) -> None: - cfg.properties['params/param1'] = FileProperties(depends_on=['params/param2'], reloads_on=['params/param2']) - cfg.properties['params/param2'] = FileProperties() - - await cfg.process_file('params/param1') - await cfg.process_file('params/param2') - await cfg.wait_complete() - - assert cfg.operation == [('load', 'params/param2'), ('load', 'params/param1')] - cfg.operation.clear() - - await cfg.process_file('params/param2') - await cfg.wait_complete() - assert cfg.operation == [('load', 'params/param2'), ('load', 'params/param1')] - cfg.operation.clear() - - await cfg.process_file('params/param1') - await cfg.wait_complete() - assert cfg.operation == [('load', 'params/param1')] - cfg.operation.clear() - - await cfg.process_file('params/param2') - await cfg.wait_complete() - assert cfg.operation == [('load', 'params/param2'), ('load', 'params/param1')] - cfg.operation.clear() - - -async def test_missing_dependencies(cfg: CfgObj, test_logs: LogCollector) -> None: - cfg.properties['params/param1'] = FileProperties(depends_on=['params/param4', 'params/param5']) - cfg.properties['params/param2'] = FileProperties(depends_on=['params/param4']) - cfg.properties['params/param3'] = FileProperties() - - await cfg.process_file('params/param1') - await cfg.process_file('params/param2') - await cfg.process_file('params/param3') - await cfg.wait_complete() - - assert cfg.operation == [('load', 'params/param3')] - - msg1 = ( - 'HABApp.files', logging.ERROR, "File depends on file that doesn't exist: params/param4" - ) - msg2 = ( - 'HABApp.files', logging.ERROR, - "File depends on files that don't exist: params/param4, params/param5" - ) - - test_logs.add_expected(*msg1) - test_logs.add_expected(*msg2) - - -# def test_missing_loads(cfg, sync_worker, event_bus: TmpEventBus, caplog): -# order = [] -# -# def process_event(event): -# order.append(event.name) -# file_load_ok(event.name) -# -# FILE_PROPS['params/param1'] = FileProperties(reloads_on=['params/param4', 'params/param5']) -# FILE_PROPS['params/param2'] = FileProperties(reloads_on=['params/param4']) -# -# event_bus.listen_events(HABApp.core.const.topics.TOPIC_FILES, process_event) -# -# process([MockFile('param1'), MockFile('param2')]) -# -# assert order == ['params/param1', 'params/param2'] -# order.clear() -# -# process([]) -# assert order == [] -# -# msg1 = ( -# 'HABApp.files', logging.WARNING, "File reloads on file that doesn't exist: params/param4" -# ) -# msg2 = ('HABApp.files', logging.WARNING, -# "File reloads on files that don't exist: params/param4, params/param5") -# -# assert msg1 in caplog.record_tuples -# assert msg2 in caplog.record_tuples -# -# -# def test_load_continue_after_missing(cfg, sync_worker, event_bus: TmpEventBus, caplog): -# order = [] -# -# def process_event(event): -# order.append(event.name) -# file_load_ok(event.name) -# -# FILE_PROPS.clear() -# FILE_PROPS['params/p1'] = FileProperties(depends_on=['params/p2'], reloads_on=[]) -# FILE_PROPS['params/p2'] = FileProperties() -# -# event_bus.listen_events(HABApp.core.const.topics.TOPIC_FILES, process_event) -# -# process([MockFile('p1')]) -# -# # File can not be loaded -# assert order == [] -# -# # Add missing file -# process([MockFile('p2')]) -# -# # Both files get loaded -# assert order == ['params/p2', 'params/p1'] diff --git a/tests/test_core/test_files/test_file_manager.py b/tests/test_core/test_files/test_file_manager.py new file mode 100644 index 00000000..bf9332c5 --- /dev/null +++ b/tests/test_core/test_files/test_file_manager.py @@ -0,0 +1,117 @@ +from collections.abc import Awaitable, Callable +from pathlib import Path +from unittest.mock import Mock + +import pytest + +import HABApp +from HABApp.core.const.topics import TOPIC_FILES +from HABApp.core.events.habapp_events import RequestFileLoadEvent, RequestFileUnloadEvent +from HABApp.core.files import FileManager +from HABApp.core.files import manager as file_manager_module +from HABApp.core.files.errors import AlreadyHandledFileError +from HABApp.core.files.file import FileProperties, FileState, HABAppFile +from HABApp.core.files.manager import log as file_manager_logger +from HABApp.core.internals import EventBus +from tests.helpers import LogCollector + + +async def test_file_watcher_event(monkeypatch, file_manager, test_logs: LogCollector) -> None: + test_logs.set_min_level(0) + + file_manager.add_folder('tests-', Path('tests/'), name='d', priority=1) + + eb = Mock(EventBus) + eb.post_event = Mock() + eb.post_event.assert_not_called() + monkeypatch.setattr(HABApp.core, 'EventBus', eb, raising=False) + + path_instance = Mock() + path_instance.is_dir = Mock(return_value=False) + monkeypatch.setattr(file_manager_module, 'Path', Mock(return_value=path_instance)) + + # Unload + path_instance.is_file = Mock(return_value=False) + await file_manager.file_watcher_event('tests/asdf') + eb.post_event.assert_called_once_with(TOPIC_FILES, RequestFileUnloadEvent('tests-asdf')) + eb.post_event.reset_mock() + + # Load + path_instance.is_file = Mock(return_value=True) + await file_manager.file_watcher_event('tests/asdf') + eb.post_event.assert_called_once_with(TOPIC_FILES, RequestFileLoadEvent('tests-asdf')) + eb.post_event.reset_mock() + + # Skip load (checksum) + file_manager._files['tests-asdf'] = f1 = HABAppFile('tests-asdf', Path('path1'), b'checksum', FileProperties()) + monkeypatch.setattr(HABAppFile, 'create_checksum', lambda x: b'checksum') + await file_manager.file_watcher_event('tests/asdf') + eb.post_event.assert_not_called() + + test_logs.add_expected( + file_manager_module.log.name, 'DEBUG', 'Skip file system event because file tests-asdf did not change') + test_logs.assert_ok() + + +def setup_file(file_manager: FileManager, *, set_state: bool = True, + on_load: Callable[[str, Path], Awaitable[None]] | None = None, + on_unload: Callable[[str, Path], Awaitable[None]] | None = None) -> HABAppFile: + + file_manager._files.clear() + file_manager._files['name1'] = f1 = HABAppFile('name1', Path('path1'), b'checksum', FileProperties()) + + if set_state: + f1._state = FileState.DEPENDENCIES_OK + + file_manager._file_handlers = () + file_manager.add_handler('myhandler', logger=file_manager_logger, prefix='n', on_load=on_load, on_unload=on_unload) + + return f1 + + +async def test_load(test_logs: LogCollector, file_manager) -> None: + setup_file(file_manager, set_state=False) + with pytest.raises(ValueError) as e: + await file_manager._do_file_load('name1') + assert str(e.value) == 'File name1 can not be loaded because current state is PENDING!' + + async def coro(name: str, path: Path) -> None: + pass + + f = setup_file(file_manager, on_load=coro) + await file_manager._do_file_load('name1') + assert f._state is FileState.LOADED + + # Error in coro -> state should be Failed + async def coro(name: str, path: Path) -> None: + raise AlreadyHandledFileError() + + f = setup_file(file_manager, on_load=coro) + await file_manager._do_file_load('name1') + assert f._state is FileState.FAILED + test_logs.assert_ok() + + +async def test_unload(test_logs: LogCollector, file_manager) -> None: + async def coro(name: str, path: Path) -> None: + pass + + # Remove should work regardless of state + f = setup_file(file_manager, set_state=False, on_unload=coro) + await file_manager._do_file_unload('name1') + assert f._state is FileState.REMOVED + assert file_manager.get_file('name1') is None + + f = setup_file(file_manager, on_unload=coro) + await file_manager._do_file_unload('name1') + assert f._state is FileState.REMOVED + assert file_manager.get_file('name1') is None + + # Error in coro -> state should be Failed + async def coro(name: str, path: Path) -> None: + raise AlreadyHandledFileError() + + f = setup_file(file_manager, on_unload=coro) + await file_manager._do_file_unload('name1') + assert f._state is FileState.FAILED + test_logs.assert_ok() diff --git a/tests/test_core/test_files/test_file_properties.py b/tests/test_core/test_files/test_file_properties.py index b6f39bee..745bfe71 100644 --- a/tests/test_core/test_files/test_file_properties.py +++ b/tests/test_core/test_files/test_file_properties.py @@ -1,12 +1,11 @@ -import pytest -from HABApp.core.files.file.file import FILES, CircularReferenceError, FileProperties, FileState, HABAppFile -from HABApp.core.files.file.properties import get_properties as get_props -from tests.helpers import LogCollector + +from HABApp.core.files.file_properties import get_file_properties as get_props def test_prop_case() -> None: - _in = '''# habapp: + _in = ''' + # habapp: # depends on: # - my_Param.yml # reloads on: @@ -31,7 +30,8 @@ def test_prop_case() -> None: def test_prop_1() -> None: - _in = '''# HABApp: + _in = ''' +# HABApp: # depends on: # - my_Param.yml # @@ -85,50 +85,3 @@ def test_prop_missing() -> None: p = get_props(_in) assert p.depends_on == [] assert p.reloads_on == [] - - -def test_deps() -> None: - FILES.clear() - FILES['name1'] = f1 = HABAppFile('name1', 'path1', FileProperties(depends_on=['name2'])) - FILES['name2'] = f2 = HABAppFile('name2', 'path2', FileProperties()) - - f1.check_properties() - f2.check_properties() - - assert f1.state is FileState.DEPENDENCIES_MISSING - assert f2.state is FileState.DEPENDENCIES_OK - - f2.state = FileState.LOADED - f1.check_dependencies() - assert f1.state is FileState.DEPENDENCIES_OK - - -def test_reloads(test_logs: LogCollector) -> None: - FILES.clear() - FILES['name1'] = f1 = HABAppFile('name1', 'path1', FileProperties(reloads_on=['name2', 'asdf'])) - FILES['name2'] = f2 = HABAppFile('name2', 'path2', FileProperties()) - - f1.check_properties() - assert f1.properties.reloads_on == ['name2', 'asdf'] - assert f2.properties.reloads_on == [] - - test_logs.add_expected('HABApp.files', 'WARNING', "File path1 reloads on file that doesn't exist: asdf") - - -def test_circ() -> None: - FILES.clear() - FILES['name1'] = f1 = HABAppFile('name1', 'path1', FileProperties(depends_on=['name2'])) - FILES['name2'] = f2 = HABAppFile('name2', 'path2', FileProperties(depends_on=['name3'])) - FILES['name3'] = f3 = HABAppFile('name3', 'path3', FileProperties(depends_on=['name1'])) - - with pytest.raises(CircularReferenceError) as e: - f1._check_circ_refs((f1.name,), 'depends_on') - assert e.value.stack == ('name1', 'name2', 'name3', 'name1') - - with pytest.raises(CircularReferenceError) as e: - f2._check_circ_refs((f2.name,), 'depends_on') - assert e.value.stack == ('name2', 'name3', 'name1', 'name2',) - - with pytest.raises(CircularReferenceError) as e: - f3._check_circ_refs((f3.name,), 'depends_on') - assert e.value.stack == ('name3', 'name1', 'name2', 'name3', ) diff --git a/tests/test_core/test_files/test_name_builder.py b/tests/test_core/test_files/test_name_builder.py new file mode 100644 index 00000000..b3371cf7 --- /dev/null +++ b/tests/test_core/test_files/test_name_builder.py @@ -0,0 +1,45 @@ +from pathlib import Path + +import pytest + +from HABApp.core.files.name_builder import FileNameBuilder + + +def test_create() -> None: + f = FileNameBuilder() + + with pytest.raises(ValueError) as e: + f.create_name('asdf') + assert str(e.value) == 'Nothing matched for path asdf' + + with pytest.raises(ValueError) as e: + f.create_path('asdf') + assert str(e.value) == 'Nothing matched for name asdf' + + f.add_folder('p1/', Path('as'), priority=1) + + assert not f.is_accepted_path('asd/asdf') + assert not f.is_accepted_name('p2/asdf') + + assert f.is_accepted_path('as/asdf') + assert f.is_accepted_name('p1/asdf') + assert f.create_path('p1/asdf') == Path('as/asdf') + assert f.create_name('as/asdf') == 'p1/asdf' + + f.add_folder('p1/', Path('as'), priority=2) + + with pytest.raises(ValueError) as e: + f.create_name(Path('as/df').as_posix()) + assert str(e.value) == 'Multiple matches for path as/df: p1/df, p1/df' + + with pytest.raises(ValueError) as e: + f.create_path('p1/df') + assert str(e.value) == 'Multiple matches for name p1/df: as/df, as/df' + + +def test_get_names() -> None: + f = FileNameBuilder() + f.add_folder('p1/', Path('fa1'), priority=1) + f.add_folder('z2/', Path('fz2'), priority=2) + + assert list(f.get_names(['p1/', 'z2/', 'z2/f', '???'])) == ['z2/', 'z2/f', 'p1/'] diff --git a/tests/test_core/test_files/test_rel_name.py b/tests/test_core/test_files/test_rel_name.py deleted file mode 100644 index b456d083..00000000 --- a/tests/test_core/test_files/test_rel_name.py +++ /dev/null @@ -1,64 +0,0 @@ -from pathlib import Path - -import pytest - -from HABApp.core.files.folders import add_folder, get_name, get_path, get_prefixes -from HABApp.core.files.folders.folders import FOLDERS - - -@pytest.fixture() -def cfg(): - FOLDERS.clear() - add_folder('rules/', Path('c:/HABApp/my_rules/'), 0) - add_folder('configs/', Path('c:/HABApp/my_config/'), 10) - add_folder('params/', Path('c:/HABApp/my_param/'), 20) - - yield None - - FOLDERS.clear() - - -def cmp(path: Path, name: str) -> None: - assert get_name(path) == name - assert get_path(name) == path - - -def test_prefix_sort(cfg) -> None: - assert get_prefixes() == ['params/', 'configs/', 'rules/'] - add_folder('params1/', Path('c:/HABApp/my_para1m/'), 50) - assert get_prefixes() == ['params1/', 'params/', 'configs/', 'rules/'] - - -def test_from_path(cfg) -> None: - cmp(Path('c:/HABApp/my_rules/rule.py'), 'rules/rule.py') - cmp(Path('c:/HABApp/my_config/params.yml'), 'configs/params.yml') - cmp(Path('c:/HABApp/my_param/cfg.yml'), 'params/cfg.yml') - - cmp(Path('c:/HABApp/my_rules/my_folder1/folder2/rule.py'), 'rules/my_folder1/folder2/rule.py') - cmp(Path('c:/HABApp/my_config/my_folder2/cfg.yml'), 'configs/my_folder2/cfg.yml') - cmp(Path('c:/HABApp/my_param/my_folder3/cfg.yml'), 'params/my_folder3/cfg.yml') - - -def test_err(cfg) -> None: - with pytest.raises(ValueError): - get_name(Path('c:/HABApp/rules/rule.py')) - - -def test_mixed() -> None: - FOLDERS.clear() - add_folder('rules/', Path('c:/HABApp/rules'), 1) - add_folder('configs/', Path('c:/HABApp/rules/my_config'), 2) - add_folder('params/', Path('c:/HABApp/rules/my_param'), 3) - - cmp(Path('c:/HABApp/rules/rule.py'), 'rules/rule.py') - cmp(Path('c:/HABApp/rules/my_config/params.yml'), 'configs/params.yml') - cmp(Path('c:/HABApp/rules/my_param/cfg.yml'), 'params/cfg.yml') - - FOLDERS.clear() - add_folder('rules/', Path('c:/HABApp/rules'), 1) - add_folder('configs/', Path('c:/HABApp/rules/my_cfg'), 2) - add_folder('params/', Path('c:/HABApp/rules/my_param'), 3) - - cmp(Path('c:/HABApp/rules/rule.py'), 'rules/rule.py') - cmp(Path('c:/HABApp/rules/my_cfg/params.yml'), 'configs/params.yml') - cmp(Path('c:/HABApp/rules/my_param/cfg.yml'), 'params/cfg.yml') diff --git a/tests/test_core/test_files/test_watcher.py b/tests/test_core/test_files/test_watcher.py index ffa54f4c..79bda3e9 100644 --- a/tests/test_core/test_files/test_watcher.py +++ b/tests/test_core/test_files/test_watcher.py @@ -1,40 +1,57 @@ -import asyncio -import time -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from unittest.mock import AsyncMock +import logging +from pathlib import PurePath +from typing import Self -from watchdog.events import FileSystemEvent +import pytest +from watchfiles import Change -import HABApp.core.files.watcher.file_watcher -from HABApp.core.files.watcher import AggregatingAsyncEventHandler -from HABApp.core.files.watcher.base_watcher import FileEndingFilter +from HABApp.core.files import HABAppFileWatcher +from HABApp.core.files import watcher as watcher_module -async def test_file_events(monkeypatch, sync_worker) -> None: +class MyPath(PurePath): - wait_time = 0.1 - monkeypatch.setattr(HABApp.core.files.watcher.file_watcher, 'DEBOUNCE_TIME', wait_time) + def __init__(self, *args) -> None: + super().__init__(*args) + self._is_dir = False + self._is_file = False - m = AsyncMock() - handler = AggregatingAsyncEventHandler(Path('folder'), m, FileEndingFilter('.tmp'), False) + def is_dir(self) -> bool: + return self._is_dir - loop = asyncio.get_event_loop() + def is_file(self) -> bool: + return self._is_file - ex = ThreadPoolExecutor(4) + def set_is_dir(self, value: bool) -> Self: + self._is_dir = value + return self - def generate_events(count: int, name: str, sleep: float) -> None: - for _ in range(count): - handler.dispatch(FileSystemEvent(name)) - time.sleep(sleep) + def set_is_file(self, value: bool) -> Self: + self._is_file = value + return self - await asyncio.gather( - loop.run_in_executor(ex, generate_events, 3, 'test/t1.tmp', wait_time), - loop.run_in_executor(ex, generate_events, 9, 'test/t2.tmp', wait_time / 2), - loop.run_in_executor(ex, generate_events, 18, 'test/t3.tmp', wait_time / 5), - ) - ex.shutdown() - await asyncio.sleep(wait_time + 0.01) - m.assert_called_once() - assert set(*m.call_args[0]) == {Path('test/t1.tmp'), Path('test/t2.tmp'), Path('test/t3.tmp')} +async def test_watcher(monkeypatch, test_logs) -> None: + logging.getLogger('HABApp.file.events').setLevel(0) + test_logs.set_min_level(0) + + f = HABAppFileWatcher() + f._watcher_task = lambda: 'ReplacedTask' + monkeypatch.setattr(watcher_module, 'create_task_from_async', lambda x: x) + + with pytest.raises(FileNotFoundError) as e: + f.add_path(MyPath('a/b/c')) + assert str(e.value) in ('Path a/b/c does not exist!', 'Path a\\b\\c does not exist') + + async def coro(text: str): + raise ValueError() + + f.watch_folder('folder1', coro, MyPath('my/folder/1').set_is_dir(True)) + + assert not f._watch_filter(Change.added, 'my/folder/2/file1') + assert not f._watch_filter(Change.added, 'my/folder/2/file1', dispatchers=f._dispatchers) + + test_logs.add_expected('HABApp.file.events', 'DEBUG', 'Added dispatcher folder1') + test_logs.add_expected('HABApp.file.events', 'DEBUG', 'Watching my\\folder\\1') + test_logs.add_expected('HABApp.file.events', 'DEBUG', 'added my/folder/2/file1') + test_logs.assert_ok() diff --git a/tests/test_core/test_lib/test_format_traceback.py b/tests/test_core/test_lib/test_format_traceback.py index c1aeecf6..be8437ee 100644 --- a/tests/test_core/test_lib/test_format_traceback.py +++ b/tests/test_core/test_lib/test_format_traceback.py @@ -9,7 +9,7 @@ from HABApp.core.const.const import PYTHON_311, PYTHON_312, PYTHON_313 from HABApp.core.const.json import dump_json, load_json from HABApp.core.lib import format_exception -from HABApp.core.lib.exceptions.format_frame import SUPPRESSED_HABAPP_PATHS, is_lib_file, is_suppressed_habapp_file +from HABApp.core.lib.exceptions.format_frame import SUPPRESSED_HABAPP_PATHS, is_suppressed_habapp_file from tests.helpers.traceback import remove_dyn_parts_from_traceback @@ -41,45 +41,6 @@ def func_obj_def_multilines() -> None: 1 / 0 -# def test_exception_format_traceback_compact_lines(): -# -# msg = exec_func(func_obj_def_multilines) -# assert msg == r''' -# File "test_core/test_lib/test_format_traceback.py", line 17 in exec_func -# -------------------------------------------------------------------------------- -# 15 | def exec_func(func) -> str: -# 16 | try: -# --> 17 | func() -# 18 | except Exception as e: -# ------------------------------------------------------------ -# e = ZeroDivisionError('division by zero') -# func = -# ------------------------------------------------------------ -# -# File "test_core/test_lib/test_format_traceback.py", line 37 in func_obj_def_multilines -# -------------------------------------------------------------------------------- -# 25 | def func_obj_def_multilines(): -# 26 | item = HABApp.core.items.Item -# 27 | a = [ -# 28 | 1, -# (...) -# 35 | 8 -# 36 | ] -# --> 37 | 1 / 0 -# ------------------------------------------------------------ -# item = -# a = [1, 2, 3, 4, 5, 6, 7, 8] -# ------------------------------------------------------------ -# -# -------------------------------------------------------------------------------- -# Traceback (most recent call last): -# File "test_core/test_lib/test_format_traceback.py", line 17, in exec_func -# func() -# File "test_core/test_lib/test_format_traceback.py", line 37, in func_obj_def_multilines -# 1 / 0 -# ZeroDivisionError: division by zero''' - - class DummyModel(BaseModel): a: int = 3 b: str = 'asdf' @@ -108,26 +69,26 @@ def test_exception_expression_remove_py310() -> None: log.setLevel(logging.WARNING) msg = exec_func(func_test_assert_none) assert msg == r''' -File "test_core/test_lib/test_format_traceback.py", line 21 in exec_func +File "test_core/test_lib/test_format_traceback.py", line x in exec_func -------------------------------------------------------------------------------- - 19 | def exec_func(func) -> str: - 20 | try: ---> 21 | func() - 22 | except Exception as e: + x | def exec_func(func) -> str: + x | try: +--> x | func() + x | except Exception as e: ------------------------------------------------------------ e = ZeroDivisionError('division by zero') func = ------------------------------------------------------------ -File "test_core/test_lib/test_format_traceback.py", line 97 in func_test_assert_none +File "test_core/test_lib/test_format_traceback.py", line x in func_test_assert_none -------------------------------------------------------------------------------- - 91 | def func_test_assert_none(a: str | None = None, b: str | None = None, c: str | int = 3) -> None: + x | def func_test_assert_none(a: str | None = None, b: str | None = None, c: str | int = 3) -> None: (...) - 94 | assert isinstance(c, (str, int)), type(c) - 95 | CONFIGURATION = '3' - 96 | my_dict = {'key_a': 'val_a'} ---> 97 | 1 / 0 - 98 | log.error('Error message') + x | assert isinstance(c, (str, int)), type(c) + x | CONFIGURATION = '3' + x | my_dict = {'key_a': 'val_a'} +--> x | 1 / 0 + x | log.error('Error message') ------------------------------------------------------------ CONFIG.a = 3 a = None @@ -142,9 +103,9 @@ def test_exception_expression_remove_py310() -> None: -------------------------------------------------------------------------------- Traceback (most recent call last): - File "test_core/test_lib/test_format_traceback.py", line 21, in exec_func + File "test_core/test_lib/test_format_traceback.py", line x, in exec_func func() - File "test_core/test_lib/test_format_traceback.py", line 97, in func_test_assert_none + File "test_core/test_lib/test_format_traceback.py", line x, in func_test_assert_none 1 / 0 ZeroDivisionError: division by zero''' @@ -156,26 +117,26 @@ def test_exception_expression_remove_py_311_312() -> None: log.setLevel(logging.WARNING) msg = exec_func(func_test_assert_none) assert msg == r''' -File "test_core/test_lib/test_format_traceback.py", line 21 in exec_func +File "test_core/test_lib/test_format_traceback.py", line x in exec_func -------------------------------------------------------------------------------- - 19 | def exec_func(func) -> str: - 20 | try: ---> 21 | func() - 22 | except Exception as e: + x | def exec_func(func) -> str: + x | try: +--> x | func() + x | except Exception as e: ------------------------------------------------------------ e = ZeroDivisionError('division by zero') func = ------------------------------------------------------------ -File "test_core/test_lib/test_format_traceback.py", line 97 in func_test_assert_none +File "test_core/test_lib/test_format_traceback.py", line x in func_test_assert_none -------------------------------------------------------------------------------- - 91 | def func_test_assert_none(a: str | None = None, b: str | None = None, c: str | int = 3) -> None: + x | def func_test_assert_none(a: str | None = None, b: str | None = None, c: str | int = 3) -> None: (...) - 94 | assert isinstance(c, (str, int)), type(c) - 95 | CONFIGURATION = '3' - 96 | my_dict = {'key_a': 'val_a'} ---> 97 | 1 / 0 - 98 | log.error('Error message') + x | assert isinstance(c, (str, int)), type(c) + x | CONFIGURATION = '3' + x | my_dict = {'key_a': 'val_a'} +--> x | 1 / 0 + x | log.error('Error message') ------------------------------------------------------------ CONFIG.a = 3 a = None @@ -190,9 +151,9 @@ def test_exception_expression_remove_py_311_312() -> None: -------------------------------------------------------------------------------- Traceback (most recent call last): - File "test_core/test_lib/test_format_traceback.py", line 21, in exec_func + File "test_core/test_lib/test_format_traceback.py", line x, in exec_func func() - File "test_core/test_lib/test_format_traceback.py", line 97, in func_test_assert_none + File "test_core/test_lib/test_format_traceback.py", line x, in func_test_assert_none 1 / 0 ~~^~~ ZeroDivisionError: division by zero''' @@ -203,26 +164,26 @@ def test_exception_expression_remove() -> None: log.setLevel(logging.WARNING) msg = exec_func(func_test_assert_none) assert msg == r''' -File "test_core/test_lib/test_format_traceback.py", line 21 in exec_func +File "test_core/test_lib/test_format_traceback.py", line x in exec_func -------------------------------------------------------------------------------- - 19 | def exec_func(func) -> str: - 20 | try: ---> 21 | func() - 22 | except Exception as e: + x | def exec_func(func) -> str: + x | try: +--> x | func() + x | except Exception as e: ------------------------------------------------------------ e = ZeroDivisionError('division by zero') func = ------------------------------------------------------------ -File "test_core/test_lib/test_format_traceback.py", line 97 in func_test_assert_none +File "test_core/test_lib/test_format_traceback.py", line x in func_test_assert_none -------------------------------------------------------------------------------- - 91 | def func_test_assert_none(a: str | None = None, b: str | None = None, c: str | int = 3) -> None: + x | def func_test_assert_none(a: str | None = None, b: str | None = None, c: str | int = 3) -> None: (...) - 94 | assert isinstance(c, (str, int)), type(c) - 95 | CONFIGURATION = '3' - 96 | my_dict = {'key_a': 'val_a'} ---> 97 | 1 / 0 - 98 | log.error('Error message') + x | assert isinstance(c, (str, int)), type(c) + x | CONFIGURATION = '3' + x | my_dict = {'key_a': 'val_a'} +--> x | 1 / 0 + x | log.error('Error message') ------------------------------------------------------------ CONFIG.a = 3 a = None @@ -237,10 +198,10 @@ def test_exception_expression_remove() -> None: -------------------------------------------------------------------------------- Traceback (most recent call last): - File "test_core/test_lib/test_format_traceback.py", line 21, in exec_func + File "test_core/test_lib/test_format_traceback.py", line x, in exec_func func() ~~~~^^ - File "test_core/test_lib/test_format_traceback.py", line 97, in func_test_assert_none + File "test_core/test_lib/test_format_traceback.py", line x, in func_test_assert_none 1 / 0 ~~^~~ ZeroDivisionError: division by zero''' @@ -273,49 +234,124 @@ def test_skip_objs(_setup_ir) -> None: log.setLevel(logging.WARNING) msg = exec_func(func_ir) assert msg == r''' -File "test_core/test_lib/test_format_traceback.py", line 21 in exec_func +File "test_core/test_lib/test_format_traceback.py", line x in exec_func -------------------------------------------------------------------------------- - 19 | def exec_func(func) -> str: - 20 | try: ---> 21 | func() - 22 | except Exception as e: + x | def exec_func(func) -> str: + x | try: +--> x | func() + x | except Exception as e: ------------------------------------------------------------ e = ItemNotFoundException('Item 1234 does not exist!') func = ------------------------------------------------------------ -File "test_core/test_lib/test_format_traceback.py", line 255 in func_ir +File "test_core/test_lib/test_format_traceback.py", line x in func_ir -------------------------------------------------------------------------------- - 249 | def func_ir() -> None: - 251 | from HABApp.core.items import Item - 252 | Items = HABApp.core.Items - 254 | Items.add_item(Item('asdf')) ---> 255 | Items.get_item('1234') + x | def func_ir() -> None: + x | from HABApp.core.items import Item + x | Items = HABApp.core.Items + x | Items.add_item(Item('asdf')) +--> x | Items.get_item('1234') -File "internals/item_registry/item_registry.py", line 31 in get_item +File "internals/item_registry/item_registry.py", line x in get_item -------------------------------------------------------------------------------- - 27 | def get_item(self, name: str) -> ItemRegistryItem: - 28 | try: - 29 | return self._items[name] - 30 | except KeyError: ---> 31 | raise ItemNotFoundException(name) from None + x | def get_item(self, name: str) -> ItemRegistryItem: + x | try: + x | return self._items[name] + x | except KeyError: +--> x | raise ItemNotFoundException(name) from None ------------------------------------------------------------ name = '1234' ------------------------------------------------------------ -------------------------------------------------------------------------------- Traceback (most recent call last): - File "test_core/test_lib/test_format_traceback.py", line 21, in exec_func + File "test_core/test_lib/test_format_traceback.py", line x, in exec_func func() ~~~~^^ - File "test_core/test_lib/test_format_traceback.py", line 255, in func_ir + File "test_core/test_lib/test_format_traceback.py", line x, in func_ir Items.get_item('1234') ~~~~~~~~~~~~~~^^^^^^^^ - File "internals/item_registry/item_registry.py", line 31, in get_item + File "internals/item_registry/item_registry.py", line x, in get_item raise ItemNotFoundException(name) from None HABApp.core.errors.ItemNotFoundException: Item 1234 does not exist!''' +def multiline_obj_name() -> None: + + class MultilineRepr: + def __repr__(self) -> str: + return '<\nmulti\nline>' + + instance = MultilineRepr() + + a = [] + + assert a == [ + 1, + 2 + ] + + +@pytest.mark.skipif(not PYTHON_313, reason='New traceback from python 3.13') +def test_multile_statements() -> None: + log.setLevel(logging.WARNING) + msg = exec_func(multiline_obj_name) + print('\n\n-') + print(msg) + print('\n\n') + assert msg == r''' +File "test_core/test_lib/test_format_traceback.py", line x in exec_func +-------------------------------------------------------------------------------- + x | def exec_func(func) -> str: + x | try: +--> x | func() + x | except Exception as e: + ------------------------------------------------------------ + e = AssertionError('assert [] == [1, 2]\n \n Right contains 2 more items, first extra item: 1\n \n Full diff:\n + []\n - [\n - 1,\n - 2,\n - ]') + func = + ------------------------------------------------------------ + +File "test_core/test_lib/test_format_traceback.py", line x in multiline_obj_name +-------------------------------------------------------------------------------- + x | def multiline_obj_name(): + (...) + x | return '<\nmulti\nline>' + x | instance = MultilineRepr() + x | a = [] +--> x | assert a == [ + x | 1, + x | 2 + x | ] + ------------------------------------------------------------ + a = [] + instance = < + multi + line> + a == [ + 1, + 2 + ] = False + ------------------------------------------------------------ + +-------------------------------------------------------------------------------- +Traceback (most recent call last): + File "test_core/test_lib/test_format_traceback.py", line x, in exec_func + func() + File "test_core/test_lib/test_format_traceback.py", line x, in multiline_obj_name + assert a == [ +AssertionError: assert [] == [1, 2] + + Right contains 2 more items, first extra item: 1 + + Full diff: + + [] + - [ + - 1, + - 2, + - ]''' + + def test_habapp_regex(pytestconfig): files = tuple(str(f) for f in (Path(pytestconfig.rootpath) / 'src' / 'HABApp').glob('**/*')) @@ -335,14 +371,5 @@ def test_regex(pytestconfig) -> None: # noqa: ARG001 assert not is_suppressed_habapp_file('/lib/HABApp/asdf') assert not is_suppressed_habapp_file('/HABApp/core/lib/asdf') assert not is_suppressed_habapp_file('/HABApp/core/lib/asdf/asdf') - - assert is_lib_file(r'\Python310\lib\runpy.py') - assert is_lib_file(r'/usr/lib/python3.10/runpy.py') - assert is_lib_file(r'/opt/habapp/lib/python3.8/site-packages/aiohttp/client.py') - assert is_lib_file(r'\Python310\lib\asyncio\tasks.py') - assert is_lib_file(r'\Python310\lib\subprocess.py') - - # Normal HABApp installation under linux - assert not is_lib_file('/opt/habapp/lib/python3.9/site-packages/HABApp/openhab/connection_logic/file.py') assert not is_suppressed_habapp_file( '/opt/habapp/lib/python3.9/site-packages/HABApp/openhab/connection_logic/file.py') diff --git a/tests/test_openhab/test_interface_sync.py b/tests/test_openhab/test_interface_sync.py index 436d5d26..c3e48df7 100644 --- a/tests/test_openhab/test_interface_sync.py +++ b/tests/test_openhab/test_interface_sync.py @@ -4,7 +4,7 @@ import pytest import HABApp.openhab.interface_sync -from HABApp.core.asyncio import AsyncContextError, async_context +from HABApp.core.asyncio import AsyncContextError from HABApp.openhab.interface_sync import ( create_item, create_link, @@ -51,7 +51,6 @@ def test_all_imported(func: Callable) -> None: (create_link, ('item', 'channel', {})), )) async def test_item_has_name(func, args) -> None: - async_context.set('Test') if func not in (post_update, send_command): with pytest.raises(AsyncContextError) as e: