diff --git a/requirements_setup.txt b/requirements_setup.txt index cf64ab68..a3c083cf 100644 --- a/requirements_setup.txt +++ b/requirements_setup.txt @@ -1,7 +1,7 @@ aiohttp == 3.11.10 pydantic == 2.10.3 bidict == 0.23.1 -watchdog == 6.0.0 +watchfiles == 1.0.3 ujson == 5.10.0 aiomqtt == 2.3.0 diff --git a/src/HABApp/__check_dependency_packages__.py b/src/HABApp/__check_dependency_packages__.py index 2aff19cc..51882269 100644 --- a/src/HABApp/__check_dependency_packages__.py +++ b/src/HABApp/__check_dependency_packages__.py @@ -19,7 +19,7 @@ def get_dependencies() -> list[str]: 'pydantic', 'stack_data', 'voluptuous', - 'watchdog', + 'watchfiles', 'ujson', 'immutables', 'javaproperties', diff --git a/src/HABApp/config/__init__.py b/src/HABApp/config/__init__.py index 60ff7186..5621c0c1 100644 --- a/src/HABApp/config/__init__.py +++ b/src/HABApp/config/__init__.py @@ -6,4 +6,4 @@ # isort: split -from .loader import load_config +from .loader import setup_habapp_configuration diff --git a/src/HABApp/config/loader.py b/src/HABApp/config/loader.py index 55b3fbf8..9b04c51f 100644 --- a/src/HABApp/config/loader.py +++ b/src/HABApp/config/loader.py @@ -5,10 +5,11 @@ import eascheduler import pydantic -import HABApp from HABApp import __version__ from HABApp.config.config import CONFIG from HABApp.config.logging import HABAppQueueHandler, load_logging_file +from HABApp.core import shutdown +from HABApp.core.internals.proxy.proxies import uses_file_manager from .debug import setup_debug from .errors import AbsolutePathExpected, InvalidConfigError @@ -19,7 +20,10 @@ log = logging.getLogger('HABApp.Config') -def load_config(config_folder: Path) -> None: +file_manager = uses_file_manager() + + +def setup_habapp_configuration(config_folder: Path) -> None: CONFIG.set_file_path(config_folder / 'config.yml') @@ -40,16 +44,14 @@ def load_config(config_folder: Path) -> None: if not loaded_logging: load_logging_cfg(logging_cfg_path) + shutdown.register(stop_queue_handlers, msg='Stop logging queue handlers', last=True) + setup_debug() - # Watch folders, so we can reload the config on the fly - filter = HABApp.core.files.watcher.FileEndingFilter('.yml') - watcher = HABApp.core.files.watcher.AggregatingAsyncEventHandler( - config_folder, config_files_changed, filter, watch_subfolders=False - ) - HABApp.core.files.watcher.add_folder_watch(watcher) + watcher = file_manager.get_file_watcher() + watcher.watch_file('config.log_file', config_file_changed, config_folder / 'logging.yml', habapp_internal=True) + watcher.watch_file('config.cfg_file', config_file_changed, config_folder / 'config.yml', habapp_internal=True) - HABApp.core.shutdown.register(stop_queue_handlers, last=True, msg='Stopping logging threads') CONFIG.habapp.logging.subscribe_for_changes(set_flush_delay) @@ -57,12 +59,12 @@ def set_flush_delay() -> None: HABAppQueueHandler.FLUSH_DELAY = CONFIG.habapp.logging.flush_every -async def config_files_changed(paths: list[Path]) -> None: - for path in paths: - if path.name == 'config.yml': - load_habapp_cfg() - if path.name == 'logging.yml': - load_logging_cfg(path) +async def config_file_changed(path: str) -> None: + file = Path(path) + if file.name == 'config.yml': + load_habapp_cfg() + if file.name == 'logging.yml': + load_logging_cfg(file) def load_habapp_cfg(do_print=False) -> None: diff --git a/src/HABApp/core/events/habapp_events.py b/src/HABApp/core/events/habapp_events.py index f971561d..34eb8a39 100644 --- a/src/HABApp/core/events/habapp_events.py +++ b/src/HABApp/core/events/habapp_events.py @@ -8,6 +8,9 @@ def __init__(self, name: str) -> None: def __repr__(self) -> str: return f'<{self.__class__.__name__} filename: {self.name}>' + def __eq__(self, other: object) -> bool: + return isinstance(other, self.__class__) and self.name == other.name + class RequestFileLoadEvent(__FileEventBase): """Request (re-) loading of the specified file diff --git a/src/HABApp/core/files/__init__.py b/src/HABApp/core/files/__init__.py index 020ab40c..43782f5b 100644 --- a/src/HABApp/core/files/__init__.py +++ b/src/HABApp/core/files/__init__.py @@ -1,8 +1,2 @@ -from . import errors - -from . import watcher -from . import file -from . import folders -from . import manager - -from .setup import setup +from .watcher import HABAppFileWatcher, FolderDispatcher, FileDispatcher +from .manager import FileManager diff --git a/src/HABApp/core/files/errors.py b/src/HABApp/core/files/errors.py index 7b9c9a20..4766579e 100644 --- a/src/HABApp/core/files/errors.py +++ b/src/HABApp/core/files/errors.py @@ -1,8 +1,5 @@ -from collections.abc import Iterable as _Iterable - - class CircularReferenceError(Exception): - def __init__(self, stack: _Iterable[str]) -> None: + def __init__(self, stack: tuple[str, ...]) -> None: self.stack = stack def __repr__(self) -> str: diff --git a/src/HABApp/core/files/file.py b/src/HABApp/core/files/file.py new file mode 100644 index 00000000..a0056408 --- /dev/null +++ b/src/HABApp/core/files/file.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from enum import Enum, auto +from hashlib import blake2b +from typing import TYPE_CHECKING, Final + +from HABApp.core.files.errors import AlreadyHandledFileError, CircularReferenceError, DependencyDoesNotExistError +from HABApp.core.files.file_properties import FileProperties +from HABApp.core.wrapper import process_exception + + +if TYPE_CHECKING: + import logging + from pathlib import Path + + from HABApp.core.files.manager import FileManager, FileTypeHandler + + +class FileState(Enum): + LOADED = auto() + FAILED = auto() + + DEPENDENCIES_OK = auto() + DEPENDENCIES_MISSING = auto() + DEPENDENCIES_ERROR = auto() + + PROPERTIES_INVALID = auto() # Properties could not be parsed + + # initial and last state + PENDING = auto() + REMOVED = auto() + + def __str__(self) -> str: + return str(self.name) + + +class HABAppFile: + + @staticmethod + def create_checksum(text: str) -> bytes: + b = blake2b() + b.update(text.encode()) + return b.digest() + + def __init__(self, name: str, path: Path, checksum: bytes, properties: FileProperties | None) -> None: + self.name: Final = name + self.path: Final = path + self.checksum: Final = checksum + self.properties: Final = properties if properties is not None else FileProperties() + self._state: FileState = FileState.PENDING if properties is not None else FileState.PROPERTIES_INVALID + + def __repr__(self) -> str: + return f'<{self.__class__.__name__} {self.name} state: {self._state}>' + + def set_state(self, new_state: FileState, manager: FileManager) -> None: + if self._state is new_state: + return None + + self._state = new_state + manager.file_state_changed(self, str(new_state)) + + def _check_circ_refs(self, stack: tuple[str, ...], prop: str, manager: FileManager) -> None: + c: list[str] = getattr(self.properties, prop) + for f in c: + _stack = stack + (f, ) + if f in stack: + raise CircularReferenceError(_stack) + + next_file = manager.get_file(f) + if next_file is not None: + next_file._check_circ_refs(_stack, prop, manager) + + def _check_properties(self, manager: FileManager, log: logging.Logger) -> None: + # check dependencies + missing = {name for name in self.properties.depends_on if manager.get_file(name) is None} + if missing: + one = len(missing) == 1 + msg = (f'File {self.path} depends on file{"" if one else "s"} that ' + f'do{"es" if one else ""}n\'t exist: {", ".join(sorted(missing))}') + raise DependencyDoesNotExistError(msg) + + # check reload + missing = {name for name in self.properties.reloads_on if manager.get_file(name) is None} + if missing: + one = len(missing) == 1 + log.warning(f'File {self.path} reloads on file{"" if one else "s"} that ' + f'do{"es" if one else ""}n\'t exist: {", ".join(sorted(missing))}') + + def check_properties(self, manager: FileManager, log: logging.Logger, *, log_msg: bool = False) -> None: + if self._state is not FileState.PENDING and self._state is not FileState.DEPENDENCIES_ERROR: + return None + + try: + self._check_properties(manager, log) + except DependencyDoesNotExistError as e: + if log_msg: + log.error(e.msg) + return self.set_state(FileState.DEPENDENCIES_ERROR, manager) + + try: + # check for circular references + self._check_circ_refs((self.name, ), 'depends_on', manager) + self._check_circ_refs((self.name, ), 'reloads_on', manager) + except CircularReferenceError as e: + log.error(f'Circular reference: {" -> ".join(e.stack)}') + return self.set_state(FileState.DEPENDENCIES_ERROR, manager) + + # Check if we can already load it + new_state = FileState.DEPENDENCIES_OK if not self.properties.depends_on else FileState.DEPENDENCIES_MISSING + self.set_state(new_state, manager) + return None + + def check_dependencies(self, manager: FileManager) -> None: + if self._state is not FileState.DEPENDENCIES_MISSING: + return None + + for name in self.properties.depends_on: + if (file := manager.get_file(name)) is None: + return None + if file._state is not FileState.LOADED: + return None + + self.set_state(FileState.DEPENDENCIES_OK, manager) + return None + + def can_be_loaded(self) -> bool: + return self._state is FileState.DEPENDENCIES_OK + + def can_be_removed(self) -> bool: + return self._state is FileState.REMOVED + + async def load(self, handler: FileTypeHandler, manager: FileManager) -> None: + if not self.can_be_loaded(): + msg = f'File {self.name} can not be loaded because current state is {self._state}!' + raise ValueError(msg) + + try: + await handler.on_load(self.name, self.path) + except Exception as e: + if not isinstance(e, AlreadyHandledFileError): + process_exception(handler.on_load, e, logger=handler.logger) + self.set_state(FileState.FAILED, manager) + return None + + self.set_state(FileState.LOADED, manager) + return None + + async def unload(self, handler: FileTypeHandler, manager: FileManager) -> None: + try: + await handler.on_unload(self.name, self.path) + except Exception as e: + if not isinstance(e, AlreadyHandledFileError): + process_exception(handler.on_unload, e, logger=handler.logger) + self.set_state(FileState.FAILED, manager) + return None + + self.set_state(FileState.REMOVED, manager) + return None + + def file_state_changed(self, file: HABAppFile, manager: FileManager) -> None: + name = file.name + if name in self.properties.reloads_on: + self.set_state(FileState.PENDING, manager) diff --git a/src/HABApp/core/files/file/__init__.py b/src/HABApp/core/files/file/__init__.py deleted file mode 100644 index e4229245..00000000 --- a/src/HABApp/core/files/file/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .file_state import FileState -from .file import HABAppFile -from .file_types import create_file, register_file_type diff --git a/src/HABApp/core/files/file/file.py b/src/HABApp/core/files/file/file.py deleted file mode 100644 index f127e3ac..00000000 --- a/src/HABApp/core/files/file/file.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import logging -from collections.abc import Awaitable, Callable -from pathlib import Path -from typing import Any - -from HABApp.core.files.errors import AlreadyHandledFileError, CircularReferenceError, DependencyDoesNotExistError -from HABApp.core.files.file.properties import FileProperties -from HABApp.core.files.manager.files import FILES, file_state_changed -from HABApp.core.wrapper import process_exception - -from . import FileState - - -log = logging.getLogger('HABApp.files') - - -class HABAppFile: - LOGGER: logging.Logger - LOAD_FUNC: Callable[[str, Path], Awaitable[Any]] - UNLOAD_FUNC: Callable[[str, Path], Awaitable[Any]] - - def __init__(self, name: str, path: Path, properties: FileProperties) -> None: - self.name: str = name - self.path: Path = path - - self.state: FileState = FileState.PENDING - self.properties: FileProperties = properties - log.debug(f'{self.name} added') - - def __repr__(self) -> str: - return f'<{self.__class__.__name__} {self.name} state: {self.state}>' - - def set_state(self, new_state: FileState) -> None: - if self.state is new_state: - return None - - self.state = new_state - log.debug(f'{self.name} changed to {self.state}') - file_state_changed(self) - - def _check_circ_refs(self, stack, prop: str) -> None: - c: list[str] = getattr(self.properties, prop) - for f in c: - _stack = stack + (f, ) - if f in stack: - raise CircularReferenceError(_stack) - - next_file = FILES.get(f) - if next_file is not None: - next_file._check_circ_refs(_stack, prop) - - def _check_properties(self) -> None: - # check dependencies - mis = set(filter(lambda x: x not in FILES, self.properties.depends_on)) - if mis: - one = len(mis) == 1 - msg = f'File {self.path} depends on file{"" if one else "s"} that ' \ - f'do{"es" if one else ""}n\'t exist: {", ".join(sorted(mis))}' - - raise DependencyDoesNotExistError(msg) - - # check reload - mis = set(filter(lambda x: x not in FILES, self.properties.reloads_on)) - if mis: - one = len(mis) == 1 - log.warning(f'File {self.path} reloads on file{"" if one else "s"} that ' - f'do{"es" if one else ""}n\'t exist: {", ".join(sorted(mis))}') - - def check_properties(self, log_msg: bool = False) -> None: - if self.state is not FileState.PENDING and self.state is not FileState.DEPENDENCIES_ERROR: - return None - - try: - self._check_properties() - except DependencyDoesNotExistError as e: - if log_msg: - log.error(e.msg) - return self.set_state(FileState.DEPENDENCIES_ERROR) - - try: - # check for circular references - self._check_circ_refs((self.name, ), 'depends_on') - self._check_circ_refs((self.name, ), 'reloads_on') - except CircularReferenceError as e: - log.error(f'Circular reference: {" -> ".join(e.stack)}') - return self.set_state(FileState.DEPENDENCIES_ERROR) - - # Check if we can already load it - self.set_state(FileState.DEPENDENCIES_OK if not self.properties.depends_on else FileState.DEPENDENCIES_MISSING) - return None - - def check_dependencies(self) -> None: - if self.state is not FileState.DEPENDENCIES_MISSING: - return None - - for name in self.properties.depends_on: - f = FILES.get(name, None) - if f is None: - return None - if f.state is not FileState.LOADED: - return None - - self.set_state(FileState.DEPENDENCIES_OK) - return None - - async def load(self) -> None: - assert self.state is FileState.DEPENDENCIES_OK, self.state - - try: - await self.__class__.LOAD_FUNC(self.name, self.path) - except Exception as e: - if not isinstance(e, AlreadyHandledFileError): - process_exception(self.__class__.LOAD_FUNC, e, logger=self.LOGGER) - self.set_state(FileState.FAILED) - return None - - self.set_state(FileState.LOADED) - return None - - async def unload(self) -> None: - try: - await self.__class__.UNLOAD_FUNC(self.name, self.path) - except Exception as e: - if not isinstance(e, AlreadyHandledFileError): - process_exception(self.__class__.UNLOAD_FUNC, e, logger=self.LOGGER) - self.set_state(FileState.FAILED) - return None - - self.set_state(FileState.REMOVED) - return None - - def file_changed(self, file: HABAppFile) -> None: - name = file.name - if name in self.properties.reloads_on: - self.set_state(FileState.PENDING) diff --git a/src/HABApp/core/files/file/file_state.py b/src/HABApp/core/files/file/file_state.py deleted file mode 100644 index 5d725a9a..00000000 --- a/src/HABApp/core/files/file/file_state.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -from enum import Enum, auto - - -class FileState(Enum): - LOADED = auto() - FAILED = auto() - - DEPENDENCIES_OK = auto() - DEPENDENCIES_MISSING = auto() - DEPENDENCIES_ERROR = auto() - - PROPERTIES_INVALID = auto() # Properties could not be parsed - - # initial and last state - PENDING = auto() - REMOVED = auto() - - def __str__(self) -> str: - return str(self.name) diff --git a/src/HABApp/core/files/file/file_types.py b/src/HABApp/core/files/file/file_types.py deleted file mode 100644 index bf206a2c..00000000 --- a/src/HABApp/core/files/file/file_types.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -import logging -from pathlib import Path - -from pydantic import ValidationError - -from HABApp.core.files.file import FileState, HABAppFile -from HABApp.core.files.file.properties import FileProperties, get_properties -from HABApp.core.logger import HABAppError - - -FILE_TYPES: dict[str, type[HABAppFile]] = {} - - -log = logging.getLogger('HABApp.files') - - -def register_file_type(prefix: str, cls: type[HABAppFile]) -> None: - assert prefix not in FILE_TYPES - - assert cls.LOGGER - assert cls.LOAD_FUNC - assert cls.UNLOAD_FUNC - - FILE_TYPES[prefix] = cls - - -def create_file(name: str, path: Path) -> HABAppFile: - for prefix, cls in FILE_TYPES.items(): - if name.startswith(prefix): - break - else: - raise ValueError(f'Unknown file type for "{name}"!') - - with path.open('r', encoding='utf-8') as f: - txt = f.read(10 * 1024) - - validation_error = True - - try: - properties = get_properties(txt) - validation_error = False - except ValidationError as e: - logger = HABAppError(log) - logger.add(f'Error while parsing properties for {name:s}:') - for line in str(e).splitlines()[1:]: - logger.add(f' {line:s}') - logger.dump() - - properties = FileProperties() - - obj = cls(name, path, properties) - if validation_error: - obj.set_state(FileState.PROPERTIES_INVALID) - - return obj diff --git a/src/HABApp/core/files/file/properties.py b/src/HABApp/core/files/file_properties.py similarity index 60% rename from src/HABApp/core/files/file/properties.py rename to src/HABApp/core/files/file_properties.py index 4ace7ccb..fb3c288e 100644 --- a/src/HABApp/core/files/file/properties.py +++ b/src/HABApp/core/files/file_properties.py @@ -12,45 +12,37 @@ class FileProperties(BaseModel): model_config = ConfigDict(extra='forbid', populate_by_name=True) -RE_START = re.compile(r'^#(\s*)HABApp\s*:', re.IGNORECASE) +RE_START = re.compile(r'^(\s*#\s*)HABApp\s*:', re.IGNORECASE) -def get_properties(_str: str) -> FileProperties: +def get_file_properties(_str: str) -> FileProperties: cfg = [] cut = 0 # extract the property string for line in _str.splitlines(): - line = line.strip() - if cut and not line: + line_strip = line.strip() + if cut and not line_strip: break - if not line: + if not line_strip: continue # break on first non-empty line that is not a comment - if line and not line.startswith('#'): + if line_strip and not line_strip.startswith('#'): break if not cut: # find out how much from the start we have to cut - m = RE_START.search(line) - if m: - cut = len(m.group(1)) + 1 + if m := RE_START.search(line): + cut = m.end(1) cfg.append(line[cut:].lower()) else: - do_break = False - for i, c in enumerate(line): - if i > cut: - break - - if c not in ('#', ' ', '\t'): - do_break = True - break - if do_break: + # If we would cut away characters it's not the yaml definition any more + # Here it's cut + 1 because it must be indented + if line[:cut + 1].strip() not in ('', '#'): break - cfg.append(line[cut:]) data = yml.load('\n'.join(cfg)) diff --git a/src/HABApp/core/files/folders/__init__.py b/src/HABApp/core/files/folders/__init__.py deleted file mode 100644 index 65c4a101..00000000 --- a/src/HABApp/core/files/folders/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .folders import add_folder, get_name, get_path, get_prefixes diff --git a/src/HABApp/core/files/folders/folders.py b/src/HABApp/core/files/folders/folders.py deleted file mode 100644 index 606be30e..00000000 --- a/src/HABApp/core/files/folders/folders.py +++ /dev/null @@ -1,71 +0,0 @@ -from pathlib import Path - -import HABApp -from HABApp.core.const.topics import TOPIC_FILES as T_FILES -from HABApp.core.events.habapp_events import RequestFileLoadEvent, RequestFileUnloadEvent -from HABApp.core.files.watcher import AggregatingAsyncEventHandler -from HABApp.core.internals import uses_post_event - - -FOLDERS: dict[str, 'ConfiguredFolder'] = {} - -post_event = uses_post_event() - - -async def _generate_file_events(files: list[Path]) -> None: - for file in files: - name = get_name(file) - post_event(T_FILES, RequestFileLoadEvent(name) if file.is_file() else RequestFileUnloadEvent(name)) - - -class ConfiguredFolder: - def __init__(self, prefix: str, folder: Path, priority: int) -> None: - self.prefix = prefix - self.folder = folder - self.priority: int = priority # priority determines the order how the files will be loaded - - def add_watch(self, file_ending: str, watch_subfolders: bool = True) -> AggregatingAsyncEventHandler: - filter = HABApp.core.files.watcher.FileEndingFilter(file_ending) - handler = AggregatingAsyncEventHandler(self.folder, _generate_file_events, filter, watch_subfolders) - HABApp.core.files.watcher.add_folder_watch(handler) - return handler - - def add_file_type(self, cls: type['HABApp.core.files.file.HABAppFile']) -> None: - HABApp.core.files.file.register_file_type(self.prefix, cls) - - -def get_prefixes() -> list[str]: - return list(map(lambda x: x.prefix, sorted(FOLDERS.values(), key=lambda x: x.priority, reverse=True))) - - -def add_folder(prefix: str, folder: Path, priority: int) -> ConfiguredFolder: - """Make a folder known - - :param prefix: HABApp file name prefix - :param folder: Folder path - :param priority: Priority (used to determine the load order) - :return: ConfiguredFolder - """ - assert prefix and prefix.endswith('/') - for obj in FOLDERS.values(): - assert obj.priority != priority - FOLDERS[prefix] = c = ConfiguredFolder(prefix, folder, priority) - return c - - -def get_name(path: Path) -> str: - path_str = path.as_posix() - for prefix, cfg in sorted(FOLDERS.items(), key=lambda x: len(x[0]), reverse=True): - folder = cfg.folder.as_posix() - if path_str.startswith(folder): - return prefix + path_str[len(folder) + 1:] - - raise ValueError(f'Path "{path_str}" is not part of the configured folders!') - - -def get_path(name: str) -> Path: - for prefix, obj in FOLDERS.items(): - if name.startswith(prefix): - return obj.folder / name[len(prefix):] - - raise ValueError(f'Prefix not found for "{name}"!') diff --git a/src/HABApp/core/files/manager.py b/src/HABApp/core/files/manager.py new file mode 100644 index 00000000..dc51b83d --- /dev/null +++ b/src/HABApp/core/files/manager.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import asyncio +import logging +from asyncio import sleep +from pathlib import Path +from time import monotonic +from typing import TYPE_CHECKING, Final + +from pydantic import ValidationError + +import HABApp +from HABApp.core.const.topics import TOPIC_FILES +from HABApp.core.files.file import HABAppFile +from HABApp.core.files.file_properties import get_file_properties +from HABApp.core.files.name_builder import FileNameBuilder +from HABApp.core.lib import SingleTask, ValueChange + + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + from re import Pattern + + from HABApp.core.events.habapp_events import RequestFileLoadEvent, RequestFileUnloadEvent + from HABApp.core.files.watcher import HABAppFileWatcher + + +log = logging.getLogger('HABApp.files') + + +class FileTypeHandler: + def __init__(self, name: str, logger: logging.Logger, *, + prefix: str, pattern: Pattern | None = None, + on_load: Callable[[str, Path], Awaitable[None]], + on_unload: Callable[[str, Path], Awaitable[None]]) -> None: + self.name: Final = name + self.logger: Final = logger + + self.prefix: Final = prefix + self.pattern: Final = pattern + + self.on_load: Final = on_load + self.on_unload: Final = on_unload + + def matches(self, name: str) -> bool: + if not name.startswith(self.prefix): + return False + + if (p := self.pattern) is not None and not p.search(name): # noqa: SIM103 + return False + + return True + + def __repr__(self) -> str: + return f'<{self.__class__.__name__} {self.name:s}>' + + +class FileManager: + def __init__(self, watcher: HABAppFileWatcher | None) -> None: + self._lock = asyncio.Lock() + self._files: Final[dict[str, HABAppFile]] = {} + self._file_names: Final = FileNameBuilder() + self._file_handlers: tuple[FileTypeHandler, ...] = () + self._task: Final = SingleTask(self._load_file_task, name='file load worker') + self._watcher: Final = watcher + + def add_folder(self, prefix: str, folder: Path, *, name: str, priority: int, pattern: Pattern | None = None) -> None: + self._file_names.add_folder(prefix, folder, priority=priority, pattern=pattern) + if self._watcher is not None: + self._watcher.watch_folder(name, self.file_watcher_event, folder) + + def get_file_watcher(self) -> HABAppFileWatcher: + if self._watcher is None: + raise ValueError() + return self._watcher + + def get_folders(self): # noqa: ANN201 + return self._file_names.get_folders() + + def add_handler(self, name: str, logger: logging.Logger, *, + prefix: str, pattern: Pattern | None = None, + on_load: Callable[[str, Path], Awaitable[None]], + on_unload: Callable[[str, Path], Awaitable[None]]) -> None: + + for h in self._file_handlers: + if h.name == name: + msg = f'Handler {name:s} already exists!' + raise ValueError(msg) + if h.prefix == prefix and h.pattern == pattern: + msg = f'Handler with prefix {prefix:s} and pattern {pattern} already exists!' + raise ValueError(msg) + + new = FileTypeHandler(name, logger, prefix=prefix, pattern=pattern, on_load=on_load, on_unload=on_unload) + self._file_handlers += (new, ) + log.debug(f'Added handler {new.name}') + + def get_file(self, name: str) -> HABAppFile | None: + return self._files.get(name) + + def file_state_changed(self, file: HABAppFile, new_state: str) -> None: + log.debug(f'{file.name} changed to {new_state:s}') + for f in self._files.values(): + f.file_state_changed(file, self) + + def _get_file_handler(self, name: str) -> FileTypeHandler: + handlers = [h for h in self._file_handlers if h.matches(name)] + if not handlers: + msg = f'No handler matched for {name:s}' + raise ValueError(msg) + + if len(handlers) > 1: + msg = f'Multiple handlers matches for {name:s}: {", ".join(str(h) for h in handlers)}' + raise ValueError(msg) + + return handlers[0] + + async def _do_file_load(self, name: str, *, aquire_lock: bool = True) -> None: + if aquire_lock: + await self._lock.acquire() + + try: + if not (file := self.get_file(name)): + return None + + await file.load(self._get_file_handler(name), manager=self) + finally: + if aquire_lock: + self._lock.release() + + async def _do_file_unload(self, name: str, *, aquire_lock: bool = True) -> None: + if aquire_lock: + await self._lock.acquire() + + try: + if not (file := self.get_file(name)): + return None + + await file.unload(self._get_file_handler(name), manager=self) + + if file.can_be_removed(): + self._files.pop(name) + finally: + if aquire_lock: + self._lock.release() + + async def _load_file_task(self) -> None: + try: + task_sleep = 0.3 + task_alive = 15 + + task_shutdown = False + last_process = monotonic() + + files_count = ValueChange[int]() + + while True: + await sleep(0) + + # wait until we have all files + while files_count.set_value(len(self._files)).changed: # noqa: ASYNC110 + await sleep(task_sleep) + + async with self._lock: + # check files for dependencies etc. + for file in self._files.values(): + file.check_properties(self, log, log_msg=task_shutdown) + file.check_dependencies(self) + + if can_be_loaded := [f.name for f in self._files.values() if f.can_be_loaded()]: + name = next(self._file_names.get_names(can_be_loaded)) + await self._do_file_load(name, aquire_lock=False) + last_process = monotonic() + + if task_shutdown: + break + task_shutdown = monotonic() - last_process > task_alive + + except Exception as e: + HABApp.core.wrapper.process_exception(self._task.name, e, logger=log) + log.debug('Worker done!') + + def __accept_event(self, event: RequestFileLoadEvent | RequestFileUnloadEvent) -> bool: + if not self._file_names.is_accepted_name(event.name): + HABApp.core.logger.log_error(log, f'Ignoring {event.__class__.__name__} for invalid name "{event.name}"') + return False + return True + + def __create_file(self, name: str) -> HABAppFile: + path = self._file_names.create_path(name) + text = path.read_text() + checksum = HABAppFile.create_checksum(text) + try: + properties = get_file_properties(text) + except ValidationError as e: + properties = None + HABApp.core.logger.log_error(log, str(e)) + + return HABAppFile(name, path, checksum, properties) + + async def event_load(self, event: RequestFileLoadEvent) -> None: + if not self.__accept_event(event): + return None + + self._task.start_if_not_running() + + name = event.name + file = self.__create_file(name) + + async with self._lock: + # file already exists -> unload first + if name in self._files: + await self._do_file_unload(name, aquire_lock=False) + + self._files[name] = file + + async def event_unload(self, event: RequestFileUnloadEvent) -> None: + if not self.__accept_event(event): + return None + + self._task.start_if_not_running() + await self._do_file_unload(event.name) + return None + + async def file_watcher_event(self, path: str) -> None: + if not self._file_names.is_accepted_path(path): + return None + + obj = Path(path) + name = self._file_names.create_name(path) + + if obj.is_dir(): + return None + + if not obj.is_file(): + HABApp.core.EventBus.post_event(TOPIC_FILES, HABApp.core.events.habapp_events.RequestFileUnloadEvent(name)) + return None + + if existing := self.get_file(name): + checksum = HABAppFile.create_checksum(obj.read_text()) + if existing.checksum == checksum: + log.debug(f'Skip file system event because file {name:s} did not change') + return None + + HABApp.core.EventBus.post_event(TOPIC_FILES, HABApp.core.events.habapp_events.RequestFileLoadEvent(name)) + return None + + def setup(self) -> None: + HABApp.core.EventBus.add_listener( + HABApp.core.internals.EventBusListener( + TOPIC_FILES, HABApp.core.internals.wrap_func(self.event_load), + HABApp.core.events.EventFilter(HABApp.core.events.habapp_events.RequestFileLoadEvent) + ) + ) + + HABApp.core.EventBus.add_listener( + HABApp.core.internals.EventBusListener( + TOPIC_FILES, HABApp.core.internals.wrap_func(self.event_unload), + HABApp.core.events.EventFilter(HABApp.core.events.habapp_events.RequestFileUnloadEvent) + ) + ) diff --git a/src/HABApp/core/files/manager/__init__.py b/src/HABApp/core/files/manager/__init__.py deleted file mode 100644 index 95f70830..00000000 --- a/src/HABApp/core/files/manager/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .files import FILES, file_state_changed -from .listen_events import setup_file_manager -from .worker import process_file diff --git a/src/HABApp/core/files/manager/files.py b/src/HABApp/core/files/manager/files.py deleted file mode 100644 index 6e5a52ce..00000000 --- a/src/HABApp/core/files/manager/files.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import TYPE_CHECKING - - -if TYPE_CHECKING: - import HABApp - -FILES: dict[str, 'HABApp.core.files.file.HABAppFile'] = {} - - -def file_state_changed(file: 'HABApp.core.files.file.HABAppFile') -> None: - for f in FILES.values(): - if f is not file: - f.file_changed(file) diff --git a/src/HABApp/core/files/manager/listen_events.py b/src/HABApp/core/files/manager/listen_events.py deleted file mode 100644 index 4c99a5c4..00000000 --- a/src/HABApp/core/files/manager/listen_events.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging - -import HABApp -from HABApp.core.const.topics import TOPIC_FILES as T_FILES -from HABApp.core.events import EventFilter -from HABApp.core.events.habapp_events import RequestFileLoadEvent, RequestFileUnloadEvent -from HABApp.core.internals import EventBusListener, uses_event_bus, wrap_func - - -log = logging.getLogger('HABApp.Files') -event_bus = uses_event_bus() - - -async def _process_event(event: RequestFileUnloadEvent | RequestFileLoadEvent) -> None: - name = event.name - await HABApp.core.files.manager.process_file(name, HABApp.core.files.folders.get_path(name)) - - -async def setup_file_manager() -> None: - # Setup events so we can process load/unload - event_bus.add_listener( - EventBusListener( - T_FILES, wrap_func(_process_event), EventFilter(RequestFileUnloadEvent) - ) - ) - event_bus.add_listener( - EventBusListener( - T_FILES, wrap_func(_process_event), EventFilter(RequestFileLoadEvent) - ) - ) diff --git a/src/HABApp/core/files/manager/worker.py b/src/HABApp/core/files/manager/worker.py deleted file mode 100644 index ee5398f7..00000000 --- a/src/HABApp/core/files/manager/worker.py +++ /dev/null @@ -1,89 +0,0 @@ -import logging -import time -from asyncio import Future, create_task, sleep -from pathlib import Path - -import HABApp -from HABApp.core.files.file import FileState -from HABApp.core.files.folders import get_prefixes - -from . import FILES - - -log = logging.getLogger('HABApp.files') - - -TASK: Future | None = None -TASK_SLEEP: float = 0.3 -TASK_DURATION: float = 15 - - -async def process_file(name: str, file: Path): - global TASK - - # unload file - if not file.is_file(): - existing = FILES.pop(name, None) - if existing is None: - return None - - await existing.unload() - log.debug(f'Removed {existing.name}') - return None - - # add file - FILES[name] = HABApp.core.files.file.create_file(name, file) - if TASK is None: - TASK = create_task(_process()) - - -async def _process() -> None: - global TASK - - prefixes = get_prefixes() - - ct = -1 - log_msg = False - last_process = time.time() - - try: - while True: - - # wait until files are stable - while ct != len(FILES): - ct = len(FILES) - await sleep(TASK_SLEEP) - - # check files for dependencies etc. - for file in FILES.values(): - file.check_properties(log_msg) - - # Load order - for prefix in prefixes: - file_loaded = False - for name in filter(lambda x: x.startswith(prefix), sorted(FILES.keys())): - file = FILES[name] - file.check_dependencies() - - if file.state is FileState.DEPENDENCIES_OK: - await file.load() - last_process = time.time() - file_loaded = True - break - if file_loaded: - break - - # if we don't have any files left to load we sleep! - if not any(map(lambda x: x.state is FileState.DEPENDENCIES_OK, FILES.values())): - await sleep(TASK_SLEEP) - - # Emit an error message during the last run - if log_msg: - break - log_msg = time.time() - last_process > TASK_DURATION - - except Exception as e: - HABApp.core.wrapper.process_exception('file load worker', e, logger=log) - finally: - TASK = None - log.debug('Worker done!') diff --git a/src/HABApp/core/files/name_builder.py b/src/HABApp/core/files/name_builder.py new file mode 100644 index 00000000..b3fe4377 --- /dev/null +++ b/src/HABApp/core/files/name_builder.py @@ -0,0 +1,99 @@ +from collections.abc import Generator, Iterable +from pathlib import Path +from re import Pattern +from typing import Final + + +class FileNameBuilderRule: + def __init__(self, prefix: str, folder: str, *, + priority: int, pattern: Pattern | None = None) -> None: + self.prefix: Final = prefix + self.folder: Final = folder + self.priority: Final = priority + self.pattern: Final = pattern + + def create_name(self, path: str) -> str | None: + if not path.startswith(folder := self.folder): + return None + + if (p := self.pattern) is not None and not p.search(path): + return None + return self.prefix + path.removeprefix(folder) + + def create_path(self, name: str) -> Path | None: + if not name.startswith(prefix := self.prefix): + return None + + if (p := self.pattern) is not None and not p.search(name): + return None + + return Path(self.folder + name.removeprefix(prefix)) + + def matches_name(self, name: str) -> bool: + return name.startswith(self.prefix) and (self.pattern is None or self.pattern.search(name)) + + +class FileNameBuilder: + def __init__(self) -> None: + self._builders: tuple[FileNameBuilderRule, ...] = () + + def add_folder(self, prefix: str, folder: Path, *, + priority: int, pattern: Pattern | None = None) -> None: + for b in self._builders: + if b.priority == priority: + msg = f'Priority {priority} already exists for {b.prefix}!' + raise ValueError(msg) + + new = FileNameBuilderRule(prefix, folder.as_posix() + '/', priority=priority, pattern=pattern) + self._builders = tuple(sorted(self._builders + (new,), key=lambda x: x.priority, reverse=True)) + + def create_name(self, path: str) -> str: + paths = [n for b in self._builders if (n := b.create_name(path)) is not None] + if not paths: + msg = f'Nothing matched for path {path:s}' + raise ValueError(msg) + + if len(paths) > 1: + msg = f'Multiple matches for path {path:s}: {", ".join(paths)}' + raise ValueError(msg) + + return paths[0] + + def create_path(self, name: str) -> Path: + paths = [p for b in self._builders if (p := b.create_path(name)) is not None] + if not paths: + msg = f'Nothing matched for name {name:s}' + raise ValueError(msg) + + if len(paths) > 1: + msg = f'Multiple matches for name {name:s}: {", ".join(p.as_posix() for p in paths)}' + raise ValueError(msg) + + return paths[0] + + def is_accepted_path(self, path: str) -> bool: + return any(b.create_name(path) is not None for b in self._builders) + + def is_accepted_name(self, path: str) -> bool: + return any(b.matches_name(path) for b in self._builders) + + def get_folders(self) -> list[str]: + ret: list[str] = [] + for b in self._builders: + if b.folder not in ret: + ret.append(b.folder) + return ret + + def get_names_with_path(self, paths: list[str]) -> list[tuple[str, Path]]: + ret = [] + for b in self._builders: + ret.extend((n, Path(p)) for p in paths if (n := b.create_name(p)) is not None) + return ret + + def get_names(self, names: Iterable[str]) -> Generator[str, None, None]: + """Get sorted names""" + names = sorted(names) + for b in self._builders: + for name in names: + if b.matches_name(name): + yield name diff --git a/src/HABApp/core/files/setup.py b/src/HABApp/core/files/setup.py deleted file mode 100644 index d361a5df..00000000 --- a/src/HABApp/core/files/setup.py +++ /dev/null @@ -1,5 +0,0 @@ -from .manager import setup_file_manager - - -async def setup() -> None: - await setup_file_manager() diff --git a/src/HABApp/core/files/watcher.py b/src/HABApp/core/files/watcher.py new file mode 100644 index 00000000..57b580d9 --- /dev/null +++ b/src/HABApp/core/files/watcher.py @@ -0,0 +1,258 @@ +import asyncio +import contextlib +import logging +import re +from asyncio import Event, Task +from collections.abc import Awaitable, Callable +from pathlib import Path +from re import Pattern +from typing import Any, Final, override + +from watchfiles import Change, DefaultFilter, awatch + +from HABApp.core.asyncio import create_task_from_async +from HABApp.core.wrapper import process_exception + + +log = logging.getLogger('HABApp.file.events') +log.setLevel(logging.INFO) + + +DEFAULT_FILTER = DefaultFilter() + +HABAPP_DISPATCHER_PREFIX: Final = 'HABAppInternal-' + + +class FileWatcherDispatcherBase: + + def __init__(self, name: str, coro: Callable[[str], Awaitable[Any]],) -> None: + self._name: Final = name + self._coro: Final = coro + + def __repr__(self) -> str: + return f'<{self.__class__.__name__} {self._name:s}>' + + def __eq__(self, other: object) -> bool: + raise NotImplementedError() + + @property + def name(self) -> str: + return self._name + + def allow(self, change: Change | None, path: str) -> bool: + raise NotImplementedError() + + async def dispatch(self, path: str) -> None: + if not self.allow(None, path): + return None + + try: + await self._coro(path) + except Exception as e: + process_exception(self._coro, e, logger=log) + + +class FolderDispatcher(FileWatcherDispatcherBase): + def __init__(self, name: str, coro: Callable[[str], Awaitable[Any]], folder: str) -> None: + super().__init__(name, coro) + self._folder: Final = folder + + @override + def __eq__(self, other: object) -> bool: + if not isinstance(other, FolderDispatcher): + return False + return self._name == other._name and self._coro is other._coro and self._folder == other._folder + + @override + def allow(self, change: Change, path: str) -> bool: + return path.startswith(self._folder) + + +class FileDispatcher(FileWatcherDispatcherBase): + def __init__(self, name: str, coro: Callable[[str], Awaitable[Any]], file: str) -> None: + super().__init__(name, coro) + self._file: Final = file + + @override + def __eq__(self, other: object) -> bool: + if not isinstance(other, FileDispatcher): + return False + return self._name == other._name and self._coro is other._coro and self._file == other._file + + @override + def allow(self, change: Change, path: str) -> bool: + return path == self._file + + +class HABAppFileWatcher: + def __init__(self) -> None: + self._dispatchers: tuple[FileWatcherDispatcherBase, ...] = () + self._paths: tuple[str, ...] = () + self._files_task: Task | None = None + self._stop_event: Final = Event() + + def __repr__(self) -> str: + return f'<{self.__class__.__name__:s}>' + + def cancel(self, dispatcher: FileWatcherDispatcherBase | str) -> None: + if isinstance(dispatcher, str): + for d in self._dispatchers: + if d.name == dispatcher: + dispatcher = d + break + else: + msg = f'No dispatcher with name "{dispatcher:s}" found' + raise ValueError(msg) + + self._dispatchers = tuple(d for d in self._dispatchers if d is not dispatcher) + + def __notify_task(self) -> None: + if self._files_task is None: + self._files_task = create_task_from_async(self._watcher_task()) + else: + self._stop_event.set() + + def watch_folder(self, name: str, coro: Callable[[str], Awaitable[Any]], folder: Path, *, + habapp_internal: bool = False) -> FolderDispatcher: + d = FolderDispatcher( + name if not habapp_internal else f'{HABAPP_DISPATCHER_PREFIX}{name}', coro, folder.as_posix() + ) + self.add_dispatcher(d) + self.add_path(folder) + return d + + def watch_file(self, name: str, coro: Callable[[str], Awaitable[Any]], file: Path, *, + habapp_internal: bool = False) -> FileDispatcher: + d = FileDispatcher( + name if not habapp_internal else f'{HABAPP_DISPATCHER_PREFIX}{name}', coro, file.as_posix() + ) + self.add_dispatcher(d) + self.add_path(file) + return d + + def add_dispatcher(self, dispatcher: FileWatcherDispatcherBase) -> None: + name = dispatcher.name.lower() + for d in self._dispatchers: + if d.name.lower() != name or d == dispatcher: + continue + msg = f'Dispatcher with name "{dispatcher.name:s}" already exists' + raise ValueError(msg) + + self._dispatchers += (dispatcher, ) + log.debug(f'Added dispatcher {dispatcher.name:s}') + self.__notify_task() + + def add_path(self, path: Path) -> None: + if path.as_posix() in self._paths: + return None + + if not path.is_dir() and not path.is_file(): + msg = f'Path {path} does not exist' + raise FileNotFoundError(msg) + + self._paths += (path.as_posix(), ) + log.debug(f'Watching {path}') + + self.__notify_task() + + def _watch_filter(self, change: Change | None, path: str, *, + dispatchers: list[FileWatcherDispatcherBase] | None = None) -> bool: + if not DEFAULT_FILTER(change, path): + return False + + if dispatchers is not None: + return any(dispatcher.allow(change, path) for dispatcher in dispatchers) + + log.debug(f'{change.name:s} {path:s}') + return any(dispatcher.allow(change, path) for dispatcher in self._dispatchers) + + async def _watcher_task(self) -> None: + delay = 1 + while self._dispatchers: + await asyncio.sleep(1) + + try: + self._stop_event.clear() + log.debug('Starting file watcher') + async for changes in awatch(*self._paths, watch_filter=self._watch_filter, stop_event=self._stop_event): + file_names = [Path(p).as_posix() for _, p in changes] + for dispatcher in self._dispatchers: + for path in file_names: + await dispatcher.dispatch(path) + + log.debug('File watcher stopped') + except Exception as e: # noqa: PERF203 + process_exception(self._watcher_task, e, logger=log) + delay *= 2 + await asyncio.sleep(delay) + + log.debug('File watcher shutdown') + + async def shutdown(self) -> None: + self._dispatchers = () + self._stop_event.set() + if self._files_task is None: + return None + + task = self._files_task + self._files_task = None + + try: + return await asyncio.wait_for(task, 2) + except asyncio.TimeoutError: + pass + + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + return None + + async def load_files(self, name_include: Pattern | str | None = None, name_exclude: Pattern | str | None = None, *, + exclude_habapp_config: bool = True) -> None: + + if isinstance(name_include, str): + name_include = re.compile(f'^{name_include}$') + if isinstance(name_exclude, str): + name_exclude = re.compile(f'^{name_exclude}$') + + dispatchers = [] + for d in self._dispatchers: + name = d.name + + if name.startswith(HABAPP_DISPATCHER_PREFIX): + if exclude_habapp_config: + continue + name = name.removeprefix(HABAPP_DISPATCHER_PREFIX) + + if exclude_habapp_config and d.name.lower().startswith(HABAPP_DISPATCHER_PREFIX): + continue + if name_include is not None and not name_include.search(name): + continue + if name_exclude is not None and name_exclude.search(name): + continue + dispatchers.append(d) + + if not dispatchers: + msg = 'No dispatchers selected!' + raise ValueError(msg) + + files: list[str] = [] + for path_str in self._paths: + if not self._watch_filter(None, path_str, dispatchers=dispatchers): + continue + path = Path(path_str) + + if path.is_file(): # noqa: PTH113 + files.append(path.as_posix()) + continue + + if path.is_dir(): + for obj in path.glob('**/*'): + obj_str = obj.as_posix() + if self._watch_filter(None, obj_str, dispatchers=dispatchers): + files.append(obj_str) + + for file in sorted(files): + for dispatcher in self._dispatchers: + if dispatcher.allow(None, file): + await dispatcher.dispatch(file) diff --git a/src/HABApp/core/files/watcher/__init__.py b/src/HABApp/core/files/watcher/__init__.py deleted file mode 100644 index bbb27003..00000000 --- a/src/HABApp/core/files/watcher/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .folder_watcher import start, remove_folder_watch, add_folder_watch -from .file_watcher import AggregatingAsyncEventHandler -from .base_watcher import FileEndingFilter diff --git a/src/HABApp/core/files/watcher/base_watcher.py b/src/HABApp/core/files/watcher/base_watcher.py deleted file mode 100644 index bc449b4f..00000000 --- a/src/HABApp/core/files/watcher/base_watcher.py +++ /dev/null @@ -1,63 +0,0 @@ -import logging -from pathlib import Path - -from watchdog.events import EVENT_TYPE_CLOSED as WD_EVENT_TYPE_CLOSED -from watchdog.events import EVENT_TYPE_CLOSED_NO_WRITE as WD_EVENT_TYPE_CLOSED_NO_WRITE -from watchdog.events import EVENT_TYPE_OPENED as WD_EVENT_TYPE_OPENED -from watchdog.events import FileSystemEvent - - -log = logging.getLogger('HABApp.file.events') -log.setLevel(logging.INFO) - - -class EventFilterBase: - def notify(self, path: str) -> bool: - raise NotImplementedError() - - -class FileEndingFilter(EventFilterBase): - def __init__(self, ending: str) -> None: - self.ending: str = ending - - def notify(self, path: str) -> bool: - return path.endswith(self.ending) - - def __repr__(self) -> str: - return f'<{self.__class__.__name__} ending: {self.ending}>' - - -class FileSystemEventHandler: - def __init__(self, folder: Path, filter: EventFilterBase, watch_subfolders: bool = False) -> None: - assert isinstance(folder, Path), type(folder) - assert watch_subfolders is True or watch_subfolders is False - - self.folder: Path = folder - self.watch_subfolders: bool = watch_subfolders - - self.filter: EventFilterBase = filter - - def dispatch(self, event: FileSystemEvent): - log.debug(event) - - # we don't process directory events - if event.is_directory: - return None - - # we don't process open and close events - if event.event_type in (WD_EVENT_TYPE_OPENED, WD_EVENT_TYPE_CLOSED, WD_EVENT_TYPE_CLOSED_NO_WRITE): - return None - - src = event.src_path - if self.filter.notify(src): - self.file_changed(src) - - # moved events have a dst, so we process it, too - if hasattr(event, 'dest_path'): - dst = event.dest_path - if self.filter.notify(dst): - self.file_changed(dst) - return None - - def file_changed(self, dst: str): - raise NotImplementedError() diff --git a/src/HABApp/core/files/watcher/file_watcher.py b/src/HABApp/core/files/watcher/file_watcher.py deleted file mode 100644 index dfe5b08a..00000000 --- a/src/HABApp/core/files/watcher/file_watcher.py +++ /dev/null @@ -1,51 +0,0 @@ -from asyncio import run_coroutine_threadsafe, sleep -from collections.abc import Awaitable, Callable -from pathlib import Path -from typing import Any - -import HABApp -from HABApp.core.wrapper import ignore_exception - -from .base_watcher import EventFilterBase, FileSystemEventHandler - - -DEBOUNCE_TIME: float = 0.6 - - -class AggregatingAsyncEventHandler(FileSystemEventHandler): - def __init__(self, folder: Path, func: Callable[[list[Path]], Awaitable[Any]], filter: EventFilterBase, - watch_subfolders: bool = False) -> None: - super().__init__(folder, filter, watch_subfolders=watch_subfolders) - - self.func = func - - self._files: set[Path] = set() - self._event_obj: object = object() - - @ignore_exception - def file_changed(self, dst: str) -> None: - # Map from thread to async - run_coroutine_threadsafe(self._event_waiter(Path(dst)), loop=HABApp.core.const.loop) - - @ignore_exception - async def _event_waiter(self, dst: Path): - self._event_obj = event_obj = object() - self._files.add(dst) - - # debounce time - await sleep(DEBOUNCE_TIME) - - # check if a new event came - if self._event_obj is not event_obj: - return None - - # Copy Path so we're done here - files = list(self._files) - self._files.clear() - - # process - await self.func(HABApp.core.lib.sort_files(files)) - - async def trigger_all(self) -> None: - files = HABApp.core.lib.list_files(self.folder, self.filter, self.watch_subfolders) - await self.func(files) diff --git a/src/HABApp/core/files/watcher/folder_watcher.py b/src/HABApp/core/files/watcher/folder_watcher.py deleted file mode 100644 index 8df80547..00000000 --- a/src/HABApp/core/files/watcher/folder_watcher.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging -from pathlib import Path -from threading import Lock - -from watchdog.observers import Observer -from watchdog.observers.api import ObservedWatch - -from HABApp.core import shutdown - -from .base_watcher import FileSystemEventHandler - - -log = logging.getLogger('HABApp.files.watcher') - -LOCK = Lock() - -OBSERVER: Observer | None = None -WATCHES: dict[str, ObservedWatch] = {} - - -def start(): - global OBSERVER - - # start only once! - assert OBSERVER is None - - OBSERVER = Observer() - OBSERVER.start() - - # register for proper shutdown - shutdown.register(OBSERVER.stop, msg='Stopping folder observer') - shutdown.register(OBSERVER.join, last=True, msg='Joining folder observer threads') - return None - - -def add_folder_watch(handler: FileSystemEventHandler) -> None: - assert OBSERVER is not None - assert isinstance(handler, FileSystemEventHandler), type(handler) - assert isinstance(handler.folder, Path) and handler.folder.is_dir() - - log.debug( - f'Adding {"recursive " if handler.watch_subfolders else ""}watcher for {handler.folder} with {handler.filter}' - ) - - with LOCK: - _folder = str(handler.folder) - assert _folder not in WATCHES - - WATCHES[_folder] = OBSERVER.schedule(handler, _folder, recursive=handler.watch_subfolders) - - -def remove_folder_watch(folder: Path) -> None: - assert OBSERVER is not None - assert isinstance(folder, Path) - - with LOCK: - OBSERVER.unschedule(WATCHES.pop(str(folder))) diff --git a/src/HABApp/core/internals/proxy/__init__.py b/src/HABApp/core/internals/proxy/__init__.py index 348ccf3d..5e4e9868 100644 --- a/src/HABApp/core/internals/proxy/__init__.py +++ b/src/HABApp/core/internals/proxy/__init__.py @@ -2,4 +2,4 @@ # isort: split -from .proxies import uses_get_item, uses_item_registry, uses_post_event, uses_event_bus, setup_internals +from .proxies import uses_get_item, uses_item_registry, uses_post_event, uses_event_bus, setup_internals, uses_file_manager diff --git a/src/HABApp/core/internals/proxy/proxies.py b/src/HABApp/core/internals/proxy/proxies.py index c49255be..9b064b7e 100644 --- a/src/HABApp/core/internals/proxy/proxies.py +++ b/src/HABApp/core/internals/proxy/proxies.py @@ -24,11 +24,17 @@ def uses_item_registry() -> 'HABApp.core.internals.ItemRegistry': return create_proxy(uses_item_registry) +def uses_file_manager() -> 'HABApp.core.files.FileManager': + return create_proxy(uses_file_manager) + + def setup_internals(ir: 'HABApp.core.internals.ItemRegistry', - eb: 'HABApp.core.internals.EventBus', final=True): + eb: 'HABApp.core.internals.EventBus', + file_manager: 'HABApp.core.files.FileManager', final=True): """Replace the proxy objects with the real thing""" replacements = { uses_item_registry: ir, uses_get_item: ir.get_item, uses_event_bus: eb, uses_post_event: eb.post_event, + uses_file_manager: file_manager, } return replace_proxies(replacements, final=final) diff --git a/src/HABApp/core/lib/__init__.py b/src/HABApp/core/lib/__init__.py index 216c938b..d17c6eff 100644 --- a/src/HABApp/core/lib/__init__.py +++ b/src/HABApp/core/lib/__init__.py @@ -1,5 +1,4 @@ from .exceptions import HINT_EXCEPTION, format_exception -from .funcs import list_files, sort_files from .instant_view import InstantView from .pending_future import PendingFuture from .priority_list import PriorityList diff --git a/src/HABApp/core/lib/funcs.py b/src/HABApp/core/lib/funcs.py index f3869dc2..f32c17d5 100644 --- a/src/HABApp/core/lib/funcs.py +++ b/src/HABApp/core/lib/funcs.py @@ -1,27 +1,11 @@ import operator as _operator -from collections.abc import Iterable -from pathlib import Path -from typing import TYPE_CHECKING +from collections.abc import Callable +from typing import Any, Final from HABApp.core.const import MISSING -if TYPE_CHECKING: - import HABApp - - -def list_files(folder: Path, file_filter: 'HABApp.core.files.watcher.file_watcher.EventFilterBase', - recursive: bool = False) -> list[Path]: - # glob is much quicker than iter_dir() - files = folder.glob('**/*' if recursive else '*') - return sorted(filter(lambda x: file_filter.notify(str(x)), files), key=lambda x: x.relative_to(folder)) - - -def sort_files(files: Iterable[Path]) -> list[Path]: - return sorted(files) - - -CMP_OPS = { +CMP_OPS: Final[dict[str, Callable[[Any, Any], bool]]] = { 'lt': _operator.lt, 'lower_than': _operator.lt, 'le': _operator.le, 'lower_equal': _operator.le, 'eq': _operator.eq, 'equal': _operator.eq, @@ -34,7 +18,7 @@ def sort_files(files: Iterable[Path]) -> list[Path]: } -def compare(value, **kwargs) -> bool: +def compare(value: Any, **kwargs) -> bool: for name, cmp_value in kwargs.items(): if cmp_value is MISSING: diff --git a/src/HABApp/core/logger.py b/src/HABApp/core/logger.py index 5b3d9e27..8ddb016b 100644 --- a/src/HABApp/core/logger.py +++ b/src/HABApp/core/logger.py @@ -19,6 +19,7 @@ def log_error(logger: logging.Logger, text: str) -> None: logger.error(line) else: logger.error(text) + post_event( _T_ERRORS, text ) diff --git a/src/HABApp/core/shutdown.py b/src/HABApp/core/shutdown.py index 09c1a82d..d86612f2 100644 --- a/src/HABApp/core/shutdown.py +++ b/src/HABApp/core/shutdown.py @@ -8,16 +8,20 @@ from dataclasses import dataclass from types import BuiltinMethodType, FunctionType, MethodType from typing import TYPE_CHECKING +from collections.abc import Awaitable from HABApp.core.asyncio import create_task from HABApp.core.const import loop if TYPE_CHECKING: - from collections.abc import Callable, Coroutine + from collections.abc import Callable from typing import Any, NoReturn +log = logging.getLogger('HABApp.Shutdown') + + @dataclass(frozen=True) class ShutdownBase: msg: str @@ -37,18 +41,18 @@ async def run(self) -> None: @dataclass(frozen=True) class ShutdownAwaitable(ShutdownBase): - func: Callable[[], Coroutine[Any, Any, Any]] + func: Callable[[], Awaitable[Any]] async def run(self) -> None: await self.func() -_REGISTERED: tuple[ShutdownBase, ...] = () +_REGISTERED: tuple[ShutdownFunction | ShutdownAwaitable, ...] = () _REQUESTED: bool = False -def register(func: Callable[[], Any], *, last: bool = False, msg: str = '') -> None: +def register(func: Callable[[], Any | Awaitable[Any]], *, last: bool = False, msg: str = '') -> None: global _REGISTERED if last is not True and last is not False: @@ -60,6 +64,17 @@ def register(func: Callable[[], Any], *, last: bool = False, msg: str = '') -> N if not msg: msg = f'{func.__module__}.{func.__name__}' + for existing in _REGISTERED: + if existing.func is func: + # If it's the same thing we don't call it multiple times + if existing.msg == msg and existing.last == last: + return None + + log.warning(f'Function {func} is already registered with a different message!') + log.warning(f' - {existing.msg:s}') + log.warning(f' - {msg:s}') + return None + if iscoroutinefunction(func): _REGISTERED += (ShutdownAwaitable(func=func, last=last, msg=msg), ) elif isinstance(func, (FunctionType, MethodType, BuiltinMethodType)): @@ -75,8 +90,6 @@ async def _shutdown() -> None: return None _REQUESTED = True - - log = logging.getLogger('HABApp.Shutdown') log.debug('Requested shutdown') objs = ( diff --git a/src/HABApp/core/wrapper.py b/src/HABApp/core/wrapper.py index c3218977..28094d1f 100644 --- a/src/HABApp/core/wrapper.py +++ b/src/HABApp/core/wrapper.py @@ -1,13 +1,13 @@ import asyncio import functools import logging -import typing -from collections.abc import Callable +from collections.abc import Awaitable, Callable from logging import Logger # noinspection PyProtectedMember from sys import _getframe as sys_get_frame from types import TracebackType +from typing import ParamSpec, TypeVar, overload from HABApp.core.const.topics import TOPIC_ERRORS, TOPIC_WARNINGS from HABApp.core.events.habapp_events import HABAppException @@ -20,6 +20,10 @@ post_event = uses_post_event() +T = TypeVar('T') # the callable/awaitable return type +P = ParamSpec('P') # the callable parameters + + def process_exception(func: Callable | str, e: Exception, do_print=False, logger: logging.Logger = log) -> None: lines = format_exception(e) @@ -39,9 +43,17 @@ def process_exception(func: Callable | str, e: Exception, post_event(TOPIC_ERRORS, HABAppException(func_name=func_name, exception=e, traceback='\n'.join(lines))) +@overload +def log_exception(func: Callable[P, T]) -> Callable[P, T]: ... + + +@overload +def log_exception(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]: ... + + def log_exception(func): # return async wrapper - if asyncio.iscoroutinefunction(func) or asyncio.iscoroutine(func): + if asyncio.iscoroutinefunction(func): @functools.wraps(func) async def a(*args, **kwargs): try: @@ -65,9 +77,17 @@ def f(*args, **kwargs): return f +@overload +def ignore_exception(func: Callable[P, T]) -> Callable[P, T]: ... + + +@overload +def ignore_exception(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]: ... + + def ignore_exception(func): # return async wrapper - if asyncio.iscoroutinefunction(func) or asyncio.iscoroutine(func): + if asyncio.iscoroutinefunction(func): @functools.wraps(func) async def a(*args, **kwargs): try: @@ -97,12 +117,14 @@ def __init__(self, logger: Logger | None = None, log_level: int = logging.ERROR, self.raised_exception = False - self.proc_tb: typing.Callable[[list], list] | None = None + self.proc_tb: Callable[[list], list] | None = None def __enter__(self) -> None: self.raised_exception = False - def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None): + def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, + exc_tb: TracebackType | None): + # no exception -> we exit gracefully if exc_type is None and exc_val is None: return True diff --git a/src/HABApp/openhab/connection/plugins/plugin_things/plugin_things.py b/src/HABApp/openhab/connection/plugins/plugin_things/plugin_things.py index 5061d3f9..cb6ddd88 100644 --- a/src/HABApp/openhab/connection/plugins/plugin_things/plugin_things.py +++ b/src/HABApp/openhab/connection/plugins/plugin_things/plugin_things.py @@ -8,8 +8,6 @@ import HABApp.openhab.events from HABApp.core.connections import BaseConnectionPlugin from HABApp.core.files.file import HABAppFile -from HABApp.core.files.folders import add_folder as add_habapp_folder -from HABApp.core.files.watcher import AggregatingAsyncEventHandler from HABApp.core.lib import PendingFuture from HABApp.core.logger import HABAppError, log_warning from HABApp.openhab.connection.connection import OpenhabConnection @@ -48,6 +46,9 @@ async def on_setup(self): if path is None: return None + log.warning('TextualThingConfig deactivated') + return None + if self.watcher is not None: return None diff --git a/src/HABApp/parameters/parameter_files.py b/src/HABApp/parameters/parameter_files.py index 023280d9..10646af0 100644 --- a/src/HABApp/parameters/parameter_files.py +++ b/src/HABApp/parameters/parameter_files.py @@ -1,10 +1,11 @@ import logging +import re import threading from pathlib import Path import HABApp from HABApp.core.files.file import HABAppFile -from HABApp.core.files.folders import add_folder as add_habapp_folder +from HABApp.core.internals.proxy import uses_file_manager from .parameters import get_parameter_file, remove_parameter_file, set_parameter_file @@ -14,6 +15,8 @@ LOCK = threading.Lock() PARAM_PREFIX = 'params/' +file_manager = uses_file_manager() + async def load_file(name: str, path: Path) -> None: with LOCK: # serialize to get proper error messages @@ -59,10 +62,12 @@ async def setup_param_files() -> bool: if path is None: return False - folder = add_habapp_folder(PARAM_PREFIX, path, 100) - folder.add_file_type(HABAppParameterFile) - watcher = folder.add_watch('.yml') - await watcher.trigger_all() + prefix = 'params/' + file_manager.add_handler('ParamFiles', log, prefix=prefix, on_load=load_file, on_unload=unload_file) + file_manager.add_folder( + prefix, path, priority=100, pattern=re.compile(r'.yml$', re.IGNORECASE), name='rules-parameters' + ) + return True diff --git a/src/HABApp/rule_manager/rule_manager.py b/src/HABApp/rule_manager/rule_manager.py index ad727787..cedd604f 100644 --- a/src/HABApp/rule_manager/rule_manager.py +++ b/src/HABApp/rule_manager/rule_manager.py @@ -1,4 +1,5 @@ import logging +import re import threading import typing from asyncio import sleep @@ -9,10 +10,8 @@ from HABApp.core import shutdown from HABApp.core.connections import Connections from HABApp.core.files.errors import AlreadyHandledFileError -from HABApp.core.files.file import HABAppFile -from HABApp.core.files.folders import add_folder as add_habapp_folder -from HABApp.core.files.watcher import AggregatingAsyncEventHandler from HABApp.core.internals import uses_item_registry +from HABApp.core.internals.proxy import uses_file_manager from HABApp.core.internals.wrapped_function import wrap_func from HABApp.core.logger import log_warning from HABApp.core.wrapper import log_exception @@ -22,6 +21,7 @@ log = logging.getLogger('HABApp.Rules') item_registry = uses_item_registry() +file_manager = uses_file_manager() class RuleManager: @@ -36,11 +36,6 @@ def __init__(self, parent) -> None: self.__load_lock = threading.Lock() self.__files_lock = threading.Lock() - # Processing - self.__process_last_sec = 60 - - self.watcher: AggregatingAsyncEventHandler | None = None - async def setup(self): # shutdown @@ -55,17 +50,17 @@ async def setup(self): shutdown.request() return None await file.check_all_rules() - return - - class HABAppRuleFile(HABAppFile): - LOGGER = log - LOAD_FUNC = self.request_file_load - UNLOAD_FUNC = self.request_file_unload path = HABApp.CONFIG.directories.rules - folder = add_habapp_folder('rules/', path, 0) - folder.add_file_type(HABAppRuleFile) - self.watcher = folder.add_watch('.py', True) + prefix = 'rules/' + + file_manager.add_handler( + self.__class__.__name__, log, prefix=prefix, + on_load=self.request_file_load, on_unload=self.request_file_unload + ) + file_manager.add_folder( + prefix, path, priority=0, pattern=re.compile(r'.py$', re.IGNORECASE), name='rules-python' + ) # Initial loading of rules HABApp.core.internals.wrap_func(self.load_rules_on_startup, logger=log).run() @@ -84,7 +79,7 @@ async def load_rules_on_startup(self): return None # trigger event for every file - await self.watcher.trigger_all() + await file_manager.get_file_watcher().load_files(name_include=r'^rules.*$') return None @log_exception diff --git a/src/HABApp/runtime/runtime.py b/src/HABApp/runtime/runtime.py index 19286b07..1b9497a3 100644 --- a/src/HABApp/runtime/runtime.py +++ b/src/HABApp/runtime/runtime.py @@ -32,22 +32,24 @@ async def start(self, config_folder: Path) -> None: # setup exception handler for the scheduler eascheduler.set_exception_handler(lambda x: process_exception('HABApp.scheduler', x)) - # Start Folder watcher! - HABApp.core.files.watcher.start() - - # Load config - HABApp.config.load_config(config_folder) + file_watcher = HABApp.core.files.HABAppFileWatcher() + shutdown.register(file_watcher.shutdown, msg='Shutdown file watcher') # replace proxy objects ir = HABApp.core.internals.ItemRegistry() eb = HABApp.core.internals.EventBus() - setup_internals(ir, eb) + file_manager = HABApp.core.files.FileManager(file_watcher) + + setup_internals(ir, eb, file_manager) assert isinstance(HABApp.core.Items, ConstProxyObj) HABApp.core.Items = ir assert isinstance(HABApp.core.EventBus, ConstProxyObj) HABApp.core.EventBus = eb - await HABApp.core.files.setup() + file_manager.setup() + + # Load config + HABApp.config.setup_habapp_configuration(config_folder) # generic HTTP await HABApp.rule.interfaces._http.create_client() diff --git a/tests/conftest.py b/tests/conftest.py index d63cbf77..e260f948 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ import pytest import HABApp +from HABApp.core.files import FileManager from HABApp.core.internals import EventBus, ItemRegistry, setup_internals from tests.helpers import LogCollector, eb, get_dummy_cfg, params, parent_rule, sync_worker from tests.helpers.log.log_matcher import AsyncDebugWarningMatcher, LogLevelMatcher @@ -59,15 +60,20 @@ def ir(): return ItemRegistry() +@pytest.fixture() +def file_manager(): + return FileManager(None) + + @pytest.fixture(autouse=True) -def clean_objs(ir: ItemRegistry, eb: EventBus, request): +def clean_objs(ir: ItemRegistry, eb: EventBus, file_manager: FileManager, request): markers = request.node.own_markers for marker in markers: if marker.name == 'no_internals': yield None return None - restore = setup_internals(ir, eb, final=False) + restore = setup_internals(ir, eb, file_manager, final=False) yield diff --git a/tests/helpers/log/log_collector.py b/tests/helpers/log/log_collector.py index c881a8e0..a9a98331 100644 --- a/tests/helpers/log/log_collector.py +++ b/tests/helpers/log/log_collector.py @@ -87,7 +87,8 @@ def update(self) -> Self: for phase in self.phases: if phase not in ALL_PYTEST_PHASES: - raise ValueError(f'Unknown pytest phase: {phase}') + msg = f'Unknown pytest phase: {phase}' + raise ValueError(msg) prev_rec = None for record in self.caplog.get_records(phase): diff --git a/tests/rule_runner/rule_runner.py b/tests/rule_runner/rule_runner.py index 11cfd862..91dfc74e 100644 --- a/tests/rule_runner/rule_runner.py +++ b/tests/rule_runner/rule_runner.py @@ -1,5 +1,4 @@ -from inspect import getmembers -from types import ModuleType, TracebackType +from types import TracebackType from astral import Observer from eascheduler.producers import prod_sun as prod_sun_module @@ -9,7 +8,7 @@ import HABApp.core.lib.exceptions.format import HABApp.rule.rule as rule_module import HABApp.rule.scheduler.job_builder as job_builder_module -from HABApp.core.asyncio import loop +from HABApp.core.files import FileManager from HABApp.core.internals import EventBus, ItemRegistry, setup_internals from HABApp.core.internals.proxy import ConstProxyObj from HABApp.core.internals.wrapped_function import wrapped_thread, wrapper @@ -17,20 +16,19 @@ from HABApp.core.lib.exceptions.format import fallback_format from HABApp.rule.rule_hook import HABAppRuleHook from HABApp.runtime import Runtime -from tests.helpers.inspect.habapp import habapp_modules -def _get_loop_modules() -> tuple[ModuleType, ...]: - ret = [] - for module in habapp_modules(): - for name, obj in getmembers(module): - if obj is loop: - ret.append(module) - assert name == 'loop' - return tuple(ret) - - -LOOP_MODULES = _get_loop_modules() +# def _get_loop_modules() -> tuple[ModuleType, ...]: +# ret = [] +# for module in habapp_modules(): +# for name, obj in getmembers(module): +# if obj is loop: +# ret.append(module) +# assert name == 'loop' +# return tuple(ret) +# +# +# LOOP_MODULES = _get_loop_modules() def suggest_rule_name(obj: object) -> str: @@ -91,7 +89,8 @@ def set_up(self) -> None: ir = ItemRegistry() eb = EventBus() - self.restore = setup_internals(ir, eb, final=False) + file_manager = FileManager(None) + self.restore = setup_internals(ir, eb, file_manager, final=False) # Scheduler self.monkeypatch.setattr(prod_sun_module, 'OBSERVER', Observer(52.51870523376821, 13.376072914752532, 10)) diff --git a/tests/test_core/test_files/test_file.py b/tests/test_core/test_files/test_file.py new file mode 100644 index 00000000..1b30ef2e --- /dev/null +++ b/tests/test_core/test_files/test_file.py @@ -0,0 +1,65 @@ +from pathlib import Path + +import pytest + +from HABApp.core.files.file import CircularReferenceError, FileProperties, FileState, HABAppFile +from HABApp.core.files.manager import log as file_manager_logger +from tests.helpers import LogCollector + + +@pytest.fixture +def files(file_manager) -> dict[str, HABAppFile]: + assert not file_manager._files + return file_manager._files + + +def test_depends(test_logs: LogCollector, files, file_manager) -> None: + files['name1'] = f1 = HABAppFile('name1', Path('path1'), b'checksum', FileProperties(depends_on=['name2'])) + files['name2'] = f2 = HABAppFile('name2', Path('path2'), b'checksum', FileProperties()) + + f1.check_properties(file_manager, file_manager_logger, log_msg=True) + f2.check_properties(file_manager, file_manager_logger, log_msg=True) + + assert f1._state is FileState.DEPENDENCIES_MISSING + assert f2._state is FileState.DEPENDENCIES_OK + + f2._state = FileState.LOADED + f1.check_dependencies(file_manager) + assert f1._state is FileState.DEPENDENCIES_OK + + files['name3'] = f3 = HABAppFile('name3', Path('path3'), b'checksum', FileProperties(depends_on=['asdf'])) + f3.check_properties(file_manager, file_manager_logger, log_msg=True) + test_logs.add_expected('HABApp.files', 'ERROR', "File path3 depends on file that doesn't exist: asdf") + + +def test_reloads(test_logs: LogCollector, files, file_manager) -> None: + files['name1'] = f1 = HABAppFile('name1', Path('path1'), b'checksum', FileProperties(reloads_on=['name2', 'asdf'])) + files['name2'] = f2 = HABAppFile('name2', Path('path2'), b'checksum', FileProperties()) + + f1.check_properties(file_manager, file_manager_logger) + assert f1.properties.reloads_on == ['name2', 'asdf'] + assert f2.properties.reloads_on == [] + + test_logs.add_expected('HABApp.files', 'WARNING', "File path1 reloads on file that doesn't exist: asdf") + + +def test_circ(test_logs: LogCollector, files, file_manager) -> None: + files['name1'] = f1 = HABAppFile('name1', Path('path1'), b'checksum', FileProperties(depends_on=['name2'])) + files['name2'] = f2 = HABAppFile('name2', Path('path2'), b'checksum', FileProperties(depends_on=['name3'])) + files['name3'] = f3 = HABAppFile('name3', Path('path3'), b'checksum', FileProperties(depends_on=['name1'])) + + with pytest.raises(CircularReferenceError) as e: + f1._check_circ_refs((f1.name,), 'depends_on', file_manager) + assert e.value.stack == ('name1', 'name2', 'name3', 'name1') + + # Check log output + f1.check_properties(file_manager, file_manager_logger) + test_logs.add_expected('HABApp.files', 'ERROR', 'Circular reference: name1 -> name2 -> name3 -> name1') + + with pytest.raises(CircularReferenceError) as e: + f2._check_circ_refs((f2.name,), 'depends_on', file_manager) + assert e.value.stack == ('name2', 'name3', 'name1', 'name2',) + + with pytest.raises(CircularReferenceError) as e: + f3._check_circ_refs((f3.name,), 'depends_on', file_manager) + assert e.value.stack == ('name3', 'name1', 'name2', 'name3', ) diff --git a/tests/test_core/test_files/test_file_dependencies.py b/tests/test_core/test_files/test_file_dependencies.py deleted file mode 100644 index ea0034f8..00000000 --- a/tests/test_core/test_files/test_file_dependencies.py +++ /dev/null @@ -1,212 +0,0 @@ -import logging -from asyncio import sleep -from pathlib import Path - -import pytest - -import HABApp -from HABApp.core.files.file.file import FileProperties, HABAppFile -from HABApp.core.files.folders import add_folder -from HABApp.core.files.folders.folders import FOLDERS -from HABApp.core.files.manager import process_file -from tests.helpers import LogCollector - - -class MockFile: - def __init__(self, name: str) -> None: - self.name = name.split('/')[1] - - def as_posix(self) -> str: - return f'/my_param/{self.name}' - - def is_file(self) -> bool: - return True - - def __repr__(self) -> str: - return f'' - - -class CfgObj: - def __init__(self) -> None: - self.properties = {} - self.operation: list[tuple[str, str]] = [] - - class TestFile(HABAppFile): - LOGGER = logging.getLogger('test') - LOAD_FUNC = self.load_file - UNLOAD_FUNC = self.unload_file - self.cls = TestFile - - async def load_file(self, name: str, path: Path) -> None: - self.operation.append(('load', name)) - - async def unload_file(self, name: str, path: Path) -> None: - self.operation.append(('unload', name)) - - async def wait_complete(self) -> None: - while HABApp.core.files.manager.worker.TASK is not None: - await sleep(0.05) - - async def process_file(self, name: str) -> None: - await process_file(name, MockFile(name)) - - def create_file(self, name, path) -> HABAppFile: - return self.cls(name, MockFile(name), self.properties[name]) - - -@pytest.fixture() -def cfg(monkeypatch): - obj = CfgObj() - - monkeypatch.setattr(HABApp.core.files.manager.worker, 'TASK_SLEEP', 0.001) - monkeypatch.setattr(HABApp.core.files.manager.worker, 'TASK_DURATION', 0.001) - monkeypatch.setattr(HABApp.core.files.file, 'create_file', obj.create_file) - - FOLDERS.clear() - add_folder('rules/', Path('c:/HABApp/my_rules/'), 0) - add_folder('configs/', Path('c:/HABApp/my_config/'), 10) - add_folder('params/', Path('c:/HABApp/my_param/'), 20) - - yield obj - - FOLDERS.clear() - - -# def test_reload_on(cfg, sync_worker, event_bus: TmpEventBus): -# order = [] -# -# def process_event(event): -# order.append(event.name) -# file_load_ok(event.name) -# -# FILE_PROPS.clear() -# FILE_PROPS['params/param1'] = FileProperties(depends_on=[], reloads_on=['params/param2']) -# FILE_PROPS['params/param2'] = FileProperties() -# -# event_bus.listen_events(HABApp.core.const.topics.TOPIC_FILES, process_event) -# -# process([MockFile('param2'), MockFile('param1')]) -# -# assert order == ['params/param1', 'params/param2', 'params/param1'] -# order.clear() -# -# process([]) -# assert order == [] -# -# process([MockFile('param2')]) -# assert order == ['params/param2', 'params/param1'] -# order.clear() -# -# process([MockFile('param1')]) -# assert order == ['params/param1'] -# order.clear() -# -# process([MockFile('param2')]) -# assert order == ['params/param2', 'params/param1'] -# order.clear() - - -async def test_reload_dep(cfg: CfgObj, caplog) -> None: - cfg.properties['params/param1'] = FileProperties(depends_on=['params/param2'], reloads_on=['params/param2']) - cfg.properties['params/param2'] = FileProperties() - - await cfg.process_file('params/param1') - await cfg.process_file('params/param2') - await cfg.wait_complete() - - assert cfg.operation == [('load', 'params/param2'), ('load', 'params/param1')] - cfg.operation.clear() - - await cfg.process_file('params/param2') - await cfg.wait_complete() - assert cfg.operation == [('load', 'params/param2'), ('load', 'params/param1')] - cfg.operation.clear() - - await cfg.process_file('params/param1') - await cfg.wait_complete() - assert cfg.operation == [('load', 'params/param1')] - cfg.operation.clear() - - await cfg.process_file('params/param2') - await cfg.wait_complete() - assert cfg.operation == [('load', 'params/param2'), ('load', 'params/param1')] - cfg.operation.clear() - - -async def test_missing_dependencies(cfg: CfgObj, test_logs: LogCollector) -> None: - cfg.properties['params/param1'] = FileProperties(depends_on=['params/param4', 'params/param5']) - cfg.properties['params/param2'] = FileProperties(depends_on=['params/param4']) - cfg.properties['params/param3'] = FileProperties() - - await cfg.process_file('params/param1') - await cfg.process_file('params/param2') - await cfg.process_file('params/param3') - await cfg.wait_complete() - - assert cfg.operation == [('load', 'params/param3')] - - msg1 = ( - 'HABApp.files', logging.ERROR, "File depends on file that doesn't exist: params/param4" - ) - msg2 = ( - 'HABApp.files', logging.ERROR, - "File depends on files that don't exist: params/param4, params/param5" - ) - - test_logs.add_expected(*msg1) - test_logs.add_expected(*msg2) - - -# def test_missing_loads(cfg, sync_worker, event_bus: TmpEventBus, caplog): -# order = [] -# -# def process_event(event): -# order.append(event.name) -# file_load_ok(event.name) -# -# FILE_PROPS['params/param1'] = FileProperties(reloads_on=['params/param4', 'params/param5']) -# FILE_PROPS['params/param2'] = FileProperties(reloads_on=['params/param4']) -# -# event_bus.listen_events(HABApp.core.const.topics.TOPIC_FILES, process_event) -# -# process([MockFile('param1'), MockFile('param2')]) -# -# assert order == ['params/param1', 'params/param2'] -# order.clear() -# -# process([]) -# assert order == [] -# -# msg1 = ( -# 'HABApp.files', logging.WARNING, "File reloads on file that doesn't exist: params/param4" -# ) -# msg2 = ('HABApp.files', logging.WARNING, -# "File reloads on files that don't exist: params/param4, params/param5") -# -# assert msg1 in caplog.record_tuples -# assert msg2 in caplog.record_tuples -# -# -# def test_load_continue_after_missing(cfg, sync_worker, event_bus: TmpEventBus, caplog): -# order = [] -# -# def process_event(event): -# order.append(event.name) -# file_load_ok(event.name) -# -# FILE_PROPS.clear() -# FILE_PROPS['params/p1'] = FileProperties(depends_on=['params/p2'], reloads_on=[]) -# FILE_PROPS['params/p2'] = FileProperties() -# -# event_bus.listen_events(HABApp.core.const.topics.TOPIC_FILES, process_event) -# -# process([MockFile('p1')]) -# -# # File can not be loaded -# assert order == [] -# -# # Add missing file -# process([MockFile('p2')]) -# -# # Both files get loaded -# assert order == ['params/p2', 'params/p1'] diff --git a/tests/test_core/test_files/test_file_manager.py b/tests/test_core/test_files/test_file_manager.py new file mode 100644 index 00000000..bf9332c5 --- /dev/null +++ b/tests/test_core/test_files/test_file_manager.py @@ -0,0 +1,117 @@ +from collections.abc import Awaitable, Callable +from pathlib import Path +from unittest.mock import Mock + +import pytest + +import HABApp +from HABApp.core.const.topics import TOPIC_FILES +from HABApp.core.events.habapp_events import RequestFileLoadEvent, RequestFileUnloadEvent +from HABApp.core.files import FileManager +from HABApp.core.files import manager as file_manager_module +from HABApp.core.files.errors import AlreadyHandledFileError +from HABApp.core.files.file import FileProperties, FileState, HABAppFile +from HABApp.core.files.manager import log as file_manager_logger +from HABApp.core.internals import EventBus +from tests.helpers import LogCollector + + +async def test_file_watcher_event(monkeypatch, file_manager, test_logs: LogCollector) -> None: + test_logs.set_min_level(0) + + file_manager.add_folder('tests-', Path('tests/'), name='d', priority=1) + + eb = Mock(EventBus) + eb.post_event = Mock() + eb.post_event.assert_not_called() + monkeypatch.setattr(HABApp.core, 'EventBus', eb, raising=False) + + path_instance = Mock() + path_instance.is_dir = Mock(return_value=False) + monkeypatch.setattr(file_manager_module, 'Path', Mock(return_value=path_instance)) + + # Unload + path_instance.is_file = Mock(return_value=False) + await file_manager.file_watcher_event('tests/asdf') + eb.post_event.assert_called_once_with(TOPIC_FILES, RequestFileUnloadEvent('tests-asdf')) + eb.post_event.reset_mock() + + # Load + path_instance.is_file = Mock(return_value=True) + await file_manager.file_watcher_event('tests/asdf') + eb.post_event.assert_called_once_with(TOPIC_FILES, RequestFileLoadEvent('tests-asdf')) + eb.post_event.reset_mock() + + # Skip load (checksum) + file_manager._files['tests-asdf'] = f1 = HABAppFile('tests-asdf', Path('path1'), b'checksum', FileProperties()) + monkeypatch.setattr(HABAppFile, 'create_checksum', lambda x: b'checksum') + await file_manager.file_watcher_event('tests/asdf') + eb.post_event.assert_not_called() + + test_logs.add_expected( + file_manager_module.log.name, 'DEBUG', 'Skip file system event because file tests-asdf did not change') + test_logs.assert_ok() + + +def setup_file(file_manager: FileManager, *, set_state: bool = True, + on_load: Callable[[str, Path], Awaitable[None]] | None = None, + on_unload: Callable[[str, Path], Awaitable[None]] | None = None) -> HABAppFile: + + file_manager._files.clear() + file_manager._files['name1'] = f1 = HABAppFile('name1', Path('path1'), b'checksum', FileProperties()) + + if set_state: + f1._state = FileState.DEPENDENCIES_OK + + file_manager._file_handlers = () + file_manager.add_handler('myhandler', logger=file_manager_logger, prefix='n', on_load=on_load, on_unload=on_unload) + + return f1 + + +async def test_load(test_logs: LogCollector, file_manager) -> None: + setup_file(file_manager, set_state=False) + with pytest.raises(ValueError) as e: + await file_manager._do_file_load('name1') + assert str(e.value) == 'File name1 can not be loaded because current state is PENDING!' + + async def coro(name: str, path: Path) -> None: + pass + + f = setup_file(file_manager, on_load=coro) + await file_manager._do_file_load('name1') + assert f._state is FileState.LOADED + + # Error in coro -> state should be Failed + async def coro(name: str, path: Path) -> None: + raise AlreadyHandledFileError() + + f = setup_file(file_manager, on_load=coro) + await file_manager._do_file_load('name1') + assert f._state is FileState.FAILED + test_logs.assert_ok() + + +async def test_unload(test_logs: LogCollector, file_manager) -> None: + async def coro(name: str, path: Path) -> None: + pass + + # Remove should work regardless of state + f = setup_file(file_manager, set_state=False, on_unload=coro) + await file_manager._do_file_unload('name1') + assert f._state is FileState.REMOVED + assert file_manager.get_file('name1') is None + + f = setup_file(file_manager, on_unload=coro) + await file_manager._do_file_unload('name1') + assert f._state is FileState.REMOVED + assert file_manager.get_file('name1') is None + + # Error in coro -> state should be Failed + async def coro(name: str, path: Path) -> None: + raise AlreadyHandledFileError() + + f = setup_file(file_manager, on_unload=coro) + await file_manager._do_file_unload('name1') + assert f._state is FileState.FAILED + test_logs.assert_ok() diff --git a/tests/test_core/test_files/test_file_properties.py b/tests/test_core/test_files/test_file_properties.py index b6f39bee..745bfe71 100644 --- a/tests/test_core/test_files/test_file_properties.py +++ b/tests/test_core/test_files/test_file_properties.py @@ -1,12 +1,11 @@ -import pytest -from HABApp.core.files.file.file import FILES, CircularReferenceError, FileProperties, FileState, HABAppFile -from HABApp.core.files.file.properties import get_properties as get_props -from tests.helpers import LogCollector + +from HABApp.core.files.file_properties import get_file_properties as get_props def test_prop_case() -> None: - _in = '''# habapp: + _in = ''' + # habapp: # depends on: # - my_Param.yml # reloads on: @@ -31,7 +30,8 @@ def test_prop_case() -> None: def test_prop_1() -> None: - _in = '''# HABApp: + _in = ''' +# HABApp: # depends on: # - my_Param.yml # @@ -85,50 +85,3 @@ def test_prop_missing() -> None: p = get_props(_in) assert p.depends_on == [] assert p.reloads_on == [] - - -def test_deps() -> None: - FILES.clear() - FILES['name1'] = f1 = HABAppFile('name1', 'path1', FileProperties(depends_on=['name2'])) - FILES['name2'] = f2 = HABAppFile('name2', 'path2', FileProperties()) - - f1.check_properties() - f2.check_properties() - - assert f1.state is FileState.DEPENDENCIES_MISSING - assert f2.state is FileState.DEPENDENCIES_OK - - f2.state = FileState.LOADED - f1.check_dependencies() - assert f1.state is FileState.DEPENDENCIES_OK - - -def test_reloads(test_logs: LogCollector) -> None: - FILES.clear() - FILES['name1'] = f1 = HABAppFile('name1', 'path1', FileProperties(reloads_on=['name2', 'asdf'])) - FILES['name2'] = f2 = HABAppFile('name2', 'path2', FileProperties()) - - f1.check_properties() - assert f1.properties.reloads_on == ['name2', 'asdf'] - assert f2.properties.reloads_on == [] - - test_logs.add_expected('HABApp.files', 'WARNING', "File path1 reloads on file that doesn't exist: asdf") - - -def test_circ() -> None: - FILES.clear() - FILES['name1'] = f1 = HABAppFile('name1', 'path1', FileProperties(depends_on=['name2'])) - FILES['name2'] = f2 = HABAppFile('name2', 'path2', FileProperties(depends_on=['name3'])) - FILES['name3'] = f3 = HABAppFile('name3', 'path3', FileProperties(depends_on=['name1'])) - - with pytest.raises(CircularReferenceError) as e: - f1._check_circ_refs((f1.name,), 'depends_on') - assert e.value.stack == ('name1', 'name2', 'name3', 'name1') - - with pytest.raises(CircularReferenceError) as e: - f2._check_circ_refs((f2.name,), 'depends_on') - assert e.value.stack == ('name2', 'name3', 'name1', 'name2',) - - with pytest.raises(CircularReferenceError) as e: - f3._check_circ_refs((f3.name,), 'depends_on') - assert e.value.stack == ('name3', 'name1', 'name2', 'name3', ) diff --git a/tests/test_core/test_files/test_name_builder.py b/tests/test_core/test_files/test_name_builder.py new file mode 100644 index 00000000..b3371cf7 --- /dev/null +++ b/tests/test_core/test_files/test_name_builder.py @@ -0,0 +1,45 @@ +from pathlib import Path + +import pytest + +from HABApp.core.files.name_builder import FileNameBuilder + + +def test_create() -> None: + f = FileNameBuilder() + + with pytest.raises(ValueError) as e: + f.create_name('asdf') + assert str(e.value) == 'Nothing matched for path asdf' + + with pytest.raises(ValueError) as e: + f.create_path('asdf') + assert str(e.value) == 'Nothing matched for name asdf' + + f.add_folder('p1/', Path('as'), priority=1) + + assert not f.is_accepted_path('asd/asdf') + assert not f.is_accepted_name('p2/asdf') + + assert f.is_accepted_path('as/asdf') + assert f.is_accepted_name('p1/asdf') + assert f.create_path('p1/asdf') == Path('as/asdf') + assert f.create_name('as/asdf') == 'p1/asdf' + + f.add_folder('p1/', Path('as'), priority=2) + + with pytest.raises(ValueError) as e: + f.create_name(Path('as/df').as_posix()) + assert str(e.value) == 'Multiple matches for path as/df: p1/df, p1/df' + + with pytest.raises(ValueError) as e: + f.create_path('p1/df') + assert str(e.value) == 'Multiple matches for name p1/df: as/df, as/df' + + +def test_get_names() -> None: + f = FileNameBuilder() + f.add_folder('p1/', Path('fa1'), priority=1) + f.add_folder('z2/', Path('fz2'), priority=2) + + assert list(f.get_names(['p1/', 'z2/', 'z2/f', '???'])) == ['z2/', 'z2/f', 'p1/'] diff --git a/tests/test_core/test_files/test_rel_name.py b/tests/test_core/test_files/test_rel_name.py deleted file mode 100644 index b456d083..00000000 --- a/tests/test_core/test_files/test_rel_name.py +++ /dev/null @@ -1,64 +0,0 @@ -from pathlib import Path - -import pytest - -from HABApp.core.files.folders import add_folder, get_name, get_path, get_prefixes -from HABApp.core.files.folders.folders import FOLDERS - - -@pytest.fixture() -def cfg(): - FOLDERS.clear() - add_folder('rules/', Path('c:/HABApp/my_rules/'), 0) - add_folder('configs/', Path('c:/HABApp/my_config/'), 10) - add_folder('params/', Path('c:/HABApp/my_param/'), 20) - - yield None - - FOLDERS.clear() - - -def cmp(path: Path, name: str) -> None: - assert get_name(path) == name - assert get_path(name) == path - - -def test_prefix_sort(cfg) -> None: - assert get_prefixes() == ['params/', 'configs/', 'rules/'] - add_folder('params1/', Path('c:/HABApp/my_para1m/'), 50) - assert get_prefixes() == ['params1/', 'params/', 'configs/', 'rules/'] - - -def test_from_path(cfg) -> None: - cmp(Path('c:/HABApp/my_rules/rule.py'), 'rules/rule.py') - cmp(Path('c:/HABApp/my_config/params.yml'), 'configs/params.yml') - cmp(Path('c:/HABApp/my_param/cfg.yml'), 'params/cfg.yml') - - cmp(Path('c:/HABApp/my_rules/my_folder1/folder2/rule.py'), 'rules/my_folder1/folder2/rule.py') - cmp(Path('c:/HABApp/my_config/my_folder2/cfg.yml'), 'configs/my_folder2/cfg.yml') - cmp(Path('c:/HABApp/my_param/my_folder3/cfg.yml'), 'params/my_folder3/cfg.yml') - - -def test_err(cfg) -> None: - with pytest.raises(ValueError): - get_name(Path('c:/HABApp/rules/rule.py')) - - -def test_mixed() -> None: - FOLDERS.clear() - add_folder('rules/', Path('c:/HABApp/rules'), 1) - add_folder('configs/', Path('c:/HABApp/rules/my_config'), 2) - add_folder('params/', Path('c:/HABApp/rules/my_param'), 3) - - cmp(Path('c:/HABApp/rules/rule.py'), 'rules/rule.py') - cmp(Path('c:/HABApp/rules/my_config/params.yml'), 'configs/params.yml') - cmp(Path('c:/HABApp/rules/my_param/cfg.yml'), 'params/cfg.yml') - - FOLDERS.clear() - add_folder('rules/', Path('c:/HABApp/rules'), 1) - add_folder('configs/', Path('c:/HABApp/rules/my_cfg'), 2) - add_folder('params/', Path('c:/HABApp/rules/my_param'), 3) - - cmp(Path('c:/HABApp/rules/rule.py'), 'rules/rule.py') - cmp(Path('c:/HABApp/rules/my_cfg/params.yml'), 'configs/params.yml') - cmp(Path('c:/HABApp/rules/my_param/cfg.yml'), 'params/cfg.yml') diff --git a/tests/test_core/test_files/test_watcher.py b/tests/test_core/test_files/test_watcher.py index ffa54f4c..79bda3e9 100644 --- a/tests/test_core/test_files/test_watcher.py +++ b/tests/test_core/test_files/test_watcher.py @@ -1,40 +1,57 @@ -import asyncio -import time -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from unittest.mock import AsyncMock +import logging +from pathlib import PurePath +from typing import Self -from watchdog.events import FileSystemEvent +import pytest +from watchfiles import Change -import HABApp.core.files.watcher.file_watcher -from HABApp.core.files.watcher import AggregatingAsyncEventHandler -from HABApp.core.files.watcher.base_watcher import FileEndingFilter +from HABApp.core.files import HABAppFileWatcher +from HABApp.core.files import watcher as watcher_module -async def test_file_events(monkeypatch, sync_worker) -> None: +class MyPath(PurePath): - wait_time = 0.1 - monkeypatch.setattr(HABApp.core.files.watcher.file_watcher, 'DEBOUNCE_TIME', wait_time) + def __init__(self, *args) -> None: + super().__init__(*args) + self._is_dir = False + self._is_file = False - m = AsyncMock() - handler = AggregatingAsyncEventHandler(Path('folder'), m, FileEndingFilter('.tmp'), False) + def is_dir(self) -> bool: + return self._is_dir - loop = asyncio.get_event_loop() + def is_file(self) -> bool: + return self._is_file - ex = ThreadPoolExecutor(4) + def set_is_dir(self, value: bool) -> Self: + self._is_dir = value + return self - def generate_events(count: int, name: str, sleep: float) -> None: - for _ in range(count): - handler.dispatch(FileSystemEvent(name)) - time.sleep(sleep) + def set_is_file(self, value: bool) -> Self: + self._is_file = value + return self - await asyncio.gather( - loop.run_in_executor(ex, generate_events, 3, 'test/t1.tmp', wait_time), - loop.run_in_executor(ex, generate_events, 9, 'test/t2.tmp', wait_time / 2), - loop.run_in_executor(ex, generate_events, 18, 'test/t3.tmp', wait_time / 5), - ) - ex.shutdown() - await asyncio.sleep(wait_time + 0.01) - m.assert_called_once() - assert set(*m.call_args[0]) == {Path('test/t1.tmp'), Path('test/t2.tmp'), Path('test/t3.tmp')} +async def test_watcher(monkeypatch, test_logs) -> None: + logging.getLogger('HABApp.file.events').setLevel(0) + test_logs.set_min_level(0) + + f = HABAppFileWatcher() + f._watcher_task = lambda: 'ReplacedTask' + monkeypatch.setattr(watcher_module, 'create_task_from_async', lambda x: x) + + with pytest.raises(FileNotFoundError) as e: + f.add_path(MyPath('a/b/c')) + assert str(e.value) in ('Path a/b/c does not exist!', 'Path a\\b\\c does not exist') + + async def coro(text: str): + raise ValueError() + + f.watch_folder('folder1', coro, MyPath('my/folder/1').set_is_dir(True)) + + assert not f._watch_filter(Change.added, 'my/folder/2/file1') + assert not f._watch_filter(Change.added, 'my/folder/2/file1', dispatchers=f._dispatchers) + + test_logs.add_expected('HABApp.file.events', 'DEBUG', 'Added dispatcher folder1') + test_logs.add_expected('HABApp.file.events', 'DEBUG', 'Watching my\\folder\\1') + test_logs.add_expected('HABApp.file.events', 'DEBUG', 'added my/folder/2/file1') + test_logs.assert_ok()