From feb5a2fc1d2e47a53db8d8913e8f69bfac6804ea Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Fri, 1 Dec 2023 11:32:47 +0100 Subject: [PATCH 01/15] 23.12.0-DEV --- docs/logging.rst | 17 +- docs/requirements.txt | 6 +- docs/util.rst | 82 +++++++++ readme.md | 7 +- requirements_setup.txt | 2 +- run/conf/logging.yml | 8 +- src/HABApp/__version__.py | 2 +- src/HABApp/config/logging/__init__.py | 2 +- src/HABApp/config/logging/default_logfile.py | 2 + src/HABApp/config/logging/handler.py | 26 +++ src/HABApp/util/__init__.py | 1 + src/HABApp/util/rate_limiter/__init__.py | 1 + src/HABApp/util/rate_limiter/limiter.py | 143 ++++++++++++++++ src/HABApp/util/rate_limiter/rate_limit.py | 73 ++++++++ src/HABApp/util/rate_limiter/registry.py | 26 +++ tests/test_utils/test_rate_limiter.py | 166 +++++++++++++++++++ 16 files changed, 551 insertions(+), 13 deletions(-) create mode 100644 src/HABApp/util/rate_limiter/__init__.py create mode 100644 src/HABApp/util/rate_limiter/limiter.py create mode 100644 src/HABApp/util/rate_limiter/rate_limit.py create mode 100644 src/HABApp/util/rate_limiter/registry.py create mode 100644 tests/test_utils/test_rate_limiter.py diff --git a/docs/logging.rst b/docs/logging.rst index 297fe85d..55f4af58 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -17,6 +17,17 @@ but the format should be pretty straight forward. | That way even if the HABApp configuration is invalid HABApp can still log the errors that have occurred. | e.g.: ``/HABApp/logs/habapp.log`` or ``c:\HABApp\logs\habapp.log`` +Provided loggers +====================================== + +The ``HABApp.config.logging`` module provides additional loggers which can be used + + +.. autoclass:: HABApp.config.logging.MidnightRotatingFileHandler + +.. autoclass:: HABApp.config.logging.CompressedMidnightRotatingFileHandler + + Example ====================================== @@ -42,7 +53,7 @@ to the file configuration under ``handlers`` in the ``logging.yml``. ... MyRuleHandler: # <-- This is the name of the handler - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.MidnightRotatingFileHandler filename: 'c:\HABApp\Logs\MyRule.log' maxBytes: 10_000_000 backupCount: 3 @@ -84,7 +95,7 @@ Full Example configuration # ----------------------------------------------------------------------------------- handlers: HABApp_default: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.MidnightRotatingFileHandler filename: 'HABApp.log' maxBytes: 10_000_000 backupCount: 3 @@ -93,7 +104,7 @@ Full Example configuration level: DEBUG MyRuleHandler: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.MidnightRotatingFileHandler filename: 'c:\HABApp\Logs\MyRule.log' # absolute filename is recommended maxBytes: 10_000_000 backupCount: 3 diff --git a/docs/requirements.txt b/docs/requirements.txt index b3f65219..8032d1f4 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ # Packages required to build the documentation -sphinx == 7.2.5 -sphinx-autodoc-typehints == 1.24.0 -sphinx_rtd_theme == 1.3.0 +sphinx == 7.2.6 +sphinx-autodoc-typehints == 1.25.2 +sphinx_rtd_theme == 2.0.0 sphinx-exec-code == 0.10 autodoc_pydantic == 2.0.1 sphinx-copybutton == 0.5.2 diff --git a/docs/util.rst b/docs/util.rst index 4e5f04ed..53f4c663 100644 --- a/docs/util.rst +++ b/docs/util.rst @@ -75,6 +75,88 @@ Converts a hsb value to the rgb color space .. autofunction:: HABApp.util.functions.hsb_to_rgb +Rate limiter +------------------------------ +A simple rate limiter implementation which can be used in rules. +The limiter is not rule bound so the same limiter can be used in multiples files. +It also works as expected across rule reloads. + +Defining limits +^^^^^^^^^^^^^^^^^^ +Limits can either be explicitly added or through a textual description. +If the limit does already exist it will not be added again. +It's possible to explicitly create the limits or through some small textual description with the following syntax: + +.. code-block:: text + + [count] [per|in|/] [count (optional)] [s|sec|second|m|min|minute|hour|h|day|month|year] [s (optional)] + +Whitespaces are ignored and can be added as desired + +Examples: + +* ``5 per minute`` +* ``20 in 15 mins`` +* ``300 / hour`` + + +Elastic expiry +^^^^^^^^^^^^^^^^^^ + +The rate limiter implements a fixed window with elastic expiry. +That means if the limit is hit the interval time will be increased by the expiry time. + +For example ``3 per minute``: + +* First hit comes ``00:00:00``. Two more hits at ``00:00:59``. + All three pass, intervall goes from ``00:00:00`` - ``00:01:00``. + Another hit comes at ``00:01:01`` an passes. The intervall now goes from ``00:01:01`` - ``00:02:01``. + +* First hit comes ``00:00:00``. Two more hits at ``00:00:30``. All three pass. + Another hit comes at ``00:00:45``, which gets rejected and the intervall now goes from ``00:00:00`` - ``00:01:45``. + A rejected hit makes the interval time longer by expiry time. If another hit comes at ``00:01:30`` it + will also get rejected and the intervall now goes from ``00:00:00`` - ``00:02:30``. + + +Example +^^^^^^^^^^^^^^^^^^ + +.. exec_code:: + + from HABApp.util import RateLimiter + + # Create or get existing, name is case insensitive + limiter = RateLimiter('MyRateLimiterName') + + # define limits, duplicate limits will only be added once + limiter.add_limit(5, 60) # add limits explicitly + limiter.parse_limits('5 per minute').parse_limits('5 in 60s', '5/60seconds') # add limits through text + + + # Test the limit without increasing the hits + for _ in range(100): + assert limiter.test_allow() + + # the limiter will allow 5 calls ... + for _ in range(5): + assert limiter.allow() + + # and reject the 6th + assert not limiter.allow() + + # It's possible to get statistics about the limiter and the corresponding windows + print(limiter.info()) + + +Documentation +^^^^^^^^^^^^^^^^^^ +.. autofunction:: HABApp.util.RateLimiter + + +.. autoclass:: HABApp.util.rate_limiter.limiter.Limiter + :members: + + Statistics ------------------------------ diff --git a/readme.md b/readme.md index 0662d034..50f941f8 100644 --- a/readme.md +++ b/readme.md @@ -127,7 +127,12 @@ MyOpenhabRule() ``` # Changelog -#### 23.11.1 (2023-11-23) +#### 23.12.0-DEV (2023-XX-XX) +- Added HABApp.util.RateLimiter +- Added CompressedMidnightRotatingFileHandler +- Updated dependencies + +#### 23.11.0 (2023-11-23) - Fix for very small float values (#425) - Fix for writing to persistence (#424) - Updated dependencies diff --git a/requirements_setup.txt b/requirements_setup.txt index 9251d495..8264b60c 100644 --- a/requirements_setup.txt +++ b/requirements_setup.txt @@ -1,4 +1,4 @@ -aiohttp == 3.9.0 +aiohttp == 3.9.1 pydantic == 2.5.2 msgspec == 0.18.4 pendulum == 2.1.2 diff --git a/run/conf/logging.yml b/run/conf/logging.yml index 6b8b363a..d4b0ce30 100644 --- a/run/conf/logging.yml +++ b/run/conf/logging.yml @@ -8,13 +8,15 @@ handlers: # There are several Handlers available: # - logging.handlers.RotatingFileHandler: # Will rotate when the file reaches a certain size (see python logging documentation for args) - # - HABApp.core.lib.handler.MidnightRotatingFileHandler: + # - HABApp.config.logging.handler.MidnightRotatingFileHandler: # Will wait until the file reaches a certain size and then will rotate on midnight + # - HABApp.config.logging.handler.CompressedMidnightRotatingFileHandler: + # Same as MidnightRotatingFileHandler but will rotate to a gzipped archive # - More handlers: # https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler HABApp_default: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.handler.MidnightRotatingFileHandler filename: 'HABApp.log' maxBytes: 1_048_576 backupCount: 3 @@ -23,7 +25,7 @@ handlers: level: DEBUG EventFile: - class: HABApp.core.lib.handler.MidnightRotatingFileHandler + class: HABApp.config.logging.handler.CompressedMidnightRotatingFileHandler filename: 'events.log' maxBytes: 1_048_576 backupCount: 3 diff --git a/src/HABApp/__version__.py b/src/HABApp/__version__.py index 3bd0717c..b0cd1b76 100644 --- a/src/HABApp/__version__.py +++ b/src/HABApp/__version__.py @@ -10,4 +10,4 @@ # Development versions contain the DEV-COUNTER postfix: # - 23.09.0.DEV-1 -__version__ = '23.11.0' +__version__ = '23.12.0.DEV-1' diff --git a/src/HABApp/config/logging/__init__.py b/src/HABApp/config/logging/__init__.py index ac466ba4..b0c0e311 100644 --- a/src/HABApp/config/logging/__init__.py +++ b/src/HABApp/config/logging/__init__.py @@ -1,4 +1,4 @@ -from .handler import MidnightRotatingFileHandler +from .handler import MidnightRotatingFileHandler, CompressedMidnightRotatingFileHandler # isort: split diff --git a/src/HABApp/config/logging/default_logfile.py b/src/HABApp/config/logging/default_logfile.py index 67f58d71..3326232a 100644 --- a/src/HABApp/config/logging/default_logfile.py +++ b/src/HABApp/config/logging/default_logfile.py @@ -18,6 +18,8 @@ def get_default_logfile() -> str: # Will rotate when the file reaches a certain size (see python logging documentation for args) # - HABApp.config.logging.MidnightRotatingFileHandler: # Will wait until the file reaches a certain size and then will rotate on midnight + # - HABApp.config.logging.CompressedMidnightRotatingFileHandler: + # Same as MidnightRotatingFileHandler but will rotate to a gzipped archive # - More handlers: # https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler diff --git a/src/HABApp/config/logging/handler.py b/src/HABApp/config/logging/handler.py index 95cb4a01..ad920d22 100644 --- a/src/HABApp/config/logging/handler.py +++ b/src/HABApp/config/logging/handler.py @@ -1,8 +1,14 @@ +import gzip +import shutil from datetime import date, datetime from logging.handlers import RotatingFileHandler +from pathlib import Path class MidnightRotatingFileHandler(RotatingFileHandler): + """A rotating file handler that checks once after midnight if the configured size has been exceeded and + then rotates the file + """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -14,3 +20,23 @@ def shouldRollover(self, record): return 0 self.last_check = date return super().shouldRollover(record) + + +class CompressedMidnightRotatingFileHandler(MidnightRotatingFileHandler): + """Same as ``MidnightRotatingFileHandler`` but rotates the file to a gzipped archive (``.gz``) + + """ + + def __init__(self, *args, **kwargs): + self.namer = self.compressed_namer + self.rotator = self.compressed_rotator + super().__init__(*args, **kwargs) + + def compressed_namer(self, default_name: str) -> str: + return default_name + ".gz" + + def compressed_rotator(self, source: str, dest: str): + src = Path(source) + with src.open('rb') as f_in, gzip.open(dest, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + src.unlink() diff --git a/src/HABApp/util/__init__.py b/src/HABApp/util/__init__.py index 2b09100d..8b56fd5a 100644 --- a/src/HABApp/util/__init__.py +++ b/src/HABApp/util/__init__.py @@ -2,6 +2,7 @@ from .statistics import Statistics from .listener_groups import EventListenerGroup from .fade import Fade +from .rate_limiter import RateLimiter from . import functions from . import multimode diff --git a/src/HABApp/util/rate_limiter/__init__.py b/src/HABApp/util/rate_limiter/__init__.py new file mode 100644 index 00000000..558dc8ea --- /dev/null +++ b/src/HABApp/util/rate_limiter/__init__.py @@ -0,0 +1 @@ +from .registry import RateLimiter diff --git a/src/HABApp/util/rate_limiter/limiter.py b/src/HABApp/util/rate_limiter/limiter.py new file mode 100644 index 00000000..0c096091 --- /dev/null +++ b/src/HABApp/util/rate_limiter/limiter.py @@ -0,0 +1,143 @@ +import re +from dataclasses import dataclass +from time import monotonic +from typing import Final, List, Tuple + +from .rate_limit import RateLimit, RateLimitInfo + + +LIMIT_REGEX = re.compile( + r""" + \s* ([1-9][0-9]*) + \s* (/|per|in) + \s* ([1-9][0-9]*)? + \s* (s|sec|second|m|min|minute|h|hour|day|month|year)s? + \s*""", + re.IGNORECASE | re.VERBOSE, +) + + +def parse_limit(text: str) -> Tuple[int, int]: + if not isinstance(text, str) or not (m := LIMIT_REGEX.fullmatch(text)): + msg = f'Invalid limit string: "{text:s}"' + raise ValueError(msg) + + count, per, factor, interval = m.groups() + + interval_secs = { + 's': 1, 'sec': 1, 'second': 1, 'm': 60, 'min': 60, 'minute': 60, 'hour': 3600, 'h': 3600, + 'day': 24 * 3600, 'month': 30 * 24 * 3600, 'year': 365 * 24 * 3600 + }[interval] + + return int(count), int(1 if factor is None else factor) * interval_secs + + +class Limiter: + def __init__(self, name: str): + self._name: Final = name + self._limits: Tuple[RateLimit, ...] = () + self._skipped = 0 + + def __repr__(self): + return f'<{self.__class__.__name__} {self._name:s}>' + + def add_limit(self, allowed: int, expiry: int) -> 'Limiter': + """Add a new rate limit + + :param allowed: How many hits are allowed + :param expiry: Interval in seconds + """ + if allowed <= 0 or not isinstance(allowed, int): + msg = f'Allowed must be an int >= 0, is {allowed} ({type(allowed)})' + raise ValueError(msg) + + if expiry <= 0 or not isinstance(expiry, int): + msg = f'Expire time must be an int >= 0, is {expiry} ({type(expiry)})' + raise ValueError(msg) + + for window in self._limits: + if window.allowed == allowed and window.expiry == expiry: + return self + + limit = RateLimit(allowed, expiry) + self._limits = tuple(sorted([*self._limits, limit], key=lambda x: x.expiry)) + return self + + def parse_limits(self, *text: str) -> 'Limiter': + """Add one or more limits in textual form, e.g. ``5 in 60s``, ``10 per hour`` or ``10/15 mins``. + If the limit does already exist it will not be added again. + + :param text: textual description of limit + """ + for limit in [parse_limit(t) for t in text]: + self.add_limit(*limit) + return self + + def allow(self) -> bool: + """Test the limit. + + :return: True if allowed, False if forbidden + """ + allow = True + clear_skipped = True + + if not self._limits: + msg = 'No limits defined!' + raise ValueError(msg) + + for limit in self._limits: + if not limit.allow(): + allow = False + # allow increments hits, if it's now 1 it was 0 before + if limit.hits != 1: + clear_skipped = False + + if clear_skipped: + self._skipped = 0 + + if not allow: + self._skipped += 1 + + return allow + + def test_allow(self) -> bool: + """Test the limit without hitting it. Calling this will not increase the hit counter. + + :return: True if allowed, False if forbidden + """ + allow = True + clear_skipped = True + + if not self._limits: + msg = 'No limits defined!' + raise ValueError(msg) + + for limit in self._limits: + if not limit.test_allow(): + allow = False + if limit.hits != 0: + clear_skipped = False + + if clear_skipped: + self._skipped = 0 + return allow + + def info(self) -> 'LimiterInfo': + """Get some info about the limiter and the defined windows + """ + now = monotonic() + remaining = max((w.stop for w in self._limits if w.hits), default=now) - now + if remaining <= 0: + remaining = 0 + + return LimiterInfo( + time_remaining=remaining, skipped=self._skipped, + limits=[limit.window_info() for limit in self._limits] + ) + + +@dataclass +class LimiterInfo: + time_remaining: float #: time remaining until skipped will reset + skipped: int #: how many entries were skipped + limits: List['RateLimitInfo'] # Info for every window diff --git a/src/HABApp/util/rate_limiter/rate_limit.py b/src/HABApp/util/rate_limiter/rate_limit.py new file mode 100644 index 00000000..500b559b --- /dev/null +++ b/src/HABApp/util/rate_limiter/rate_limit.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass +from time import monotonic +from typing import Final + + +@dataclass +class RateLimitInfo: + time_remaining: float #: Time remaining until this window will reset + hits: int #: Hits + skips: int #: Skips + limit: int #: Boundary + + @property + def hits_remaining(self) -> int: + return self.limit - self.hits + + +class RateLimit: + def __init__(self, allowed: int, expiry: int): + super().__init__() + assert allowed > 0, allowed + assert expiry > 0, expiry + + self.expiry: Final = expiry + self.allowed: Final = allowed + + self.start: float = -1.0 + self.stop: float = -1.0 + self.hits: int = 0 + self.skips: int = 0 + + def __repr__(self): + return (f'<{self.__class__.__name__} hits={self.hits:d}/{self.allowed:d} ' + f'expiry={self.expiry:d}s window={self.stop - self.start:.0f}s>') + + def allow(self) -> bool: + now = monotonic() + + if self.stop < now: + self.hits = 0 + self.skips = 0 + self.start = now + self.stop = now + self.expiry + + self.hits += 1 + if self.hits <= self.allowed: + return True + + self.skips += 1 + self.hits = self.allowed + self.stop = now + self.expiry + return False + + def test_allow(self) -> bool: + now = monotonic() + + if self.hits and self.stop < now: + self.hits = 0 + self.skips = 0 + + return self.hits < self.allowed + + def window_info(self) -> RateLimitInfo: + if self.hits <= 0: + remaining = self.expiry + else: + remaining = self.stop - monotonic() + if remaining <= 0: + remaining = 0 + + return RateLimitInfo( + time_remaining=remaining, hits=self.hits, skips=self.skips, limit=self.allowed + ) diff --git a/src/HABApp/util/rate_limiter/registry.py b/src/HABApp/util/rate_limiter/registry.py new file mode 100644 index 00000000..048c7e40 --- /dev/null +++ b/src/HABApp/util/rate_limiter/registry.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from threading import Lock + +from .limiter import Limiter + + +LOCK = Lock() + +_LIMITERS: dict[str, Limiter] = {} + + +def RateLimiter(name: str) -> Limiter: + """Create a new rate limiter or return an already existing one with a given name. + + :param name: case insensitive name of limiter + :return: Rate limiter object + """ + + key = name.lower() + + with LOCK: + if (obj := _LIMITERS.get(key)) is None: + _LIMITERS[key] = obj = Limiter(name) + + return obj diff --git a/tests/test_utils/test_rate_limiter.py b/tests/test_utils/test_rate_limiter.py new file mode 100644 index 00000000..820ec820 --- /dev/null +++ b/tests/test_utils/test_rate_limiter.py @@ -0,0 +1,166 @@ +import pytest + +import HABApp.util.rate_limiter.limiter as limiter_module +import HABApp.util.rate_limiter.rate_limit as rate_limit_module +import HABApp.util.rate_limiter.registry as registry_module +from HABApp.util.rate_limiter.limiter import Limiter, RateLimit, parse_limit + + +@pytest.mark.parametrize( + 'unit,factor', ( + ('s', 1), ('sec', 1), ('second', 1), + ('m', 60), ('min', 60), ('minute', 60), + ('h', 3600), ('hour', 3600), + ('day', 24 * 3600), ('month', 30 * 24 * 3600), ('year', 365 * 24 * 3600) + ) +) +def test_parse(unit: str, factor: int): + assert parse_limit(f' 1 per {unit} ') == (1, factor) + assert parse_limit(f' 1 / {unit} ') == (1, factor) + assert parse_limit(f'3 per {unit}') == (3, factor) + assert parse_limit(f'3 in {unit}') == (3, factor) + + for ctr in (1, 12, 375, 5533): + assert parse_limit(f'{ctr:d} in 5{unit}') == (ctr, 5 * factor) + assert parse_limit(f'{ctr:d} in 5{unit}s') == (ctr, 5 * factor) + + with pytest.raises(ValueError) as e: + parse_limit('asdf') + assert str(e.value) == 'Invalid limit string: "asdf"' + + +def test_window(monkeypatch): + time = 0 + monkeypatch.setattr(rate_limit_module, 'monotonic', lambda: time) + + limit = RateLimit(5, 3) + assert str(limit) == '' + assert limit.test_allow() + + assert limit.allow() + assert str(limit) == '' + + for _ in range(4): + assert limit.allow() + + assert str(limit) == '' + + # Limit is full, stop gets moved further + time = 1 + assert not limit.allow() + assert str(limit) == '' + + # move out of interval + time = 4.1 + assert limit.allow() + assert limit.hits == 1 + assert str(limit) == '' + + +def test_window_test_allow(monkeypatch): + time = 0 + monkeypatch.setattr(rate_limit_module, 'monotonic', lambda: time) + + limit = RateLimit(5, 3) + limit.hits = 5 + limit.stop = 2.99999 + assert not limit.test_allow() + + # expiry when out of window + time = 3 + assert limit.test_allow() + assert not limit.hits + + +def test_limiter_add(monkeypatch): + limiter = Limiter('test') + limiter.add_limit(3, 5).add_limit(3, 5).parse_limits('3 in 5s') + assert len(limiter._limits) == 1 + + +def test_limiter_info(monkeypatch): + time = 0 + monkeypatch.setattr(rate_limit_module, 'monotonic', lambda: time) + monkeypatch.setattr(limiter_module, 'monotonic', lambda: time) + + limiter = Limiter('test') + + info = limiter.info() + assert info.time_remaining == 0 + assert info.skipped == 0 + + with pytest.raises(ValueError): + limiter.allow() + + with pytest.raises(ValueError): + limiter.test_allow() + + limiter.add_limit(3, 3) + + info = limiter.info() + assert info.time_remaining == 0 + assert info.skipped == 0 + + w_info = info.limits[0] + assert w_info.limit == 3 + assert w_info.skips == 0 + assert w_info.time_remaining == 3 + assert w_info.hits == 0 + + limiter.allow() + time = 2 + limiter.allow() + + info = limiter.info() + assert info.time_remaining == 1 + assert info.skipped == 0 + + w_info = info.limits[0] + assert w_info.limit == 3 + assert w_info.skips == 0 + assert w_info.time_remaining == 1 + assert w_info.hits == 2 + + # add a longer limiter - this one should now define the time_remaining + limiter.add_limit(4, 5) + limiter.allow() + + info = limiter.info() + assert info.time_remaining == 5 + assert info.skipped == 0 + + w_info = info.limits[0] + assert w_info.limit == 3 + assert w_info.skips == 0 + assert w_info.time_remaining == 1 + assert w_info.hits == 3 + + w_info = info.limits[1] + assert w_info.limit == 4 + assert w_info.skips == 0 + assert w_info.time_remaining == 5 + assert w_info.hits == 1 + + time += 5.0001 + + info = limiter.info() + assert info.time_remaining == 0 + + w_info = info.limits[0] + assert w_info.limit == 3 + assert w_info.skips == 0 + assert w_info.time_remaining == 0 + assert w_info.hits == 3 + + w_info = info.limits[1] + assert w_info.limit == 4 + assert w_info.skips == 0 + assert w_info.time_remaining == 0 + assert w_info.hits == 1 + + +def test_registry(monkeypatch): + monkeypatch.setattr(registry_module, '_LIMITERS', {}) + + obj = registry_module.RateLimiter('test') + assert obj is registry_module.RateLimiter('TEST') From 277465746f15cc809fab33f73b857a7595817c7f Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Tue, 5 Dec 2023 07:16:13 +0100 Subject: [PATCH 02/15] Added info about installation of a version --- docs/installation.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 1b8a30d6..7b85c7bb 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -59,8 +59,8 @@ Installation After the installation take a look how to configure HABApp. A default configuration will be created on the first start. -Upgrading -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Upgrade to the latest version +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #. Stop HABApp #. Activate the virtual environment @@ -87,6 +87,16 @@ Upgrading #. Observe the logs for errors in case there were changes +Installation of a certain version +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Installing a certain version of HABApp requires the same steps used for installation or upgrading HABApp. +However the final ``python3 -m pip install`` command is now different and contains the version number:: + + python3 -m pip install HABApp==23.12.0 + +The complete list of available versions can be found on `pypi `_. + Autostart after reboot ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Check where habapp is installed:: From 0d299eac15ca2d1518517e21e65e273b95a9efed Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Tue, 19 Dec 2023 09:39:54 +0100 Subject: [PATCH 03/15] Small improvement for RGB and HSB types --- readme.md | 1 + requirements.txt | 4 +- requirements_setup.txt | 6 +- requirements_tests.txt | 2 +- src/HABApp/core/types/color.py | 70 ++++++++++++++++-------- tests/test_core/test_types/test_color.py | 18 +++++- 6 files changed, 70 insertions(+), 31 deletions(-) diff --git a/readme.md b/readme.md index 50f941f8..ae2eb018 100644 --- a/readme.md +++ b/readme.md @@ -131,6 +131,7 @@ MyOpenhabRule() - Added HABApp.util.RateLimiter - Added CompressedMidnightRotatingFileHandler - Updated dependencies +- Small improvement for RGB and HSB types #### 23.11.0 (2023-11-23) - Fix for very small float values (#425) diff --git a/requirements.txt b/requirements.txt index 9bcc4000..aa857f7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,8 @@ # ----------------------------------------------------------------------------- # Packages for source formatting # ----------------------------------------------------------------------------- -pre-commit >= 3.5, < 3.6 -ruff >= 0.1.6, < 0.2 +pre-commit >= 3.6, < 4 +ruff >= 0.1.8, < 0.2 # ----------------------------------------------------------------------------- # Packages for other developement tasks diff --git a/requirements_setup.txt b/requirements_setup.txt index 8264b60c..a634b771 100644 --- a/requirements_setup.txt +++ b/requirements_setup.txt @@ -1,10 +1,10 @@ aiohttp == 3.9.1 pydantic == 2.5.2 -msgspec == 0.18.4 +msgspec == 0.18.5 pendulum == 2.1.2 bidict == 0.22.1 watchdog == 3.0.0 -ujson == 5.8.0 +ujson == 5.9.0 aiomqtt == 1.2.1 immutables == 0.20 @@ -15,7 +15,7 @@ colorama == 0.4.6 voluptuous == 0.14.1 -typing-extensions == 4.8.0 +typing-extensions == 4.9.0 aiohttp-sse-client == 0.2.1 diff --git a/requirements_tests.txt b/requirements_tests.txt index 34746f45..c6dc925e 100644 --- a/requirements_tests.txt +++ b/requirements_tests.txt @@ -8,4 +8,4 @@ # ----------------------------------------------------------------------------- packaging == 23.2 pytest == 7.4.3 -pytest-asyncio == 0.21.1 +pytest-asyncio == 0.23.2 diff --git a/src/HABApp/core/types/color.py b/src/HABApp/core/types/color.py index 52d4be76..66dec2e4 100644 --- a/src/HABApp/core/types/color.py +++ b/src/HABApp/core/types/color.py @@ -1,6 +1,6 @@ from colorsys import hsv_to_rgb as _hsv_to_rgb from colorsys import rgb_to_hsv as _rgb_to_hsv -from typing import Tuple, Union, Optional +from typing import Optional, Tuple, Union from typing_extensions import Self @@ -90,19 +90,30 @@ def __str__(self): def __eq__(self, other): if isinstance(other, self.__class__): return self._r == other._r and self._g == other._g and self._b == other._b - elif isinstance(other, HSB): + if isinstance(other, HSB): return self == self.__class__.from_hsb(other) - else: - return NotImplemented - - def __getitem__(self, item: int) -> int: - if item == 0: - return self._r - if item == 1: - return self._g - if item == 2: - return self._b - raise IndexError() + return NotImplemented + + def __getitem__(self, item: Union[int, str]) -> int: + if isinstance(item, int): + if item == 0: + return self._r + if item == 1: + return self._g + if item == 2: + return self._b + raise IndexError() + + if isinstance(item, str): + if item in ('r', 'red'): + return self._r + if item in ('g', 'green'): + return self._g + if item in ('b', 'blue'): + return self._b + raise KeyError() + + raise TypeError() # ------------------------------------------------------------------------------------------------------------------ # Conversions @@ -227,17 +238,28 @@ def __eq__(self, other): return self._hue == other._hue and \ self._saturation == other._saturation and \ self._brightness == other._brightness - else: - return NotImplemented - - def __getitem__(self, item: int) -> float: - if item == 0: - return self._hue - if item == 1: - return self._saturation - if item == 2: - return self._brightness - raise IndexError() + return NotImplemented + + def __getitem__(self, item: Union[int, str]) -> float: + if isinstance(item, int): + if item == 0: + return self._hue + if item == 1: + return self._saturation + if item == 2: + return self._brightness + raise IndexError() + + if isinstance(item, str): + if item in ('h', 'hue'): + return self._hue + if item in ('s', 'saturation'): + return self._saturation + if item in ('b', 'brightness'): + return self._brightness + raise KeyError() + + raise TypeError() # ------------------------------------------------------------------------------------------------------------------ # Conversions diff --git a/tests/test_core/test_types/test_color.py b/tests/test_core/test_types/test_color.py index f464a86a..375b04f5 100644 --- a/tests/test_core/test_types/test_color.py +++ b/tests/test_core/test_types/test_color.py @@ -1,6 +1,6 @@ import pytest -from HABApp.core.types.color import RGB, HSB +from HABApp.core.types.color import HSB, RGB def test_rgb(): @@ -10,8 +10,16 @@ def test_rgb(): assert rgb.b == rgb.blue == 3 assert rgb[0] == 1 + assert rgb['r'] == 1 + assert rgb['red'] == 1 + assert rgb[1] == 2 + assert rgb['g'] == 2 + assert rgb['green'] == 2 + assert rgb[2] == 3 + assert rgb['b'] == 3 + assert rgb['blue'] == 3 r, g, b = rgb assert r == rgb.r @@ -69,8 +77,16 @@ def test_hsb(): assert hsb.b == hsb.brightness == 3 assert hsb[0] == 1 + assert hsb['h'] == 1 + assert hsb['hue'] == 1 + assert hsb[1] == 2 + assert hsb['s'] == 2 + assert hsb['saturation'] == 2 + assert hsb[2] == 3 + assert hsb['b'] == 3 + assert hsb['brightness'] == 3 h, s, b = hsb assert h == hsb.h From 223d7b455d6685719be5ff26d0918834de5b09a8 Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Wed, 20 Dec 2023 08:02:16 +0100 Subject: [PATCH 04/15] Added toggle for SwitchItem --- readme.md | 2 ++ requirements.txt | 4 +-- src/HABApp/core/const/hints.py | 8 +++++ src/HABApp/core/errors.py | 22 ++++++++++++ src/HABApp/openhab/items/base_item.py | 5 ++- src/HABApp/openhab/items/commands.py | 29 +++++++-------- src/HABApp/openhab/items/contact_item.py | 6 ++-- src/HABApp/openhab/items/dimmer_item.py | 29 +++++++++------ src/HABApp/openhab/items/number_item.py | 14 +++++++- .../openhab/items/rollershutter_item.py | 20 +++++++---- src/HABApp/openhab/items/switch_item.py | 35 +++++++++++++------ src/HABApp/openhab/map_items.py | 13 +++---- .../test_openhab/test_items/test_commands.py | 12 +++++-- tests/test_openhab/test_items/test_contact.py | 10 ++++++ tests/test_openhab/test_items/test_dimmer.py | 22 ++++++++++++ tests/test_openhab/test_items/test_number.py | 19 ++++++++++ .../test_items/test_rollershutter.py | 14 ++++++++ tests/test_openhab/test_items/test_switch.py | 21 +++++++++++ 18 files changed, 229 insertions(+), 56 deletions(-) create mode 100644 tests/test_openhab/test_items/test_dimmer.py create mode 100644 tests/test_openhab/test_items/test_rollershutter.py create mode 100644 tests/test_openhab/test_items/test_switch.py diff --git a/readme.md b/readme.md index ae2eb018..ba865437 100644 --- a/readme.md +++ b/readme.md @@ -132,6 +132,8 @@ MyOpenhabRule() - Added CompressedMidnightRotatingFileHandler - Updated dependencies - Small improvement for RGB and HSB types +- Small improvements for openHAB items +- Added toggle for SwitchItem #### 23.11.0 (2023-11-23) - Fix for very small float values (#425) diff --git a/requirements.txt b/requirements.txt index aa857f7a..9b86d6f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,8 @@ # ----------------------------------------------------------------------------- # Packages for source formatting # ----------------------------------------------------------------------------- -pre-commit >= 3.6, < 4 -ruff >= 0.1.8, < 0.2 +pre-commit == 3.5.0 +ruff == 0.1.8 # ----------------------------------------------------------------------------- # Packages for other developement tasks diff --git a/src/HABApp/core/const/hints.py b/src/HABApp/core/const/hints.py index 9fe059e1..ffdf34bb 100644 --- a/src/HABApp/core/const/hints.py +++ b/src/HABApp/core/const/hints.py @@ -1,6 +1,7 @@ from typing import Any as __Any from typing import Awaitable as __Awaitable from typing import Callable as __Callable +from typing import Protocol as __Protocol from typing import Type as __Type from .const import PYTHON_310 as __IS_GE_PYTHON_310 @@ -16,3 +17,10 @@ TYPE_FUNC_ASYNC: TypeAlias = __Callable[..., __Awaitable[__Any]] TYPE_EVENT_CALLBACK: TypeAlias = __Callable[[__Any], __Any] + + +# noinspection PyPropertyDefinition +class HasNameAttr(__Protocol): + @property + def name(self) -> str: + ... diff --git a/src/HABApp/core/errors.py b/src/HABApp/core/errors.py index 59128461..fa87be13 100644 --- a/src/HABApp/core/errors.py +++ b/src/HABApp/core/errors.py @@ -1,3 +1,6 @@ +from HABApp.core.const.hints import HasNameAttr as _HasNameAttr + + class HABAppException(Exception): pass @@ -33,3 +36,22 @@ class ContextBoundObjectIsAlreadyLinkedError(HABAppException): class ContextBoundObjectIsAlreadyUnlinkedError(HABAppException): pass + + +# ---------------------------------------------------------------------------------------------------------------------- +# Value errors +# ---------------------------------------------------------------------------------------------------------------------- +class HABAppValueError(ValueError, HABAppException): + pass + + +class ItemValueIsNoneError(HABAppValueError): + @classmethod + def from_item(cls, item: _HasNameAttr): + return cls(f'Item value is None (item "{item.name:s}")') + + +class InvalidItemValue(HABAppValueError): + @classmethod + def from_item(cls, item: _HasNameAttr, value): + return cls(f'Invalid value for {item.__class__.__name__} {item.name:s}: {value}') diff --git a/src/HABApp/openhab/items/base_item.py b/src/HABApp/openhab/items/base_item.py index af772e69..bc855c7d 100644 --- a/src/HABApp/openhab/items/base_item.py +++ b/src/HABApp/openhab/items/base_item.py @@ -1,5 +1,5 @@ import datetime -from typing import Any, FrozenSet, Mapping, NamedTuple, Optional, TypeVar, Type +from typing import Any, FrozenSet, Mapping, NamedTuple, Optional, Type from immutables import Map @@ -110,5 +110,4 @@ def get_persistence_data(self, persistence: Optional[str] = None, ) -HINT_OPENHAB_ITEM = TypeVar('HINT_OPENHAB_ITEM', bound=OpenhabItem) -HINT_TYPE_OPENHAB_ITEM = Type[HINT_OPENHAB_ITEM] +HINT_TYPE_OPENHAB_ITEM = Type[OpenhabItem] diff --git a/src/HABApp/openhab/items/commands.py b/src/HABApp/openhab/items/commands.py index 38fa68a7..5452ef4e 100644 --- a/src/HABApp/openhab/items/commands.py +++ b/src/HABApp/openhab/items/commands.py @@ -1,46 +1,47 @@ +from HABApp.core.const.hints import HasNameAttr as _HasNameAttr from HABApp.openhab.definitions import OnOffValue, UpDownValue from HABApp.openhab.interface_sync import send_command class OnOffCommand: - def is_on(self) -> bool: + def is_on(self: _HasNameAttr) -> bool: """Test value against on-value""" raise NotImplementedError() - def is_off(self) -> bool: + def is_off(self: _HasNameAttr) -> bool: """Test value against off-value""" raise NotImplementedError() - def on(self): + def on(self: _HasNameAttr): """Command item on""" - send_command(self, OnOffValue.ON) + send_command(self.name, OnOffValue.ON) - def off(self): + def off(self: _HasNameAttr): """Command item off""" - send_command(self, OnOffValue.OFF) + send_command(self.name, OnOffValue.OFF) class PercentCommand: - def percent(self, value: float): + def percent(self: _HasNameAttr, value: float): """Command to value (in percent)""" assert 0 <= value <= 100, value - send_command(self, str(value)) + send_command(self.name, str(value)) class UpDownCommand: - def up(self): + def up(self: _HasNameAttr): """Command up""" - send_command(self, UpDownValue.UP) + send_command(self.name, UpDownValue.UP) - def down(self): + def down(self: _HasNameAttr): """Command down""" - send_command(self, UpDownValue.DOWN) + send_command(self.name, UpDownValue.DOWN) - def is_up(self) -> bool: + def is_up(self: _HasNameAttr) -> bool: """Test value against on-value""" raise NotImplementedError() - def is_down(self) -> bool: + def is_down(self: _HasNameAttr) -> bool: """Test value against off-value""" raise NotImplementedError() diff --git a/src/HABApp/openhab/items/contact_item.py b/src/HABApp/openhab/items/contact_item.py index be01e1d5..5714b9fc 100644 --- a/src/HABApp/openhab/items/contact_item.py +++ b/src/HABApp/openhab/items/contact_item.py @@ -5,6 +5,7 @@ from ...core.const import MISSING from ..errors import SendCommandNotSupported from HABApp.openhab.interface_sync import post_update +from ...core.errors import InvalidItemValue if TYPE_CHECKING: Optional = Optional @@ -40,8 +41,9 @@ def set_value(self, new_value) -> bool: if isinstance(new_value, OpenClosedValue): new_value = new_value.value - if new_value is not None and new_value != OPEN and new_value != CLOSED: - raise ValueError(f'Invalid value for ContactItem: {new_value}') + if new_value not in (OPEN, CLOSED, None): + raise InvalidItemValue.from_item(self, new_value) + return super().set_value(new_value) def is_open(self) -> bool: diff --git a/src/HABApp/openhab/items/dimmer_item.py b/src/HABApp/openhab/items/dimmer_item.py index 260a3e34..8e419a11 100644 --- a/src/HABApp/openhab/items/dimmer_item.py +++ b/src/HABApp/openhab/items/dimmer_item.py @@ -1,9 +1,12 @@ -from typing import Union, TYPE_CHECKING, Optional, FrozenSet, Mapping +from typing import TYPE_CHECKING, FrozenSet, Mapping, Optional, Union -from HABApp.openhab.items.base_item import OpenhabItem, MetaData +from HABApp.openhab.items.base_item import MetaData, OpenhabItem from HABApp.openhab.items.commands import OnOffCommand, PercentCommand + +from ...core.errors import InvalidItemValue, ItemValueIsNoneError from ..definitions import OnOffValue, PercentValue + if TYPE_CHECKING: Union = Union Optional = Optional @@ -39,15 +42,13 @@ def set_value(self, new_value) -> bool: new_value = new_value.value # Percent is 0 ... 100 - if isinstance(new_value, (int, float)): - assert 0 <= new_value <= 100, new_value - else: - assert new_value is None, new_value + if isinstance(new_value, (int, float)) and (0 <= new_value <= 100): + return super().set_value(new_value) - return super().set_value(new_value) + if new_value is None: + return super().set_value(new_value) - def __str__(self): - return self.value + raise InvalidItemValue.from_item(self, new_value) def is_on(self) -> bool: """Test value against on-value""" @@ -55,4 +56,12 @@ def is_on(self) -> bool: def is_off(self) -> bool: """Test value against off-value""" - return not bool(self.value) + return self.value is not None and not self.value + + def __str__(self): + return self.value + + def __bool__(self): + if self.value is None: + raise ItemValueIsNoneError.from_item(self) + return self.is_on() diff --git a/src/HABApp/openhab/items/number_item.py b/src/HABApp/openhab/items/number_item.py index a3f37b92..4b46969d 100644 --- a/src/HABApp/openhab/items/number_item.py +++ b/src/HABApp/openhab/items/number_item.py @@ -2,6 +2,7 @@ from HABApp.openhab.items.base_item import OpenhabItem, MetaData from ..definitions import QuantityValue +from ...core.errors import ItemValueIsNoneError, InvalidItemValue if TYPE_CHECKING: Union = Union @@ -41,4 +42,15 @@ def set_value(self, new_value) -> bool: if isinstance(new_value, QuantityValue): return super().set_value(new_value.value) - return super().set_value(new_value) + if isinstance(new_value, (int, float)): + return super().set_value(new_value) + + if new_value is None: + return super().set_value(new_value) + + raise InvalidItemValue.from_item(self, new_value) + + def __bool__(self): + if self.value is None: + raise ItemValueIsNoneError.from_item(self) + return bool(self.value) diff --git a/src/HABApp/openhab/items/rollershutter_item.py b/src/HABApp/openhab/items/rollershutter_item.py index 0d892a6f..71717881 100644 --- a/src/HABApp/openhab/items/rollershutter_item.py +++ b/src/HABApp/openhab/items/rollershutter_item.py @@ -1,8 +1,10 @@ -from typing import TYPE_CHECKING, Optional, FrozenSet, Mapping, Union +from typing import TYPE_CHECKING, FrozenSet, Mapping, Optional, Union + +from HABApp.core.errors import InvalidItemValue +from HABApp.openhab.definitions import PercentValue, UpDownValue +from HABApp.openhab.items.base_item import MetaData, OpenhabItem +from HABApp.openhab.items.commands import PercentCommand, UpDownCommand -from HABApp.openhab.items.base_item import OpenhabItem, MetaData -from HABApp.openhab.items.commands import UpDownCommand, PercentCommand -from ..definitions import UpDownValue, PercentValue if TYPE_CHECKING: Union = Union @@ -38,8 +40,14 @@ def set_value(self, new_value) -> bool: elif isinstance(new_value, PercentValue): new_value = new_value.value - assert isinstance(new_value, (int, float)) or new_value is None, new_value - return super().set_value(new_value) + # Position is 0 ... 100 + if isinstance(new_value, (int, float)) and (0 <= new_value <= 100): + return super().set_value(new_value) + + if new_value is None: + return super().set_value(new_value) + + raise InvalidItemValue.from_item(self, new_value) def is_up(self) -> bool: return self.value <= 0 diff --git a/src/HABApp/openhab/items/switch_item.py b/src/HABApp/openhab/items/switch_item.py index a5687f9f..add0db75 100644 --- a/src/HABApp/openhab/items/switch_item.py +++ b/src/HABApp/openhab/items/switch_item.py @@ -1,9 +1,11 @@ -from typing import TYPE_CHECKING, Tuple, Optional, FrozenSet, Mapping +from typing import TYPE_CHECKING, Final, FrozenSet, Mapping, Optional, Tuple +from HABApp.core.errors import ItemValueIsNoneError, InvalidItemValue from HABApp.openhab.definitions import OnOffValue -from HABApp.openhab.items.base_item import OpenhabItem, MetaData +from HABApp.openhab.items.base_item import MetaData, OpenhabItem from HABApp.openhab.items.commands import OnOffCommand + if TYPE_CHECKING: Tuple = Tuple Optional = Optional @@ -12,8 +14,8 @@ MetaData = MetaData -ON = OnOffValue.ON -OFF = OnOffValue.OFF +ON: Final = OnOffValue.ON +OFF: Final = OnOffValue.OFF class SwitchItem(OpenhabItem, OnOffCommand): @@ -28,7 +30,6 @@ class SwitchItem(OpenhabItem, OnOffCommand): :ivar Mapping[str, MetaData] metadata: |oh_item_desc_metadata| """ - @staticmethod def _state_from_oh_str(state: str): if state != ON and state != OFF: @@ -40,8 +41,9 @@ def set_value(self, new_value) -> bool: if isinstance(new_value, OnOffValue): new_value = new_value.value - if new_value is not None and new_value != ON and new_value != OFF: - raise ValueError(f'Invalid value for SwitchItem {self.name}: {new_value}') + if new_value not in (ON, OFF, None): + raise InvalidItemValue.from_item(self, new_value) + return super().set_value(new_value) def is_on(self) -> bool: @@ -52,15 +54,26 @@ def is_off(self) -> bool: """Test value against off-value""" return self.value == OFF + def toggle(self): + """Toggle the switch. Turns the switch on when off or off when currently on.""" + if self.value == ON: + self.off() + elif self.value == OFF: + self.on() + elif self.value is None: + raise ItemValueIsNoneError.from_item(self) + else: + raise InvalidItemValue.from_item(self, self.value) + def __str__(self): return self.value def __eq__(self, other): if isinstance(other, SwitchItem): return self.value == other.value - elif isinstance(other, str): + if isinstance(other, str): return self.value == other - elif isinstance(other, int): + if isinstance(other, int): if other and self.is_on(): return True if not other and self.is_off(): @@ -70,4 +83,6 @@ def __eq__(self, other): return NotImplemented def __bool__(self): - return self.is_on() + if self.value is None: + raise ItemValueIsNoneError.from_item(self) + return self.value == ON diff --git a/src/HABApp/openhab/map_items.py b/src/HABApp/openhab/map_items.py index d08ed681..a65db9ef 100644 --- a/src/HABApp/openhab/map_items.py +++ b/src/HABApp/openhab/map_items.py @@ -3,13 +3,14 @@ from immutables import Map -import HABApp from HABApp.core.wrapper import process_exception from HABApp.openhab.definitions.values import QuantityValue -from HABApp.openhab.items import ColorItem, ContactItem, DatetimeItem, DimmerItem, GroupItem, ImageItem, LocationItem, \ - NumberItem, PlayerItem, RollershutterItem, StringItem, SwitchItem, CallItem -from HABApp.openhab.items.base_item import HINT_TYPE_OPENHAB_ITEM -from HABApp.openhab.items.base_item import MetaData +from HABApp.openhab.items import ( + CallItem, ColorItem, ContactItem, DatetimeItem, DimmerItem, GroupItem, ImageItem, + LocationItem, NumberItem, PlayerItem, RollershutterItem, StringItem, SwitchItem, +) +from HABApp.openhab.items.base_item import HINT_TYPE_OPENHAB_ITEM, MetaData, OpenhabItem + log = logging.getLogger('HABApp.openhab.items') @@ -34,7 +35,7 @@ def map_item(name: str, type: str, value: Optional[str], label: Optional[str], tags: FrozenSet[str], groups: FrozenSet[str], metadata: Optional[Dict[str, Dict[str, Any]]]) -> \ - Optional['HABApp.openhab.items.OpenhabItem']: + Optional[OpenhabItem]: try: assert isinstance(type, str) assert value is None or isinstance(value, str) diff --git a/tests/test_openhab/test_items/test_commands.py b/tests/test_openhab/test_items/test_commands.py index 599517ca..88031b67 100644 --- a/tests/test_openhab/test_items/test_commands.py +++ b/tests/test_openhab/test_items/test_commands.py @@ -2,15 +2,20 @@ import pytest -from HABApp.openhab.definitions import OnOffValue, UpDownValue, OpenClosedValue +from HABApp import __version__ +from HABApp.openhab.definitions import OnOffValue, OpenClosedValue, UpDownValue from HABApp.openhab.items import ContactItem -from HABApp.openhab.items.commands import UpDownCommand, OnOffCommand +from HABApp.openhab.items.commands import OnOffCommand, UpDownCommand from HABApp.openhab.map_items import _items as item_dict @pytest.mark.parametrize("cls", [cls for cls in item_dict.values() if issubclass(cls, OnOffCommand)]) def test_OnOff(cls): c = cls('item_name') + assert not c.is_on() + if not __version__.startswith('23.12.0'): + assert not c.is_off() + c.set_value(OnOffValue('ON')) assert c.is_on() assert not c.is_off() @@ -37,6 +42,9 @@ def test_UpDown(cls): @pytest.mark.parametrize("cls", (ContactItem, )) def test_OpenClosed(cls: typing.Type[ContactItem]): c = cls('item_name') + assert not c.is_closed() + assert not c.is_open() + c.set_value(OpenClosedValue.OPEN) assert c.is_open() assert not c.is_closed() diff --git a/tests/test_openhab/test_items/test_contact.py b/tests/test_openhab/test_items/test_contact.py index f551a4f8..638e4661 100644 --- a/tests/test_openhab/test_items/test_contact.py +++ b/tests/test_openhab/test_items/test_contact.py @@ -1,5 +1,6 @@ import pytest +from HABApp.core.errors import InvalidItemValue from HABApp.openhab.errors import SendCommandNotSupported from HABApp.openhab.items import ContactItem @@ -11,3 +12,12 @@ def test_send_command(): c.oh_send_command('asdf') assert str(e.value) == 'ContactItem does not support send command! See openHAB documentation for details.' + + +def test_switch_set_value(): + ContactItem('').set_value(None) + ContactItem('').set_value('OPEN') + ContactItem('').set_value('CLOSED') + + with pytest.raises(InvalidItemValue): + ContactItem('item_name').set_value('asdf') diff --git a/tests/test_openhab/test_items/test_dimmer.py b/tests/test_openhab/test_items/test_dimmer.py new file mode 100644 index 00000000..fee6a32b --- /dev/null +++ b/tests/test_openhab/test_items/test_dimmer.py @@ -0,0 +1,22 @@ +import pytest + +from HABApp.core.errors import InvalidItemValue, ItemValueIsNoneError +from HABApp.openhab.items import DimmerItem + + +def test_dimmer_item_bool(): + with pytest.raises(ItemValueIsNoneError): + assert not DimmerItem('asdf') + + assert not DimmerItem('asdf', 0) + assert DimmerItem('asdf', 1) + + +def test_dimmer_set_value(): + DimmerItem('').set_value(None) + DimmerItem('').set_value(0) + DimmerItem('').set_value(100) + DimmerItem('').set_value(55.55) + + with pytest.raises(InvalidItemValue): + DimmerItem('item_name').set_value('asdf') diff --git a/tests/test_openhab/test_items/test_number.py b/tests/test_openhab/test_items/test_number.py index efecef75..a9b75a51 100644 --- a/tests/test_openhab/test_items/test_number.py +++ b/tests/test_openhab/test_items/test_number.py @@ -1,5 +1,7 @@ +import pytest from immutables import Map +from HABApp.core.errors import InvalidItemValue, ItemValueIsNoneError from HABApp.openhab.items import NumberItem from HABApp.openhab.items.base_item import MetaData @@ -7,3 +9,20 @@ def test_number_item_unit(): assert NumberItem('test', 1).unit is None assert NumberItem('test', 1, metadata=Map(unit=MetaData('°C'))).unit == '°C' + + +def test_number_item_bool(): + with pytest.raises(ItemValueIsNoneError): + assert not NumberItem('asdf') + + assert not NumberItem('asdf', 0) + assert NumberItem('asdf', 1) + + +def test_number_set_value(): + NumberItem('').set_value(None) + NumberItem('').set_value(1) + NumberItem('').set_value(-3.3) + + with pytest.raises(InvalidItemValue): + NumberItem('item_name').set_value('asdf') diff --git a/tests/test_openhab/test_items/test_rollershutter.py b/tests/test_openhab/test_items/test_rollershutter.py new file mode 100644 index 00000000..a6b0938c --- /dev/null +++ b/tests/test_openhab/test_items/test_rollershutter.py @@ -0,0 +1,14 @@ +import pytest + +from HABApp.core.errors import InvalidItemValue +from HABApp.openhab.items import RollershutterItem + + +def test_dimmer_set_value(): + RollershutterItem('').set_value(None) + RollershutterItem('').set_value(0) + RollershutterItem('').set_value(100) + RollershutterItem('').set_value(55.55) + + with pytest.raises(InvalidItemValue): + RollershutterItem('item_name').set_value('asdf') diff --git a/tests/test_openhab/test_items/test_switch.py b/tests/test_openhab/test_items/test_switch.py new file mode 100644 index 00000000..6bc7c9f1 --- /dev/null +++ b/tests/test_openhab/test_items/test_switch.py @@ -0,0 +1,21 @@ +import pytest + +from HABApp.core.errors import InvalidItemValue, ItemValueIsNoneError +from HABApp.openhab.items import SwitchItem + + +def test_switch_item_bool(): + with pytest.raises(ItemValueIsNoneError): + assert SwitchItem('test') + + assert not SwitchItem('test', 'OFF') + assert SwitchItem('test', 'ON') + + +def test_switch_set_value(): + SwitchItem('').set_value(None) + SwitchItem('').set_value('ON') + SwitchItem('').set_value('OFF') + + with pytest.raises(InvalidItemValue): + SwitchItem('item_name').set_value('asdf') From 710a5ae21418fa89cb5dcc8a3a9cb7e299d83ff7 Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:26:15 +0100 Subject: [PATCH 05/15] Update rate limiter --- docs/util.rst | 32 ++- src/HABApp/util/functions/min_max.py | 10 +- src/HABApp/util/rate_limiter/limiter.py | 95 ++++--- .../util/rate_limiter/limits/__init__.py | 3 + src/HABApp/util/rate_limiter/limits/base.py | 73 ++++++ .../util/rate_limiter/limits/fixed_window.py | 51 ++++ .../util/rate_limiter/limits/leaky_bucket.py | 50 ++++ src/HABApp/util/rate_limiter/parser.py | 27 ++ src/HABApp/util/rate_limiter/rate_limit.py | 73 ------ tests/test_utils/test_rate_limiter.py | 244 +++++++++++------- 10 files changed, 440 insertions(+), 218 deletions(-) create mode 100644 src/HABApp/util/rate_limiter/limits/__init__.py create mode 100644 src/HABApp/util/rate_limiter/limits/base.py create mode 100644 src/HABApp/util/rate_limiter/limits/fixed_window.py create mode 100644 src/HABApp/util/rate_limiter/limits/leaky_bucket.py create mode 100644 src/HABApp/util/rate_limiter/parser.py delete mode 100644 src/HABApp/util/rate_limiter/rate_limit.py diff --git a/docs/util.rst b/docs/util.rst index 53f4c663..560bbf43 100644 --- a/docs/util.rst +++ b/docs/util.rst @@ -81,6 +81,7 @@ A simple rate limiter implementation which can be used in rules. The limiter is not rule bound so the same limiter can be used in multiples files. It also works as expected across rule reloads. + Defining limits ^^^^^^^^^^^^^^^^^^ Limits can either be explicitly added or through a textual description. @@ -100,10 +101,10 @@ Examples: * ``300 / hour`` -Elastic expiry -^^^^^^^^^^^^^^^^^^ +Fixed window elastic expiry algorithm +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The rate limiter implements a fixed window with elastic expiry. +This algorithm implements a fixed window with elastic expiry. That means if the limit is hit the interval time will be increased by the expiry time. For example ``3 per minute``: @@ -118,6 +119,14 @@ For example ``3 per minute``: will also get rejected and the intervall now goes from ``00:00:00`` - ``00:02:30``. +Leaky bucket algorithm +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The leaky bucket algorithm is based on the analogy of a bucket that leaks at a constant rate. +As long as the bucket is not full the hits will pass. If the bucket overflows the hits will get rejected. +Since the bucket leaks at a constant rate it will gradually get empty again thus allowing hits to pass again. + + Example ^^^^^^^^^^^^^^^^^^ @@ -128,10 +137,12 @@ Example # Create or get existing, name is case insensitive limiter = RateLimiter('MyRateLimiterName') - # define limits, duplicate limits will only be added once + # define limits, duplicate limits of the same algorithm will only be added once limiter.add_limit(5, 60) # add limits explicitly limiter.parse_limits('5 per minute').parse_limits('5 in 60s', '5/60seconds') # add limits through text + # add additional limit with leaky bucket algorithm + limiter.add_limit(10, 120, algorithm='leaky_bucket') # Test the limit without increasing the hits for _ in range(100): @@ -155,6 +166,19 @@ Documentation .. autoclass:: HABApp.util.rate_limiter.limiter.Limiter :members: + :inherited-members: + +.. autoclass:: HABApp.util.rate_limiter.limiter.LimiterInfo + :members: + :inherited-members: + +.. autoclass:: HABApp.util.rate_limiter.limiter.FixedWindowElasticExpiryLimitInfo + :members: + :inherited-members: + +.. autoclass:: HABApp.util.rate_limiter.limiter.LeakyBucketLimitInfo + :members: + :inherited-members: Statistics diff --git a/src/HABApp/util/functions/min_max.py b/src/HABApp/util/functions/min_max.py index d03284dc..92ab567b 100644 --- a/src/HABApp/util/functions/min_max.py +++ b/src/HABApp/util/functions/min_max.py @@ -2,8 +2,9 @@ from builtins import min as _min -def max(*args, default=None): - """Behaves like the built in max function but ignores any ``None`` values. e.g. ``max([1, None, 2]) == 2``. +# noinspection PyShadowingBuiltins +def max(*args, default=None): # noqa: A001 + """Behaves like the built-in max function but ignores any ``None`` values. e.g. ``max([1, None, 2]) == 2``. If the iterable is empty ``default`` will be returned. :param args: Single iterable or 1..n arguments @@ -16,8 +17,9 @@ def max(*args, default=None): ) -def min(*args, default=None): - """Behaves like the built in min function but ignores any ``None`` values. e.g. ``min([1, None, 2]) == 1``. +# noinspection PyShadowingBuiltins +def min(*args, default=None): # noqa: A001 + """Behaves like the built-in min function but ignores any ``None`` values. e.g. ``min([1, None, 2]) == 1``. If the iterable is empty ``default`` will be returned. :param args: Single iterable or 1..n arguments diff --git a/src/HABApp/util/rate_limiter/limiter.py b/src/HABApp/util/rate_limiter/limiter.py index 0c096091..cbed1ae3 100644 --- a/src/HABApp/util/rate_limiter/limiter.py +++ b/src/HABApp/util/rate_limiter/limiter.py @@ -1,82 +1,84 @@ -import re from dataclasses import dataclass -from time import monotonic -from typing import Final, List, Tuple +from typing import Final, List, Literal, Tuple, TypeAlias, Union -from .rate_limit import RateLimit, RateLimitInfo +from HABApp.core.const.const import StrEnum - -LIMIT_REGEX = re.compile( - r""" - \s* ([1-9][0-9]*) - \s* (/|per|in) - \s* ([1-9][0-9]*)? - \s* (s|sec|second|m|min|minute|h|hour|day|month|year)s? - \s*""", - re.IGNORECASE | re.VERBOSE, +from .limits import ( + BaseRateLimit, + FixedWindowElasticExpiryLimit, + FixedWindowElasticExpiryLimitInfo, + LeakyBucketLimit, + LeakyBucketLimitInfo, ) +from .parser import parse_limit -def parse_limit(text: str) -> Tuple[int, int]: - if not isinstance(text, str) or not (m := LIMIT_REGEX.fullmatch(text)): - msg = f'Invalid limit string: "{text:s}"' - raise ValueError(msg) - - count, per, factor, interval = m.groups() +class LimitTypeEnum(StrEnum): + LEAKY_BUCKET = 'leaky_bucket' + FIXED_WINDOW_ELASTIC_EXPIRY = 'fixed_window_elastic_expiry' - interval_secs = { - 's': 1, 'sec': 1, 'second': 1, 'm': 60, 'min': 60, 'minute': 60, 'hour': 3600, 'h': 3600, - 'day': 24 * 3600, 'month': 30 * 24 * 3600, 'year': 365 * 24 * 3600 - }[interval] - return int(count), int(1 if factor is None else factor) * interval_secs +LIMITER_ALGORITHM_HINT: TypeAlias = Literal[LimitTypeEnum.LEAKY_BUCKET, LimitTypeEnum.FIXED_WINDOW_ELASTIC_EXPIRY] class Limiter: def __init__(self, name: str): self._name: Final = name - self._limits: Tuple[RateLimit, ...] = () - self._skipped = 0 + self._limits: Tuple[BaseRateLimit, ...] = () + self._skips = 0 def __repr__(self): return f'<{self.__class__.__name__} {self._name:s}>' - def add_limit(self, allowed: int, expiry: int) -> 'Limiter': + def add_limit(self, allowed: int, interval: int, + algorithm: LIMITER_ALGORITHM_HINT = LimitTypeEnum.FIXED_WINDOW_ELASTIC_EXPIRY) -> 'Limiter': """Add a new rate limit :param allowed: How many hits are allowed - :param expiry: Interval in seconds + :param interval: Interval in seconds + :param algorithm: Which algorithm should this limit use """ if allowed <= 0 or not isinstance(allowed, int): msg = f'Allowed must be an int >= 0, is {allowed} ({type(allowed)})' raise ValueError(msg) - if expiry <= 0 or not isinstance(expiry, int): - msg = f'Expire time must be an int >= 0, is {expiry} ({type(expiry)})' + if interval <= 0 or not isinstance(interval, int): + msg = f'Expire time must be an int >= 0, is {interval} ({type(interval)})' raise ValueError(msg) + algo = LimitTypeEnum(algorithm) + if algo is LimitTypeEnum.FIXED_WINDOW_ELASTIC_EXPIRY: + cls = FixedWindowElasticExpiryLimit + elif algo is LimitTypeEnum.LEAKY_BUCKET: + cls = LeakyBucketLimit + else: + raise ValueError() + + # Check if we have already added an algorithm with these parameters for window in self._limits: - if window.allowed == allowed and window.expiry == expiry: + if isinstance(window, cls) and window.allowed == allowed and window.interval == interval: return self - limit = RateLimit(allowed, expiry) - self._limits = tuple(sorted([*self._limits, limit], key=lambda x: x.expiry)) + limit = cls(allowed, interval) + self._limits = tuple(sorted([*self._limits, limit], key=lambda x: x.interval)) return self - def parse_limits(self, *text: str) -> 'Limiter': + def parse_limits(self, *text: str, + algorithm: LIMITER_ALGORITHM_HINT = LimitTypeEnum.FIXED_WINDOW_ELASTIC_EXPIRY) -> 'Limiter': """Add one or more limits in textual form, e.g. ``5 in 60s``, ``10 per hour`` or ``10/15 mins``. If the limit does already exist it will not be added again. :param text: textual description of limit + :param algorithm: Which algorithm should these limits use """ for limit in [parse_limit(t) for t in text]: - self.add_limit(*limit) + self.add_limit(*limit, algorithm=algorithm) return self def allow(self) -> bool: """Test the limit. - :return: True if allowed, False if forbidden + :return: ``True`` if allowed, ``False`` if forbidden """ allow = True clear_skipped = True @@ -93,17 +95,17 @@ def allow(self) -> bool: clear_skipped = False if clear_skipped: - self._skipped = 0 + self._skips = 0 if not allow: - self._skipped += 1 + self._skips += 1 return allow def test_allow(self) -> bool: """Test the limit without hitting it. Calling this will not increase the hit counter. - :return: True if allowed, False if forbidden + :return: ``True`` if allowed, ``False`` if forbidden """ allow = True clear_skipped = True @@ -119,25 +121,20 @@ def test_allow(self) -> bool: clear_skipped = False if clear_skipped: - self._skipped = 0 + self._skips = 0 return allow def info(self) -> 'LimiterInfo': """Get some info about the limiter and the defined windows """ - now = monotonic() - remaining = max((w.stop for w in self._limits if w.hits), default=now) - now - if remaining <= 0: - remaining = 0 return LimiterInfo( - time_remaining=remaining, skipped=self._skipped, - limits=[limit.window_info() for limit in self._limits] + skips=self._skips, + limits=[limit.info() for limit in self._limits] ) @dataclass class LimiterInfo: - time_remaining: float #: time remaining until skipped will reset - skipped: int #: how many entries were skipped - limits: List['RateLimitInfo'] # Info for every window + skips: int #: How many entries were skipped + limits: List[Union[FixedWindowElasticExpiryLimitInfo, LeakyBucketLimitInfo]] #: Info for every limit diff --git a/src/HABApp/util/rate_limiter/limits/__init__.py b/src/HABApp/util/rate_limiter/limits/__init__.py new file mode 100644 index 00000000..2d7e52ce --- /dev/null +++ b/src/HABApp/util/rate_limiter/limits/__init__.py @@ -0,0 +1,3 @@ +from .base import BaseRateLimit +from .fixed_window import FixedWindowElasticExpiryLimit, FixedWindowElasticExpiryLimitInfo +from .leaky_bucket import LeakyBucketLimit, LeakyBucketLimitInfo diff --git a/src/HABApp/util/rate_limiter/limits/base.py b/src/HABApp/util/rate_limiter/limits/base.py new file mode 100644 index 00000000..ba68c156 --- /dev/null +++ b/src/HABApp/util/rate_limiter/limits/base.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass +from typing import Final + + +@dataclass +class BaseRateLimitInfo: + hits: int #: Hits + skips: int #: Skips + limit: int #: Boundary + + @property + def hits_remaining(self) -> int: + return self.limit - self.hits + + +class BaseRateLimit: + def __init__(self, allowed: int, interval: int): + super().__init__() + assert allowed > 0, allowed + assert interval > 0, interval + + self.interval: Final = interval + self.allowed: Final = allowed + + self.hits: int = 0 + self.skips: int = 0 + + def repr_text(self) -> str: + return '' + + def __repr__(self): + return ( + f'<{self.__class__.__name__} hits={self.hits:d}/{self.allowed:d} interval={self.interval:d}s ' + f'{self.repr_text():s}>' + ) + + def do_test_allow(self): + raise NotImplementedError() + + def do_allow(self): + raise NotImplementedError() + + def do_deny(self): + raise NotImplementedError() + + def info(self) -> BaseRateLimitInfo: + raise NotImplementedError() + + def allow(self, weight: int = 1) -> bool: + if not isinstance(weight, int) or weight <= 0: + msg = f'weight must be an int > 0, is {weight}' + raise ValueError(msg) + + self.do_allow() + + self.hits += weight + if self.hits <= self.allowed: + return True + + self.skips += 1 + self.hits = self.allowed + + if self.do_deny: + self.do_deny() + return False + + def test_allow(self, weight: int = 1) -> bool: + if not isinstance(weight, int) or weight <= 0: + msg = f'weight must be an int > 0, is {weight}' + raise ValueError(msg) + + self.do_test_allow() + return self.hits + weight <= self.allowed diff --git a/src/HABApp/util/rate_limiter/limits/fixed_window.py b/src/HABApp/util/rate_limiter/limits/fixed_window.py new file mode 100644 index 00000000..48b15ba7 --- /dev/null +++ b/src/HABApp/util/rate_limiter/limits/fixed_window.py @@ -0,0 +1,51 @@ +from dataclasses import dataclass +from time import monotonic + +from .base import BaseRateLimit, BaseRateLimitInfo + + +@dataclass +class FixedWindowElasticExpiryLimitInfo(BaseRateLimitInfo): + time_remaining: float #: Time remaining until this window will reset + + +class FixedWindowElasticExpiryLimit(BaseRateLimit): + def __init__(self, allowed: int, interval: int): + super().__init__(allowed, interval) + + self.start: float = -1.0 + self.stop: float = -1.0 + + def repr_text(self): + return f'window={self.stop - self.start:.0f}s' + + def do_test_allow(self): + if self.stop <= monotonic(): + self.hits = 0 + self.skips = 0 + + def do_allow(self): + now = monotonic() + + if self.stop <= now: + self.hits = 0 + self.skips = 0 + self.start = now + self.stop = now + self.interval + + def do_deny(self): + self.stop = monotonic() + self.interval + + def info(self) -> FixedWindowElasticExpiryLimitInfo: + self.do_test_allow() + + remaining = self.stop - monotonic() + if remaining <= 0: + remaining = 0 + + if not remaining and not self.hits: + remaining = self.interval + + return FixedWindowElasticExpiryLimitInfo( + time_remaining=remaining, hits=self.hits, skips=self.skips, limit=self.allowed + ) diff --git a/src/HABApp/util/rate_limiter/limits/leaky_bucket.py b/src/HABApp/util/rate_limiter/limits/leaky_bucket.py new file mode 100644 index 00000000..9f61ed7b --- /dev/null +++ b/src/HABApp/util/rate_limiter/limits/leaky_bucket.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass +from time import monotonic +from typing import Final + +from .base import BaseRateLimit, BaseRateLimitInfo + + +@dataclass +class LeakyBucketLimitInfo(BaseRateLimitInfo): + time_remaining: float #: Time remaining until the next drop + + +class LeakyBucketLimit(BaseRateLimit): + def __init__(self, allowed: int, interval: int): + super().__init__(allowed, interval) + + self.drop_interval: Final = interval / allowed + self.next_drop: float = -1.0 + + def repr_text(self): + return f'drop_interval={self.drop_interval:.1f}s' + + def do_test_allow(self): + + while self.next_drop <= monotonic(): + self.hits -= 1 + self.next_drop += self.drop_interval + + if self.hits <= 0: + # out of drop interval -> reset stats + if self.next_drop <= monotonic(): + self.next_drop = monotonic() + self.drop_interval + self.skips = 0 + + self.hits = 0 + break + + do_allow = do_test_allow + do_deny = None + + def info(self) -> LeakyBucketLimitInfo: + self.do_test_allow() + + remaining = self.next_drop - monotonic() + if remaining <= 0: + remaining = 0 + + return LeakyBucketLimitInfo( + time_remaining=remaining, hits=self.hits, skips=self.skips, limit=self.allowed + ) diff --git a/src/HABApp/util/rate_limiter/parser.py b/src/HABApp/util/rate_limiter/parser.py new file mode 100644 index 00000000..44e8b4a1 --- /dev/null +++ b/src/HABApp/util/rate_limiter/parser.py @@ -0,0 +1,27 @@ +import re +from typing import Tuple + +LIMIT_REGEX = re.compile( + r""" + \s* ([1-9][0-9]*) + \s* (/|per|in) + \s* ([1-9][0-9]*)? + \s* (s|sec|second|m|min|minute|h|hour|day|month|year)s? + \s*""", + re.IGNORECASE | re.VERBOSE, +) + + +def parse_limit(text: str) -> Tuple[int, int]: + if not isinstance(text, str) or not (m := LIMIT_REGEX.fullmatch(text)): + msg = f'Invalid limit string: "{text:s}"' + raise ValueError(msg) + + count, per, factor, interval = m.groups() + + interval_secs = { + 's': 1, 'sec': 1, 'second': 1, 'm': 60, 'min': 60, 'minute': 60, 'hour': 3600, 'h': 3600, + 'day': 24 * 3600, 'month': 30 * 24 * 3600, 'year': 365 * 24 * 3600 + }[interval] + + return int(count), int(1 if factor is None else factor) * interval_secs diff --git a/src/HABApp/util/rate_limiter/rate_limit.py b/src/HABApp/util/rate_limiter/rate_limit.py deleted file mode 100644 index 500b559b..00000000 --- a/src/HABApp/util/rate_limiter/rate_limit.py +++ /dev/null @@ -1,73 +0,0 @@ -from dataclasses import dataclass -from time import monotonic -from typing import Final - - -@dataclass -class RateLimitInfo: - time_remaining: float #: Time remaining until this window will reset - hits: int #: Hits - skips: int #: Skips - limit: int #: Boundary - - @property - def hits_remaining(self) -> int: - return self.limit - self.hits - - -class RateLimit: - def __init__(self, allowed: int, expiry: int): - super().__init__() - assert allowed > 0, allowed - assert expiry > 0, expiry - - self.expiry: Final = expiry - self.allowed: Final = allowed - - self.start: float = -1.0 - self.stop: float = -1.0 - self.hits: int = 0 - self.skips: int = 0 - - def __repr__(self): - return (f'<{self.__class__.__name__} hits={self.hits:d}/{self.allowed:d} ' - f'expiry={self.expiry:d}s window={self.stop - self.start:.0f}s>') - - def allow(self) -> bool: - now = monotonic() - - if self.stop < now: - self.hits = 0 - self.skips = 0 - self.start = now - self.stop = now + self.expiry - - self.hits += 1 - if self.hits <= self.allowed: - return True - - self.skips += 1 - self.hits = self.allowed - self.stop = now + self.expiry - return False - - def test_allow(self) -> bool: - now = monotonic() - - if self.hits and self.stop < now: - self.hits = 0 - self.skips = 0 - - return self.hits < self.allowed - - def window_info(self) -> RateLimitInfo: - if self.hits <= 0: - remaining = self.expiry - else: - remaining = self.stop - monotonic() - if remaining <= 0: - remaining = 0 - - return RateLimitInfo( - time_remaining=remaining, hits=self.hits, skips=self.skips, limit=self.allowed - ) diff --git a/tests/test_utils/test_rate_limiter.py b/tests/test_utils/test_rate_limiter.py index 820ec820..87c68de0 100644 --- a/tests/test_utils/test_rate_limiter.py +++ b/tests/test_utils/test_rate_limiter.py @@ -1,18 +1,48 @@ +import re + import pytest -import HABApp.util.rate_limiter.limiter as limiter_module -import HABApp.util.rate_limiter.rate_limit as rate_limit_module +import HABApp.util.rate_limiter.limits.fixed_window as fixed_window_module +import HABApp.util.rate_limiter.limits.leaky_bucket as leaky_bucket_module import HABApp.util.rate_limiter.registry as registry_module -from HABApp.util.rate_limiter.limiter import Limiter, RateLimit, parse_limit +from HABApp.util.rate_limiter.limiter import ( + FixedWindowElasticExpiryLimit, + FixedWindowElasticExpiryLimitInfo, + LeakyBucketLimit, + LeakyBucketLimitInfo, + Limiter, + parse_limit, +) +from HABApp.util.rate_limiter.parser import LIMIT_REGEX + + +class MockedMonotonic: + def __init__(self): + self.time = 0 + + def get_time(self): + return self.time + + def __iadd__(self, other): + self.time += other + return self + + +@pytest.fixture() +def time(monkeypatch) -> MockedMonotonic: + m = MockedMonotonic() + monkeypatch.setattr(fixed_window_module, 'monotonic', m.get_time) + monkeypatch.setattr(leaky_bucket_module, 'monotonic', m.get_time) + return m @pytest.mark.parametrize( - 'unit,factor', ( + 'unit,factor', [ ('s', 1), ('sec', 1), ('second', 1), ('m', 60), ('min', 60), ('minute', 60), ('h', 3600), ('hour', 3600), ('day', 24 * 3600), ('month', 30 * 24 * 3600), ('year', 365 * 24 * 3600) - ) + ] ) def test_parse(unit: str, factor: int): assert parse_limit(f' 1 per {unit} ') == (1, factor) @@ -29,138 +59,176 @@ def test_parse(unit: str, factor: int): assert str(e.value) == 'Invalid limit string: "asdf"' -def test_window(monkeypatch): - time = 0 - monkeypatch.setattr(rate_limit_module, 'monotonic', lambda: time) +def test_regex_all_units(): + m = re.search(r'\(([^)]+)\)s\?', LIMIT_REGEX.pattern) + values = m.group(1) - limit = RateLimit(5, 3) - assert str(limit) == '' - assert limit.test_allow() + for unit in values.split('|'): + parse_limit(f'1 in 3 {unit}') + parse_limit(f'1 in 3 {unit}s') + + +def test_fixed_window(time): + + limit = FixedWindowElasticExpiryLimit(5, 3) + assert str(limit) == '' + for _ in range(10): + assert limit.test_allow() assert limit.allow() - assert str(limit) == '' + assert str(limit) == '' for _ in range(4): assert limit.allow() - assert str(limit) == '' + assert str(limit) == '' # Limit is full, stop gets moved further - time = 1 + time += 1 assert not limit.allow() - assert str(limit) == '' + assert str(limit) == '' # move out of interval - time = 4.1 + time += 3.1 + assert limit.allow() + assert limit.hits == 1 + assert str(limit) == '' + + +def test_leaky_bucket(time): + limit = LeakyBucketLimit(4, 2) + assert str(limit) == '' + for _ in range(10): + assert limit.test_allow() + + assert limit.allow() + assert limit.hits == 1 + + assert limit.allow() + assert limit.hits == 2 + + assert limit.allow() + assert limit.allow() + assert not limit.allow() + assert not limit.allow() + assert limit.hits == 4 + + time += 0.5 + assert limit.test_allow() + assert limit.hits == 3 + + time += 1.7 + assert limit.test_allow() + assert limit.hits == 0 + + assert limit.allow() + assert limit.hits == 1 + + time += 0.3 + assert limit.test_allow() + assert limit.hits == 0 + + time += 1 assert limit.allow() + time += 0.4999 + + limit.test_allow() assert limit.hits == 1 - assert str(limit) == '' + time += 0.0001 + limit.test_allow() + assert limit.hits == 0 -def test_window_test_allow(monkeypatch): - time = 0 - monkeypatch.setattr(rate_limit_module, 'monotonic', lambda: time) - limit = RateLimit(5, 3) +def test_window_test_allow(time): + + limit = FixedWindowElasticExpiryLimit(5, 3) limit.hits = 5 limit.stop = 2.99999 assert not limit.test_allow() # expiry when out of window - time = 3 + time += 3 assert limit.test_allow() assert not limit.hits -def test_limiter_add(monkeypatch): +def test_limiter_add(time): limiter = Limiter('test') limiter.add_limit(3, 5).add_limit(3, 5).parse_limits('3 in 5s') assert len(limiter._limits) == 1 -def test_limiter_info(monkeypatch): - time = 0 - monkeypatch.setattr(rate_limit_module, 'monotonic', lambda: time) - monkeypatch.setattr(limiter_module, 'monotonic', lambda: time) +def test_fixed_window_info(time): + limit = FixedWindowElasticExpiryLimit(5, 3) + Info = FixedWindowElasticExpiryLimitInfo - limiter = Limiter('test') + assert limit.info() == Info(hits=0, skips=0, limit=5, time_remaining=3) - info = limiter.info() - assert info.time_remaining == 0 - assert info.skipped == 0 + limit.allow() + assert limit.info() == Info(hits=1, skips=0, limit=5, time_remaining=3) + limit.allow(4) + assert limit.info() == Info(hits=5, skips=0, limit=5, time_remaining=3) + limit.allow() + assert limit.info() == Info(hits=5, skips=1, limit=5, time_remaining=3) - with pytest.raises(ValueError): - limiter.allow() + time += 1 + assert limit.info() == Info(hits=5, skips=1, limit=5, time_remaining=2) - with pytest.raises(ValueError): - limiter.test_allow() + time += 3 + assert limit.info() == Info(hits=0, skips=0, limit=5, time_remaining=3) - limiter.add_limit(3, 3) + assert not limit.test_allow(6) + assert limit.info() == Info(hits=0, skips=0, limit=5, time_remaining=3) - info = limiter.info() - assert info.time_remaining == 0 - assert info.skipped == 0 - w_info = info.limits[0] - assert w_info.limit == 3 - assert w_info.skips == 0 - assert w_info.time_remaining == 3 - assert w_info.hits == 0 +def test_leaky_bucket_info(time): + limit = LeakyBucketLimit(2, 2) + Info = LeakyBucketLimitInfo - limiter.allow() - time = 2 - limiter.allow() + assert limit.info() == Info(hits=0, skips=0, limit=2, time_remaining=1) - info = limiter.info() - assert info.time_remaining == 1 - assert info.skipped == 0 - - w_info = info.limits[0] - assert w_info.limit == 3 - assert w_info.skips == 0 - assert w_info.time_remaining == 1 - assert w_info.hits == 2 - # add a longer limiter - this one should now define the time_remaining - limiter.add_limit(4, 5) - limiter.allow() +def test_registry(monkeypatch): + monkeypatch.setattr(registry_module, '_LIMITERS', {}) - info = limiter.info() - assert info.time_remaining == 5 - assert info.skipped == 0 + obj = registry_module.RateLimiter('Test') + assert obj is registry_module.RateLimiter('TEST') + assert obj is registry_module.RateLimiter('test') - w_info = info.limits[0] - assert w_info.limit == 3 - assert w_info.skips == 0 - assert w_info.time_remaining == 1 - assert w_info.hits == 3 - w_info = info.limits[1] - assert w_info.limit == 4 - assert w_info.skips == 0 - assert w_info.time_remaining == 5 - assert w_info.hits == 1 +def test_limiter(time): - time += 5.0001 + limiter = Limiter('Test') + assert limiter.__repr__() == '' info = limiter.info() - assert info.time_remaining == 0 + assert info.skips == 0 + + with pytest.raises(ValueError): + limiter.allow() - w_info = info.limits[0] - assert w_info.limit == 3 - assert w_info.skips == 0 - assert w_info.time_remaining == 0 - assert w_info.hits == 3 + limiter.add_limit(2, 1).add_limit(2, 2) - w_info = info.limits[1] - assert w_info.limit == 4 - assert w_info.skips == 0 - assert w_info.time_remaining == 0 - assert w_info.hits == 1 + assert limiter.allow() + assert limiter.allow() + time += 0.5 + assert not limiter.allow() + time += 1 + assert not limiter.allow() -def test_registry(monkeypatch): - monkeypatch.setattr(registry_module, '_LIMITERS', {}) + assert limiter.info().skips == 2 + time += 2 - obj = registry_module.RateLimiter('test') - assert obj is registry_module.RateLimiter('TEST') + assert limiter.test_allow() + assert limiter.info().skips == 0 + + assert limiter.allow() + assert limiter.allow() + assert not limiter.allow() + assert limiter.info().skips == 1 + + time += 2 + assert limiter.allow() + assert limiter.info().skips == 0 From 0f718556cae91a14acca05893c79d497feee2528 Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:30:10 +0100 Subject: [PATCH 06/15] . --- src/HABApp/util/rate_limiter/limiter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/HABApp/util/rate_limiter/limiter.py b/src/HABApp/util/rate_limiter/limiter.py index cbed1ae3..487e0c1a 100644 --- a/src/HABApp/util/rate_limiter/limiter.py +++ b/src/HABApp/util/rate_limiter/limiter.py @@ -1,7 +1,7 @@ from dataclasses import dataclass -from typing import Final, List, Literal, Tuple, TypeAlias, Union +from typing import Final, List, Literal, Tuple, Union -from HABApp.core.const.const import StrEnum +from HABApp.core.const.const import PYTHON_310, StrEnum from .limits import ( BaseRateLimit, @@ -13,6 +13,12 @@ from .parser import parse_limit +if PYTHON_310: + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + + class LimitTypeEnum(StrEnum): LEAKY_BUCKET = 'leaky_bucket' FIXED_WINDOW_ELASTIC_EXPIRY = 'fixed_window_elastic_expiry' From 942e21085adcd3ddb8bda495c244a820f7e7aa8d Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:14:04 +0100 Subject: [PATCH 07/15] update action --- .github/workflows/publish-pypi.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 15a12ece..2aac5900 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -7,11 +7,16 @@ jobs: build-n-publish: name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI runs-on: ubuntu-latest + environment: release + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write steps: - uses: actions/checkout@v3 with: ref: master + - name: Set up Python 3.10 uses: actions/setup-python@v4 with: @@ -27,7 +32,4 @@ jobs: python setup.py sdist bdist_wheel - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - user: __token__ - password: ${{ secrets.pypi_api_key }} + uses: pypa/gh-action-pypi-publish@release/v1 From 7cfff81f9233ded3a1239710c83676969d7052cd Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:16:29 +0100 Subject: [PATCH 08/15] update action --- .github/workflows/publish-dockerhub.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-dockerhub.yml b/.github/workflows/publish-dockerhub.yml index 1ea4c7db..ff3c08a6 100644 --- a/.github/workflows/publish-dockerhub.yml +++ b/.github/workflows/publish-dockerhub.yml @@ -9,6 +9,8 @@ on: jobs: buildx: runs-on: ubuntu-latest + environment: release + steps: - uses: actions/checkout@v2 with: From ba73243d7a7626ba928877752fd0fcfee12d23aa Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Thu, 21 Dec 2023 09:08:40 +0100 Subject: [PATCH 09/15] update --- docs/util.rst | 6 +- src/HABApp/util/rate_limiter/limiter.py | 91 +++++++++---------- src/HABApp/util/rate_limiter/limits/base.py | 5 +- .../util/rate_limiter/limits/fixed_window.py | 4 +- .../util/rate_limiter/limits/leaky_bucket.py | 4 +- tests/test_utils/test_rate_limiter.py | 21 ++++- 6 files changed, 76 insertions(+), 55 deletions(-) diff --git a/docs/util.rst b/docs/util.rst index 560bbf43..5f18a0cb 100644 --- a/docs/util.rst +++ b/docs/util.rst @@ -138,11 +138,15 @@ Example limiter = RateLimiter('MyRateLimiterName') # define limits, duplicate limits of the same algorithm will only be added once + # These lines all define the same limit so it'll result in only one limiter added limiter.add_limit(5, 60) # add limits explicitly limiter.parse_limits('5 per minute').parse_limits('5 in 60s', '5/60seconds') # add limits through text # add additional limit with leaky bucket algorithm - limiter.add_limit(10, 120, algorithm='leaky_bucket') + limiter.add_limit(10, 100, algorithm='leaky_bucket') + + # add additional limit with fixed window elastic expiry algorithm + limiter.add_limit(10, 100, algorithm='fixed_window_elastic_expiry') # Test the limit without increasing the hits for _ in range(100): diff --git a/src/HABApp/util/rate_limiter/limiter.py b/src/HABApp/util/rate_limiter/limiter.py index 487e0c1a..1d743b89 100644 --- a/src/HABApp/util/rate_limiter/limiter.py +++ b/src/HABApp/util/rate_limiter/limiter.py @@ -1,17 +1,10 @@ from dataclasses import dataclass -from typing import Final, List, Literal, Tuple, Union - -from HABApp.core.const.const import PYTHON_310, StrEnum - -from .limits import ( - BaseRateLimit, - FixedWindowElasticExpiryLimit, - FixedWindowElasticExpiryLimitInfo, - LeakyBucketLimit, - LeakyBucketLimitInfo, -) -from .parser import parse_limit +from typing import Final, List, Literal, Tuple, Union, get_args +from HABApp.core.const.const import PYTHON_310 +from HABApp.util.rate_limiter.limits import BaseRateLimit, FixedWindowElasticExpiryLimit, \ + FixedWindowElasticExpiryLimitInfo, LeakyBucketLimit, LeakyBucketLimitInfo +from HABApp.util.rate_limiter.parser import parse_limit if PYTHON_310: from typing import TypeAlias @@ -19,12 +12,16 @@ from typing_extensions import TypeAlias -class LimitTypeEnum(StrEnum): - LEAKY_BUCKET = 'leaky_bucket' - FIXED_WINDOW_ELASTIC_EXPIRY = 'fixed_window_elastic_expiry' +_LITERAL_LEAKY_BUCKET = Literal['leaky_bucket'] +_LITERAL_FIXED_WINDOW_ELASTIC_EXPIRY = Literal['fixed_window_elastic_expiry'] + +LIMITER_ALGORITHM_HINT: TypeAlias = Literal[_LITERAL_LEAKY_BUCKET, _LITERAL_FIXED_WINDOW_ELASTIC_EXPIRY] -LIMITER_ALGORITHM_HINT: TypeAlias = Literal[LimitTypeEnum.LEAKY_BUCKET, LimitTypeEnum.FIXED_WINDOW_ELASTIC_EXPIRY] +def _check_arg(name: str, value, allow_0=False): + if not isinstance(value, int) or ((value <= 0) if not allow_0 else (value < 0)): + msg = f'Parameter {name:s} must be an int >{"=" if allow_0 else ""} 0, is {value} ({type(value)})' + raise ValueError(msg) class Limiter: @@ -36,66 +33,70 @@ def __init__(self, name: str): def __repr__(self): return f'<{self.__class__.__name__} {self._name:s}>' - def add_limit(self, allowed: int, interval: int, - algorithm: LIMITER_ALGORITHM_HINT = LimitTypeEnum.FIXED_WINDOW_ELASTIC_EXPIRY) -> 'Limiter': + def add_limit(self, allowed: int, interval: int, *, + hits: int = 0, + algorithm: LIMITER_ALGORITHM_HINT = 'leaky_bucket') -> 'Limiter': """Add a new rate limit :param allowed: How many hits are allowed :param interval: Interval in seconds + :param hits: How many hits the limit already has when it gets initially created :param algorithm: Which algorithm should this limit use """ - if allowed <= 0 or not isinstance(allowed, int): - msg = f'Allowed must be an int >= 0, is {allowed} ({type(allowed)})' - raise ValueError(msg) - - if interval <= 0 or not isinstance(interval, int): - msg = f'Expire time must be an int >= 0, is {interval} ({type(interval)})' + _check_arg('allowed', allowed) + _check_arg('interval', interval) + _check_arg('hits', hits, allow_0=True) + if not hits <= allowed: + msg = f'Parameter hits must be <= parameter allowed! {hits:d} <= {allowed:d}!' raise ValueError(msg) - algo = LimitTypeEnum(algorithm) - if algo is LimitTypeEnum.FIXED_WINDOW_ELASTIC_EXPIRY: - cls = FixedWindowElasticExpiryLimit - elif algo is LimitTypeEnum.LEAKY_BUCKET: + if algorithm == get_args(_LITERAL_LEAKY_BUCKET)[0]: cls = LeakyBucketLimit + elif algorithm == get_args(_LITERAL_FIXED_WINDOW_ELASTIC_EXPIRY)[0]: + cls = FixedWindowElasticExpiryLimit else: - raise ValueError() + msg = f'Unknown algorithm "{algorithm}"' + raise ValueError(msg) # Check if we have already added an algorithm with these parameters for window in self._limits: if isinstance(window, cls) and window.allowed == allowed and window.interval == interval: return self - limit = cls(allowed, interval) + limit = cls(allowed, interval, hits=hits) self._limits = tuple(sorted([*self._limits, limit], key=lambda x: x.interval)) return self def parse_limits(self, *text: str, - algorithm: LIMITER_ALGORITHM_HINT = LimitTypeEnum.FIXED_WINDOW_ELASTIC_EXPIRY) -> 'Limiter': + hits: int = 0, + algorithm: LIMITER_ALGORITHM_HINT = 'leaky_bucket') -> 'Limiter': """Add one or more limits in textual form, e.g. ``5 in 60s``, ``10 per hour`` or ``10/15 mins``. If the limit does already exist it will not be added again. :param text: textual description of limit + :param hits: How many hits the limit already has when it gets initially created :param algorithm: Which algorithm should these limits use """ for limit in [parse_limit(t) for t in text]: - self.add_limit(*limit, algorithm=algorithm) + self.add_limit(*limit, hits=hits, algorithm=algorithm) return self def allow(self) -> bool: - """Test the limit. + """Test the limit(s). :return: ``True`` if allowed, ``False`` if forbidden """ - allow = True - clear_skipped = True - if not self._limits: msg = 'No limits defined!' raise ValueError(msg) + clear_skipped = True + for limit in self._limits: if not limit.allow(): - allow = False + self._skips += 1 + return False + # allow increments hits, if it's now 1 it was 0 before if limit.hits != 1: clear_skipped = False @@ -103,32 +104,30 @@ def allow(self) -> bool: if clear_skipped: self._skips = 0 - if not allow: - self._skips += 1 - - return allow + return True def test_allow(self) -> bool: - """Test the limit without hitting it. Calling this will not increase the hit counter. + """Test the limit(s) without hitting it. Calling this will not increase the hit counter. :return: ``True`` if allowed, ``False`` if forbidden """ - allow = True - clear_skipped = True if not self._limits: msg = 'No limits defined!' raise ValueError(msg) + clear_skipped = True + for limit in self._limits: if not limit.test_allow(): - allow = False + return False + if limit.hits != 0: clear_skipped = False if clear_skipped: self._skips = 0 - return allow + return True def info(self) -> 'LimiterInfo': """Get some info about the limiter and the defined windows diff --git a/src/HABApp/util/rate_limiter/limits/base.py b/src/HABApp/util/rate_limiter/limits/base.py index ba68c156..ef8ae87d 100644 --- a/src/HABApp/util/rate_limiter/limits/base.py +++ b/src/HABApp/util/rate_limiter/limits/base.py @@ -14,15 +14,16 @@ def hits_remaining(self) -> int: class BaseRateLimit: - def __init__(self, allowed: int, interval: int): + def __init__(self, allowed: int, interval: int, hits: int = 0): super().__init__() assert allowed > 0, allowed assert interval > 0, interval + assert 0 <= hits <= allowed self.interval: Final = interval self.allowed: Final = allowed - self.hits: int = 0 + self.hits: int = hits self.skips: int = 0 def repr_text(self) -> str: diff --git a/src/HABApp/util/rate_limiter/limits/fixed_window.py b/src/HABApp/util/rate_limiter/limits/fixed_window.py index 48b15ba7..95cc4328 100644 --- a/src/HABApp/util/rate_limiter/limits/fixed_window.py +++ b/src/HABApp/util/rate_limiter/limits/fixed_window.py @@ -10,8 +10,8 @@ class FixedWindowElasticExpiryLimitInfo(BaseRateLimitInfo): class FixedWindowElasticExpiryLimit(BaseRateLimit): - def __init__(self, allowed: int, interval: int): - super().__init__(allowed, interval) + def __init__(self, allowed: int, interval: int, hits: int = 0): + super().__init__(allowed, interval, hits) self.start: float = -1.0 self.stop: float = -1.0 diff --git a/src/HABApp/util/rate_limiter/limits/leaky_bucket.py b/src/HABApp/util/rate_limiter/limits/leaky_bucket.py index 9f61ed7b..ed91eeb9 100644 --- a/src/HABApp/util/rate_limiter/limits/leaky_bucket.py +++ b/src/HABApp/util/rate_limiter/limits/leaky_bucket.py @@ -11,8 +11,8 @@ class LeakyBucketLimitInfo(BaseRateLimitInfo): class LeakyBucketLimit(BaseRateLimit): - def __init__(self, allowed: int, interval: int): - super().__init__(allowed, interval) + def __init__(self, allowed: int, interval: int, hits: int = 0): + super().__init__(allowed, interval, hits) self.drop_interval: Final = interval / allowed self.next_drop: float = -1.0 diff --git a/tests/test_utils/test_rate_limiter.py b/tests/test_utils/test_rate_limiter.py index 87c68de0..b68aa401 100644 --- a/tests/test_utils/test_rate_limiter.py +++ b/tests/test_utils/test_rate_limiter.py @@ -59,7 +59,7 @@ def test_parse(unit: str, factor: int): assert str(e.value) == 'Invalid limit string: "asdf"' -def test_regex_all_units(): +def test_parse_regex_all_units(): m = re.search(r'\(([^)]+)\)s\?', LIMIT_REGEX.pattern) values = m.group(1) @@ -158,6 +158,22 @@ def test_limiter_add(time): limiter.add_limit(3, 5).add_limit(3, 5).parse_limits('3 in 5s') assert len(limiter._limits) == 1 + with pytest.raises(ValueError) as e: + limiter.add_limit(0, 5) + assert str(e.value) == "Parameter allowed must be an int > 0, is 0 ()" + + with pytest.raises(ValueError) as e: + limiter.add_limit(1, 0.5) + assert str(e.value) == "Parameter interval must be an int > 0, is 0.5 ()" + + with pytest.raises(ValueError) as e: + limiter.add_limit(3, 5, hits=-1) + assert str(e.value) == "Parameter hits must be an int >= 0, is -1 ()" + + with pytest.raises(ValueError) as e: + limiter.add_limit(3, 5, hits=5) + assert str(e.value) == "Parameter hits must be <= parameter allowed! 5 <= 3!" + def test_fixed_window_info(time): limit = FixedWindowElasticExpiryLimit(5, 3) @@ -208,7 +224,8 @@ def test_limiter(time): with pytest.raises(ValueError): limiter.allow() - limiter.add_limit(2, 1).add_limit(2, 2) + limiter.add_limit( + 2, 1, algorithm='fixed_window_elastic_expiry').add_limit(2, 2, algorithm='fixed_window_elastic_expiry') assert limiter.allow() assert limiter.allow() From b470a4d0de0284f862b0d6d0f52ef89e0c208a2e Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Fri, 22 Dec 2023 05:48:11 +0100 Subject: [PATCH 10/15] update --- src/HABApp/util/rate_limiter/limiter.py | 18 +++++++++--------- tests/test_utils/test_rate_limiter.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/HABApp/util/rate_limiter/limiter.py b/src/HABApp/util/rate_limiter/limiter.py index 1d743b89..f47f0527 100644 --- a/src/HABApp/util/rate_limiter/limiter.py +++ b/src/HABApp/util/rate_limiter/limiter.py @@ -34,20 +34,20 @@ def __repr__(self): return f'<{self.__class__.__name__} {self._name:s}>' def add_limit(self, allowed: int, interval: int, *, - hits: int = 0, + initial_hits: int = 0, algorithm: LIMITER_ALGORITHM_HINT = 'leaky_bucket') -> 'Limiter': """Add a new rate limit :param allowed: How many hits are allowed :param interval: Interval in seconds - :param hits: How many hits the limit already has when it gets initially created + :param initial_hits: How many hits the limit already has when it gets initially created :param algorithm: Which algorithm should this limit use """ _check_arg('allowed', allowed) _check_arg('interval', interval) - _check_arg('hits', hits, allow_0=True) - if not hits <= allowed: - msg = f'Parameter hits must be <= parameter allowed! {hits:d} <= {allowed:d}!' + _check_arg('hits', initial_hits, allow_0=True) + if not initial_hits <= allowed: + msg = f'Parameter hits must be <= parameter allowed! {initial_hits:d} <= {allowed:d}!' raise ValueError(msg) if algorithm == get_args(_LITERAL_LEAKY_BUCKET)[0]: @@ -63,22 +63,22 @@ def add_limit(self, allowed: int, interval: int, *, if isinstance(window, cls) and window.allowed == allowed and window.interval == interval: return self - limit = cls(allowed, interval, hits=hits) + limit = cls(allowed, interval, hits=initial_hits) self._limits = tuple(sorted([*self._limits, limit], key=lambda x: x.interval)) return self def parse_limits(self, *text: str, - hits: int = 0, + initial_hits: int = 0, algorithm: LIMITER_ALGORITHM_HINT = 'leaky_bucket') -> 'Limiter': """Add one or more limits in textual form, e.g. ``5 in 60s``, ``10 per hour`` or ``10/15 mins``. If the limit does already exist it will not be added again. :param text: textual description of limit - :param hits: How many hits the limit already has when it gets initially created + :param initial_hits: How many hits the limit already has when it gets initially created :param algorithm: Which algorithm should these limits use """ for limit in [parse_limit(t) for t in text]: - self.add_limit(*limit, hits=hits, algorithm=algorithm) + self.add_limit(*limit, initial_hits=initial_hits, algorithm=algorithm) return self def allow(self) -> bool: diff --git a/tests/test_utils/test_rate_limiter.py b/tests/test_utils/test_rate_limiter.py index b68aa401..844bb72d 100644 --- a/tests/test_utils/test_rate_limiter.py +++ b/tests/test_utils/test_rate_limiter.py @@ -167,11 +167,11 @@ def test_limiter_add(time): assert str(e.value) == "Parameter interval must be an int > 0, is 0.5 ()" with pytest.raises(ValueError) as e: - limiter.add_limit(3, 5, hits=-1) + limiter.add_limit(3, 5, initial_hits=-1) assert str(e.value) == "Parameter hits must be an int >= 0, is -1 ()" with pytest.raises(ValueError) as e: - limiter.add_limit(3, 5, hits=5) + limiter.add_limit(3, 5, initial_hits=5) assert str(e.value) == "Parameter hits must be <= parameter allowed! 5 <= 3!" From cd78048c39d0f9173fb178826c1844e3e6c3c73b Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Wed, 27 Dec 2023 06:11:46 +0100 Subject: [PATCH 11/15] added total skips --- src/HABApp/util/rate_limiter/limiter.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/HABApp/util/rate_limiter/limiter.py b/src/HABApp/util/rate_limiter/limiter.py index f47f0527..c34a092a 100644 --- a/src/HABApp/util/rate_limiter/limiter.py +++ b/src/HABApp/util/rate_limiter/limiter.py @@ -28,7 +28,13 @@ class Limiter: def __init__(self, name: str): self._name: Final = name self._limits: Tuple[BaseRateLimit, ...] = () - self._skips = 0 + self._skips: int = 0 + self._skips_total: int = 0 + + @property + def total_skips(self) -> int: + """A user counter to track skips which can be manually reset""" + return self._skips_total def __repr__(self): return f'<{self.__class__.__name__} {self._name:s}>' @@ -95,6 +101,7 @@ def allow(self) -> bool: for limit in self._limits: if not limit.allow(): self._skips += 1 + self._skips_total += 1 return False # allow increments hits, if it's now 1 it was 0 before @@ -134,12 +141,18 @@ def info(self) -> 'LimiterInfo': """ return LimiterInfo( - skips=self._skips, + skips=self._skips, total_skips=self._skips_total, limits=[limit.info() for limit in self._limits] ) + def reset(self) -> 'Limiter': + """Reset the manual skip counter""" + self._skips_total = 0 + return self + @dataclass class LimiterInfo: - skips: int #: How many entries were skipped + skips: int #: How many entries were skipped in the active interval(s) + total_skips: int #: How many entries were skipped in total limits: List[Union[FixedWindowElasticExpiryLimitInfo, LeakyBucketLimitInfo]] #: Info for every limit From d60b92bcf86c70c60917e0d1907cb6be2e2ab3e6 Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Mon, 8 Jan 2024 06:02:22 +0100 Subject: [PATCH 12/15] update --- requirements.txt | 4 ++-- requirements_setup.txt | 4 ++-- requirements_tests.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9b86d6f2..0df97ba0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,8 @@ # ----------------------------------------------------------------------------- # Packages for source formatting # ----------------------------------------------------------------------------- -pre-commit == 3.5.0 -ruff == 0.1.8 +pre-commit == 3.6.0 +ruff == 0.1.11 # ----------------------------------------------------------------------------- # Packages for other developement tasks diff --git a/requirements_setup.txt b/requirements_setup.txt index a634b771..f635f60b 100644 --- a/requirements_setup.txt +++ b/requirements_setup.txt @@ -1,7 +1,6 @@ aiohttp == 3.9.1 -pydantic == 2.5.2 +pydantic == 2.5.3 msgspec == 0.18.5 -pendulum == 2.1.2 bidict == 0.22.1 watchdog == 3.0.0 ujson == 5.9.0 @@ -10,6 +9,7 @@ aiomqtt == 1.2.1 immutables == 0.20 eascheduler == 0.1.11 easyconfig == 0.3.1 +pendulum == 2.1.2 stack_data == 0.6.3 colorama == 0.4.6 diff --git a/requirements_tests.txt b/requirements_tests.txt index c6dc925e..9451f3ee 100644 --- a/requirements_tests.txt +++ b/requirements_tests.txt @@ -7,5 +7,5 @@ # Packages to run source tests # ----------------------------------------------------------------------------- packaging == 23.2 -pytest == 7.4.3 -pytest-asyncio == 0.23.2 +pytest == 7.4.4 +pytest-asyncio == 0.23.3 From df74ef4e6ef2d0a043a0d763af4005dcb09d5b76 Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Mon, 8 Jan 2024 06:05:48 +0100 Subject: [PATCH 13/15] update --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0df97ba0..57c6b701 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # ----------------------------------------------------------------------------- # Packages for source formatting # ----------------------------------------------------------------------------- -pre-commit == 3.6.0 +pre-commit == 3.5.0 # 3.6.0 requires python >= 3.10 ruff == 0.1.11 # ----------------------------------------------------------------------------- From d3f6826a15a65a1b5c9449ad79598e71dc1ac556 Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Mon, 8 Jan 2024 09:22:54 +0100 Subject: [PATCH 14/15] update --- docs/util.rst | 24 +++++++++++ readme.md | 2 +- .../rules/openhab/test_persistence.py | 41 +++++++++++++++++-- src/HABApp/__version__.py | 2 +- src/HABApp/mqtt/connection/handler.py | 3 ++ src/HABApp/openhab/connection/connection.py | 2 + .../openhab/connection/handler/handler.py | 4 +- .../openhab/connection/handler/helper.py | 7 +++- src/HABApp/openhab/connection/plugins/out.py | 4 +- .../definitions/helpers/persistence_data.py | 1 + src/HABApp/util/rate_limiter/limiter.py | 4 +- .../util/rate_limiter/limits/leaky_bucket.py | 2 +- .../test_openhab/test_items/test_commands.py | 2 +- .../test_plugins/test_load_items.py | 4 +- 14 files changed, 88 insertions(+), 14 deletions(-) diff --git a/docs/util.rst b/docs/util.rst index 5f18a0cb..1499c06c 100644 --- a/docs/util.rst +++ b/docs/util.rst @@ -162,6 +162,30 @@ Example # It's possible to get statistics about the limiter and the corresponding windows print(limiter.info()) + # There is a counter that keeps track of the total skips that can be reset + print('Counter:') + print(limiter.total_skips) + limiter.reset() # Can be reset + print(limiter.total_skips) + + +Recommendation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Limiting external requests to an external API works well with the leaky bucket algorithm (maybe with some initial hits). +For limiting notifications the best results can be achieved by combining both algorithms. +Fixed window elastic expiry will notify but block until an issue is resolved, +that's why it's more suited for small intervals. Leaky bucket will allow hits even while the issue persists, +that's why it's more suited for larger intervals. + +.. exec_code:: + + from HABApp.util import RateLimiter + + limiter = RateLimiter('MyNotifications') + limiter.parse_limits('5 in 1 minute', algorithm='fixed_window_elastic_expiry') + limiter.parse_limits("20 in 1 hour", algorithm='leaky_bucket') + Documentation ^^^^^^^^^^^^^^^^^^ diff --git a/readme.md b/readme.md index ba865437..42badb74 100644 --- a/readme.md +++ b/readme.md @@ -127,7 +127,7 @@ MyOpenhabRule() ``` # Changelog -#### 23.12.0-DEV (2023-XX-XX) +#### 24.01.0-DEV (2024-XX-XX) - Added HABApp.util.RateLimiter - Added CompressedMidnightRotatingFileHandler - Updated dependencies diff --git a/run/conf_testing/rules/openhab/test_persistence.py b/run/conf_testing/rules/openhab/test_persistence.py index 983d204f..e5d5cb0a 100644 --- a/run/conf_testing/rules/openhab/test_persistence.py +++ b/run/conf_testing/rules/openhab/test_persistence.py @@ -1,11 +1,16 @@ from __future__ import annotations from datetime import datetime, timedelta -from typing import Final, Any +from typing import TYPE_CHECKING, Any, Final -from HABApp.openhab.definitions.helpers import OpenhabPersistenceData +from HABAppTests import ItemWaiter, TestBaseRule + +from HABApp.core.connections import Connections from HABApp.openhab.items import NumberItem -from HABAppTests import TestBaseRule + + +if TYPE_CHECKING: + from HABApp.openhab.definitions.helpers import OpenhabPersistenceData class TestPersistenceBase(TestBaseRule): @@ -20,6 +25,9 @@ def __init__(self, service_name: str, item_name: str): def set_up(self): i = NumberItem.get_item(self.item_name) + if i.value is None: + i.oh_post_update(0) + ItemWaiter(self.item_name).wait_for_state(0) i.oh_post_update(int(i.value) + 1 if i.value < 10 else 0) def test_service_available(self): @@ -64,3 +72,30 @@ def test_get(self): TestMapDB() + + +class TestInMemory(TestPersistenceBase): + + def __init__(self): + super().__init__('inmemory', 'RRD4J_Item') + + if Connections.get('openhab').context.version >= (4, 1): + self.add_test('InMemory', self.test_in_memory) + else: + print('Skip "TestInMemory" because of no InMemoryDb') + + def test_in_memory(self): + now = datetime.now().replace(microsecond=0) + t1 = now - timedelta(milliseconds=100) + t2 = now + timedelta(milliseconds=100) + + self.set_persistence_data(t1, 5) + self.set_persistence_data(now, 6) + self.set_persistence_data(t2, 7) + value = self.get_persistence_data(now - timedelta(milliseconds=200), now + timedelta(milliseconds=200)) + + objs = value.get_data() + assert objs == {t1.timestamp(): 5, now.timestamp(): 6, t2.timestamp(): 7} + + +TestInMemory() diff --git a/src/HABApp/__version__.py b/src/HABApp/__version__.py index b0cd1b76..bae0faf9 100644 --- a/src/HABApp/__version__.py +++ b/src/HABApp/__version__.py @@ -10,4 +10,4 @@ # Development versions contain the DEV-COUNTER postfix: # - 23.09.0.DEV-1 -__version__ = '23.12.0.DEV-1' +__version__ = '24.01.0.DEV-1' diff --git a/src/HABApp/mqtt/connection/handler.py b/src/HABApp/mqtt/connection/handler.py index ebb63b94..4c0be234 100644 --- a/src/HABApp/mqtt/connection/handler.py +++ b/src/HABApp/mqtt/connection/handler.py @@ -72,6 +72,9 @@ async def on_disconnected(self, connection: MqttConnection, context: CONTEXT_TYP assert context is not None connection.log.info('Disconnected') + # remove this check when https://github.com/sbtinstruments/aiomqtt/pull/249 gets merged + if not context._lock.locked(): + await context._lock.acquire() await context.__aexit__(None, None, None) diff --git a/src/HABApp/openhab/connection/connection.py b/src/HABApp/openhab/connection/connection.py index 00a3d825..54bf6ee5 100644 --- a/src/HABApp/openhab/connection/connection.py +++ b/src/HABApp/openhab/connection/connection.py @@ -33,6 +33,8 @@ class OpenhabContext: session: aiohttp.ClientSession session_options: dict[str, Any] + workaround_small_floats: bool + CONTEXT_TYPE: TypeAlias = Optional[OpenhabContext] diff --git a/src/HABApp/openhab/connection/handler/handler.py b/src/HABApp/openhab/connection/handler/handler.py index 2bba83e7..6958547c 100644 --- a/src/HABApp/openhab/connection/handler/handler.py +++ b/src/HABApp/openhab/connection/handler/handler.py @@ -172,7 +172,9 @@ async def on_connecting(self, connection: OpenhabConnection): version=vers, is_oh3=vers < (4, 0), waited_for_openhab=False, created_items={}, created_things={}, - session=self.session, session_options=self.options + session=self.session, session_options=self.options, + + workaround_small_floats=vers < (4, 1) ) # during startup we get OpenhabCredentialsInvalidError even though credentials are correct diff --git a/src/HABApp/openhab/connection/handler/helper.py b/src/HABApp/openhab/connection/handler/helper.py index eccf3fc7..07a6a2a4 100644 --- a/src/HABApp/openhab/connection/handler/helper.py +++ b/src/HABApp/openhab/connection/handler/helper.py @@ -4,14 +4,17 @@ from typing import Any from HABApp.core.items import BaseValueItem -from HABApp.core.types import RGB, HSB +from HABApp.core.types import HSB, RGB -def convert_to_oh_type(obj: Any) -> str: +def convert_to_oh_type(obj: Any, scientific_floats=False) -> str: if isinstance(obj, (str, int, bool)): return str(obj) if isinstance(obj, float): + if scientific_floats: + return str(obj) + v = str(obj) if 'e-' not in v: return v diff --git a/src/HABApp/openhab/connection/plugins/out.py b/src/HABApp/openhab/connection/plugins/out.py index eaa46fb4..21a2c2e3 100644 --- a/src/HABApp/openhab/connection/plugins/out.py +++ b/src/HABApp/openhab/connection/plugins/out.py @@ -75,6 +75,8 @@ async def queue_worker(self): queue: Final = self.queue to_str: Final = convert_to_oh_type + scientific_floats = not self.plugin_connection.context.workaround_small_floats + while True: try: while True: @@ -84,7 +86,7 @@ async def queue_worker(self): item = item._name if not isinstance(state, str): - state = to_str(state) + state = to_str(state, scientific_floats=scientific_floats) if is_cmd: await post(f'/rest/items/{item:s}', data=state) diff --git a/src/HABApp/openhab/definitions/helpers/persistence_data.py b/src/HABApp/openhab/definitions/helpers/persistence_data.py index 03e78163..5cbc5613 100644 --- a/src/HABApp/openhab/definitions/helpers/persistence_data.py +++ b/src/HABApp/openhab/definitions/helpers/persistence_data.py @@ -3,6 +3,7 @@ from HABApp.openhab.definitions.rest import ItemHistoryResp + OPTIONAL_DT = Optional[datetime] diff --git a/src/HABApp/util/rate_limiter/limiter.py b/src/HABApp/util/rate_limiter/limiter.py index c34a092a..7b6aaead 100644 --- a/src/HABApp/util/rate_limiter/limiter.py +++ b/src/HABApp/util/rate_limiter/limiter.py @@ -33,7 +33,7 @@ def __init__(self, name: str): @property def total_skips(self) -> int: - """A user counter to track skips which can be manually reset""" + """A counter to track skips which can be manually reset""" return self._skips_total def __repr__(self): @@ -146,7 +146,7 @@ def info(self) -> 'LimiterInfo': ) def reset(self) -> 'Limiter': - """Reset the manual skip counter""" + """Reset the skip counter""" self._skips_total = 0 return self diff --git a/src/HABApp/util/rate_limiter/limits/leaky_bucket.py b/src/HABApp/util/rate_limiter/limits/leaky_bucket.py index ed91eeb9..fe39bb2e 100644 --- a/src/HABApp/util/rate_limiter/limits/leaky_bucket.py +++ b/src/HABApp/util/rate_limiter/limits/leaky_bucket.py @@ -15,7 +15,7 @@ def __init__(self, allowed: int, interval: int, hits: int = 0): super().__init__(allowed, interval, hits) self.drop_interval: Final = interval / allowed - self.next_drop: float = -1.0 + self.next_drop: float = monotonic() + self.drop_interval def repr_text(self): return f'drop_interval={self.drop_interval:.1f}s' diff --git a/tests/test_openhab/test_items/test_commands.py b/tests/test_openhab/test_items/test_commands.py index 88031b67..99de6c8c 100644 --- a/tests/test_openhab/test_items/test_commands.py +++ b/tests/test_openhab/test_items/test_commands.py @@ -13,7 +13,7 @@ def test_OnOff(cls): c = cls('item_name') assert not c.is_on() - if not __version__.startswith('23.12.0'): + if not __version__.startswith('24.01.0'): assert not c.is_off() c.set_value(OnOffValue('ON')) diff --git a/tests/test_openhab/test_plugins/test_load_items.py b/tests/test_openhab/test_plugins/test_load_items.py index 8a124adf..d548da7d 100644 --- a/tests/test_openhab/test_plugins/test_load_items.py +++ b/tests/test_openhab/test_plugins/test_load_items.py @@ -78,7 +78,9 @@ async def test_item_sync(monkeypatch, ir: ItemRegistry, test_logs): waited_for_openhab=False, created_items={}, created_things={}, - session=None, session_options=None + session=None, session_options=None, + + workaround_small_floats=False ) # initial item create await LoadOpenhabItemsPlugin().on_connected(context) From 6a7db6e30b460768cea1c4a0b14a21dc06e41616 Mon Sep 17 00:00:00 2001 From: spacemanspiff2007 <10754716+spacemanspiff2007@users.noreply.github.com> Date: Mon, 8 Jan 2024 10:10:24 +0100 Subject: [PATCH 15/15] update --- readme.md | 2 +- src/HABApp/__version__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 42badb74..ca924f6a 100644 --- a/readme.md +++ b/readme.md @@ -127,7 +127,7 @@ MyOpenhabRule() ``` # Changelog -#### 24.01.0-DEV (2024-XX-XX) +#### 24.01.0 (2024-01-08) - Added HABApp.util.RateLimiter - Added CompressedMidnightRotatingFileHandler - Updated dependencies diff --git a/src/HABApp/__version__.py b/src/HABApp/__version__.py index bae0faf9..a97afa18 100644 --- a/src/HABApp/__version__.py +++ b/src/HABApp/__version__.py @@ -8,6 +8,6 @@ # - 23.10.0 # # Development versions contain the DEV-COUNTER postfix: -# - 23.09.0.DEV-1 +# - 24.01.0.DEV-1 -__version__ = '24.01.0.DEV-1' +__version__ = '24.01.0'