From 1be66428d35e21eadae0461dd8b136696fbc1d73 Mon Sep 17 00:00:00 2001 From: Max KvR Date: Mon, 9 Sep 2024 13:32:28 +0200 Subject: [PATCH 1/7] feat: standardized CLI --- examples/metricq_client.py | 6 +- examples/metricq_get_history.py | 5 +- examples/metricq_get_history_raw.py | 5 +- examples/metricq_pandas.py | 5 +- examples/metricq_sink.py | 5 +- examples/metricq_source.py | 6 +- examples/metricq_synchronous_source.py | 6 +- metricq/cli/__init__.py | 17 +++ metricq/cli/command.py | 96 ++++++++++++++ metricq/cli/params.py | 171 +++++++++++++++++++++++++ setup.cfg | 7 +- 11 files changed, 304 insertions(+), 25 deletions(-) create mode 100644 metricq/cli/__init__.py create mode 100644 metricq/cli/command.py create mode 100644 metricq/cli/params.py diff --git a/examples/metricq_client.py b/examples/metricq_client.py index 130c3f06..579be272 100755 --- a/examples/metricq_client.py +++ b/examples/metricq_client.py @@ -43,10 +43,10 @@ import logging import aiomonitor # type: ignore -import click import click_log # type: ignore import metricq +from metricq.cli import metricq_command logger = metricq.get_logger() click_log.basic_config(logger) @@ -66,9 +66,7 @@ async def run(server: str, token: str) -> None: await client.stopped() -@click.command() -@click.option("--server", default="amqp://admin:admin@localhost/") -@click.option("--token", default="client-py-example") +@metricq_command(default_token="client-py-example") @click_log.simple_verbosity_option(logger) # type: ignore def main(server: str, token: str) -> None: asyncio.run(run(server, token)) diff --git a/examples/metricq_get_history.py b/examples/metricq_get_history.py index b7f3c65e..377def9f 100755 --- a/examples/metricq_get_history.py +++ b/examples/metricq_get_history.py @@ -36,6 +36,7 @@ import click_log # type: ignore import metricq +from metricq.cli import metricq_command logger = metricq.get_logger() @@ -98,9 +99,7 @@ async def aget_history( click.echo(aggregate) -@click.command() -@click.option("--server", default="amqp://localhost/") -@click.option("--token", default="history-py-dummy") +@metricq_command(default_token="history-py-dummy") @click.option("--metric", default=None) @click.option("--list-metrics", is_flag=True) @click.option("--list-metadata", is_flag=True) diff --git a/examples/metricq_get_history_raw.py b/examples/metricq_get_history_raw.py index 3fe2a282..7167daa2 100755 --- a/examples/metricq_get_history_raw.py +++ b/examples/metricq_get_history_raw.py @@ -35,6 +35,7 @@ import click_log # type: ignore import metricq +from metricq.cli import metricq_command from metricq.history_client import HistoryRequestType logger = metricq.get_logger() @@ -83,9 +84,7 @@ async def aget_history(server: str, token: str, metric: str) -> None: await client.stop(None) -@click.command() -@click.option("--server", default="amqp://localhost/") -@click.option("--token", default="history-py-dummy") +@metricq_command(default_token="history-py-dummy") @click.argument("metric") @click_log.simple_verbosity_option(logger) # type: ignore def get_history(server: str, token: str, metric: str) -> None: diff --git a/examples/metricq_pandas.py b/examples/metricq_pandas.py index 1096f079..7985c62f 100755 --- a/examples/metricq_pandas.py +++ b/examples/metricq_pandas.py @@ -35,6 +35,7 @@ import click_log # type: ignore import metricq +from metricq.cli import metricq_command from metricq.pandas import PandasHistoryClient logger = metricq.get_logger() @@ -81,9 +82,7 @@ async def aget_history(server: str, token: str, metric: str) -> None: click.echo("----------") -@click.command() -@click.option("--server", default="amqp://localhost/") -@click.option("--token", default="history-py-dummy") +@metricq_command(default_token="history-py-dummy") @click.option("--metric", default="example.quantity") @click_log.simple_verbosity_option(logger) # type: ignore def get_history(server: str, token: str, metric: str) -> None: diff --git a/examples/metricq_sink.py b/examples/metricq_sink.py index 272487d6..a1a2f8f0 100755 --- a/examples/metricq_sink.py +++ b/examples/metricq_sink.py @@ -35,6 +35,7 @@ import metricq from metricq import Metric +from metricq.cli import metricq_command from metricq.logging import get_logger logger = get_logger() @@ -76,9 +77,7 @@ async def on_data( ) -@click.command() -@click.option("--server", default="amqp://localhost/") -@click.option("--token", default="sink-py-dummy") +@metricq_command(default_token="sink-py-dummy") @click.option("-m", "--metrics", multiple=True, required=True) @click_log.simple_verbosity_option(logger) # type: ignore def source(server: str, token: str, metrics: list[Metric]) -> None: diff --git a/examples/metricq_source.py b/examples/metricq_source.py index 7004834c..c7a189fc 100755 --- a/examples/metricq_source.py +++ b/examples/metricq_source.py @@ -31,10 +31,10 @@ import random from typing import Any -import click import click_log # type: ignore import metricq +from metricq.cli import metricq_command from metricq.logging import get_logger logger = get_logger() @@ -74,9 +74,7 @@ async def update(self) -> None: ) -@click.command() -@click.option("--server", default="amqp://localhost/") -@click.option("--token", default="source-py-dummy") +@metricq_command(default_token="source-py-dummy") @click_log.simple_verbosity_option(logger) # type: ignore def source(server: str, token: str) -> None: src = DummySource(token=token, url=server) diff --git a/examples/metricq_synchronous_source.py b/examples/metricq_synchronous_source.py index 6050915d..715cc763 100755 --- a/examples/metricq_synchronous_source.py +++ b/examples/metricq_synchronous_source.py @@ -33,10 +33,10 @@ import random import time -import click import click_log # type: ignore from metricq import SynchronousSource, Timestamp, get_logger +from metricq.cli import metricq_command logger = get_logger() @@ -47,9 +47,7 @@ ) -@click.command() -@click.option("--server", default="amqp://localhost/") -@click.option("--token", default="source-py-dummy") +@metricq_command(default_token="source-py-dummy") @click_log.simple_verbosity_option(logger) # type: ignore def synchronous_source(server: str, token: str) -> None: ssource = SynchronousSource(token=token, url=server) diff --git a/metricq/cli/__init__.py b/metricq/cli/__init__.py new file mode 100644 index 00000000..f9c59d0e --- /dev/null +++ b/metricq/cli/__init__.py @@ -0,0 +1,17 @@ +from .command import metricq_command +from .params import ( + ChoiceParam, + CommandLineChoice, + DurationParam, + TemplateStringParam, + TimestampParam, +) + +__all__ = [ + "ChoiceParam", + "CommandLineChoice", + "DurationParam", + "TemplateStringParam", + "TimestampParam", + "metricq_command", +] diff --git a/metricq/cli/command.py b/metricq/cli/command.py new file mode 100644 index 00000000..82abd86d --- /dev/null +++ b/metricq/cli/command.py @@ -0,0 +1,96 @@ +import logging +from typing import Callable, cast + +import click +import click_log # type: ignore +from click import option +from dotenv import find_dotenv, load_dotenv + +from .. import get_logger +from .params import FC, TemplateStringParam + +# We do not interpolate (i.e. replace ${VAR} with corresponding environment variables). +# That is because we want to be able to interpolate ourselves for metrics and tokens +# using the same syntax. If it was only ${USER} for the token, we could use the +# override functionality, but most unfortunately there is no standard environment +# variable for the hostname. Even $HOST on zsh is not actually part of the environment. +# ``override=false`` just means that environment variables have priority over the +# env files. +load_dotenv(dotenv_path=find_dotenv(".metricq"), interpolate=False, override=False) + + +def metricq_server_option() -> Callable[[FC], FC]: + return option( + "--server", + type=TemplateStringParam(), + metavar="URL", + required=True, + help="MetricQ server URL.", + ) + + +def metricq_token_option(default: str) -> Callable[[FC], FC]: + return option( + "--token", + type=TemplateStringParam(), + metavar="CLIENT_TOKEN", + default=default, + show_default=True, + help="A token to identify this client on the MetricQ network.", + ) + + +def metricq_metric_option(default: Optional[str] = None) -> Callable[[FC], FC]: + return option( + "--metric", + type=MetricParam(), + metavar="METRIC", + show_default=True, + required=default is None, + default=default, + help="Use the -–metric parameter to specify which metric the program should use", + ) + + +def get_metric_command_logger() -> logging.Logger: + logger = get_logger() + logger.setLevel(logging.WARNING) + click_log.basic_config(logger) + + return logger + + +def metricq_command( + default_token: str, client_version: str | None = None +) -> Callable[[FC], click.Command]: + logger = get_metric_command_logger() + + log_decorator = cast( + Callable[[FC], FC], click_log.simple_verbosity_option(logger, default="warning") + ) + context_settings = {"auto_envvar_prefix": "METRICQ"} + epilog = ( + "All options can be passed as environment variables prefixed with 'METRICQ_'." + "I.e., 'METRICQ_SERVER=amqps://...'.\n" + "\n" + "You can also create a '.metricq' file in the current or home directory that " + "contains environment variable settings in the same format.\n" + "\n" + "Some options, including server and token, can contain placeholders for $USER " + "and $HOST." + ) + + def decorator(func: FC) -> click.Command: + return click.version_option(version=client_version)( + log_decorator( + metricq_token_option(default_token)( + metricq_server_option()( + click.command(context_settings=context_settings, epilog=epilog)( + func + ) + ) + ) + ) + ) + + return decorator diff --git a/metricq/cli/params.py b/metricq/cli/params.py new file mode 100644 index 00000000..455a2a23 --- /dev/null +++ b/metricq/cli/params.py @@ -0,0 +1,171 @@ +import re +from contextlib import suppress +from getpass import getuser +from socket import gethostname +from string import Template +from typing import Any, Generic, List, Optional, Type, TypeVar, Union, cast + +import click +from click import Context, Parameter, ParamType, option + +from ..timeseries import Timedelta, Timestamp + +_C = TypeVar("_C", covariant=True) + + +def camelcase_to_kebabcase(camelcase: str) -> str: + # Match empty string preceeding uppercase character, but not at the start + # of the word. Replace with '-' and make lowercase to get kebab-case word. + return re.sub(r"(? str: + return "".join(part.title() for part in kebabcase.split("-")) + + +class CommandLineChoice: + @classmethod + def as_choice_list(cls) -> List[str]: + return [ + camelcase_to_kebabcase(name) for name in getattr(cls, "__members__").keys() + ] + + def as_choice(self) -> str: + return camelcase_to_kebabcase(getattr(self, "name")) + + @classmethod + def default(cls: Type[_C]) -> Optional[_C]: + return None + + @classmethod + def from_choice(cls: Type[_C], option: str) -> _C: + member_name = kebabcase_to_camelcase(option.lower()) + return cast(_C, getattr(cls, "__members__")[member_name]) + + +ChoiceType = TypeVar("ChoiceType", bound=CommandLineChoice) + + +class ChoiceParam(Generic[ChoiceType], ParamType): + def __init__(self, cls: Type[ChoiceType], name: str): + self.cls = cls + self.name = name + + def get_metavar(self, param: Parameter) -> str: + return f"({'|'.join(self.cls.as_choice_list())})" + + def convert( + self, + value: Union[str, ChoiceType], + param: Optional[Parameter], + ctx: Optional[Context], + ) -> Optional[ChoiceType]: + if value is None: + return None + + try: + if isinstance(value, str): + return self.cls.from_choice(value) + else: + return value + except (KeyError, ValueError): + self.fail( + f"unknown choice {value!r}, expected: {', '.join(self.cls.as_choice_list())}", + param=param, + ctx=ctx, + ) + + +class DurationParam(ParamType): + name = "duration" + + def __init__(self, default: Optional[Timedelta]): + self.default = default + + def convert( + self, + value: Union[str, Timedelta], + param: Optional[Parameter], + ctx: Optional[Context], + ) -> Optional[Timedelta]: + if value is None: + return None + elif isinstance(value, str): + try: + return Timedelta.from_string(value) + except ValueError: + self.fail( + 'expected a duration: "[]"', + param=param, + ctx=ctx, + ) + else: + return value + + +class TimestampParam(ParamType): + """ + Convert strings to ``metricq.Timestamp`` objects. + + Accepts the following string inputs + - ISO-8601 timestamp (with timezone) + - Past Duration, e.g., '-10h' from now + - Posix timestamp, float seconds since 1.1.1970 midnight. (UTC) + - 'now' + - 'epoch', i.e., 1.1.1970 midnight + """ + + name = "timestamp" + + @staticmethod + def _convert(value: str) -> Timestamp: + if value == "now": + return Timestamp.now() + if value == "epoch": + return Timestamp.from_posix_seconds(0) + if value.startswith("-"): + # Plus because the minus makes negative timedelta + return Timestamp.now() + Timedelta.from_string(value) + with suppress(ValueError): + return Timestamp.from_posix_seconds(float(value)) + + return Timestamp.from_iso8601(value) + + def convert( + self, value: Any, param: Optional[Parameter], ctx: Optional[Context] + ) -> Optional[Timestamp]: + if value is None: + return None + elif isinstance(value, Timestamp): + return value + elif isinstance(value, str): + try: + return self._convert(value) + except ValueError: + self.fail( + "expected an ISO-8601 timestamp (e.g. '2012-12-21T00:00:00Z'), " + "POSIX timestamp, 'now', 'epoch', or a past duration (e.g. '-10h')", + param=param, + ctx=ctx, + ) + else: + self.fail("unexpected type to convert to TimeStamp", param=param, ctx=ctx) + + +class TemplateStringParam(ParamType): + name = "text" + mapping: dict[str, str] + + def __init__(self) -> None: + self.mapping = {} + with suppress(Exception): + self.mapping["USER"] = getuser() + with suppress(Exception): + self.mapping["HOST"] = gethostname() + + def convert( + self, value: Any, param: Optional[Parameter], ctx: Optional[Context] + ) -> str: + if not isinstance(value, str): + raise TypeError("expected a string type for TemplateStringParam") + return Template(value).safe_substitute(self.mapping) diff --git a/setup.cfg b/setup.cfg index 4fd874b9..b497c3ed 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,9 +62,14 @@ dev = %(examples)s %(typing)s %(docs)s + %(cli)s tox pandas = pandas ~= 2.0.1 +cli = + click + click-log + python-dotenv~=1.0.0 [flake8] application-import-names = @@ -126,5 +131,5 @@ deps = .[lint] commands = flake8 . [testenv:mypy] -deps = .[typing] +deps = .[typing, cli] commands = mypy --strict metricq examples tests setup.py From ef61fac5ae48bdbe2d79017c210268c1ca6f9310 Mon Sep 17 00:00:00 2001 From: Max KvR Date: Wed, 2 Oct 2024 13:44:37 +0200 Subject: [PATCH 2/7] feat: @metric_input as a standardized way to input --metric values --- examples/metricq_get_history.py | 3 ++- examples/metricq_pandas.py | 4 ++-- metricq/cli/__init__.py | 4 +++- metricq/cli/params.py | 29 ++++++++++++++++++++++++-- metricq/cli/types.py | 5 +++++ metricq/cli/{command.py => wrapper.py} | 3 ++- 6 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 metricq/cli/types.py rename metricq/cli/{command.py => wrapper.py} (98%) diff --git a/examples/metricq_get_history.py b/examples/metricq_get_history.py index 377def9f..ee5fbcb4 100755 --- a/examples/metricq_get_history.py +++ b/examples/metricq_get_history.py @@ -37,6 +37,7 @@ import metricq from metricq.cli import metricq_command +from metricq.cli.params import metric_input logger = metricq.get_logger() @@ -100,7 +101,7 @@ async def aget_history( @metricq_command(default_token="history-py-dummy") -@click.option("--metric", default=None) +@metric_input() @click.option("--list-metrics", is_flag=True) @click.option("--list-metadata", is_flag=True) @click_log.simple_verbosity_option(logger) # type: ignore diff --git a/examples/metricq_pandas.py b/examples/metricq_pandas.py index 7985c62f..26a1cf5f 100755 --- a/examples/metricq_pandas.py +++ b/examples/metricq_pandas.py @@ -35,7 +35,7 @@ import click_log # type: ignore import metricq -from metricq.cli import metricq_command +from metricq.cli import metric_input, metricq_command from metricq.pandas import PandasHistoryClient logger = metricq.get_logger() @@ -83,7 +83,7 @@ async def aget_history(server: str, token: str, metric: str) -> None: @metricq_command(default_token="history-py-dummy") -@click.option("--metric", default="example.quantity") +@metric_input(default="example.quantity") @click_log.simple_verbosity_option(logger) # type: ignore def get_history(server: str, token: str, metric: str) -> None: asyncio.run(aget_history(server, token, metric)) diff --git a/metricq/cli/__init__.py b/metricq/cli/__init__.py index f9c59d0e..e3777b26 100644 --- a/metricq/cli/__init__.py +++ b/metricq/cli/__init__.py @@ -1,11 +1,12 @@ -from .command import metricq_command from .params import ( ChoiceParam, CommandLineChoice, DurationParam, TemplateStringParam, TimestampParam, + metric_input, ) +from .wrapper import metricq_command __all__ = [ "ChoiceParam", @@ -14,4 +15,5 @@ "TemplateStringParam", "TimestampParam", "metricq_command", + "metric_input", ] diff --git a/metricq/cli/params.py b/metricq/cli/params.py index 455a2a23..92bbd8a1 100644 --- a/metricq/cli/params.py +++ b/metricq/cli/params.py @@ -3,12 +3,12 @@ from getpass import getuser from socket import gethostname from string import Template -from typing import Any, Generic, List, Optional, Type, TypeVar, Union, cast +from typing import Any, Generic, List, Optional, Type, TypeVar, Union, cast, Callable -import click from click import Context, Parameter, ParamType, option from ..timeseries import Timedelta, Timestamp +from .types import FC _C = TypeVar("_C", covariant=True) @@ -169,3 +169,28 @@ def convert( if not isinstance(value, str): raise TypeError("expected a string type for TemplateStringParam") return Template(value).safe_substitute(self.mapping) + + +class MetricInputParam(ParamType): + pattern = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+\.)+[a-zA-Z][a-zA-Z0-9_]+") + + def convert( + self, value: Any, param: Optional[Parameter], ctx: Optional[Context] + ) -> str: + if not isinstance(value, str): + raise TypeError("expected a string type for the metric input") + if not self.pattern.match(value): + raise ValueError(f"Invalid metric format: '{value}'.") + return value + + +def metric_input(default: Optional[str] = None) -> Callable[[FC], FC]: + return option( + "--metric", + type=MetricInputParam(), + metavar="METRIC", + show_default=True, + required=default is None, + default=default, + help="Use the -–metric parameter to specify which metric the program should use", + ) diff --git a/metricq/cli/types.py b/metricq/cli/types.py new file mode 100644 index 00000000..43e54bce --- /dev/null +++ b/metricq/cli/types.py @@ -0,0 +1,5 @@ +from typing import Any, Callable, TypeVar, Union + +import click + +FC = TypeVar("FC", bound=Union[Callable[..., Any], click.Command]) diff --git a/metricq/cli/command.py b/metricq/cli/wrapper.py similarity index 98% rename from metricq/cli/command.py rename to metricq/cli/wrapper.py index 82abd86d..2db95839 100644 --- a/metricq/cli/command.py +++ b/metricq/cli/wrapper.py @@ -7,7 +7,8 @@ from dotenv import find_dotenv, load_dotenv from .. import get_logger -from .params import FC, TemplateStringParam +from .params import TemplateStringParam +from .types import FC # We do not interpolate (i.e. replace ${VAR} with corresponding environment variables). # That is because we want to be able to interpolate ourselves for metrics and tokens From 294d65f4bc6be65acd52eb26294394302f638e8a Mon Sep 17 00:00:00 2001 From: Max Kutschka <2004maximilian@web.de> Date: Tue, 15 Oct 2024 11:47:54 +0200 Subject: [PATCH 3/7] Add cli dependencies to test dependencies --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index b497c3ed..c7875234 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,7 @@ lint = check-manifest pre-commit test = + %(cli)s pytest pytest-asyncio pytest-mock From c895f8695d8bd7daa4de1b7e27218ec9345f62e4 Mon Sep 17 00:00:00 2001 From: Max Kutschka <2004maximilian@web.de> Date: Thu, 17 Oct 2024 13:28:49 +0200 Subject: [PATCH 4/7] implementing the new changes to the examples --- examples/metricq_client.py | 8 -------- examples/metricq_get_history.py | 18 ++++-------------- examples/metricq_get_history_raw.py | 16 ++-------------- examples/metricq_pandas.py | 16 +++------------- examples/metricq_sink.py | 9 --------- examples/metricq_source.py | 10 ---------- examples/metricq_synchronous_source.py | 17 ++++------------- metricq/cli/__init__.py | 12 +++++++++--- metricq/cli/params.py | 21 ++++----------------- metricq/cli/wrapper.py | 5 +++-- 10 files changed, 29 insertions(+), 103 deletions(-) diff --git a/examples/metricq_client.py b/examples/metricq_client.py index 579be272..7108078b 100755 --- a/examples/metricq_client.py +++ b/examples/metricq_client.py @@ -40,20 +40,13 @@ `telnet localhost 50101` (or `netcat`), inspect tasks and run code in a REPL. """ import asyncio -import logging import aiomonitor # type: ignore -import click_log # type: ignore import metricq from metricq.cli import metricq_command logger = metricq.get_logger() -click_log.basic_config(logger) -logger.setLevel("INFO") -logger.handlers[0].formatter = logging.Formatter( - fmt="%(asctime)s [%(levelname)-8s] [%(name)-20s] %(message)s" -) async def run(server: str, token: str) -> None: @@ -67,7 +60,6 @@ async def run(server: str, token: str) -> None: @metricq_command(default_token="client-py-example") -@click_log.simple_verbosity_option(logger) # type: ignore def main(server: str, token: str) -> None: asyncio.run(run(server, token)) diff --git a/examples/metricq_get_history.py b/examples/metricq_get_history.py index ee5fbcb4..71e21310 100755 --- a/examples/metricq_get_history.py +++ b/examples/metricq_get_history.py @@ -28,26 +28,17 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import asyncio -import logging import pprint from datetime import timedelta import click -import click_log # type: ignore import metricq from metricq.cli import metricq_command -from metricq.cli.params import metric_input +from metricq.cli.wrapper import metricq_metric_option +from metricq.logging import get_logger -logger = metricq.get_logger() - -click_log.basic_config(logger) -logger.setLevel("INFO") -# Use this if we ever use threads -# logger.handlers[0].formatter = logging.Formatter(fmt='%(asctime)s %(threadName)-16s %(levelname)-8s %(message)s') -logger.handlers[0].formatter = logging.Formatter( - fmt="%(asctime)s [%(levelname)-8s] [%(name)-20s] %(message)s" -) +logger = get_logger() async def aget_history( @@ -101,10 +92,9 @@ async def aget_history( @metricq_command(default_token="history-py-dummy") -@metric_input() +@metricq_metric_option() @click.option("--list-metrics", is_flag=True) @click.option("--list-metadata", is_flag=True) -@click_log.simple_verbosity_option(logger) # type: ignore def get_history( server: str, token: str, metric: str, list_metrics: bool, list_metadata: bool ) -> None: diff --git a/examples/metricq_get_history_raw.py b/examples/metricq_get_history_raw.py index 7167daa2..d4372b10 100755 --- a/examples/metricq_get_history_raw.py +++ b/examples/metricq_get_history_raw.py @@ -28,26 +28,15 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import asyncio -import logging from datetime import timedelta import click -import click_log # type: ignore import metricq from metricq.cli import metricq_command +from metricq.cli.wrapper import metricq_metric_option from metricq.history_client import HistoryRequestType -logger = metricq.get_logger() - -click_log.basic_config(logger) -logger.setLevel("INFO") -# Use this if we ever use threads -# logger.handlers[0].formatter = logging.Formatter(fmt='%(asctime)s %(threadName)-16s %(levelname)-8s %(message)s') -logger.handlers[0].formatter = logging.Formatter( - fmt="%(asctime)s [%(levelname)-8s] [%(name)-20s] %(message)s" -) - async def aget_history(server: str, token: str, metric: str) -> None: client = metricq.HistoryClient(token=token, url=server) @@ -85,8 +74,7 @@ async def aget_history(server: str, token: str, metric: str) -> None: @metricq_command(default_token="history-py-dummy") -@click.argument("metric") -@click_log.simple_verbosity_option(logger) # type: ignore +@metricq_metric_option() def get_history(server: str, token: str, metric: str) -> None: asyncio.run(aget_history(server, token, metric)) diff --git a/examples/metricq_pandas.py b/examples/metricq_pandas.py index 26a1cf5f..05eec51e 100755 --- a/examples/metricq_pandas.py +++ b/examples/metricq_pandas.py @@ -28,26 +28,17 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import asyncio -import logging from datetime import timedelta import click -import click_log # type: ignore import metricq -from metricq.cli import metric_input, metricq_command +from metricq.cli import metricq_command +from metricq.cli.wrapper import metricq_metric_option from metricq.pandas import PandasHistoryClient logger = metricq.get_logger() -click_log.basic_config(logger) -logger.setLevel("INFO") -# Use this if we ever use threads -# logger.handlers[0].formatter = logging.Formatter(fmt='%(asctime)s %(threadName)-16s %(levelname)-8s %(message)s') -logger.handlers[0].formatter = logging.Formatter( - fmt="%(asctime)s [%(levelname)-8s] [%(name)-20s] %(message)s" -) - async def aget_history(server: str, token: str, metric: str) -> None: async with PandasHistoryClient(token=token, url=server) as client: @@ -83,8 +74,7 @@ async def aget_history(server: str, token: str, metric: str) -> None: @metricq_command(default_token="history-py-dummy") -@metric_input(default="example.quantity") -@click_log.simple_verbosity_option(logger) # type: ignore +@metricq_metric_option(default="example.quantity") def get_history(server: str, token: str, metric: str) -> None: asyncio.run(aget_history(server, token, metric)) diff --git a/examples/metricq_sink.py b/examples/metricq_sink.py index a1a2f8f0..847e3950 100755 --- a/examples/metricq_sink.py +++ b/examples/metricq_sink.py @@ -27,11 +27,9 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import logging from typing import Any import click -import click_log # type: ignore import metricq from metricq import Metric @@ -40,12 +38,6 @@ logger = get_logger() -click_log.basic_config(logger) -logger.setLevel("INFO") -logger.handlers[0].formatter = logging.Formatter( - fmt="%(asctime)s [%(levelname)-8s] [%(name)-20s] %(message)s" -) - # To implement a MetricQ Sink, subclass metricq.Sink class DummySink(metricq.Sink): @@ -79,7 +71,6 @@ async def on_data( @metricq_command(default_token="sink-py-dummy") @click.option("-m", "--metrics", multiple=True, required=True) -@click_log.simple_verbosity_option(logger) # type: ignore def source(server: str, token: str, metrics: list[Metric]) -> None: # Initialize the DummySink class with a list of metrics given on the # command line. diff --git a/examples/metricq_source.py b/examples/metricq_source.py index c7a189fc..edd9e635 100755 --- a/examples/metricq_source.py +++ b/examples/metricq_source.py @@ -27,24 +27,15 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import logging import random from typing import Any -import click_log # type: ignore - import metricq from metricq.cli import metricq_command from metricq.logging import get_logger logger = get_logger() -click_log.basic_config(logger) -logger.setLevel("INFO") -logger.handlers[0].formatter = logging.Formatter( - fmt="%(asctime)s [%(levelname)-8s] [%(name)-20s] %(message)s" -) - class DummySource(metricq.IntervalSource): def __init__(self, *args: Any, **kwargs: Any): @@ -75,7 +66,6 @@ async def update(self) -> None: @metricq_command(default_token="source-py-dummy") -@click_log.simple_verbosity_option(logger) # type: ignore def source(server: str, token: str) -> None: src = DummySource(token=token, url=server) src.run() diff --git a/examples/metricq_synchronous_source.py b/examples/metricq_synchronous_source.py index 715cc763..32b8c7eb 100755 --- a/examples/metricq_synchronous_source.py +++ b/examples/metricq_synchronous_source.py @@ -29,27 +29,18 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import logging + import random import time -import click_log # type: ignore - -from metricq import SynchronousSource, Timestamp, get_logger +from metricq import SynchronousSource, Timestamp from metricq.cli import metricq_command - -logger = get_logger() - -click_log.basic_config(logger) -logger.setLevel("INFO") -logger.handlers[0].formatter = logging.Formatter( - fmt="%(asctime)s [%(levelname)-8s] [%(name)-20s] %(message)s" -) +from metricq.logging import get_logger @metricq_command(default_token="source-py-dummy") -@click_log.simple_verbosity_option(logger) # type: ignore def synchronous_source(server: str, token: str) -> None: + logger = get_logger() ssource = SynchronousSource(token=token, url=server) ssource.declare_metrics( { diff --git a/metricq/cli/__init__.py b/metricq/cli/__init__.py index e3777b26..c393106e 100644 --- a/metricq/cli/__init__.py +++ b/metricq/cli/__init__.py @@ -4,9 +4,13 @@ DurationParam, TemplateStringParam, TimestampParam, - metric_input, ) -from .wrapper import metricq_command +from .wrapper import ( + metricq_command, + metricq_metric_option, + metricq_server_option, + metricq_token_option, +) __all__ = [ "ChoiceParam", @@ -15,5 +19,7 @@ "TemplateStringParam", "TimestampParam", "metricq_command", - "metric_input", + "metricq_metric_option", + "metricq_server_option", + "metricq_token_option", ] diff --git a/metricq/cli/params.py b/metricq/cli/params.py index 92bbd8a1..766acbfb 100644 --- a/metricq/cli/params.py +++ b/metricq/cli/params.py @@ -3,12 +3,11 @@ from getpass import getuser from socket import gethostname from string import Template -from typing import Any, Generic, List, Optional, Type, TypeVar, Union, cast, Callable +from typing import Any, Generic, List, Optional, Type, TypeVar, Union, cast -from click import Context, Parameter, ParamType, option +from click import Context, Parameter, ParamType from ..timeseries import Timedelta, Timestamp -from .types import FC _C = TypeVar("_C", covariant=True) @@ -171,26 +170,14 @@ def convert( return Template(value).safe_substitute(self.mapping) -class MetricInputParam(ParamType): +class MetricParam(ParamType): pattern = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+\.)+[a-zA-Z][a-zA-Z0-9_]+") def convert( self, value: Any, param: Optional[Parameter], ctx: Optional[Context] ) -> str: if not isinstance(value, str): - raise TypeError("expected a string type for the metric input") + raise TypeError("expected a string type for the MetricParam") if not self.pattern.match(value): raise ValueError(f"Invalid metric format: '{value}'.") return value - - -def metric_input(default: Optional[str] = None) -> Callable[[FC], FC]: - return option( - "--metric", - type=MetricInputParam(), - metavar="METRIC", - show_default=True, - required=default is None, - default=default, - help="Use the -–metric parameter to specify which metric the program should use", - ) diff --git a/metricq/cli/wrapper.py b/metricq/cli/wrapper.py index 2db95839..31e9c30c 100644 --- a/metricq/cli/wrapper.py +++ b/metricq/cli/wrapper.py @@ -1,5 +1,5 @@ import logging -from typing import Callable, cast +from typing import Callable, Optional, cast import click import click_log # type: ignore @@ -7,7 +7,7 @@ from dotenv import find_dotenv, load_dotenv from .. import get_logger -from .params import TemplateStringParam +from .params import MetricParam, TemplateStringParam from .types import FC # We do not interpolate (i.e. replace ${VAR} with corresponding environment variables). @@ -53,6 +53,7 @@ def metricq_metric_option(default: Optional[str] = None) -> Callable[[FC], FC]: ) + def get_metric_command_logger() -> logging.Logger: logger = get_logger() logger.setLevel(logging.WARNING) From 1868d62bb64653312d9ea239cfe665d3cbe8b8c3 Mon Sep 17 00:00:00 2001 From: Max Kutschka <2004maximilian@web.de> Date: Tue, 22 Oct 2024 13:48:31 +0200 Subject: [PATCH 5/7] feat: metricq_metric_option now allows multiple metrics --- examples/metricq_sink.py | 7 ++++--- examples/metricq_synchronous_source.py | 3 ++- metricq/cli/__init__.py | 2 ++ metricq/cli/wrapper.py | 16 ++++++++++++---- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/examples/metricq_sink.py b/examples/metricq_sink.py index 847e3950..98226e49 100755 --- a/examples/metricq_sink.py +++ b/examples/metricq_sink.py @@ -34,6 +34,7 @@ import metricq from metricq import Metric from metricq.cli import metricq_command +from metricq.cli.wrapper import metricq_metric_option from metricq.logging import get_logger logger = get_logger() @@ -70,11 +71,11 @@ async def on_data( @metricq_command(default_token="sink-py-dummy") -@click.option("-m", "--metrics", multiple=True, required=True) -def source(server: str, token: str, metrics: list[Metric]) -> None: +@metricq_metric_option(multiple=True) +def source(server: str, token: str, metric: list[Metric]) -> None: # Initialize the DummySink class with a list of metrics given on the # command line. - sink = DummySink(metrics=metrics, token=token, url=server) + sink = DummySink(metrics=metric, token=token, url=server) # Run the sink. This call will block until the connection is closed. sink.run() diff --git a/examples/metricq_synchronous_source.py b/examples/metricq_synchronous_source.py index 32b8c7eb..930cb984 100755 --- a/examples/metricq_synchronous_source.py +++ b/examples/metricq_synchronous_source.py @@ -37,10 +37,11 @@ from metricq.cli import metricq_command from metricq.logging import get_logger +logger = get_logger() + @metricq_command(default_token="source-py-dummy") def synchronous_source(server: str, token: str) -> None: - logger = get_logger() ssource = SynchronousSource(token=token, url=server) ssource.declare_metrics( { diff --git a/metricq/cli/__init__.py b/metricq/cli/__init__.py index c393106e..2fdc524b 100644 --- a/metricq/cli/__init__.py +++ b/metricq/cli/__init__.py @@ -2,6 +2,7 @@ ChoiceParam, CommandLineChoice, DurationParam, + MetricParam, TemplateStringParam, TimestampParam, ) @@ -18,6 +19,7 @@ "DurationParam", "TemplateStringParam", "TimestampParam", + "MetricParam", "metricq_command", "metricq_metric_option", "metricq_server_option", diff --git a/metricq/cli/wrapper.py b/metricq/cli/wrapper.py index 31e9c30c..dc8097f6 100644 --- a/metricq/cli/wrapper.py +++ b/metricq/cli/wrapper.py @@ -41,19 +41,27 @@ def metricq_token_option(default: str) -> Callable[[FC], FC]: ) -def metricq_metric_option(default: Optional[str] = None) -> Callable[[FC], FC]: +def metricq_metric_option( + default: Optional[str] = None, multiple: bool = False +) -> Callable[[FC], FC]: + response_default = default if (default is None or not multiple) else [default] + help = "Use the -–metric / -m parameter to specify which metric the program should use." + if default: + help += "You can also specify multpile metrics." + return option( "--metric", + "-m", type=MetricParam(), metavar="METRIC", show_default=True, required=default is None, - default=default, - help="Use the -–metric parameter to specify which metric the program should use", + default=response_default, + multiple=multiple, + help=help, ) - def get_metric_command_logger() -> logging.Logger: logger = get_logger() logger.setLevel(logging.WARNING) From 8c7bdd91f45cace6544bed25180c37735469db53 Mon Sep 17 00:00:00 2001 From: Max Kutschka <2004maximilian@web.de> Date: Tue, 22 Oct 2024 13:57:45 +0200 Subject: [PATCH 6/7] bug: fixed logger global scope From 7eb526acc4c0b4a64166ea980a60c14fce6f962e Mon Sep 17 00:00:00 2001 From: Max Kutschka <2004maximilian@web.de> Date: Tue, 5 Nov 2024 14:51:02 +0100 Subject: [PATCH 7/7] feat: Syslog handler and formatter for the metricq_command --- metricq/cli/syslog.py | 30 ++++++++++++++++++++++++++++++ metricq/cli/wrapper.py | 40 +++++++++++++++++++++++++++++++++++----- 2 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 metricq/cli/syslog.py diff --git a/metricq/cli/syslog.py b/metricq/cli/syslog.py new file mode 100644 index 00000000..aad034b4 --- /dev/null +++ b/metricq/cli/syslog.py @@ -0,0 +1,30 @@ +import logging +import socket +import time +from logging.handlers import SysLogHandler + + +class SyslogFormatter(logging.Formatter): + def __init__(self, *args, name: str = "metricq", **kwargs): # type: ignore + super().__init__(*args, **kwargs) + self.program = name + + def format(self, record: logging.LogRecord) -> str: + timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(record.created)) + hostname = socket.gethostname() + pid = record.process + program = self.program + # Custom Formatter based on rfc3164 + # Format the header as " TIMESTAMP HOSTNAME PROGRAM[PID]: MESSAGE" + # is already beeing set by the SysLogHanlder, we only need to add the rest + syslog_header = f"{timestamp} {hostname} {program}[{pid}]: " + message = super().format(record) + return syslog_header + message + + +def get_syslog_handler(address: str) -> SysLogHandler: + if ":" in address: + ip, port = address.split(":") + return SysLogHandler(address=(ip, int(port))) + else: + return SysLogHandler(address=address) diff --git a/metricq/cli/wrapper.py b/metricq/cli/wrapper.py index dc8097f6..246bfea4 100644 --- a/metricq/cli/wrapper.py +++ b/metricq/cli/wrapper.py @@ -1,13 +1,14 @@ import logging -from typing import Callable, Optional, cast +from typing import Any, Callable, Optional, cast import click import click_log # type: ignore -from click import option +from click import Context, option from dotenv import find_dotenv, load_dotenv from .. import get_logger from .params import MetricParam, TemplateStringParam +from .syslog import SyslogFormatter, get_syslog_handler from .types import FC # We do not interpolate (i.e. replace ${VAR} with corresponding environment variables). @@ -41,6 +42,32 @@ def metricq_token_option(default: str) -> Callable[[FC], FC]: ) +def metricq_syslog_option() -> Callable[[FC], FC]: + """ + Exposes the --syslog option to a click program. + is_flag=False and flag_value="localhost:514" allowes the programm to accept --syslog and --syslog {url} values. + """ + + def enable_syslog(ctx: Context, param: Any | None, value: Optional[str]) -> None: + if value is not None: + logger = get_logger() + + program_name = ctx.params.get("token", "metricq.program") + + handler = get_syslog_handler(value) + handler.setFormatter(SyslogFormatter(name=program_name)) + logger.addHandler(handler) + + return option( + "--syslog", + help="Enable syslog logging by specifying the a Unix socket or host:port for the logger. If --syslog is set but no value is specified, the default of localhost:514 will be used.", + callback=enable_syslog, + expose_value=True, + is_flag=False, + flag_value="localhost:514", + ) + + def metricq_metric_option( default: Optional[str] = None, multiple: bool = False ) -> Callable[[FC], FC]: @@ -71,7 +98,8 @@ def get_metric_command_logger() -> logging.Logger: def metricq_command( - default_token: str, client_version: str | None = None + default_token: str, + client_version: str | None = None, ) -> Callable[[FC], click.Command]: logger = get_metric_command_logger() @@ -95,8 +123,10 @@ def decorator(func: FC) -> click.Command: log_decorator( metricq_token_option(default_token)( metricq_server_option()( - click.command(context_settings=context_settings, epilog=epilog)( - func + metricq_syslog_option()( + click.command( + context_settings=context_settings, epilog=epilog + )(func) ) ) )