From 96f79e41063fffa1d4adc8ab0d0f70d3b1360a3a Mon Sep 17 00:00:00 2001 From: shivam Date: Wed, 8 Mar 2023 09:19:50 +0530 Subject: [PATCH] Move argparse code into separate argparse modules --- kcidb/__init__.py | 20 ++-- kcidb/argparse.py | 270 +++++++++++++++++++++++++++++++++++++++++++ kcidb/db/__init__.py | 202 ++------------------------------ kcidb/db/argparse.py | 197 +++++++++++++++++++++++++++++++ kcidb/misc.py | 229 +----------------------------------- kcidb/mq/__init__.py | 177 ++-------------------------- kcidb/mq/argparse.py | 186 +++++++++++++++++++++++++++++ 7 files changed, 679 insertions(+), 602 deletions(-) create mode 100644 kcidb/argparse.py create mode 100644 kcidb/db/argparse.py create mode 100644 kcidb/mq/argparse.py diff --git a/kcidb/__init__.py b/kcidb/__init__.py index c36268db..b7d99a13 100644 --- a/kcidb/__init__.py +++ b/kcidb/__init__.py @@ -5,7 +5,7 @@ import logging from kcidb.misc import LIGHT_ASSERTS # Silence flake8 "imported but unused" warning -from kcidb import io, db, mq, orm, oo, monitor, tests, unittest, misc # noqa +from kcidb import io, db, mq, orm, oo, monitor, tests, unittest, misc, argparse # noqa # Module's logger @@ -184,7 +184,7 @@ def submit_main(): sys.excepthook = misc.log_and_print_excepthook description = \ 'kcidb-submit - Submit Kernel CI reports, print submission IDs' - parser = misc.ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) parser.add_argument( '-p', '--project', help='ID of the Google Cloud project containing the message queue', @@ -234,8 +234,8 @@ def schema_main(): """Execute the kcidb-schema command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-schema - Output current or older I/O JSON schema' - parser = misc.OutputArgumentParser(description=description) - misc.argparse_schema_add_args(parser, "output") + parser = argparse.OutputArgumentParser(description=description) + argparse.schema_add_args(parser, "output") args = parser.parse_args() misc.json_dump(args.schema_version.json, sys.stdout, indent=args.indent, seq=args.seq) @@ -245,8 +245,8 @@ def validate_main(): """Execute the kcidb-validate command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-validate - Validate I/O JSON data' - parser = misc.OutputArgumentParser(description=description) - misc.argparse_schema_add_args(parser, "validate against") + parser = argparse.OutputArgumentParser(description=description) + argparse.schema_add_args(parser, "validate against") args = parser.parse_args() misc.json_dump_stream( ( @@ -261,8 +261,8 @@ def upgrade_main(): """Execute the kcidb-upgrade command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-upgrade - Upgrade I/O JSON data to current schema' - parser = misc.OutputArgumentParser(description=description) - misc.argparse_schema_add_args(parser, "upgrade") + parser = argparse.OutputArgumentParser(description=description) + argparse.schema_add_args(parser, "upgrade") args = parser.parse_args() misc.json_dump_stream( ( @@ -277,7 +277,7 @@ def count_main(): """Execute the kcidb-count command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-count - Count number of objects in I/O JSON data' - parser = misc.ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) parser.parse_args() for data in misc.json_load_stream_fd(sys.stdin.fileno()): @@ -289,7 +289,7 @@ def merge_main(): """Execute the kcidb-merge command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-merge - Upgrade and merge I/O data sets' - parser = misc.OutputArgumentParser(description=description) + parser = argparse.OutputArgumentParser(description=description) args = parser.parse_args() sources = [ diff --git a/kcidb/argparse.py b/kcidb/argparse.py new file mode 100644 index 00000000..6e0170ec --- /dev/null +++ b/kcidb/argparse.py @@ -0,0 +1,270 @@ +import math +import re +import os +import atexit +import tempfile +import sys +import traceback +import itertools +import argparse +import logging +import json +import argparse +from textwrap import indent + + +import dateutil.parser +try: # Python 3.9 + from importlib import metadata +except ImportError: # Python 3.6 + import importlib_metadata as metadata +from google.cloud import secretmanager +import jq +import kcidb.io as io + +# Module's logger +LOGGER = logging.getLogger(__name__) + +# A dictionary of names of logging levels and their values +LOGGING_LEVEL_MAP = { + name: value + for name, value in logging.__dict__.items() + if name.isalpha() and name.isupper() and isinstance(value, int) and value +} +# Logging level disabling all logging +LOGGING_LEVEL_MAP["NONE"] = max(LOGGING_LEVEL_MAP.values()) + 1 +# Sort levels highest->lowest +# I don't see it, pylint: disable=unnecessary-comprehension +LOGGING_LEVEL_MAP = { + k: v + for k, v in sorted(LOGGING_LEVEL_MAP.items(), + key=lambda i: i[1], reverse=True) +} + +def logging_setup(level): + """ + Setup logging: set root logger log level and disable irrelevant logging. + + Args: + level: Logging level for the root logger. + """ + assert isinstance(level, int) + logging.getLogger().setLevel(level) + # TODO Consider separate arguments for controlling the below + logging.getLogger("urllib3").setLevel(LOGGING_LEVEL_MAP["NONE"]) + logging.getLogger("google").setLevel(LOGGING_LEVEL_MAP["NONE"]) + + +class ArgumentParser(argparse.ArgumentParser): + """ + KCIDB command-line argument parser handling common arguments. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding common arguments. + + Args: + args: Positional arguments to initialize ArgumentParser with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + self.add_argument( + '--version', + action='version', + version=f"Version {metadata.version('kcidb')}" + ) + self.add_argument( + '-l', '--log-level', + metavar="LEVEL", + default="NONE", + choices=LOGGING_LEVEL_MAP.keys(), + help='Limit logging to LEVEL (%(choices)s). Default is NONE.' + ) + + def parse_args(self, args=None, namespace=None): + """ + Parse arguments, including common ones, apply ones affecting global + state. + + Args: + args: List of strings to parse. The default is taken from + sys.argv. + namespace: An object to take the attributes. The default is a new + empty argparse.Namespace object. + + Returns: + Namespace populated with arguments. + """ + args = super().parse_args(args=args, namespace=namespace) + logging.basicConfig() + logging_setup(LOGGING_LEVEL_MAP[args.log_level]) + return args + + +def non_negative_int(string): + """ + Parse a non-negative integer out of a string. + Matches the argparse type function interface. + + Args: + string: The string to parse. + + Returns: + The non-negative integer parsed out of the string. + + Raises: + argparse.ArgumentTypeError: the string wasn't representing a + non-negative integer. + """ + if not re.fullmatch("[0-9]+", string): + raise argparse.ArgumentTypeError( + f'{repr(string)} is not a positive integer, nor zero' + ) + return int(string) + + +def non_negative_int_or_inf(string): + """ + Parse a non-negative integer or a positive infinity out of a string. + Matches the argparse type function interface. + + Args: + string: The string to parse. + + Returns: + The non-negative integer, or positive infinity parsed out of the + string. + + Raises: + argparse.ArgumentTypeError: the string wasn't representing a + non-negative integer or infinity. + """ + try: + value = float(string) + if value != math.inf: + value = non_negative_int(string) + except (ValueError, argparse.ArgumentTypeError) as exc: + raise argparse.ArgumentTypeError( + f'{repr(string)} is not zero, a positive integer, or infinity' + ) from exc + return value + + +def output_add_args(parser): + """ + Add JSON output arguments to a command-line argument parser. + + Args: + parser: The parser to add arguments to. + """ + parser.add_argument( + '--indent', + metavar="NUMBER", + type=non_negative_int, + help='Pretty-print JSON using NUMBER of spaces for indenting. ' + 'Print single-line if zero. Default is 4.', + default=4, + required=False + ) + parser.add_argument( + '--seq', + help='Prefix JSON output with the RS character, to match ' + 'RFC 7464 and "application/json-seq" media type.', + action='store_true' + ) + + +class OutputArgumentParser(ArgumentParser): + """ + Command-line argument parser for tools outputting JSON. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding JSON output arguments. + + Args: + args: Positional arguments to initialize ArgumentParser with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + output_add_args(self) + + +class SplitOutputArgumentParser(OutputArgumentParser): + """ + Command-line argument parser for tools supporting split-report output. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding split-report output arguments. + + Args: + args: Positional arguments to initialize ArgumentParser with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + self.add_argument( + '-o', '--objects-per-report', + metavar="NUMBER", + type=non_negative_int, + help='Put maximum NUMBER of objects into each output ' + 'report, or all, if zero. Default is zero.', + default=0, + required=False + ) + + +def schema_add_args(parser, version_verb): + """ + Add schema selection arguments to a command-line argument parser. + + Args: + parser: The parser to add arguments to. + version_verb: The verb to apply to the schema in the version + argument description. + """ + def schema_version(string): + """ + Lookup a schema version object using a major version number string. + Matches the argparse type function interface. + + Args: + string: The string representing the major version number of the + schema to lookup. + + Returns: + The looked up schema version object. + + Raises: + ValueError if the string was not representing a positive integer + number, or the schema with the supplied major version number was + not found. + """ + if not re.fullmatch("0*[1-9][0-9]*", string): + raise argparse.ArgumentTypeError( + f"Invalid major version number: {string!r}" + ) + major = int(string) + # It's OK, pylint: disable=redefined-outer-name + version = io.SCHEMA + while version and version.major != major: + version = version.previous + if version is None: + raise argparse.ArgumentTypeError( + f"No schema version found for major number {major}" + ) + return version + + parser.add_argument( + 'schema_version', + metavar="SCHEMA_VERSION", + type=schema_version, + help=f"{version_verb.capitalize()} the schema with the specified " + f"major version. Default is the current schema version " + f"(currently {io.SCHEMA.major}).", + nargs='?', + default=io.SCHEMA + ) diff --git a/kcidb/db/__init__.py b/kcidb/db/__init__.py index 52142379..02dd542b 100644 --- a/kcidb/db/__init__.py +++ b/kcidb/db/__init__.py @@ -365,198 +365,12 @@ def load(self, data): self.driver.load(data) -class DBHelpAction(argparse.Action): - """Argparse action outputting database string help and exiting.""" - def __init__(self, - option_strings, - dest=argparse.SUPPRESS, - default=argparse.SUPPRESS, - help=None): - super().__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - print("KCIDB has several database drivers for both actual and " - "virtual databases.\n" - "You can specify a particular driver to use, and its " - "parameters, using the\n" - "-d/--database option.\n" - "\n" - "The format of the option value is [:], " - "where is the\n" - "name of the driver, and is a (sometimes optional) " - "driver-specific\n" - "parameter string.\n" - "\n" - "For example, \"-d bigquery:kernelci-production.kcidb_01\" " - "requests the use of\n" - "the \"bigquery\" database driver with the parameter string\n" - "\"kernelci-production.kcidb_01\", from which the driver " - "extracts the Google\n" - "Cloud project \"kernelci-production\" and the dataset " - "\"kcidb_01\" to connect to.\n" - "\n" - "Available drivers and format of their parameter strings " - "follow.\n") - for name, driver in DRIVER_TYPES.items(): - print(f"\n{name!r} driver\n" + - "-" * (len(name) + 9) + "\n" + - driver.get_doc()) - parser.exit() - - -def argparse_add_args(parser, database=None): - """ - Add common database arguments to an argument parser. - - Args: - parser: The parser to add arguments to. - database: The default database specification to use. - """ - assert database is None or isinstance(database, str) - parser.add_argument( - '-d', '--database', - help=("Specify DATABASE to use, formatted as :. " + - "Use --database-help for more details." + - ("" if database is None - else f"Default is {database!r}.")), - default=database, - required=(database is None) - ) - parser.add_argument( - '--database-help', - action=DBHelpAction, - help='Print documentation on database specification strings and exit.' - ) - - -class ArgumentParser(kcidb.misc.ArgumentParser): - """ - Command-line argument parser with common database arguments added. - """ - - def __init__(self, *args, database=None, **kwargs): - """ - Initialize the parser, adding common database arguments. - - Args: - args: Positional arguments to initialize ArgumentParser - with. - database: The default database specification to use, or None to - make database specification required. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - argparse_add_args(self, database=database) - - -class OutputArgumentParser(kcidb.misc.OutputArgumentParser): - """ - Command-line argument parser for tools outputting JSON, - with common database arguments added. - """ - - def __init__(self, *args, database=None, **kwargs): - """ - Initialize the parser, adding JSON output arguments. - - Args: - args: Positional arguments to initialize ArgumentParser - with. - database: The default database specification to use, or None to - make database specification required. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - argparse_add_args(self, database=database) - - -class SplitOutputArgumentParser(kcidb.misc.SplitOutputArgumentParser): - """ - Command-line argument parser for tools outputting split-report streams, - with common database arguments added. - """ - - def __init__(self, *args, database=None, **kwargs): - """ - Initialize the parser, adding split-report output arguments. - - Args: - args: Positional arguments to initialize ArgumentParser - with. - database: The default database specification to use, or None to - make database specification required. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - argparse_add_args(self, database=database) - - -# No, it's OK, pylint: disable=too-many-ancestors -class QueryArgumentParser(SplitOutputArgumentParser): - """ - Command-line argument parser with common database query arguments added. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding common database query arguments. - - Args: - args: Positional arguments to initialize the parent - SplitOutputArgumentParser with. - kwargs: Keyword arguments to initialize the parent - SplitOutputArgumentParser with. - """ - super().__init__(*args, **kwargs) - - self.add_argument( - '-c', '--checkout-id', - metavar="ID", - default=[], - help='ID of a checkout to match', - dest="checkout_ids", - action='append', - ) - self.add_argument( - '-b', '--build-id', - metavar="ID", - default=[], - help='ID of a build to match', - dest="build_ids", - action='append', - ) - self.add_argument( - '-t', '--test-id', - metavar="ID", - default=[], - help='ID of a test to match', - dest="test_ids", - action='append', - ) - - self.add_argument( - '--parents', - help='Match parents of matching objects', - action='store_true' - ) - self.add_argument( - '--children', - help='Match children of matching objects', - action='store_true' - ) - - def dump_main(): """Execute the kcidb-db-dump command-line tool""" sys.excepthook = kcidb.misc.log_and_print_excepthook description = \ 'kcidb-db-dump - Dump all data from Kernel CI report database' - parser = SplitOutputArgumentParser(description=description) + parser = argparse.SplitOutputArgumentParser(description=description) args = parser.parse_args() client = Client(args.database) if not client.is_initialized(): @@ -572,7 +386,7 @@ def query_main(): sys.excepthook = kcidb.misc.log_and_print_excepthook description = \ "kcidb-db-query - Query objects from Kernel CI report database" - parser = QueryArgumentParser(description=description) + parser = argparse.QueryArgumentParser(description=description) args = parser.parse_args() client = Client(args.database) if not client.is_initialized(): @@ -595,7 +409,7 @@ def load_main(): sys.excepthook = kcidb.misc.log_and_print_excepthook description = \ 'kcidb-db-load - Load reports into Kernel CI report database' - parser = ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) args = parser.parse_args() client = Client(args.database) if not client.is_initialized(): @@ -611,7 +425,7 @@ def schemas_main(): sys.excepthook = kcidb.misc.log_and_print_excepthook description = 'kcidb-db-schemas - List available database schemas ' \ '(as : )' - parser = ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) args = parser.parse_args() client = Client(args.database) curr_version = client.get_schema()[0] if client.is_initialized() else None @@ -633,7 +447,7 @@ def init_main(): """Execute the kcidb-db-init command-line tool""" sys.excepthook = kcidb.misc.log_and_print_excepthook description = 'kcidb-db-init - Initialize a Kernel CI report database' - parser = ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) parser.add_argument( '--ignore-initialized', help='Do not fail if the database is already initialized.', @@ -665,7 +479,7 @@ def upgrade_main(): """Execute the kcidb-db-upgrade command-line tool""" sys.excepthook = kcidb.misc.log_and_print_excepthook description = 'kcidb-db-upgrade - Upgrade database schema' - parser = ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) parser.add_argument( '-s', '--schema', metavar="VERSION", @@ -701,7 +515,7 @@ def cleanup_main(): """Execute the kcidb-db-cleanup command-line tool""" sys.excepthook = kcidb.misc.log_and_print_excepthook description = 'kcidb-db-cleanup - Cleanup a Kernel CI report database' - parser = ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) parser.add_argument( '--ignore-not-initialized', help='Do not fail if the database is not initialized.', @@ -730,7 +544,7 @@ def empty_main(): sys.excepthook = kcidb.misc.log_and_print_excepthook description = 'kcidb-db-empty - Remove all data from a ' \ 'Kernel CI report database' - parser = ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description) args = parser.parse_args() client = Client(args.database) if client.is_initialized(): diff --git a/kcidb/db/argparse.py b/kcidb/db/argparse.py new file mode 100644 index 00000000..173cfd41 --- /dev/null +++ b/kcidb/db/argparse.py @@ -0,0 +1,197 @@ +import sys +import logging +import argparse +import datetime +import kcidb.io as io +import kcidb.orm +import kcidb.misc +from kcidb.misc import LIGHT_ASSERTS +from kcidb.db import abstract, schematic, mux, \ + bigquery, postgresql, sqlite, json, null, misc # noqa: F401 + + + +class DBHelpAction(argparse.Action): + """Argparse action outputting database string help and exiting.""" + def __init__(self, + option_strings, + dest=argparse.SUPPRESS, + default=argparse.SUPPRESS, + help=None): + super().__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + print("KCIDB has several database drivers for both actual and " + "virtual databases.\n" + "You can specify a particular driver to use, and its " + "parameters, using the\n" + "-d/--database option.\n" + "\n" + "The format of the option value is [:], " + "where is the\n" + "name of the driver, and is a (sometimes optional) " + "driver-specific\n" + "parameter string.\n" + "\n" + "For example, \"-d bigquery:kernelci-production.kcidb_01\" " + "requests the use of\n" + "the \"bigquery\" database driver with the parameter string\n" + "\"kernelci-production.kcidb_01\", from which the driver " + "extracts the Google\n" + "Cloud project \"kernelci-production\" and the dataset " + "\"kcidb_01\" to connect to.\n" + "\n" + "Available drivers and format of their parameter strings " + "follow.\n") + for name, driver in DRIVER_TYPES.items(): + print(f"\n{name!r} driver\n" + + "-" * (len(name) + 9) + "\n" + + driver.get_doc()) + parser.exit() + + +def argparse_add_args(parser, database=None): + """ + Add common database arguments to an argument parser. + + Args: + parser: The parser to add arguments to. + database: The default database specification to use. + """ + assert database is None or isinstance(database, str) + parser.add_argument( + '-d', '--database', + help=("Specify DATABASE to use, formatted as :. " + + "Use --database-help for more details." + + ("" if database is None + else f"Default is {database!r}.")), + default=database, + required=(database is None) + ) + parser.add_argument( + '--database-help', + action=DBHelpAction, + help='Print documentation on database specification strings and exit.' + ) + + +class ArgumentParser(kcidb.misc.ArgumentParser): + """ + Command-line argument parser with common database arguments added. + """ + + def __init__(self, *args, database=None, **kwargs): + """ + Initialize the parser, adding common database arguments. + + Args: + args: Positional arguments to initialize ArgumentParser + with. + database: The default database specification to use, or None to + make database specification required. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + argparse_add_args(self, database=database) + + +class OutputArgumentParser(kcidb.misc.OutputArgumentParser): + """ + Command-line argument parser for tools outputting JSON, + with common database arguments added. + """ + + def __init__(self, *args, database=None, **kwargs): + """ + Initialize the parser, adding JSON output arguments. + + Args: + args: Positional arguments to initialize ArgumentParser + with. + database: The default database specification to use, or None to + make database specification required. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + argparse_add_args(self, database=database) + + +class SplitOutputArgumentParser(kcidb.misc.SplitOutputArgumentParser): + """ + Command-line argument parser for tools outputting split-report streams, + with common database arguments added. + """ + + def __init__(self, *args, database=None, **kwargs): + """ + Initialize the parser, adding split-report output arguments. + + Args: + args: Positional arguments to initialize ArgumentParser + with. + database: The default database specification to use, or None to + make database specification required. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + argparse_add_args(self, database=database) + + +# No, it's OK, pylint: disable=too-many-ancestors +class QueryArgumentParser(SplitOutputArgumentParser): + """ + Command-line argument parser with common database query arguments added. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding common database query arguments. + + Args: + args: Positional arguments to initialize the parent + SplitOutputArgumentParser with. + kwargs: Keyword arguments to initialize the parent + SplitOutputArgumentParser with. + """ + super().__init__(*args, **kwargs) + + self.add_argument( + '-c', '--checkout-id', + metavar="ID", + default=[], + help='ID of a checkout to match', + dest="checkout_ids", + action='append', + ) + self.add_argument( + '-b', '--build-id', + metavar="ID", + default=[], + help='ID of a build to match', + dest="build_ids", + action='append', + ) + self.add_argument( + '-t', '--test-id', + metavar="ID", + default=[], + help='ID of a test to match', + dest="test_ids", + action='append', + ) + + self.add_argument( + '--parents', + help='Match parents of matching objects', + action='store_true' + ) + self.add_argument( + '--children', + help='Match children of matching objects', + action='store_true' + ) diff --git a/kcidb/misc.py b/kcidb/misc.py index 1fd2dd69..89d912c4 100644 --- a/kcidb/misc.py +++ b/kcidb/misc.py @@ -12,6 +12,7 @@ import logging import json from textwrap import indent + import dateutil.parser try: # Python 3.9 from importlib import metadata @@ -44,19 +45,6 @@ LIGHT_ASSERTS = not os.environ.get("KCIDB_HEAVY_ASSERTS", "") -def logging_setup(level): - """ - Setup logging: set root logger log level and disable irrelevant logging. - - Args: - level: Logging level for the root logger. - """ - assert isinstance(level, int) - logging.getLogger().setLevel(level) - # TODO Consider separate arguments for controlling the below - logging.getLogger("urllib3").setLevel(LOGGING_LEVEL_MAP["NONE"]) - logging.getLogger("google").setLevel(LOGGING_LEVEL_MAP["NONE"]) - def format_exception_stack(exc): """ @@ -102,102 +90,6 @@ def log_and_print_excepthook(type, value, tb): print(format_exception_stack(value), file=sys.stderr) -class ArgumentParser(argparse.ArgumentParser): - """ - KCIDB command-line argument parser handling common arguments. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding common arguments. - - Args: - args: Positional arguments to initialize ArgumentParser with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - self.add_argument( - '--version', - action='version', - version=f"Version {metadata.version('kcidb')}" - ) - self.add_argument( - '-l', '--log-level', - metavar="LEVEL", - default="NONE", - choices=LOGGING_LEVEL_MAP.keys(), - help='Limit logging to LEVEL (%(choices)s). Default is NONE.' - ) - - def parse_args(self, args=None, namespace=None): - """ - Parse arguments, including common ones, apply ones affecting global - state. - - Args: - args: List of strings to parse. The default is taken from - sys.argv. - namespace: An object to take the attributes. The default is a new - empty argparse.Namespace object. - - Returns: - Namespace populated with arguments. - """ - args = super().parse_args(args=args, namespace=namespace) - logging.basicConfig() - logging_setup(LOGGING_LEVEL_MAP[args.log_level]) - return args - - -def non_negative_int(string): - """ - Parse a non-negative integer out of a string. - Matches the argparse type function interface. - - Args: - string: The string to parse. - - Returns: - The non-negative integer parsed out of the string. - - Raises: - argparse.ArgumentTypeError: the string wasn't representing a - non-negative integer. - """ - if not re.fullmatch("[0-9]+", string): - raise argparse.ArgumentTypeError( - f'{repr(string)} is not a positive integer, nor zero' - ) - return int(string) - - -def non_negative_int_or_inf(string): - """ - Parse a non-negative integer or a positive infinity out of a string. - Matches the argparse type function interface. - - Args: - string: The string to parse. - - Returns: - The non-negative integer, or positive infinity parsed out of the - string. - - Raises: - argparse.ArgumentTypeError: the string wasn't representing a - non-negative integer or infinity. - """ - try: - value = float(string) - if value != math.inf: - value = non_negative_int(string) - except (ValueError, argparse.ArgumentTypeError) as exc: - raise argparse.ArgumentTypeError( - f'{repr(string)} is not zero, a positive integer, or infinity' - ) from exc - return value - - def iso_timestamp(string): """ Parse an ISO-8601 timestamp out of a string, assuming local timezone, if @@ -247,125 +139,6 @@ def version(string): return int(match.group(1)), int(match.group(2)) -def argparse_output_add_args(parser): - """ - Add JSON output arguments to a command-line argument parser. - - Args: - parser: The parser to add arguments to. - """ - parser.add_argument( - '--indent', - metavar="NUMBER", - type=non_negative_int, - help='Pretty-print JSON using NUMBER of spaces for indenting. ' - 'Print single-line if zero. Default is 4.', - default=4, - required=False - ) - parser.add_argument( - '--seq', - help='Prefix JSON output with the RS character, to match ' - 'RFC 7464 and "application/json-seq" media type.', - action='store_true' - ) - - -class OutputArgumentParser(ArgumentParser): - """ - Command-line argument parser for tools outputting JSON. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding JSON output arguments. - - Args: - args: Positional arguments to initialize ArgumentParser with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - argparse_output_add_args(self) - - -class SplitOutputArgumentParser(OutputArgumentParser): - """ - Command-line argument parser for tools supporting split-report output. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding split-report output arguments. - - Args: - args: Positional arguments to initialize ArgumentParser with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - self.add_argument( - '-o', '--objects-per-report', - metavar="NUMBER", - type=non_negative_int, - help='Put maximum NUMBER of objects into each output ' - 'report, or all, if zero. Default is zero.', - default=0, - required=False - ) - - -def argparse_schema_add_args(parser, version_verb): - """ - Add schema selection arguments to a command-line argument parser. - - Args: - parser: The parser to add arguments to. - version_verb: The verb to apply to the schema in the version - argument description. - """ - def schema_version(string): - """ - Lookup a schema version object using a major version number string. - Matches the argparse type function interface. - - Args: - string: The string representing the major version number of the - schema to lookup. - - Returns: - The looked up schema version object. - - Raises: - ValueError if the string was not representing a positive integer - number, or the schema with the supplied major version number was - not found. - """ - if not re.fullmatch("0*[1-9][0-9]*", string): - raise argparse.ArgumentTypeError( - f"Invalid major version number: {string!r}" - ) - major = int(string) - # It's OK, pylint: disable=redefined-outer-name - version = io.SCHEMA - while version and version.major != major: - version = version.previous - if version is None: - raise argparse.ArgumentTypeError( - f"No schema version found for major number {major}" - ) - return version - - parser.add_argument( - 'schema_version', - metavar="SCHEMA_VERSION", - type=schema_version, - help=f"{version_verb.capitalize()} the schema with the specified " - f"major version. Default is the current schema version " - f"(currently {io.SCHEMA.major}).", - nargs='?', - default=io.SCHEMA - ) - - def json_load_stream_fd(stream_fd, chunk_size=4*1024*1024): """ Load a series of JSON values from a stream file descriptor. diff --git a/kcidb/mq/__init__.py b/kcidb/mq/__init__.py index 8090faa9..b901a023 100644 --- a/kcidb/mq/__init__.py +++ b/kcidb/mq/__init__.py @@ -10,6 +10,7 @@ import email import email.message import email.policy +import argparse from abc import ABC, abstractmethod from google.cloud import pubsub from google.api_core.exceptions import DeadlineExceeded @@ -598,170 +599,6 @@ def __init__(self, *args, **kwargs): self.parser = email.parser.Parser(policy=email.policy.SMTPUTF8) -def argparse_add_args(parser): - """ - Add common message queue arguments to an argument parser. - - Args: - parser: The parser to add arguments to. - """ - parser.add_argument( - '-p', '--project', - help='ID of the Google Cloud project with the message queue', - required=True - ) - parser.add_argument( - '-t', '--topic', - help='Name of the message queue topic', - required=True - ) - - -class ArgumentParser(misc.ArgumentParser): - """ - Command-line argument parser with common message queue arguments added. - """ - - def __init__(self, *args, **kwargs): - """ - Initialize the parser, adding common message queue arguments. - - Args: - args: Positional arguments to initialize ArgumentParser - with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - argparse_add_args(self) - - -def argparse_publisher_add_args(parser, data_name): - """ - Add message queue publisher arguments to an argument parser. - - Args: - parser: The parser to add arguments to. - data_name: Name of the message queue data. - """ - argparse_add_args(parser) - subparsers = parser.add_subparsers(dest="command", - title="Available commands", - metavar="COMMAND", - parser_class=argparse.ArgumentParser) - subparsers.required = True - parser.subparsers = {} - description = f"Initialize {data_name} publisher setup" - parser.subparsers["init"] = subparsers.add_parser( - name="init", help=description, description=description - ) - description = \ - f"Publish {data_name} from standard input, print publishing IDs" - parser.subparsers["publish"] = subparsers.add_parser( - name="publish", help=description, description=description - ) - description = f"Cleanup {data_name} publisher setup" - parser.subparsers["cleanup"] = subparsers.add_parser( - name="cleanup", help=description, description=description - ) - - -class PublisherArgumentParser(misc.ArgumentParser): - """ - Command-line argument parser with common message queue arguments added. - """ - - def __init__(self, data_name, *args, **kwargs): - """ - Initialize the parser, adding common message queue arguments. - - Args: - data_name: Name of the message queue data. - args: Positional arguments to initialize ArgumentParser - with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - self.subparsers = {} - argparse_publisher_add_args(self, data_name) - - -def argparse_subscriber_add_args(parser, data_name): - """ - Add message queue subscriber arguments to an argument parser. - - Args: - parser: The parser to add arguments to. - data_name: Name of the message queue data. - """ - argparse_add_args(parser) - parser.add_argument( - '-s', '--subscription', - help='Name of the subscription', - required=True - ) - subparsers = parser.add_subparsers(dest="command", - title="Available commands", - metavar="COMMAND", - parser_class=argparse.ArgumentParser) - subparsers.required = True - parser.subparsers = {} - - description = f"Initialize {data_name} subscriber setup" - parser.subparsers["init"] = subparsers.add_parser( - name="init", help=description, description=description - ) - - description = \ - f"Pull {data_name} with a subscriber, print to standard output" - parser.subparsers["pull"] = subparsers.add_parser( - name="pull", help=description, description=description - ) - parser.subparsers["pull"].add_argument( - '--timeout', - metavar="SECONDS", - type=float, - help='Wait the specified number of SECONDS for a message, ' - 'or "inf" for infinity. Default is "inf".', - default=math.inf, - required=False - ) - parser.subparsers["pull"].add_argument( - '-m', - '--messages', - metavar="NUMBER", - type=misc.non_negative_int_or_inf, - help='Pull maximum NUMBER of messages, or "inf" for infinity. ' - 'Default is 1.', - default=1, - required=False - ) - - description = f"Cleanup {data_name} subscriber setup" - parser.subparsers["cleanup"] = subparsers.add_parser( - name="cleanup", help=description, description=description - ) - - -class SubscriberArgumentParser(misc.ArgumentParser): - """ - Command-line argument parser with message queue subscriber arguments - added. - """ - - def __init__(self, data_name, *args, **kwargs): - """ - Initialize the parser, adding message queue subscriber arguments. - - Args: - data_name: Name of the message queue data. - args: Positional arguments to initialize ArgumentParser - with. - kwargs: Keyword arguments to initialize ArgumentParser with. - """ - super().__init__(*args, **kwargs) - self.subparsers = {} - argparse_subscriber_add_args(self, data_name) - def io_publisher_main(): """Execute the kcidb-mq-io-publisher command-line tool""" @@ -769,7 +606,7 @@ def io_publisher_main(): description = \ 'kcidb-mq-io-publisher - ' \ 'Kernel CI I/O data publisher management tool' - parser = PublisherArgumentParser("I/O data", description=description) + parser = argparse.PublisherArgumentParser("I/O data", description=description) args = parser.parse_args() publisher = IOPublisher(args.project, args.topic) if args.command == "init": @@ -793,7 +630,7 @@ def io_subscriber_main(): description = \ 'kcidb-mq-io-subscriber - ' \ 'Kernel CI I/O data subscriber management tool' - parser = SubscriberArgumentParser("I/O data", description=description) + parser = argparse.SubscriberArgumentParser("I/O data", description=description) misc.argparse_output_add_args(parser.subparsers["pull"]) args = parser.parse_args() subscriber = IOSubscriber(args.project, args.topic, args.subscription) @@ -815,7 +652,7 @@ def pattern_publisher_main(): description = \ 'kcidb-mq-pattern-publisher - ' \ 'Kernel CI ORM pattern publisher management tool' - parser = PublisherArgumentParser("ORM patterns", description=description) + parser = argparse.PublisherArgumentParser("ORM patterns", description=description) parser.subparsers["publish"].add_argument( '--pattern-help', action=kcidb.orm.PatternHelpAction, @@ -846,7 +683,7 @@ def pattern_subscriber_main(): description = \ 'kcidb-mq-pattern-subscriber - ' \ 'Kernel CI ORM pattern subscriber management tool' - parser = SubscriberArgumentParser("ORM patterns", description=description) + parser = argparse.SubscriberArgumentParser("ORM patterns", description=description) args = parser.parse_args() subscriber = ORMPatternSubscriber(args.project, args.topic, args.subscription) @@ -870,7 +707,7 @@ def email_publisher_main(): description = \ 'kcidb-mq-email-publisher - ' \ 'Kernel CI email queue publisher management tool' - parser = PublisherArgumentParser("email", description=description) + parser = argparse.PublisherArgumentParser("email", description=description) args = parser.parse_args() publisher = EmailPublisher(args.project, args.topic) if args.command == "init": @@ -888,7 +725,7 @@ def email_subscriber_main(): description = \ 'kcidb-mq-email-subscriber - ' \ 'Kernel CI email queue subscriber management tool' - parser = SubscriberArgumentParser("email", description=description) + parser = argparse.SubscriberArgumentParser("email", description=description) args = parser.parse_args() subscriber = EmailSubscriber(args.project, args.topic, args.subscription) if args.command == "init": diff --git a/kcidb/mq/argparse.py b/kcidb/mq/argparse.py new file mode 100644 index 00000000..d10ab2a3 --- /dev/null +++ b/kcidb/mq/argparse.py @@ -0,0 +1,186 @@ + +import math +import datetime +import json +import logging +import threading +import sys +import argparse +import email +import email.message +import email.policy +from abc import ABC, abstractmethod +from google.cloud import pubsub +from google.api_core.exceptions import DeadlineExceeded +import kcidb.io as io +import kcidb.orm +from kcidb import misc +from kcidb.misc import LIGHT_ASSERTS + + +# Module's logger +LOGGER = logging.getLogger(__name__) + +def argparse_add_args(parser): + """ + Add common message queue arguments to an argument parser. + + Args: + parser: The parser to add arguments to. + """ + parser.add_argument( + '-p', '--project', + help='ID of the Google Cloud project with the message queue', + required=True + ) + parser.add_argument( + '-t', '--topic', + help='Name of the message queue topic', + required=True + ) + + +class ArgumentParser(misc.ArgumentParser): + """ + Command-line argument parser with common message queue arguments added. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize the parser, adding common message queue arguments. + + Args: + args: Positional arguments to initialize ArgumentParser + with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + argparse_add_args(self) + + +def argparse_publisher_add_args(parser, data_name): + """ + Add message queue publisher arguments to an argument parser. + + Args: + parser: The parser to add arguments to. + data_name: Name of the message queue data. + """ + argparse_add_args(parser) + subparsers = parser.add_subparsers(dest="command", + title="Available commands", + metavar="COMMAND", + parser_class=argparse.ArgumentParser) + subparsers.required = True + parser.subparsers = {} + description = f"Initialize {data_name} publisher setup" + parser.subparsers["init"] = subparsers.add_parser( + name="init", help=description, description=description + ) + description = \ + f"Publish {data_name} from standard input, print publishing IDs" + parser.subparsers["publish"] = subparsers.add_parser( + name="publish", help=description, description=description + ) + description = f"Cleanup {data_name} publisher setup" + parser.subparsers["cleanup"] = subparsers.add_parser( + name="cleanup", help=description, description=description + ) + + +class PublisherArgumentParser(misc.ArgumentParser): + """ + Command-line argument parser with common message queue arguments added. + """ + + def __init__(self, data_name, *args, **kwargs): + """ + Initialize the parser, adding common message queue arguments. + + Args: + data_name: Name of the message queue data. + args: Positional arguments to initialize ArgumentParser + with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + self.subparsers = {} + argparse_publisher_add_args(self, data_name) + + +def argparse_subscriber_add_args(parser, data_name): + """ + Add message queue subscriber arguments to an argument parser. + + Args: + parser: The parser to add arguments to. + data_name: Name of the message queue data. + """ + argparse_add_args(parser) + parser.add_argument( + '-s', '--subscription', + help='Name of the subscription', + required=True + ) + subparsers = parser.add_subparsers(dest="command", + title="Available commands", + metavar="COMMAND", + parser_class=argparse.ArgumentParser) + subparsers.required = True + parser.subparsers = {} + + description = f"Initialize {data_name} subscriber setup" + parser.subparsers["init"] = subparsers.add_parser( + name="init", help=description, description=description + ) + + description = \ + f"Pull {data_name} with a subscriber, print to standard output" + parser.subparsers["pull"] = subparsers.add_parser( + name="pull", help=description, description=description + ) + parser.subparsers["pull"].add_argument( + '--timeout', + metavar="SECONDS", + type=float, + help='Wait the specified number of SECONDS for a message, ' + 'or "inf" for infinity. Default is "inf".', + default=math.inf, + required=False + ) + parser.subparsers["pull"].add_argument( + '-m', + '--messages', + metavar="NUMBER", + type=misc.non_negative_int_or_inf, + help='Pull maximum NUMBER of messages, or "inf" for infinity. ' + 'Default is 1.', + default=1, + required=False + ) + + description = f"Cleanup {data_name} subscriber setup" + parser.subparsers["cleanup"] = subparsers.add_parser( + name="cleanup", help=description, description=description + ) + + +class SubscriberArgumentParser(misc.ArgumentParser): + """ + Command-line argument parser with message queue subscriber arguments + added. + """ + + def __init__(self, data_name, *args, **kwargs): + """ + Initialize the parser, adding message queue subscriber arguments. + + Args: + data_name: Name of the message queue data. + args: Positional arguments to initialize ArgumentParser + with. + kwargs: Keyword arguments to initialize ArgumentParser with. + """ + super().__init__(*args, **kwargs) + self.subparsers = {} + argparse_subscriber_add_args(self, data_name)